Merge pull request #838 from AnishSarkar22/fix/docker

feat: docker-compose and docker CI pipeline enhancements

.env.example

@@ -1,41 +0,0 @@
-# Docker Specific Env's Only - Can skip if needed
-
-# Celery Config
-REDIS_PORT=6379
-FLOWER_PORT=5555
-
-# Frontend Configuration
-FRONTEND_PORT=3000
-NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000 (Default: http://localhost:8000)
-NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL or GOOGLE (Default: LOCAL)
-NEXT_PUBLIC_ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING (Default: DOCLING)
-# Backend Configuration
-BACKEND_PORT=8000
-# Auth type for backend login flow (Default: LOCAL)
-# Set to GOOGLE if using Google OAuth
-AUTH_TYPE=LOCAL
-# Frontend URL used by backend for CORS allowed origins and OAuth redirects
-# Must match the URL your browser uses to access the frontend
-NEXT_FRONTEND_URL=http://localhost:3000
-
-# Database Configuration
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=postgres
-POSTGRES_DB=surfsense
-POSTGRES_PORT=5432
-
-# Electric-SQL Configuration
-ELECTRIC_PORT=5133
-# PostgreSQL host for Electric connection
-# - 'db' for Docker PostgreSQL (service name in docker-compose)
-# - 'host.docker.internal' for local PostgreSQL (recommended when Electric runs in Docker)
-#   Note: host.docker.internal works on Docker Desktop (Mac/Windows) and can be enabled on Linux
-POSTGRES_HOST=db
-ELECTRIC_DB_USER=electric
-ELECTRIC_DB_PASSWORD=electric_password
-NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133
-
-# pgAdmin Configuration
-PGADMIN_PORT=5050
-PGADMIN_DEFAULT_EMAIL=admin@surfsense.com
-PGADMIN_DEFAULT_PASSWORD=surfsense

.github/workflows/docker_build.yaml

@@ -1,6 +1,12 @@
-name: Build and Push Docker Image
+name: Build and Push Docker Images
 
 on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'surfsense_backend/**'
+      - 'surfsense_web/**'
   workflow_dispatch:
     inputs:
       branch:
@@ -8,6 +14,10 @@ on:
         required: false
         default: ''
 
+concurrency:
+  group: docker-build
+  cancel-in-progress: false
+
 permissions:
   contents: write
   packages: write
@@ -28,33 +38,28 @@ jobs:
       - name: Read app version and calculate next Docker build version
         id: tag_version
         run: |
-          # Read version from pyproject.toml
           APP_VERSION=$(grep -E '^version = ' surfsense_backend/pyproject.toml | sed 's/version = "\(.*\)"/\1/')
           echo "App version from pyproject.toml: $APP_VERSION"
-          
+
           if [ -z "$APP_VERSION" ]; then
             echo "Error: Could not read version from surfsense_backend/pyproject.toml"
             exit 1
           fi
-          
-          # Fetch all tags
+
           git fetch --tags
-          
-          # Find the latest docker build tag for this app version (format: APP_VERSION.BUILD_NUMBER)
-          # Tags follow pattern: 0.0.11.1, 0.0.11.2, etc.
+
           LATEST_BUILD_TAG=$(git tag --list "${APP_VERSION}.*" --sort='-v:refname' | head -n 1)
-          
+
           if [ -z "$LATEST_BUILD_TAG" ]; then
             echo "No previous Docker build tag found for version ${APP_VERSION}. Starting with ${APP_VERSION}.1"
             NEXT_VERSION="${APP_VERSION}.1"
           else
             echo "Latest Docker build tag found: $LATEST_BUILD_TAG"
-            # Extract the build number (4th component)
             BUILD_NUMBER=$(echo "$LATEST_BUILD_TAG" | rev | cut -d. -f1 | rev)
             NEXT_BUILD=$((BUILD_NUMBER + 1))
             NEXT_VERSION="${APP_VERSION}.${NEXT_BUILD}"
           fi
-          
+
           echo "Calculated next Docker version: $NEXT_VERSION"
           echo "next_version=$NEXT_VERSION" >> $GITHUB_OUTPUT
 
@@ -78,67 +83,35 @@ jobs:
           git ls-remote --tags origin | grep "refs/tags/${{ steps.tag_version.outputs.next_version }}" || (echo "Tag push verification failed!" && exit 1)
           echo "Tag successfully pushed."
 
-  # Build for AMD64 on native x64 runner
-  build_amd64:
-    runs-on: ubuntu-latest
+  build:
     needs: tag_release
+    runs-on: ${{ matrix.os }}
     permissions:
       packages: write
       contents: read
-    outputs:
-      digest: ${{ steps.build.outputs.digest }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [linux/amd64, linux/arm64]
+        image: [backend, web]
+        include:
+          - platform: linux/amd64
+            suffix: amd64
+            os: ubuntu-latest
+          - platform: linux/arm64
+            suffix: arm64
+            os: ubuntu-24.04-arm
+          - image: backend
+            name: surfsense-backend
+            context: ./surfsense_backend
+            file: ./surfsense_backend/Dockerfile
+          - image: web
+            name: surfsense-web
+            context: ./surfsense_web
+            file: ./surfsense_web/Dockerfile
     env:
-      REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/surfsense
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
+      REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }}
 
-      - name: Set lowercase image name
-        id: image
-        run: echo "name=${REGISTRY_IMAGE,,}" >> $GITHUB_OUTPUT
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Free up disk space
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /opt/ghc
-          sudo rm -rf /usr/local/share/boost
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-          docker system prune -af
-
-      - name: Build and push AMD64 image
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./Dockerfile.allinone
-          push: true
-          tags: ${{ steps.image.outputs.name }}:${{ needs.tag_release.outputs.new_tag }}-amd64
-          platforms: linux/amd64
-          cache-from: type=gha,scope=amd64
-          cache-to: type=gha,mode=max,scope=amd64
-          provenance: false
-
-  # Build for ARM64 on native arm64 runner (no QEMU emulation!)
-  build_arm64:
-    runs-on: ubuntu-24.04-arm
-    needs: tag_release
-    permissions:
-      packages: write
-      contents: read
-    outputs:
-      digest: ${{ steps.build.outputs.digest }}
-    env:
-      REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/surfsense
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -165,28 +138,41 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
           docker system prune -af
 
-      - name: Build and push ARM64 image
+      - name: Build and push ${{ matrix.name }} (${{ matrix.suffix }})
         id: build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
-          context: .
-          file: ./Dockerfile.allinone
+          context: ${{ matrix.context }}
+          file: ${{ matrix.file }}
           push: true
-          tags: ${{ steps.image.outputs.name }}:${{ needs.tag_release.outputs.new_tag }}-arm64
-          platforms: linux/arm64
-          cache-from: type=gha,scope=arm64
-          cache-to: type=gha,mode=max,scope=arm64
+          tags: ${{ steps.image.outputs.name }}:${{ needs.tag_release.outputs.new_tag }}-${{ matrix.suffix }}
+          platforms: ${{ matrix.platform }}
+          cache-from: type=gha,scope=${{ matrix.image }}-${{ matrix.suffix }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.image }}-${{ matrix.suffix }}
           provenance: false
+          build-args: |
+            ${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__' || '' }}
+            ${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__' || '' }}
+            ${{ matrix.image == 'web' && 'NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__' || '' }}
+            ${{ matrix.image == 'web' && 'NEXT_PUBLIC_ELECTRIC_URL=__NEXT_PUBLIC_ELECTRIC_URL__' || '' }}
+            ${{ matrix.image == 'web' && 'NEXT_PUBLIC_ELECTRIC_AUTH_MODE=__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__' || '' }}
+            ${{ matrix.image == 'web' && 'NEXT_PUBLIC_DEPLOYMENT_MODE=__NEXT_PUBLIC_DEPLOYMENT_MODE__' || '' }}
 
-  # Create multi-arch manifest combining both platform images
   create_manifest:
     runs-on: ubuntu-latest
-    needs: [tag_release, build_amd64, build_arm64]
+    needs: [tag_release, build]
     permissions:
       packages: write
       contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: surfsense-backend
+          - name: surfsense-web
     env:
-      REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/surfsense
+      REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }}
+
     steps:
       - name: Set lowercase image name
         id: image
@@ -203,28 +189,31 @@ jobs:
         run: |
           VERSION_TAG="${{ needs.tag_release.outputs.new_tag }}"
           IMAGE="${{ steps.image.outputs.name }}"
-          
-          # Create manifest for version tag
+          APP_VERSION=$(echo "$VERSION_TAG" | rev | cut -d. -f2- | rev)
+
           docker manifest create ${IMAGE}:${VERSION_TAG} \
             ${IMAGE}:${VERSION_TAG}-amd64 \
             ${IMAGE}:${VERSION_TAG}-arm64
-          
+
           docker manifest push ${IMAGE}:${VERSION_TAG}
-          
-          # Create/update latest tag if on default branch
+
           if [[ "${{ github.ref }}" == "refs/heads/${{ github.event.repository.default_branch }}" ]] || [[ "${{ github.event.inputs.branch }}" == "${{ github.event.repository.default_branch }}" ]]; then
+            docker manifest create ${IMAGE}:${APP_VERSION} \
+              ${IMAGE}:${VERSION_TAG}-amd64 \
+              ${IMAGE}:${VERSION_TAG}-arm64
+
+            docker manifest push ${IMAGE}:${APP_VERSION}
+
             docker manifest create ${IMAGE}:latest \
               ${IMAGE}:${VERSION_TAG}-amd64 \
               ${IMAGE}:${VERSION_TAG}-arm64
-            
+
             docker manifest push ${IMAGE}:latest
           fi
 
-      - name: Clean up architecture-specific tags (optional)
-        continue-on-error: true
+      - name: Summary
         run: |
-          # Note: GHCR doesn't support tag deletion via API easily
-          # The arch-specific tags will remain but users should use the main tags
-          echo "Multi-arch manifest created successfully!"
-          echo "Users should pull: ${{ steps.image.outputs.name }}:${{ needs.tag_release.outputs.new_tag }}"
-          echo "Or for latest: ${{ steps.image.outputs.name }}:latest"
+          echo "Multi-arch manifest created for ${{ matrix.name }}!"
+          echo "Versioned: ${{ steps.image.outputs.name }}:${{ needs.tag_release.outputs.new_tag }}"
+          echo "App version: ${{ steps.image.outputs.name }}:$(echo '${{ needs.tag_release.outputs.new_tag }}' | rev | cut -d. -f2- | rev)"
+          echo "Latest:    ${{ steps.image.outputs.name }}:latest"

Dockerfile.allinone

@@ -1,289 +0,0 @@
-# SurfSense All-in-One Docker Image
-# This image bundles PostgreSQL+pgvector, Redis, Electric SQL, Backend, and Frontend
-# Usage: docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 -v surfsense-data:/data --name surfsense ghcr.io/modsetter/surfsense:latest
-#
-# Included Services (all run locally by default):
-# - PostgreSQL 14 + pgvector (vector database)
-# - Redis (task queue)
-# - Electric SQL (real-time sync)
-# - Docling (document processing, CPU-only, OCR disabled)
-# - Kokoro TTS (local text-to-speech for podcasts)
-# - Faster-Whisper (local speech-to-text for audio files)
-# - Playwright Chromium (web scraping)
-#
-# Note: This is the CPU-only version. A :cuda tagged image with GPU support
-# will be available in the future for faster AI inference.
-
-# ====================
-# Stage 1: Get Electric SQL Binary
-# ====================
-FROM electricsql/electric:latest AS electric-builder
-
-# ====================
-# Stage 2: Build Frontend
-# ====================
-FROM node:20-alpine AS frontend-builder
-
-WORKDIR /app
-
-# Install pnpm
-RUN corepack enable pnpm
-
-# Copy package files
-COPY surfsense_web/package.json surfsense_web/pnpm-lock.yaml* ./
-COPY surfsense_web/source.config.ts ./
-COPY surfsense_web/content ./content
-
-# Install dependencies (skip postinstall which requires all source files)
-RUN pnpm install --frozen-lockfile --ignore-scripts
-
-# Copy source
-COPY surfsense_web/ ./
-
-# Run fumadocs-mdx postinstall now that source files are available
-RUN pnpm fumadocs-mdx
-
-# Build with placeholder values that will be replaced at runtime
-# These unique strings allow runtime substitution via entrypoint script
-ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__
-ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__
-ENV NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__
-ENV NEXT_PUBLIC_ELECTRIC_URL=__NEXT_PUBLIC_ELECTRIC_URL__
-ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__
-ENV NEXT_PUBLIC_DEPLOYMENT_MODE=__NEXT_PUBLIC_DEPLOYMENT_MODE__
-
-# Build
-RUN pnpm run build
-
-# ====================
-# Stage 3: Runtime Image
-# ====================
-FROM ubuntu:22.04 AS runtime
-
-# Prevent interactive prompts
-ENV DEBIAN_FRONTEND=noninteractive
-
-# Install system dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    # PostgreSQL
-    postgresql-14 \
-    postgresql-contrib-14 \
-    # Build tools for pgvector
-    build-essential \
-    postgresql-server-dev-14 \
-    git \
-    # Redis
-    redis-server \
-    # Node.js prerequisites
-    curl \
-    ca-certificates \
-    gnupg \
-    # Backend dependencies
-    gcc \
-    wget \
-    unzip \
-    dos2unix \
-    # For PPAs
-    software-properties-common \
-    # ============================
-    # Local TTS (Kokoro) dependencies
-    # ============================
-    espeak-ng \
-    libespeak-ng1 \
-    # ============================
-    # Local STT (Faster-Whisper) dependencies
-    # ============================
-    ffmpeg \
-    # ============================
-    # Audio processing (soundfile)
-    # ============================
-    libsndfile1 \
-    # ============================
-    # Image/OpenCV dependencies (for Docling)
-    # ============================
-    libgl1 \
-    libglib2.0-0 \
-    libsm6 \
-    libxext6 \
-    libxrender1 \
-    # ============================
-    # Playwright browser dependencies
-    # ============================
-    libnspr4 \
-    libnss3 \
-    libatk1.0-0 \
-    libatk-bridge2.0-0 \
-    libcups2 \
-    libxkbcommon0 \
-    libatspi2.0-0 \
-    libxcomposite1 \
-    libxdamage1 \
-    libxrandr2 \
-    libgbm1 \
-    libcairo2 \
-    libpango-1.0-0 \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install Pandoc 3.x from GitHub (apt ships 2.9 which has broken table rendering).
-RUN ARCH=$(dpkg --print-architecture) && \
-    wget -qO /tmp/pandoc.deb "https://github.com/jgm/pandoc/releases/download/3.9/pandoc-3.9-1-${ARCH}.deb" && \
-    dpkg -i /tmp/pandoc.deb && \
-    rm /tmp/pandoc.deb
-
-
-# Install Node.js 20.x (for running frontend)
-RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
-    && apt-get install -y nodejs \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install Python 3.12 from deadsnakes PPA
-RUN add-apt-repository ppa:deadsnakes/ppa -y \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends \
-    python3.12 \
-    python3.12-venv \
-    python3.12-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-# Set Python 3.12 as default
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.12 1 \
-    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
-
-# Install pip for Python 3.12
-RUN python3.12 -m ensurepip --upgrade \
-    && python3.12 -m pip install --upgrade pip
-
-# Install supervisor via pip (system package incompatible with Python 3.12)
-RUN pip install --no-cache-dir supervisor
-
-# Build and install pgvector
-RUN cd /tmp \
-    && git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git \
-    && cd pgvector \
-    && make \
-    && make install \
-    && rm -rf /tmp/pgvector
-
-# Update certificates
-RUN update-ca-certificates
-
-# Create data directories
-RUN mkdir -p /data/postgres /data/redis /data/surfsense \
-    && chown -R postgres:postgres /data/postgres
-
-# ====================
-# Copy Frontend Build
-# ====================
-WORKDIR /app/frontend
-
-# Copy only the standalone build (not node_modules)
-COPY --from=frontend-builder /app/.next/standalone ./
-COPY --from=frontend-builder /app/.next/static ./.next/static
-COPY --from=frontend-builder /app/public ./public
-
-COPY surfsense_web/content/docs /app/surfsense_web/content/docs
-
-# ====================
-# Copy Electric SQL Release
-# ====================
-COPY --from=electric-builder /app /app/electric-release
-
-# ====================
-# Setup Backend
-# ====================
-WORKDIR /app/backend
-
-# Copy backend dependency files
-COPY surfsense_backend/pyproject.toml surfsense_backend/uv.lock ./
-
-# Install PyTorch CPU-only (Docling needs it but OCR is disabled, no GPU needed)
-RUN pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu
-
-# Install python dependencies
-RUN pip install --no-cache-dir certifi pip-system-certs uv \
-    && uv pip install --system --no-cache-dir -e .
-
-# Set SSL environment variables
-RUN CERTIFI_PATH=$(python -c "import certifi; print(certifi.where())") \
-    && echo "export SSL_CERT_FILE=$CERTIFI_PATH" >> /etc/profile.d/ssl.sh \
-    && echo "export REQUESTS_CA_BUNDLE=$CERTIFI_PATH" >> /etc/profile.d/ssl.sh
-
-# Note: EasyOCR models NOT downloaded - OCR is disabled in docling_service.py
-# GPU support will be added in a future :cuda tagged image
-
-# Install Playwright browsers
-RUN pip install --no-cache-dir playwright \
-    && playwright install chromium \
-    && rm -rf /root/.cache/ms-playwright/ffmpeg* 
-
-# Copy backend source
-COPY surfsense_backend/ ./
-
-# ====================
-# Configuration
-# ====================
-WORKDIR /app
-
-# Copy supervisor configuration
-COPY scripts/docker/supervisor-allinone.conf /etc/supervisor/conf.d/surfsense.conf
-
-# Copy entrypoint script
-COPY scripts/docker/entrypoint-allinone.sh /app/entrypoint.sh
-RUN dos2unix /app/entrypoint.sh && chmod +x /app/entrypoint.sh
-
-# PostgreSQL initialization script
-COPY scripts/docker/init-postgres.sh /app/init-postgres.sh
-RUN dos2unix /app/init-postgres.sh && chmod +x /app/init-postgres.sh
-
-# Clean up build dependencies to reduce image size
-RUN apt-get purge -y build-essential postgresql-server-dev-14 \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-# Environment variables with defaults
-ENV POSTGRES_USER=surfsense
-ENV POSTGRES_PASSWORD=surfsense
-ENV POSTGRES_DB=surfsense
-ENV DATABASE_URL=postgresql+asyncpg://surfsense:surfsense@localhost:5432/surfsense
-ENV CELERY_BROKER_URL=redis://localhost:6379/0
-ENV CELERY_RESULT_BACKEND=redis://localhost:6379/0
-ENV CELERY_TASK_DEFAULT_QUEUE=surfsense
-ENV PYTHONPATH=/app/backend
-ENV NEXT_FRONTEND_URL=http://localhost:3000
-ENV AUTH_TYPE=LOCAL
-ENV ETL_SERVICE=DOCLING
-ENV EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
-
-# Frontend configuration (can be overridden at runtime)
-# These are injected into the Next.js build at container startup
-ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000
-ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL
-ENV NEXT_PUBLIC_ETL_SERVICE=DOCLING
-
-# Daytona Sandbox (cloud code execution — no local server needed)
-ENV DAYTONA_SANDBOX_ENABLED=FALSE
-# DAYTONA_API_KEY, DAYTONA_API_URL, DAYTONA_TARGET: set at runtime for production.
-
-# Electric SQL configuration (ELECTRIC_DATABASE_URL is built dynamically by entrypoint from these values)
-ENV ELECTRIC_DB_USER=electric
-ENV ELECTRIC_DB_PASSWORD=electric_password
-# Note: ELECTRIC_DATABASE_URL is NOT set here - entrypoint builds it dynamically from ELECTRIC_DB_USER/PASSWORD
-ENV ELECTRIC_INSECURE=true
-ENV ELECTRIC_WRITE_TO_PG_MODE=direct
-ENV ELECTRIC_PORT=5133
-ENV PORT=5133
-ENV NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133
-ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure
-
-# Data volume
-VOLUME ["/data"]
-
-# Expose ports (Frontend: 3000, Backend: 8000, Electric: 5133)
-EXPOSE 3000 8000 5133
-
-# Health check
-HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \
-    CMD curl -f http://localhost:3000 || exit 1
-
-# Run entrypoint
-CMD ["/app/entrypoint.sh"]

README.es.md

@@ -81,13 +81,16 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
 
 Ejecuta SurfSense en tu propia infraestructura para control total de datos y privacidad.
 
-**Inicio Rápido (Docker en un solo comando):**
+**Requisitos previos:** [Docker](https://docs.docker.com/get-docker/) (con [Docker Compose](https://docs.docker.com/compose/install/)) debe estar instalado y en ejecución.
+
+> [!NOTE]
+> Usuarios de Windows: instalen [WSL](https://learn.microsoft.com/en-us/windows/wsl/install) primero y ejecuten el siguiente comando en la terminal de Ubuntu.
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 -v surfsense-data:/data --name surfsense --restart unless-stopped ghcr.io/modsetter/surfsense:latest
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
 ```
 
-Después de iniciar, abre [http://localhost:3000](http://localhost:3000) en tu navegador.
+El script de instalación configura [Watchtower](https://github.com/nicholas-fedor/watchtower) automáticamente para actualizaciones diarias. Para omitirlo, agrega la bandera `--no-watchtower`.
 
 Para Docker Compose, instalación manual y otras opciones de despliegue, consulta la [documentación](https://www.surfsense.com/docs/).
 

README.hi.md

@@ -81,13 +81,16 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
 
 पूर्ण डेटा नियंत्रण और गोपनीयता के लिए SurfSense को अपने स्वयं के बुनियादी ढांचे पर चलाएं।
 
-**त्वरित शुरुआत (Docker एक कमांड में):**
+**आवश्यकताएँ:** [Docker](https://docs.docker.com/get-docker/) ([Docker Compose](https://docs.docker.com/compose/install/) सहित) इंस्टॉल और चालू होना चाहिए।
+
+> [!NOTE]
+> Windows उपयोगकर्ता: पहले [WSL](https://learn.microsoft.com/en-us/windows/wsl/install) इंस्टॉल करें और नीचे दिया गया कमांड Ubuntu टर्मिनल में चलाएं।
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 -v surfsense-data:/data --name surfsense --restart unless-stopped ghcr.io/modsetter/surfsense:latest
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
 ```
 
-शुरू करने के बाद, अपने ब्राउज़र में [http://localhost:3000](http://localhost:3000) खोलें।
+इंस्टॉल स्क्रिप्ट दैनिक ऑटो-अपडेट के लिए स्वचालित रूप से [Watchtower](https://github.com/nicholas-fedor/watchtower) सेटअप करती है। इसे छोड़ने के लिए, `--no-watchtower` फ्लैग जोड़ें।
 
 Docker Compose, मैनुअल इंस्टॉलेशन और अन्य डिप्लॉयमेंट विकल्पों के लिए, [डॉक्स](https://www.surfsense.com/docs/) देखें।
 

README.md

@@ -81,21 +81,18 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
 
 Run SurfSense on your own infrastructure for full data control and privacy.
 
-**Quick Start (Docker one-liner):**
+**Prerequisites:** [Docker](https://docs.docker.com/get-docker/) (with [Docker Compose](https://docs.docker.com/compose/install/)) must be installed and running.
+
+> [!NOTE]
+> Windows users: install [WSL](https://learn.microsoft.com/en-us/windows/wsl/install) first and run the command below in the Ubuntu terminal.
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 -v surfsense-data:/data --name surfsense --restart unless-stopped ghcr.io/modsetter/surfsense:latest
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
 ```
 
-After starting, open [http://localhost:3000](http://localhost:3000) in your browser.
+The install script sets up [Watchtower](https://github.com/nicholas-fedor/watchtower) automatically for daily auto-updates. To skip it, add the `--no-watchtower` flag.
 
-**Update (Automatic updates with Watchtower):**
-
-```bash
-docker run --rm -v /var/run/docker.sock:/var/run/docker.sock nickfedor/watchtower --run-once surfsense
-```
-
-For Docker Compose, manual installation, and other deployment options, check the [docs](https://www.surfsense.com/docs/).
+For Docker Compose, manual installation, and other deployment options, see the [docs](https://www.surfsense.com/docs/).
 
 ### How to Realtime Collaborate (Beta)
 

README.pt-BR.md

@@ -81,13 +81,16 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
 
 Execute o SurfSense na sua própria infraestrutura para controle total de dados e privacidade.
 
-**Início Rápido (Docker em um único comando):**
+**Pré-requisitos:** [Docker](https://docs.docker.com/get-docker/) (com [Docker Compose](https://docs.docker.com/compose/install/)) deve estar instalado e em execução.
+
+> [!NOTE]
+> Usuários do Windows: instalem o [WSL](https://learn.microsoft.com/en-us/windows/wsl/install) primeiro e executem o comando abaixo no terminal do Ubuntu.
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 -v surfsense-data:/data --name surfsense --restart unless-stopped ghcr.io/modsetter/surfsense:latest
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
 ```
 
-Após iniciar, abra [http://localhost:3000](http://localhost:3000) no seu navegador.
+O script de instalação configura o [Watchtower](https://github.com/nicholas-fedor/watchtower) automaticamente para atualizações diárias. Para pular, adicione a flag `--no-watchtower`.
 
 Para Docker Compose, instalação manual e outras opções de implantação, consulte a [documentação](https://www.surfsense.com/docs/).
 

README.zh-CN.md

@@ -81,13 +81,16 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
 
 在您自己的基础设施上运行 SurfSense，实现完全的数据控制和隐私保护。
 
-**快速开始（Docker 一行命令）：**
+**前置条件：** 需要安装并运行 [Docker](https://docs.docker.com/get-docker/)（含 [Docker Compose](https://docs.docker.com/compose/install/)）。
+
+> [!NOTE]
+> Windows 用户：请先安装 [WSL](https://learn.microsoft.com/en-us/windows/wsl/install)，然后在 Ubuntu 终端中运行以下命令。
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 -v surfsense-data:/data --name surfsense --restart unless-stopped ghcr.io/modsetter/surfsense:latest
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
 ```
 
-启动后，在浏览器中打开 [http://localhost:3000](http://localhost:3000)。
+安装脚本会自动配置 [Watchtower](https://github.com/nicholas-fedor/watchtower) 以实现每日自动更新。如需跳过，请添加 `--no-watchtower` 参数。
 
 如需 Docker Compose、手动安装及其他部署方式，请查看[文档](https://www.surfsense.com/docs/)。
 

docker-compose.quickstart.yml

@@ -1,80 +0,0 @@
-# SurfSense Quick Start Docker Compose
-# 
-# This is a simplified docker-compose for quick local deployment using pre-built images.
-# For production or customized deployments, use the main docker-compose.yml
-#
-# Usage:
-#   1. (Optional) Create a .env file with your configuration
-#   2. Run: docker compose -f docker-compose.quickstart.yml up -d
-#   3. Access SurfSense at http://localhost:3000
-#
-# All Environment Variables are Optional:
-#   - SECRET_KEY: JWT secret key (auto-generated and persisted if not set)
-#   - EMBEDDING_MODEL: Embedding model to use (default: sentence-transformers/all-MiniLM-L6-v2)
-#   - ETL_SERVICE: Document parsing service - DOCLING, UNSTRUCTURED, or LLAMACLOUD (default: DOCLING)
-#   - TTS_SERVICE: Text-to-speech service for podcasts (default: local/kokoro)
-#   - STT_SERVICE: Speech-to-text service with model size (default: local/base)
-#   - FIRECRAWL_API_KEY: For web crawling features
-
-version: "3.8"
-
-services:
-  # All-in-one SurfSense container
-  surfsense:
-    image: ghcr.io/modsetter/surfsense:latest
-    container_name: surfsense
-    ports:
-      - "${FRONTEND_PORT:-3000}:3000"
-      - "${BACKEND_PORT:-8000}:8000"
-    volumes:
-      - surfsense-data:/data
-    environment:
-      # Authentication (auto-generated if not set)
-      - SECRET_KEY=${SECRET_KEY:-}
-      
-      # Auth Configuration
-      - AUTH_TYPE=${AUTH_TYPE:-LOCAL}
-      - GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID:-}
-      - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-}
-      
-      # AI/ML Configuration
-      - EMBEDDING_MODEL=${EMBEDDING_MODEL:-sentence-transformers/all-MiniLM-L6-v2}
-      - RERANKERS_ENABLED=${RERANKERS_ENABLED:-FALSE}
-      - RERANKERS_MODEL_NAME=${RERANKERS_MODEL_NAME:-}
-      - RERANKERS_MODEL_TYPE=${RERANKERS_MODEL_TYPE:-}
-      
-      # Document Processing
-      - ETL_SERVICE=${ETL_SERVICE:-DOCLING}
-      - UNSTRUCTURED_API_KEY=${UNSTRUCTURED_API_KEY:-}
-      - LLAMA_CLOUD_API_KEY=${LLAMA_CLOUD_API_KEY:-}
-      
-      # Audio Services
-      - TTS_SERVICE=${TTS_SERVICE:-local/kokoro}
-      - TTS_SERVICE_API_KEY=${TTS_SERVICE_API_KEY:-}
-      - STT_SERVICE=${STT_SERVICE:-local/base}
-      - STT_SERVICE_API_KEY=${STT_SERVICE_API_KEY:-}
-      
-      # Web Crawling
-      - FIRECRAWL_API_KEY=${FIRECRAWL_API_KEY:-}
-      
-      # Optional Features
-      - REGISTRATION_ENABLED=${REGISTRATION_ENABLED:-TRUE}
-      - SCHEDULE_CHECKER_INTERVAL=${SCHEDULE_CHECKER_INTERVAL:-1m}
-      
-      # LangSmith Observability (optional)
-      - LANGSMITH_TRACING=${LANGSMITH_TRACING:-false}
-      - LANGSMITH_ENDPOINT=${LANGSMITH_ENDPOINT:-}
-      - LANGSMITH_API_KEY=${LANGSMITH_API_KEY:-}
-      - LANGSMITH_PROJECT=${LANGSMITH_PROJECT:-}
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:3000", "&&", "curl", "-f", "http://localhost:8000/docs"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 120s
-
-volumes:
-  surfsense-data:
-    name: surfsense-data
-

docker-compose.yml

@@ -1,172 +0,0 @@
-version: "3.8"
-
-services:
-  db:
-    image: ankane/pgvector:latest
-    ports:
-      - "${POSTGRES_PORT:-5432}:5432"
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-      - ./scripts/docker/postgresql.conf:/etc/postgresql/postgresql.conf:ro
-      - ./scripts/docker/init-electric-user.sh:/docker-entrypoint-initdb.d/init-electric-user.sh:ro
-    environment:
-      - POSTGRES_USER=${POSTGRES_USER:-postgres}
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
-      - POSTGRES_DB=${POSTGRES_DB:-surfsense}
-      - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric}
-      - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
-    command: postgres -c config_file=/etc/postgresql/postgresql.conf
-
-  pgadmin:
-    image: dpage/pgadmin4
-    ports:
-      - "${PGADMIN_PORT:-5050}:80"
-    environment:
-      - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL:-admin@surfsense.com}
-      - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD:-surfsense}
-    volumes:
-      - pgadmin_data:/var/lib/pgadmin
-    depends_on:
-      - db
-
-  redis:
-    image: redis:7-alpine
-    ports:
-      - "${REDIS_PORT:-6379}:6379"
-    volumes:
-      - redis_data:/data
-    command: redis-server --appendonly yes
-
-  backend:
-    build: ./surfsense_backend
-    # image: ghcr.io/modsetter/surfsense_backend:latest
-    ports:
-      - "${BACKEND_PORT:-8000}:8000"
-    volumes:
-      - ./surfsense_backend/app:/app/app
-      - shared_temp:/tmp
-      # Uncomment and edit the line below to enable Obsidian vault indexing
-      # - /path/to/your/obsidian/vault:/obsidian-vault:ro
-    env_file:
-      - ./surfsense_backend/.env
-    environment:
-      - DATABASE_URL=postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-surfsense}
-      - CELERY_BROKER_URL=redis://redis:${REDIS_PORT:-6379}/0
-      - CELERY_RESULT_BACKEND=redis://redis:${REDIS_PORT:-6379}/0
-      - REDIS_APP_URL=redis://redis:${REDIS_PORT:-6379}/0
-      # Queue name isolation - prevents task collision if Redis is shared with other apps
-      - CELERY_TASK_DEFAULT_QUEUE=surfsense
-      - PYTHONPATH=/app
-      - UVICORN_LOOP=asyncio
-      - UNSTRUCTURED_HAS_PATCHED_LOOP=1
-      - LANGCHAIN_TRACING_V2=false
-      - LANGSMITH_TRACING=false
-      - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric}
-      - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
-      - AUTH_TYPE=${AUTH_TYPE:-LOCAL}
-      - NEXT_FRONTEND_URL=${NEXT_FRONTEND_URL:-http://localhost:3000}
-      # Daytona Sandbox – uncomment and set credentials to enable cloud code execution
-      # - DAYTONA_SANDBOX_ENABLED=TRUE
-      # - DAYTONA_API_KEY=${DAYTONA_API_KEY:-}
-      # - DAYTONA_API_URL=${DAYTONA_API_URL:-https://app.daytona.io/api}
-      # - DAYTONA_TARGET=${DAYTONA_TARGET:-us}
-    depends_on:
-      - db
-      - redis
-
-  # Run these services separately in production
-  # celery_worker:
-  #   build: ./surfsense_backend
-  #   # image: ghcr.io/modsetter/surfsense_backend:latest
-  #   command: celery -A app.celery_app worker --loglevel=info --concurrency=1 --pool=solo
-  #   volumes:
-  #     - ./surfsense_backend:/app
-  #     - shared_temp:/tmp
-  #   env_file:
-  #     - ./surfsense_backend/.env
-  #   environment:
-  #     - DATABASE_URL=postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-surfsense}
-  #     - CELERY_BROKER_URL=redis://redis:${REDIS_PORT:-6379}/0
-  #     - CELERY_RESULT_BACKEND=redis://redis:${REDIS_PORT:-6379}/0
-  #     - PYTHONPATH=/app
-  #   depends_on:
-  #     - db
-  #     - redis
-  #     - backend
-
-  # celery_beat:
-  #   build: ./surfsense_backend
-  #   # image: ghcr.io/modsetter/surfsense_backend:latest
-  #   command: celery -A app.celery_app beat --loglevel=info
-  #   volumes:
-  #     - ./surfsense_backend:/app
-  #     - shared_temp:/tmp
-  #   env_file:
-  #     - ./surfsense_backend/.env
-  #   environment:
-  #     - DATABASE_URL=postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-surfsense}
-  #     - CELERY_BROKER_URL=redis://redis:${REDIS_PORT:-6379}/0
-  #     - CELERY_RESULT_BACKEND=redis://redis:${REDIS_PORT:-6379}/0
-  #     - PYTHONPATH=/app
-  #   depends_on:
-  #     - db
-  #     - redis
-  #     - celery_worker
-
-  # flower:
-  #   build: ./surfsense_backend
-  #   # image: ghcr.io/modsetter/surfsense_backend:latest
-  #   command: celery -A app.celery_app flower --port=5555
-  #   ports:
-  #     - "${FLOWER_PORT:-5555}:5555"
-  #   env_file:
-  #     - ./surfsense_backend/.env
-  #   environment:
-  #     - CELERY_BROKER_URL=redis://redis:${REDIS_PORT:-6379}/0
-  #     - CELERY_RESULT_BACKEND=redis://redis:${REDIS_PORT:-6379}/0
-  #     - PYTHONPATH=/app
-  #   depends_on:
-  #     - redis
-  #     - celery_worker
-
-  electric:
-    image: electricsql/electric:latest
-    ports:
-      - "${ELECTRIC_PORT:-5133}:3000"
-    environment:
-      - DATABASE_URL=${ELECTRIC_DATABASE_URL:-postgresql://${ELECTRIC_DB_USER:-electric}:${ELECTRIC_DB_PASSWORD:-electric_password}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-surfsense}?sslmode=disable}
-      - ELECTRIC_INSECURE=true
-      - ELECTRIC_WRITE_TO_PG_MODE=direct
-    restart: unless-stopped
-    # depends_on:
-    #   - db
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:3000/v1/health"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
-  frontend:
-    build:
-      context: ./surfsense_web
-      # image: ghcr.io/modsetter/surfsense_ui:latest
-      args:
-        NEXT_PUBLIC_FASTAPI_BACKEND_URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:8000}
-        NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:-LOCAL}
-        NEXT_PUBLIC_ETL_SERVICE: ${NEXT_PUBLIC_ETL_SERVICE:-DOCLING}
-    ports:
-      - "${FRONTEND_PORT:-3000}:3000"
-    env_file:
-      - ./surfsense_web/.env
-    environment:
-      - NEXT_PUBLIC_ELECTRIC_URL=${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}
-      - NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure
-    depends_on:
-      - backend
-      - electric
-
-volumes:
-  postgres_data:
-  pgadmin_data:
-  redis_data:
-  shared_temp:

docker/.env.example

@@ -0,0 +1,256 @@
+# ==============================================================================
+# SurfSense Docker Configuration
+# ==============================================================================
+# Database, Redis, and internal service wiring are handled automatically.
+# ==============================================================================
+
+# SurfSense version (use "latest", a clean version like "0.0.14", or a specific build like "0.0.14.1")
+SURFSENSE_VERSION=latest
+
+# ------------------------------------------------------------------------------
+# Core Settings
+# ------------------------------------------------------------------------------
+
+# REQUIRED: Generate a secret key with: openssl rand -base64 32
+SECRET_KEY=replace_me_with_a_random_string
+
+# Auth type: LOCAL (email/password) or GOOGLE (OAuth)
+AUTH_TYPE=LOCAL
+
+# Allow new user registrations (TRUE or FALSE)
+# REGISTRATION_ENABLED=TRUE
+
+# Document parsing service: DOCLING, UNSTRUCTURED, or LLAMACLOUD
+ETL_SERVICE=DOCLING
+
+# Embedding model for vector search
+# Local: sentence-transformers/all-MiniLM-L6-v2
+# OpenAI: openai://text-embedding-ada-002 (set OPENAI_API_KEY below)
+# Cohere: cohere://embed-english-light-v3.0 (set COHERE_API_KEY below)
+EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+
+# ------------------------------------------------------------------------------
+# Ports (change to avoid conflicts with other services on your machine)
+# ------------------------------------------------------------------------------
+
+# BACKEND_PORT=8000
+# FRONTEND_PORT=3000
+# ELECTRIC_PORT=5133
+# FLOWER_PORT=5555
+
+# ==============================================================================
+# DEV COMPOSE ONLY (docker-compose.dev.yml)
+# You only need them only if you are running `docker-compose.dev.yml`.
+# ==============================================================================
+
+# -- pgAdmin (database GUI) --
+# PGADMIN_PORT=5050
+# PGADMIN_DEFAULT_EMAIL=admin@surfsense.com
+# PGADMIN_DEFAULT_PASSWORD=surfsense
+
+# -- Redis exposed port (dev only; Redis is internal-only in prod) --
+# REDIS_PORT=6379
+
+# -- Frontend Build Args --
+# In dev, the frontend is built from source and these are passed as build args.
+# In prod, they are automatically derived from AUTH_TYPE, ETL_SERVICE, and the port settings above.
+# NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL
+# NEXT_PUBLIC_ETL_SERVICE=DOCLING
+# NEXT_PUBLIC_DEPLOYMENT_MODE=self-hosted
+# NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure
+
+# ------------------------------------------------------------------------------
+# Custom Domain / Reverse Proxy
+# ------------------------------------------------------------------------------
+# ONLY set these if you are serving SurfSense on a real domain via a reverse
+# proxy (e.g. Caddy, Nginx, Cloudflare Tunnel).
+# For standard localhost deployments, leave all of these commented out —
+# they are automatically derived from the port settings above.
+#
+# NEXT_FRONTEND_URL=https://app.yourdomain.com
+# BACKEND_URL=https://api.yourdomain.com
+# NEXT_PUBLIC_FASTAPI_BACKEND_URL=https://api.yourdomain.com
+# NEXT_PUBLIC_ELECTRIC_URL=https://electric.yourdomain.com
+
+
+# ------------------------------------------------------------------------------
+# Database (defaults work out of the box, change for security)
+# ------------------------------------------------------------------------------
+
+# DB_USER=surfsense
+# DB_PASSWORD=surfsense
+# DB_NAME=surfsense
+# DB_HOST=db
+# DB_PORT=5432
+
+# SSL mode for database connections: disable, require, verify-ca, verify-full
+# DB_SSLMODE=disable
+
+# Full DATABASE_URL override — when set, takes precedence over the individual
+# DB_USER / DB_PASSWORD / DB_NAME / DB_HOST / DB_PORT settings above.
+# Use this for managed databases (AWS RDS, GCP Cloud SQL, Supabase, etc.)
+# DATABASE_URL=postgresql+asyncpg://user:password@your-rds-host:5432/surfsense?sslmode=require
+
+# ------------------------------------------------------------------------------
+# Redis (defaults work out of the box)
+# ------------------------------------------------------------------------------
+# Full Redis URL override for Celery broker, result backend, and app cache.
+# Use this for managed Redis (AWS ElastiCache, Redis Cloud, etc.)
+# Supports auth: redis://:password@host:port/0
+# Supports TLS: rediss://:password@host:6380/0
+# REDIS_URL=redis://redis:6379/0
+
+# ------------------------------------------------------------------------------
+# Electric SQL (real-time sync credentials)
+# ------------------------------------------------------------------------------
+# These must match on the db, backend, and electric services.
+# Change for security; defaults work out of the box.
+
+# ELECTRIC_DB_USER=electric
+# ELECTRIC_DB_PASSWORD=electric_password
+# Full override for the Electric → Postgres connection URL.
+# Leave commented out to use the Docker-managed `db` container (default).
+# Uncomment and set `db` to `host.docker.internal` when pointing Electric at a local Postgres instance (e.g. Postgres.app on macOS):
+# ELECTRIC_DATABASE_URL=postgresql://electric:electric_password@db:5432/surfsense?sslmode=disable
+
+# ------------------------------------------------------------------------------
+# TTS & STT (Text-to-Speech / Speech-to-Text)
+# ------------------------------------------------------------------------------
+
+# Local Kokoro TTS (default) or LiteLLM provider
+TTS_SERVICE=local/kokoro
+# TTS_SERVICE_API_KEY=
+# TTS_SERVICE_API_BASE=
+
+# Local Faster-Whisper STT: local/MODEL_SIZE (tiny, base, small, medium, large-v3)
+STT_SERVICE=local/base
+# Or use LiteLLM: openai/whisper-1
+# STT_SERVICE_API_KEY=
+# STT_SERVICE_API_BASE=
+
+# ------------------------------------------------------------------------------
+# Rerankers (optional, disabled by default)
+# ------------------------------------------------------------------------------
+
+# RERANKERS_ENABLED=TRUE
+# RERANKERS_MODEL_NAME=ms-marco-MiniLM-L-12-v2
+# RERANKERS_MODEL_TYPE=flashrank
+
+# ------------------------------------------------------------------------------
+# Google OAuth (only if AUTH_TYPE=GOOGLE)
+# ------------------------------------------------------------------------------
+
+# GOOGLE_OAUTH_CLIENT_ID=
+# GOOGLE_OAUTH_CLIENT_SECRET=
+
+# ------------------------------------------------------------------------------
+# Connector OAuth Keys (uncomment connectors you want to use)
+# ------------------------------------------------------------------------------
+
+# -- Google Connectors --
+# GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback
+# GOOGLE_GMAIL_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/gmail/connector/callback
+# GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/drive/connector/callback
+
+# -- Notion --
+# NOTION_CLIENT_ID=
+# NOTION_CLIENT_SECRET=
+# NOTION_REDIRECT_URI=http://localhost:8000/api/v1/auth/notion/connector/callback
+
+# -- Slack --
+# SLACK_CLIENT_ID=
+# SLACK_CLIENT_SECRET=
+# SLACK_REDIRECT_URI=http://localhost:8000/api/v1/auth/slack/connector/callback
+
+# -- Discord --
+# DISCORD_CLIENT_ID=
+# DISCORD_CLIENT_SECRET=
+# DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callback
+# DISCORD_BOT_TOKEN=
+
+# -- Atlassian (Jira & Confluence) --
+# ATLASSIAN_CLIENT_ID=
+# ATLASSIAN_CLIENT_SECRET=
+# JIRA_REDIRECT_URI=http://localhost:8000/api/v1/auth/jira/connector/callback
+# CONFLUENCE_REDIRECT_URI=http://localhost:8000/api/v1/auth/confluence/connector/callback
+
+# -- Linear --
+# LINEAR_CLIENT_ID=
+# LINEAR_CLIENT_SECRET=
+# LINEAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/linear/connector/callback
+
+# -- ClickUp --
+# CLICKUP_CLIENT_ID=
+# CLICKUP_CLIENT_SECRET=
+# CLICKUP_REDIRECT_URI=http://localhost:8000/api/v1/auth/clickup/connector/callback
+
+# -- Airtable --
+# AIRTABLE_CLIENT_ID=
+# AIRTABLE_CLIENT_SECRET=
+# AIRTABLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/airtable/connector/callback
+
+# -- Microsoft Teams --
+# TEAMS_CLIENT_ID=
+# TEAMS_CLIENT_SECRET=
+# TEAMS_REDIRECT_URI=http://localhost:8000/api/v1/auth/teams/connector/callback
+
+# -- Composio --
+# COMPOSIO_API_KEY=
+# COMPOSIO_ENABLED=TRUE
+# COMPOSIO_REDIRECT_URI=http://localhost:8000/api/v1/auth/composio/connector/callback
+
+# ------------------------------------------------------------------------------
+# Daytona Sandbox (optional — cloud code execution for the deep agent)
+# ------------------------------------------------------------------------------
+# Set DAYTONA_SANDBOX_ENABLED=TRUE and provide credentials to give the agent
+# an isolated code execution environment via the Daytona cloud API.
+# DAYTONA_SANDBOX_ENABLED=FALSE
+# DAYTONA_API_KEY=
+# DAYTONA_API_URL=https://app.daytona.io/api
+# DAYTONA_TARGET=us
+
+# ------------------------------------------------------------------------------
+# External API Keys (optional)
+# ------------------------------------------------------------------------------
+
+# Firecrawl (web scraping)
+# FIRECRAWL_API_KEY=
+
+# Unstructured (if ETL_SERVICE=UNSTRUCTURED)
+# UNSTRUCTURED_API_KEY=
+
+# LlamaCloud (if ETL_SERVICE=LLAMACLOUD)
+# LLAMA_CLOUD_API_KEY=
+
+# ------------------------------------------------------------------------------
+# Observability (optional)
+# ------------------------------------------------------------------------------
+
+# LANGSMITH_TRACING=true
+# LANGSMITH_ENDPOINT=https://api.smith.langchain.com
+# LANGSMITH_API_KEY=
+# LANGSMITH_PROJECT=surfsense
+
+# ------------------------------------------------------------------------------
+# Advanced (optional)
+# ------------------------------------------------------------------------------
+
+# Periodic connector sync interval (default: 5m)
+# SCHEDULE_CHECKER_INTERVAL=5m
+
+# JWT token lifetimes
+# ACCESS_TOKEN_LIFETIME_SECONDS=86400
+# REFRESH_TOKEN_LIFETIME_SECONDS=1209600
+
+# Pages limit per user for ETL (default: unlimited)
+# PAGES_LIMIT=500
+
+# Connector indexing lock TTL in seconds (default: 28800 = 8 hours)
+# CONNECTOR_INDEXING_LOCK_TTL_SECONDS=28800
+
+# Residential proxy for web crawling
+# RESIDENTIAL_PROXY_USERNAME=
+# RESIDENTIAL_PROXY_PASSWORD=
+# RESIDENTIAL_PROXY_HOSTNAME=
+# RESIDENTIAL_PROXY_LOCATION=
+# RESIDENTIAL_PROXY_TYPE=1

docker/docker-compose.dev.yml

@@ -0,0 +1,206 @@
+# =============================================================================
+# SurfSense — Development Docker Compose
+# =============================================================================
+# Usage (from repo root):
+#   docker compose -f docker/docker-compose.dev.yml up --build
+#
+# This file builds from source and includes dev tools like pgAdmin.
+# For production with prebuilt images, use docker/docker-compose.yml instead.
+# =============================================================================
+
+name: surfsense
+
+services:
+  db:
+    image: pgvector/pgvector:pg17
+    ports:
+      - "${POSTGRES_PORT:-5432}:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+      - ./postgresql.conf:/etc/postgresql/postgresql.conf:ro
+      - ./scripts/init-electric-user.sh:/docker-entrypoint-initdb.d/init-electric-user.sh:ro
+    environment:
+      - POSTGRES_USER=${DB_USER:-postgres}
+      - POSTGRES_PASSWORD=${DB_PASSWORD:-postgres}
+      - POSTGRES_DB=${DB_NAME:-surfsense}
+      - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric}
+      - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
+    command: postgres -c config_file=/etc/postgresql/postgresql.conf
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME:-surfsense}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  pgadmin:
+    image: dpage/pgadmin4
+    ports:
+      - "${PGADMIN_PORT:-5050}:80"
+    environment:
+      - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL:-admin@surfsense.com}
+      - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD:-surfsense}
+    volumes:
+      - pgadmin_data:/var/lib/pgadmin
+    depends_on:
+      - db
+
+  redis:
+    image: redis:8-alpine
+    ports:
+      - "${REDIS_PORT:-6379}:6379"
+    volumes:
+      - redis_data:/data
+    command: redis-server --appendonly yes
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  backend:
+    build: ../surfsense_backend
+    ports:
+      - "${BACKEND_PORT:-8000}:8000"
+    volumes:
+      - ../surfsense_backend/app:/app/app
+      - shared_temp:/shared_tmp
+    env_file:
+      - ../surfsense_backend/.env
+    environment:
+      - DATABASE_URL=${DATABASE_URL:-postgresql+asyncpg://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}}
+      - CELERY_BROKER_URL=${REDIS_URL:-redis://redis:6379/0}
+      - CELERY_RESULT_BACKEND=${REDIS_URL:-redis://redis:6379/0}
+      - REDIS_APP_URL=${REDIS_URL:-redis://redis:6379/0}
+      - CELERY_TASK_DEFAULT_QUEUE=surfsense
+      - PYTHONPATH=/app
+      - UVICORN_LOOP=asyncio
+      - UNSTRUCTURED_HAS_PATCHED_LOOP=1
+      - LANGCHAIN_TRACING_V2=false
+      - LANGSMITH_TRACING=false
+      - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric}
+      - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
+      - AUTH_TYPE=${AUTH_TYPE:-LOCAL}
+      - NEXT_FRONTEND_URL=${NEXT_FRONTEND_URL:-http://localhost:3000}
+      # Daytona Sandbox – uncomment and set credentials to enable cloud code execution
+      # - DAYTONA_SANDBOX_ENABLED=TRUE
+      # - DAYTONA_API_KEY=${DAYTONA_API_KEY:-}
+      # - DAYTONA_API_URL=${DAYTONA_API_URL:-https://app.daytona.io/api}
+      # - DAYTONA_TARGET=${DAYTONA_TARGET:-us}
+      - SERVICE_ROLE=api
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 15s
+      timeout: 5s
+      retries: 30
+      start_period: 200s
+
+  celery_worker:
+    build: ../surfsense_backend
+    volumes:
+      - ../surfsense_backend/app:/app/app
+      - shared_temp:/shared_tmp
+    env_file:
+      - ../surfsense_backend/.env
+    environment:
+      - DATABASE_URL=${DATABASE_URL:-postgresql+asyncpg://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}}
+      - CELERY_BROKER_URL=${REDIS_URL:-redis://redis:6379/0}
+      - CELERY_RESULT_BACKEND=${REDIS_URL:-redis://redis:6379/0}
+      - REDIS_APP_URL=${REDIS_URL:-redis://redis:6379/0}
+      - CELERY_TASK_DEFAULT_QUEUE=surfsense
+      - PYTHONPATH=/app
+      - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric}
+      - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
+      - SERVICE_ROLE=worker
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      backend:
+        condition: service_healthy
+
+  celery_beat:
+    build: ../surfsense_backend
+    env_file:
+      - ../surfsense_backend/.env
+    environment:
+      - DATABASE_URL=${DATABASE_URL:-postgresql+asyncpg://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}}
+      - CELERY_BROKER_URL=${REDIS_URL:-redis://redis:6379/0}
+      - CELERY_RESULT_BACKEND=${REDIS_URL:-redis://redis:6379/0}
+      - CELERY_TASK_DEFAULT_QUEUE=surfsense
+      - PYTHONPATH=/app
+      - SERVICE_ROLE=beat
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      celery_worker:
+        condition: service_started
+
+  # flower:
+  #   build: ../surfsense_backend
+  #   ports:
+  #     - "${FLOWER_PORT:-5555}:5555"
+  #   env_file:
+  #     - ../surfsense_backend/.env
+  #   environment:
+  #     - CELERY_BROKER_URL=${REDIS_URL:-redis://redis:6379/0}
+  #     - CELERY_RESULT_BACKEND=${REDIS_URL:-redis://redis:6379/0}
+  #     - PYTHONPATH=/app
+  #   command: celery -A app.celery_app flower --port=5555
+  #   depends_on:
+  #     - redis
+  #     - celery_worker
+
+  electric:
+    image: electricsql/electric:1.4.10
+    ports:
+      - "${ELECTRIC_PORT:-5133}:3000"
+    # depends_on:
+    #   - db
+    environment:
+      - DATABASE_URL=${ELECTRIC_DATABASE_URL:-postgresql://${ELECTRIC_DB_USER:-electric}:${ELECTRIC_DB_PASSWORD:-electric_password}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}}
+      - ELECTRIC_INSECURE=true
+      - ELECTRIC_WRITE_TO_PG_MODE=direct
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/v1/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  frontend:
+    build:
+      context: ../surfsense_web
+      args:
+        NEXT_PUBLIC_FASTAPI_BACKEND_URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:8000}
+        NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:-LOCAL}
+        NEXT_PUBLIC_ETL_SERVICE: ${NEXT_PUBLIC_ETL_SERVICE:-DOCLING}
+        NEXT_PUBLIC_ELECTRIC_URL: ${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}
+        NEXT_PUBLIC_ELECTRIC_AUTH_MODE: ${NEXT_PUBLIC_ELECTRIC_AUTH_MODE:-insecure}
+        NEXT_PUBLIC_DEPLOYMENT_MODE: ${NEXT_PUBLIC_DEPLOYMENT_MODE:-self-hosted}
+    ports:
+      - "${FRONTEND_PORT:-3000}:3000"
+    env_file:
+      - ../surfsense_web/.env
+    depends_on:
+      backend:
+        condition: service_healthy
+      electric:
+        condition: service_healthy
+
+volumes:
+  postgres_data:
+    name: surfsense-postgres
+  pgadmin_data:
+    name: surfsense-pgadmin
+  redis_data:
+    name: surfsense-redis
+  shared_temp:
+    name: surfsense-shared-temp

docker/docker-compose.yml

@@ -0,0 +1,195 @@
+# =============================================================================
+# SurfSense — Production Docker Compose
+# Docs: https://docs.surfsense.com/docs/docker-installation
+# =============================================================================
+# Usage:
+#   1. Copy .env.example to .env and edit the required values
+#   2. docker compose up -d
+# =============================================================================
+
+name: surfsense
+
+services:
+  db:
+    image: pgvector/pgvector:pg17
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+      - ./postgresql.conf:/etc/postgresql/postgresql.conf:ro
+      - ./scripts/init-electric-user.sh:/docker-entrypoint-initdb.d/init-electric-user.sh:ro
+    environment:
+      POSTGRES_USER: ${DB_USER:-surfsense}
+      POSTGRES_PASSWORD: ${DB_PASSWORD:-surfsense}
+      POSTGRES_DB: ${DB_NAME:-surfsense}
+      ELECTRIC_DB_USER: ${ELECTRIC_DB_USER:-electric}
+      ELECTRIC_DB_PASSWORD: ${ELECTRIC_DB_PASSWORD:-electric_password}
+    command: postgres -c config_file=/etc/postgresql/postgresql.conf
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-surfsense} -d ${DB_NAME:-surfsense}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  redis:
+    image: redis:8-alpine
+    volumes:
+      - redis_data:/data
+    command: redis-server --appendonly yes
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  backend:
+    image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
+    ports:
+      - "${BACKEND_PORT:-8000}:8000"
+    volumes:
+      - shared_temp:/shared_tmp
+    env_file:
+      - .env
+    environment:
+      DATABASE_URL: ${DATABASE_URL:-postgresql+asyncpg://${DB_USER:-surfsense}:${DB_PASSWORD:-surfsense}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}}
+      CELERY_BROKER_URL: ${REDIS_URL:-redis://redis:6379/0}
+      CELERY_RESULT_BACKEND: ${REDIS_URL:-redis://redis:6379/0}
+      REDIS_APP_URL: ${REDIS_URL:-redis://redis:6379/0}
+      CELERY_TASK_DEFAULT_QUEUE: surfsense
+      PYTHONPATH: /app
+      UVICORN_LOOP: asyncio
+      UNSTRUCTURED_HAS_PATCHED_LOOP: "1"
+      ELECTRIC_DB_USER: ${ELECTRIC_DB_USER:-electric}
+      ELECTRIC_DB_PASSWORD: ${ELECTRIC_DB_PASSWORD:-electric_password}
+      NEXT_FRONTEND_URL: ${NEXT_FRONTEND_URL:-http://localhost:${FRONTEND_PORT:-3000}}
+      # Daytona Sandbox – uncomment and set credentials to enable cloud code execution
+      # DAYTONA_SANDBOX_ENABLED: "TRUE"
+      # DAYTONA_API_KEY: ${DAYTONA_API_KEY:-}
+      # DAYTONA_API_URL: ${DAYTONA_API_URL:-https://app.daytona.io/api}
+      # DAYTONA_TARGET: ${DAYTONA_TARGET:-us}
+      SERVICE_ROLE: api
+    labels:
+      - "com.centurylinklabs.watchtower.enable=true"
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 15s
+      timeout: 5s
+      retries: 30
+      start_period: 200s
+
+  celery_worker:
+    image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
+    volumes:
+      - shared_temp:/shared_tmp
+    env_file:
+      - .env
+    environment:
+      DATABASE_URL: ${DATABASE_URL:-postgresql+asyncpg://${DB_USER:-surfsense}:${DB_PASSWORD:-surfsense}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}}
+      CELERY_BROKER_URL: ${REDIS_URL:-redis://redis:6379/0}
+      CELERY_RESULT_BACKEND: ${REDIS_URL:-redis://redis:6379/0}
+      REDIS_APP_URL: ${REDIS_URL:-redis://redis:6379/0}
+      CELERY_TASK_DEFAULT_QUEUE: surfsense
+      PYTHONPATH: /app
+      ELECTRIC_DB_USER: ${ELECTRIC_DB_USER:-electric}
+      ELECTRIC_DB_PASSWORD: ${ELECTRIC_DB_PASSWORD:-electric_password}
+      SERVICE_ROLE: worker
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      backend:
+        condition: service_healthy
+    labels:
+      - "com.centurylinklabs.watchtower.enable=true"
+    restart: unless-stopped
+
+  celery_beat:
+    image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
+    env_file:
+      - .env
+    environment:
+      DATABASE_URL: ${DATABASE_URL:-postgresql+asyncpg://${DB_USER:-surfsense}:${DB_PASSWORD:-surfsense}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}}
+      CELERY_BROKER_URL: ${REDIS_URL:-redis://redis:6379/0}
+      CELERY_RESULT_BACKEND: ${REDIS_URL:-redis://redis:6379/0}
+      CELERY_TASK_DEFAULT_QUEUE: surfsense
+      PYTHONPATH: /app
+      SERVICE_ROLE: beat
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      celery_worker:
+        condition: service_started
+    labels:
+      - "com.centurylinklabs.watchtower.enable=true"
+    restart: unless-stopped
+
+  # flower:
+  #   image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
+  #   ports:
+  #     - "${FLOWER_PORT:-5555}:5555"
+  #   env_file:
+  #     - .env
+  #   environment:
+  #     CELERY_BROKER_URL: ${REDIS_URL:-redis://redis:6379/0}
+  #     CELERY_RESULT_BACKEND: ${REDIS_URL:-redis://redis:6379/0}
+  #     PYTHONPATH: /app
+  #   command: celery -A app.celery_app flower --port=5555
+  #   depends_on:
+  #     - redis
+  #     - celery_worker
+  #   restart: unless-stopped
+
+  electric:
+    image: electricsql/electric:1.4.10
+    ports:
+      - "${ELECTRIC_PORT:-5133}:3000"
+    environment:
+      DATABASE_URL: ${ELECTRIC_DATABASE_URL:-postgresql://${ELECTRIC_DB_USER:-electric}:${ELECTRIC_DB_PASSWORD:-electric_password}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}}
+      ELECTRIC_INSECURE: "true"
+      ELECTRIC_WRITE_TO_PG_MODE: direct
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/v1/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  frontend:
+    image: ghcr.io/modsetter/surfsense-web:${SURFSENSE_VERSION:-latest}
+    ports:
+      - "${FRONTEND_PORT:-3000}:3000"
+    environment:
+      NEXT_PUBLIC_FASTAPI_BACKEND_URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:${BACKEND_PORT:-8000}}
+      NEXT_PUBLIC_ELECTRIC_URL: ${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:${ELECTRIC_PORT:-5133}}
+      NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${AUTH_TYPE:-LOCAL}
+      NEXT_PUBLIC_ETL_SERVICE: ${ETL_SERVICE:-DOCLING}
+      NEXT_PUBLIC_DEPLOYMENT_MODE: ${DEPLOYMENT_MODE:-self-hosted}
+      NEXT_PUBLIC_ELECTRIC_AUTH_MODE: ${NEXT_PUBLIC_ELECTRIC_AUTH_MODE:-insecure}
+    labels:
+      - "com.centurylinklabs.watchtower.enable=true"
+    depends_on:
+      backend:
+        condition: service_healthy
+      electric:
+        condition: service_healthy
+    restart: unless-stopped
+
+volumes:
+  postgres_data:
+    name: surfsense-postgres
+  redis_data:
+    name: surfsense-redis
+  shared_temp:
+    name: surfsense-shared-temp

docker/scripts/init-electric-user.sh

@@ -1,26 +1,9 @@
 #!/bin/sh
-# ============================================================================
-# Electric SQL User Initialization Script (docker-compose only)
-# ============================================================================
-# This script is ONLY used when running via docker-compose.
-#
-# How it works:
-# - docker-compose.yml mounts this script into the PostgreSQL container's
-#   /docker-entrypoint-initdb.d/ directory
-# - PostgreSQL automatically executes scripts in that directory on first
-#   container initialization
-#
-# For local PostgreSQL users (non-Docker), this script is NOT used.
-# Instead, the Electric user is created by Alembic migration 66
-# (66_add_notifications_table_and_electric_replication.py).
-#
-# Both approaches are idempotent (use IF NOT EXISTS), so running both
-# will not cause conflicts.
-# ============================================================================
+# Creates the Electric SQL replication user on first DB initialization.
+# Idempotent — safe to run alongside Alembic migration 66.
 
 set -e
 
-# Use environment variables with defaults
 ELECTRIC_DB_USER="${ELECTRIC_DB_USER:-electric}"
 ELECTRIC_DB_PASSWORD="${ELECTRIC_DB_PASSWORD:-electric_password}"
 
@@ -43,7 +26,6 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
     ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $ELECTRIC_DB_USER;
     ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO $ELECTRIC_DB_USER;
 
-    -- Create the publication for Electric SQL (if not exists)
     DO \$\$
     BEGIN
         IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN

docker/scripts/install.sh

@@ -0,0 +1,340 @@
+#!/usr/bin/env bash
+# =============================================================================
+# SurfSense — One-line Install Script
+#
+#
+# Usage: curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
+#
+# Flags:
+#   --no-watchtower              Skip automatic Watchtower setup
+#   --watchtower-interval=SECS   Check interval in seconds (default: 86400 = 24h)
+#
+# Handles two cases automatically:
+#   1. Fresh install        — no prior SurfSense data detected
+#   2. Migration from the legacy all-in-one container (surfsense-data volume)
+#      Downloads and runs migrate-database.sh --yes, then restores the dump
+#      into the new PostgreSQL 17 stack. The user runs one command for both.
+#
+# If you used custom database credentials in the old all-in-one container, run
+# migrate-database.sh manually first (with --db-user / --db-password flags),
+# then re-run this script:
+#   curl -fsSL .../docker/scripts/migrate-database.sh | bash -s -- --db-user X --db-password Y
+# =============================================================================
+
+set -euo pipefail
+
+main() {
+
+REPO_RAW="https://raw.githubusercontent.com/MODSetter/SurfSense/main"
+INSTALL_DIR="./surfsense"
+OLD_VOLUME="surfsense-data"
+DUMP_FILE="./surfsense_migration_backup.sql"
+KEY_FILE="./surfsense_migration_secret.key"
+MIGRATION_DONE_FILE="${INSTALL_DIR}/.migration_done"
+MIGRATION_MODE=false
+SETUP_WATCHTOWER=true
+WATCHTOWER_INTERVAL=86400
+WATCHTOWER_CONTAINER="watchtower"
+
+# ── Parse flags ─────────────────────────────────────────────────────────────
+for arg in "$@"; do
+    case "$arg" in
+        --no-watchtower) SETUP_WATCHTOWER=false ;;
+        --watchtower-interval=*) WATCHTOWER_INTERVAL="${arg#*=}" ;;
+    esac
+done
+
+CYAN='\033[1;36m'
+YELLOW='\033[1;33m'
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+BOLD='\033[1m'
+NC='\033[0m'
+
+info()    { printf "${CYAN}[SurfSense]${NC} %s\n"        "$1"; }
+success() { printf "${GREEN}[SurfSense]${NC} %s\n"       "$1"; }
+warn()    { printf "${YELLOW}[SurfSense]${NC} %s\n"      "$1"; }
+error()   { printf "${RED}[SurfSense]${NC} ERROR: %s\n"  "$1" >&2; exit 1; }
+step()    { printf "\n${BOLD}${CYAN}── %s${NC}\n"        "$1"; }
+
+# ── Pre-flight checks ────────────────────────────────────────────────────────
+
+step "Checking prerequisites"
+
+command -v docker >/dev/null 2>&1 \
+    || error "Docker is not installed. Install it at: https://docs.docker.com/get-docker/"
+success "Docker found."
+
+docker info >/dev/null 2>&1 < /dev/null \
+    || error "Docker daemon is not running. Please start Docker and try again."
+success "Docker daemon is running."
+
+if docker compose version >/dev/null 2>&1 < /dev/null; then
+    DC="docker compose"
+elif command -v docker-compose >/dev/null 2>&1; then
+    DC="docker-compose"
+else
+    error "Docker Compose is not installed. Install it at: https://docs.docker.com/compose/install/"
+fi
+success "Docker Compose found ($DC)."
+
+# ── Wait-for-postgres helper ─────────────────────────────────────────────────
+wait_for_pg() {
+    local db_user="$1"
+    local max_attempts=45
+    local attempt=0
+
+    info "Waiting for PostgreSQL to accept connections..."
+    until (cd "${INSTALL_DIR}" && ${DC} exec -T db pg_isready -U "${db_user}" -q 2>/dev/null) < /dev/null; do
+        attempt=$((attempt + 1))
+        if [[ $attempt -ge $max_attempts ]]; then
+            error "PostgreSQL did not become ready after $((max_attempts * 2)) seconds.\nCheck logs: cd ${INSTALL_DIR} && ${DC} logs db"
+        fi
+        printf "."
+        sleep 2
+    done
+    printf "\n"
+    success "PostgreSQL is ready."
+}
+
+# ── Download files ───────────────────────────────────────────────────────────
+
+step "Downloading SurfSense files"
+info "Installation directory: ${INSTALL_DIR}"
+mkdir -p "${INSTALL_DIR}/scripts"
+
+FILES=(
+    "docker/docker-compose.yml:docker-compose.yml"
+    "docker/.env.example:.env.example"
+    "docker/postgresql.conf:postgresql.conf"
+    "docker/scripts/init-electric-user.sh:scripts/init-electric-user.sh"
+    "docker/scripts/migrate-database.sh:scripts/migrate-database.sh"
+)
+
+for entry in "${FILES[@]}"; do
+    src="${entry%%:*}"
+    dest="${entry##*:}"
+    info "Downloading ${dest}..."
+    curl -fsSL "${REPO_RAW}/${src}" -o "${INSTALL_DIR}/${dest}" \
+        || error "Failed to download ${dest}. Check your internet connection and try again."
+done
+
+chmod +x "${INSTALL_DIR}/scripts/init-electric-user.sh"
+chmod +x "${INSTALL_DIR}/scripts/migrate-database.sh"
+success "All files downloaded to ${INSTALL_DIR}/"
+
+# ── Legacy all-in-one detection ──────────────────────────────────────────────
+# Detect surfsense-data volume → migration mode.
+# If a dump already exists (from a previous partial run) skip extraction and
+# go straight to restore — this makes re-runs safe and idempotent.
+
+if docker volume ls --format '{{.Name}}' 2>/dev/null < /dev/null | grep -q "^${OLD_VOLUME}$" \
+   && [[ ! -f "${MIGRATION_DONE_FILE}" ]]; then
+    MIGRATION_MODE=true
+
+    if [[ -f "${DUMP_FILE}" ]]; then
+        step "Migration mode — using existing dump (skipping extraction)"
+        info "Found existing dump: ${DUMP_FILE}"
+        info "Skipping data extraction — proceeding directly to restore."
+        info "To force a fresh extraction, remove the dump first: rm ${DUMP_FILE}"
+    else
+        step "Migration mode — legacy all-in-one container detected"
+        warn "Volume '${OLD_VOLUME}' found. Your data will be migrated automatically."
+        warn "PostgreSQL is being upgraded from version 14 to 17."
+        warn "Your original data will NOT be deleted."
+        printf "\n"
+        info "Running data extraction (migrate-database.sh --yes)..."
+        info "Full extraction log: ./surfsense-migration.log"
+        printf "\n"
+
+        # Run extraction non-interactively. On failure the error from
+        # migrate-database.sh is printed and install.sh exits here.
+        bash "${INSTALL_DIR}/scripts/migrate-database.sh" --yes < /dev/null \
+            || error "Data extraction failed. See ./surfsense-migration.log for details.\nYou can also run migrate-database.sh manually with custom flags:\n  bash ${INSTALL_DIR}/scripts/migrate-database.sh --db-user X --db-password Y"
+
+        printf "\n"
+        success "Data extraction complete. Proceeding with installation and restore."
+    fi
+fi
+
+# ── Set up .env ──────────────────────────────────────────────────────────────
+
+step "Configuring environment"
+
+if [ ! -f "${INSTALL_DIR}/.env" ]; then
+    cp "${INSTALL_DIR}/.env.example" "${INSTALL_DIR}/.env"
+
+    if $MIGRATION_MODE && [[ -f "${KEY_FILE}" ]]; then
+        SECRET_KEY=$(cat "${KEY_FILE}" | tr -d '[:space:]')
+        success "Using SECRET_KEY recovered from legacy container."
+    else
+        SECRET_KEY=$(openssl rand -base64 32 2>/dev/null \
+            || head -c 32 /dev/urandom | base64 | tr -d '\n')
+        success "Generated new random SECRET_KEY."
+    fi
+
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        sed -i '' "s|SECRET_KEY=replace_me_with_a_random_string|SECRET_KEY=${SECRET_KEY}|" "${INSTALL_DIR}/.env"
+    else
+        sed -i "s|SECRET_KEY=replace_me_with_a_random_string|SECRET_KEY=${SECRET_KEY}|" "${INSTALL_DIR}/.env"
+    fi
+    info "Created ${INSTALL_DIR}/.env"
+else
+    warn ".env already exists — keeping your existing configuration."
+fi
+
+# ── Start containers ─────────────────────────────────────────────────────────
+
+if $MIGRATION_MODE; then
+    # Read DB credentials from .env (fall back to defaults from docker-compose.yml)
+    DB_USER=$(grep '^DB_USER=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true)
+    DB_PASS=$(grep '^DB_PASSWORD=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true)
+    DB_NAME=$(grep '^DB_NAME=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true)
+    DB_USER="${DB_USER:-surfsense}"
+    DB_PASS="${DB_PASS:-surfsense}"
+    DB_NAME="${DB_NAME:-surfsense}"
+
+    step "Starting PostgreSQL 17"
+    (cd "${INSTALL_DIR}" && ${DC} up -d db) < /dev/null
+    wait_for_pg "${DB_USER}"
+
+    step "Restoring database"
+    [[ -f "${DUMP_FILE}" ]] \
+        || error "Dump file '${DUMP_FILE}' not found. The migration script may have failed.\n  Check: ./surfsense-migration.log\n  Or run manually: bash ${INSTALL_DIR}/scripts/migrate-database.sh --yes"
+    info "Restoring dump into PostgreSQL 17 — this may take a while for large databases..."
+
+    RESTORE_ERR="/tmp/surfsense_restore_err.log"
+    (cd "${INSTALL_DIR}" && ${DC} exec -T \
+        -e PGPASSWORD="${DB_PASS}" \
+        db psql -U "${DB_USER}" -d "${DB_NAME}" \
+        >/dev/null 2>"${RESTORE_ERR}") < "${DUMP_FILE}" || true
+
+    # Surface real errors; ignore benign "already exists" noise from pg_dump headers
+    FATAL_ERRORS=$(grep -i "^ERROR:" "${RESTORE_ERR}" \
+        | grep -iv "already exists" \
+        | grep -iv "multiple primary keys" \
+        || true)
+
+    if [[ -n "${FATAL_ERRORS}" ]]; then
+        warn "Restore completed with errors (may be harmless pg_dump header noise):"
+        printf "%s\n" "${FATAL_ERRORS}"
+        warn "If SurfSense behaves incorrectly, inspect manually:"
+        warn "  cd ${INSTALL_DIR} && ${DC} exec db psql -U ${DB_USER} -d ${DB_NAME} < ${DUMP_FILE}"
+    else
+        success "Database restored with no fatal errors."
+    fi
+
+    # Smoke test — verify tables are present
+    TABLE_COUNT=$(
+        cd "${INSTALL_DIR}" && ${DC} exec -T \
+            -e PGPASSWORD="${DB_PASS}" \
+            db psql -U "${DB_USER}" -d "${DB_NAME}" -t \
+            -c "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public';" \
+            2>/dev/null < /dev/null | tr -d ' \n' || echo "0"
+    )
+    if [[ "${TABLE_COUNT}" == "0" || -z "${TABLE_COUNT}" ]]; then
+        warn "Smoke test: no tables found after restore."
+        warn "The restore may have failed silently. Check: cd ${INSTALL_DIR} && ${DC} logs db"
+    else
+        success "Smoke test passed: ${TABLE_COUNT} table(s) restored successfully."
+        touch "${MIGRATION_DONE_FILE}"
+    fi
+
+    step "Starting all SurfSense services"
+    (cd "${INSTALL_DIR}" && ${DC} up -d) < /dev/null
+    success "All services started."
+
+    # Key file is no longer needed — SECRET_KEY is now in .env
+    rm -f "${KEY_FILE}"
+
+else
+    step "Starting SurfSense"
+    (cd "${INSTALL_DIR}" && ${DC} up -d) < /dev/null
+    success "All services started."
+fi
+
+# ── Watchtower (auto-update) ─────────────────────────────────────────────────
+
+if $SETUP_WATCHTOWER; then
+    step "Setting up Watchtower (auto-updates every $((WATCHTOWER_INTERVAL / 3600))h)"
+
+    WT_STATE=$(docker inspect -f '{{.State.Running}}' "${WATCHTOWER_CONTAINER}" 2>/dev/null < /dev/null || echo "missing")
+
+    if [[ "${WT_STATE}" == "true" ]]; then
+        success "Watchtower is already running — skipping."
+    else
+        if [[ "${WT_STATE}" != "missing" ]]; then
+            info "Removing stopped Watchtower container..."
+            docker rm -f "${WATCHTOWER_CONTAINER}" >/dev/null 2>&1 < /dev/null || true
+        fi
+        docker run -d \
+            --name "${WATCHTOWER_CONTAINER}" \
+            --restart unless-stopped \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            nickfedor/watchtower \
+            --label-enable \
+            --interval "${WATCHTOWER_INTERVAL}" >/dev/null 2>&1 < /dev/null \
+            && success "Watchtower started — labeled SurfSense containers will auto-update." \
+            || warn "Could not start Watchtower. You can set it up manually or use: docker compose pull && docker compose up -d"
+    fi
+else
+    info "Skipping Watchtower setup (--no-watchtower flag)."
+fi
+
+# ── Done ─────────────────────────────────────────────────────────────────────
+
+echo ""
+printf '\033[1;37m'
+cat << 'EOF'
+
+
+ .d8888b.                    .d888 .d8888b.                                      
+d88P  Y88b                  d88P" d88P  Y88b                                     
+Y88b.                       888   Y88b.                                          
+ "Y888b.   888  888 888d888 888888 "Y888b.    .d88b.  88888b.  .d8888b   .d88b.  
+    "Y88b. 888  888 888P"   888       "Y88b. d8P  Y8b 888 "88b 88K      d8P  Y8b 
+      "888 888  888 888     888         "888 88888888 888  888 "Y8888b. 88888888 
+Y88b  d88P Y88b 888 888     888   Y88b  d88P Y8b.     888  888      X88 Y8b.     
+ "Y8888P"   "Y88888 888     888    "Y8888P"   "Y8888  888  888  88888P'  "Y8888  
+
+
+EOF
+if [[ "${SURFSENSE_VERSION:-latest}" == "latest" ]]; then
+    _version_display="latest"
+else
+    _version_display="v${SURFSENSE_VERSION}"
+fi
+printf "         Your personal AI-powered search engine  ${YELLOW}[%s]${NC}\n" "${_version_display}"
+printf "${CYAN}══════════════════════════════════════════════════════════════${NC}\n\n"
+
+info "  Frontend:  http://localhost:3000"
+info "  Backend:   http://localhost:8000"
+info "  API Docs:  http://localhost:8000/docs"
+info ""
+info "  Config:    ${INSTALL_DIR}/.env"
+info "  Logs:      cd ${INSTALL_DIR} && ${DC} logs -f"
+info "  Stop:      cd ${INSTALL_DIR} && ${DC} down"
+info "  Update:    cd ${INSTALL_DIR} && ${DC} pull && ${DC} up -d"
+info ""
+
+if $SETUP_WATCHTOWER; then
+    info "  Watchtower: auto-updates every $((WATCHTOWER_INTERVAL / 3600))h (stop: docker rm -f ${WATCHTOWER_CONTAINER})"
+else
+    warn "  Watchtower skipped. For auto-updates, re-run without --no-watchtower."
+fi
+info ""
+
+if $MIGRATION_MODE; then
+    warn "  Migration complete! Open frontend and verify your data."
+    warn "  Once verified, clean up the legacy volume and migration files:"
+    warn "    docker volume rm ${OLD_VOLUME}"
+    warn "    rm ${DUMP_FILE}"
+    warn "    rm ${MIGRATION_DONE_FILE}"
+else
+    warn "  First startup may take a few minutes while images are pulled."
+    warn "  Edit ${INSTALL_DIR}/.env to configure API keys, OAuth, etc."
+fi
+
+} # end main()
+
+main "$@"

docker/scripts/migrate-database.sh

@@ -0,0 +1,335 @@
+#!/usr/bin/env bash
+# =============================================================================
+# SurfSense — Database Migration Script
+#
+# Extracts data from the legacy all-in-one surfsense-data volume (PostgreSQL 14)
+# and saves it as a SQL dump + SECRET_KEY file ready for install.sh to restore.
+#
+# Usage:
+#   bash migrate-database.sh [options]
+#
+# Options:
+#   --db-user USER        Old PostgreSQL username   (default: surfsense)
+#   --db-password PASS    Old PostgreSQL password   (default: surfsense)
+#   --db-name NAME        Old PostgreSQL database   (default: surfsense)
+#   --yes / -y            Skip all confirmation prompts
+#   --help / -h           Show this help
+#
+# Prerequisites:
+#   - Docker installed and running
+#   - The legacy surfsense-data volume must exist
+#   - ~500 MB free disk space for the dump file
+#
+# What this script does:
+#   1. Stops any container using surfsense-data (to prevent corruption)
+#   2. Starts a temporary PG14 container against the old volume
+#   3. Dumps the database to ./surfsense_migration_backup.sql
+#   4. Recovers the SECRET_KEY to ./surfsense_migration_secret.key
+#   5. Exits — leaving installation to install.sh
+#
+# What this script does NOT do:
+#   - Delete the original surfsense-data volume (do this manually after verifying)
+#   - Install the new SurfSense stack (install.sh handles that automatically)
+#
+# Note:
+#   install.sh downloads and runs this script automatically when it detects the
+#   legacy surfsense-data volume. You only need to run this script manually if
+#   you have custom database credentials (--db-user / --db-password / --db-name)
+#   or if the automatic migration inside install.sh fails at the extraction step.
+# =============================================================================
+
+set -euo pipefail
+
+# ── Colours ──────────────────────────────────────────────────────────────────
+CYAN='\033[1;36m'
+YELLOW='\033[1;33m'
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+BOLD='\033[1m'
+NC='\033[0m'
+
+# ── Logging — tee everything to a log file ───────────────────────────────────
+LOG_FILE="./surfsense-migration.log"
+exec > >(tee -a "${LOG_FILE}") 2>&1
+
+# ── Output helpers ────────────────────────────────────────────────────────────
+info()    { printf "${CYAN}[SurfSense]${NC} %s\n"        "$1"; }
+success() { printf "${GREEN}[SurfSense]${NC} %s\n"       "$1"; }
+warn()    { printf "${YELLOW}[SurfSense]${NC} %s\n"      "$1"; }
+error()   { printf "${RED}[SurfSense]${NC} ERROR: %s\n"  "$1" >&2; exit 1; }
+step()    { printf "\n${BOLD}${CYAN}── Step %s: %s${NC}\n" "$1" "$2"; }
+
+# ── Constants ─────────────────────────────────────────────────────────────────
+OLD_VOLUME="surfsense-data"
+TEMP_CONTAINER="surfsense-pg14-migration"
+DUMP_FILE="./surfsense_migration_backup.sql"
+KEY_FILE="./surfsense_migration_secret.key"
+PG14_IMAGE="pgvector/pgvector:pg14"
+
+# ── Defaults ──────────────────────────────────────────────────────────────────
+OLD_DB_USER="surfsense"
+OLD_DB_PASSWORD="surfsense"
+OLD_DB_NAME="surfsense"
+AUTO_YES=false
+
+# ── Argument parsing ──────────────────────────────────────────────────────────
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --db-user)      OLD_DB_USER="$2";     shift 2 ;;
+        --db-password)  OLD_DB_PASSWORD="$2"; shift 2 ;;
+        --db-name)      OLD_DB_NAME="$2";     shift 2 ;;
+        --yes|-y)       AUTO_YES=true;        shift   ;;
+        --help|-h)
+            grep '^#' "$0" | grep -v '^#!/' | sed 's/^# \{0,1\}//'
+            exit 0
+            ;;
+        *) error "Unknown option: $1  — run with --help for usage." ;;
+    esac
+done
+
+# ── Confirmation helper ───────────────────────────────────────────────────────
+confirm() {
+    if $AUTO_YES; then return 0; fi
+    printf "${YELLOW}[SurfSense]${NC} %s [y/N] " "$1"
+    read -r reply
+    [[ "$reply" =~ ^[Yy]$ ]] || { warn "Aborted."; exit 0; }
+}
+
+# ── Cleanup trap — always remove the temp container ──────────────────────────
+cleanup() {
+    local exit_code=$?
+    if docker ps -a --format '{{.Names}}' 2>/dev/null < /dev/null | grep -q "^${TEMP_CONTAINER}$"; then
+        info "Cleaning up temporary container '${TEMP_CONTAINER}'..."
+        docker stop "${TEMP_CONTAINER}" >/dev/null 2>&1 < /dev/null || true
+        docker rm   "${TEMP_CONTAINER}" >/dev/null 2>&1 < /dev/null || true
+    fi
+    if [[ $exit_code -ne 0 ]]; then
+        printf "\n${RED}[SurfSense]${NC} Migration data extraction failed (exit code %s).\n" "${exit_code}" >&2
+        printf "${RED}[SurfSense]${NC} Full log: %s\n" "${LOG_FILE}" >&2
+        printf "${YELLOW}[SurfSense]${NC} Your original data in '${OLD_VOLUME}' is untouched.\n" >&2
+    fi
+}
+trap cleanup EXIT
+
+# ── Wait-for-postgres helper ──────────────────────────────────────────────────
+wait_for_pg() {
+    local container="$1"
+    local user="$2"
+    local label="${3:-PostgreSQL}"
+    local max_attempts=45
+    local attempt=0
+
+    info "Waiting for ${label} to accept connections..."
+    until docker exec "${container}" pg_isready -U "${user}" -q 2>/dev/null < /dev/null; do
+        attempt=$((attempt + 1))
+        if [[ $attempt -ge $max_attempts ]]; then
+            error "${label} did not become ready after $((max_attempts * 2)) seconds. Check: docker logs ${container}"
+        fi
+        printf "."
+        sleep 2
+    done
+    printf "\n"
+    success "${label} is ready."
+}
+
+info "Migrating data from legacy database (PostgreSQL 14 → 17)"
+
+# ── Step 0: Pre-flight checks ─────────────────────────────────────────────────
+step "0" "Pre-flight checks"
+
+# Docker CLI
+command -v docker >/dev/null 2>&1 \
+    || error "Docker is not installed. Install it at: https://docs.docker.com/get-docker/"
+
+# Docker daemon
+docker info >/dev/null 2>&1 < /dev/null \
+    || error "Docker daemon is not running. Please start Docker and try again."
+
+# Old volume must exist
+docker volume ls --format '{{.Name}}' < /dev/null | grep -q "^${OLD_VOLUME}$" \
+    || error "Legacy volume '${OLD_VOLUME}' not found.\n       Are you sure you ran the old all-in-one SurfSense container?"
+success "Found legacy volume: ${OLD_VOLUME}"
+
+# Detect and stop any container currently using the old volume
+# (mounting a live PG volume into a second container causes the new container's
+#  entrypoint to chown the data files, breaking the running container's access)
+OLD_CONTAINER=$(docker ps --filter "volume=${OLD_VOLUME}" --format '{{.Names}}' < /dev/null | head -n1 || true)
+if [[ -n "${OLD_CONTAINER}" ]]; then
+    warn "Container '${OLD_CONTAINER}' is running and using the '${OLD_VOLUME}' volume."
+    warn "It must be stopped before migration to prevent data file corruption."
+    confirm "Stop '${OLD_CONTAINER}' now and proceed with data extraction?"
+    docker stop "${OLD_CONTAINER}" >/dev/null 2>&1 < /dev/null \
+        || error "Failed to stop '${OLD_CONTAINER}'. Try: docker stop ${OLD_CONTAINER}"
+    success "Container '${OLD_CONTAINER}' stopped."
+fi
+
+# Bail out if a dump already exists — don't overwrite a previous successful run
+if [[ -f "${DUMP_FILE}" ]]; then
+    warn "Dump file '${DUMP_FILE}' already exists."
+    warn "If a previous extraction succeeded, just run install.sh now."
+    warn "To re-extract, remove the file first: rm ${DUMP_FILE}"
+    error "Aborting to avoid overwriting an existing dump."
+fi
+
+# Clean up any stale temp container from a previous failed run
+if docker ps -a --format '{{.Names}}' < /dev/null | grep -q "^${TEMP_CONTAINER}$"; then
+    warn "Stale migration container '${TEMP_CONTAINER}' found — removing it."
+    docker stop "${TEMP_CONTAINER}" >/dev/null 2>&1 < /dev/null || true
+    docker rm   "${TEMP_CONTAINER}" >/dev/null 2>&1 < /dev/null || true
+fi
+
+# Disk space (warn if < 500 MB free)
+if command -v df >/dev/null 2>&1; then
+    FREE_KB=$(df -k . | awk 'NR==2 {print $4}')
+    FREE_MB=$(( FREE_KB / 1024 ))
+    if [[ $FREE_MB -lt 500 ]]; then
+        warn "Low disk space: ${FREE_MB} MB free. At least 500 MB recommended for the dump."
+        confirm "Continue anyway?"
+    else
+        success "Disk space: ${FREE_MB} MB free."
+    fi
+fi
+
+success "All pre-flight checks passed."
+
+# ── Confirmation prompt ───────────────────────────────────────────────────────
+printf "\n${BOLD}Extraction plan:${NC}\n"
+printf "  Source volume   : ${YELLOW}%s${NC}  (PG14 data at /data/postgres)\n" "${OLD_VOLUME}"
+printf "  Old credentials : user=${YELLOW}%s${NC}  db=${YELLOW}%s${NC}\n" "${OLD_DB_USER}" "${OLD_DB_NAME}"
+printf "  Dump saved to   : ${YELLOW}%s${NC}\n" "${DUMP_FILE}"
+printf "  SECRET_KEY to   : ${YELLOW}%s${NC}\n" "${KEY_FILE}"
+printf "  Log file        : ${YELLOW}%s${NC}\n\n" "${LOG_FILE}"
+confirm "Start data extraction? (Your original data will not be deleted or modified.)"
+
+# ── Step 1: Start temporary PostgreSQL 14 container ──────────────────────────
+step "1" "Starting temporary PostgreSQL 14 container"
+
+info "Pulling ${PG14_IMAGE}..."
+docker pull "${PG14_IMAGE}" >/dev/null 2>&1 < /dev/null \
+    || warn "Could not pull ${PG14_IMAGE} — using cached image if available."
+
+# Detect the UID that owns the existing data files and run the temp container
+# as that user. This prevents the official postgres image entrypoint from
+# running as root and doing `chown -R postgres /data/postgres`, which would
+# re-own the files to UID 999 and break any subsequent access by the original
+# container's postgres process (which may run as a different UID).
+DATA_UID=$(docker run --rm -v "${OLD_VOLUME}:/data" alpine \
+    stat -c '%u' /data/postgres 2>/dev/null < /dev/null || echo "")
+if [[ -z "${DATA_UID}" || "${DATA_UID}" == "0" ]]; then
+    warn "Could not detect data directory UID — falling back to default (may chown files)."
+    USER_FLAG=""
+else
+    info "Data directory owned by UID ${DATA_UID} — starting temp container as that user."
+    USER_FLAG="--user ${DATA_UID}"
+fi
+
+docker run -d \
+    --name "${TEMP_CONTAINER}" \
+    -v "${OLD_VOLUME}:/data" \
+    -e PGDATA=/data/postgres \
+    -e POSTGRES_USER="${OLD_DB_USER}" \
+    -e POSTGRES_PASSWORD="${OLD_DB_PASSWORD}" \
+    -e POSTGRES_DB="${OLD_DB_NAME}" \
+    ${USER_FLAG} \
+    "${PG14_IMAGE}" >/dev/null < /dev/null
+
+success "Temporary container '${TEMP_CONTAINER}' started."
+wait_for_pg "${TEMP_CONTAINER}" "${OLD_DB_USER}" "PostgreSQL 14"
+
+# ── Step 2: Dump the database ─────────────────────────────────────────────────
+step "2" "Dumping PostgreSQL 14 database"
+
+info "Running pg_dump — this may take a while for large databases..."
+
+if ! docker exec \
+        -e PGPASSWORD="${OLD_DB_PASSWORD}" \
+        "${TEMP_CONTAINER}" \
+        pg_dump -U "${OLD_DB_USER}" --no-password "${OLD_DB_NAME}" \
+        > "${DUMP_FILE}" 2>/tmp/surfsense_pgdump_err < /dev/null; then
+    cat /tmp/surfsense_pgdump_err >&2
+    error "pg_dump failed. See above for details."
+fi
+
+# Validate: non-empty file
+[[ -s "${DUMP_FILE}" ]] \
+    || error "Dump file '${DUMP_FILE}' is empty. Something went wrong with pg_dump."
+
+# Validate: looks like a real PG dump
+grep -q "PostgreSQL database dump" "${DUMP_FILE}" \
+    || error "Dump file does not contain a valid PostgreSQL dump header — the file may be corrupt."
+
+# Validate: sanity-check line count
+DUMP_LINES=$(wc -l < "${DUMP_FILE}" | tr -d ' ')
+[[ $DUMP_LINES -ge 10 ]] \
+    || error "Dump has only ${DUMP_LINES} lines — suspiciously small. Aborting."
+
+DUMP_SIZE=$(du -sh "${DUMP_FILE}" 2>/dev/null | cut -f1)
+success "Dump complete: ${DUMP_SIZE} (${DUMP_LINES} lines) → ${DUMP_FILE}"
+
+# Stop the temp container (trap will also handle it on unexpected exit)
+info "Stopping temporary PostgreSQL 14 container..."
+docker stop "${TEMP_CONTAINER}" >/dev/null 2>&1 < /dev/null || true
+docker rm   "${TEMP_CONTAINER}" >/dev/null 2>&1 < /dev/null || true
+success "Temporary container removed."
+
+# ── Step 3: Recover SECRET_KEY ────────────────────────────────────────────────
+step "3" "Recovering SECRET_KEY"
+
+RECOVERED_KEY=""
+
+if docker run --rm -v "${OLD_VOLUME}:/data" alpine \
+        sh -c 'test -f /data/.secret_key && cat /data/.secret_key' \
+        2>/dev/null < /dev/null | grep -q .; then
+    RECOVERED_KEY=$(
+        docker run --rm -v "${OLD_VOLUME}:/data" alpine \
+            cat /data/.secret_key 2>/dev/null < /dev/null | tr -d '[:space:]'
+    )
+    success "Recovered SECRET_KEY from '${OLD_VOLUME}'."
+else
+    warn "No SECRET_KEY file found at /data/.secret_key in '${OLD_VOLUME}'."
+    warn "This means the all-in-one container was launched with SECRET_KEY set as an explicit env var."
+    if $AUTO_YES; then
+        # Non-interactive (called from install.sh) — auto-generate rather than hanging on read
+        RECOVERED_KEY=$(openssl rand -base64 32 2>/dev/null \
+            || head -c 32 /dev/urandom | base64 | tr -d '\n')
+        warn "Non-interactive mode: generated a new SECRET_KEY automatically."
+        warn "All active browser sessions will be logged out after migration."
+        warn "To restore your original key, update SECRET_KEY in ./surfsense/.env afterwards."
+    else
+        printf "${YELLOW}[SurfSense]${NC} Enter the SECRET_KEY from your old container's environment\n"
+        printf "${YELLOW}[SurfSense]${NC} (press Enter to generate a new one — existing sessions will be invalidated): "
+        read -r RECOVERED_KEY
+        if [[ -z "${RECOVERED_KEY}" ]]; then
+            RECOVERED_KEY=$(openssl rand -base64 32 2>/dev/null \
+                || head -c 32 /dev/urandom | base64 | tr -d '\n')
+            warn "Generated a new SECRET_KEY. All active browser sessions will be logged out after migration."
+        fi
+    fi
+fi
+
+# Save SECRET_KEY to a file for install.sh to pick up
+printf '%s' "${RECOVERED_KEY}" > "${KEY_FILE}"
+success "SECRET_KEY saved to ${KEY_FILE}"
+
+# ── Done ──────────────────────────────────────────────────────────────────────
+printf "\n${GREEN}${BOLD}"
+printf "══════════════════════════════════════════════════════════════\n"
+printf "  Data extraction complete!\n"
+printf "══════════════════════════════════════════════════════════════\n"
+printf "${NC}\n"
+
+success "Dump file : ${DUMP_FILE}  (${DUMP_SIZE})"
+success "Secret key: ${KEY_FILE}"
+printf "\n"
+info "Next step — run install.sh from this same directory:"
+printf "\n"
+printf "${CYAN}  curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash${NC}\n"
+printf "\n"
+info "install.sh will detect the dump, restore your data into PostgreSQL 17,"
+info "and start the full SurfSense stack automatically."
+printf "\n"
+warn "Keep both files until you have verified the migration:"
+warn "  ${DUMP_FILE}"
+warn "  ${KEY_FILE}"
+warn "Full log saved to: ${LOG_FILE}"
+printf "\n"

scripts/docker/entrypoint-allinone.sh

@@ -1,244 +0,0 @@
-#!/bin/bash
-set -e
-
-echo "==========================================="
-echo "  🏄 SurfSense All-in-One Container"
-echo "==========================================="
-
-# Create log directory
-mkdir -p /var/log/supervisor
-
-# ================================================
-# Ensure data directory exists
-# ================================================
-mkdir -p /data
-
-# ================================================
-# Generate SECRET_KEY if not provided
-# ================================================
-if [ -z "$SECRET_KEY" ]; then
-    # Generate a random secret key and persist it
-    if [ -f /data/.secret_key ]; then
-        export SECRET_KEY=$(cat /data/.secret_key)
-        echo "✅ Using existing SECRET_KEY from persistent storage"
-    else
-        export SECRET_KEY=$(python3 -c "import secrets; print(secrets.token_urlsafe(32))")
-        echo "$SECRET_KEY" > /data/.secret_key
-        chmod 600 /data/.secret_key
-        echo "✅ Generated new SECRET_KEY (saved for persistence)"
-    fi
-fi
-
-# ================================================
-# Set default TTS/STT services if not provided
-# ================================================
-if [ -z "$TTS_SERVICE" ]; then
-    export TTS_SERVICE="local/kokoro"
-    echo "✅ Using default TTS_SERVICE: local/kokoro"
-fi
-
-if [ -z "$STT_SERVICE" ]; then
-    export STT_SERVICE="local/base"
-    echo "✅ Using default STT_SERVICE: local/base"
-fi
-
-# ================================================
-# Set Electric SQL configuration
-# ================================================
-export ELECTRIC_DB_USER="${ELECTRIC_DB_USER:-electric}"
-export ELECTRIC_DB_PASSWORD="${ELECTRIC_DB_PASSWORD:-electric_password}"
-if [ -z "$ELECTRIC_DATABASE_URL" ]; then
-    export ELECTRIC_DATABASE_URL="postgresql://${ELECTRIC_DB_USER}:${ELECTRIC_DB_PASSWORD}@localhost:5432/${POSTGRES_DB:-surfsense}?sslmode=disable"
-    echo "✅ Electric SQL URL configured dynamically"
-else
-    # Ensure sslmode=disable is in the URL if not already present
-    if [[ "$ELECTRIC_DATABASE_URL" != *"sslmode="* ]]; then
-        # Add sslmode=disable (handle both cases: with or without existing query params)
-        if [[ "$ELECTRIC_DATABASE_URL" == *"?"* ]]; then
-            export ELECTRIC_DATABASE_URL="${ELECTRIC_DATABASE_URL}&sslmode=disable"
-        else
-            export ELECTRIC_DATABASE_URL="${ELECTRIC_DATABASE_URL}?sslmode=disable"
-        fi
-    fi
-    echo "✅ Electric SQL URL configured from environment"
-fi
-
-# Set Electric SQL port
-export ELECTRIC_PORT="${ELECTRIC_PORT:-5133}"
-export PORT="${ELECTRIC_PORT}"
-
-# ================================================
-# Initialize PostgreSQL if needed
-# ================================================
-if [ ! -f /data/postgres/PG_VERSION ]; then
-    echo "📦 Initializing PostgreSQL database..."
-    
-    # Initialize PostgreSQL data directory
-    chown -R postgres:postgres /data/postgres
-    chmod 700 /data/postgres
-    
-    # Initialize with UTF8 encoding (required for proper text handling)
-    su - postgres -c "/usr/lib/postgresql/14/bin/initdb -D /data/postgres --encoding=UTF8 --locale=C.UTF-8"
-    
-    # Configure PostgreSQL for connections
-    echo "host all all 0.0.0.0/0 md5" >> /data/postgres/pg_hba.conf
-    echo "local all all trust" >> /data/postgres/pg_hba.conf
-    echo "listen_addresses='*'" >> /data/postgres/postgresql.conf
-    
-    # Enable logical replication for Electric SQL
-    echo "wal_level = logical" >> /data/postgres/postgresql.conf
-    echo "max_replication_slots = 10" >> /data/postgres/postgresql.conf
-    echo "max_wal_senders = 10" >> /data/postgres/postgresql.conf
-    
-    # Start PostgreSQL temporarily to create database and user
-    su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres -l /tmp/postgres_init.log start"
-    
-    # Wait for PostgreSQL to be ready
-    sleep 5
-    
-    # Create user and database
-    su - postgres -c "psql -c \"CREATE USER ${POSTGRES_USER:-surfsense} WITH PASSWORD '${POSTGRES_PASSWORD:-surfsense}' SUPERUSER;\""
-    su - postgres -c "psql -c \"CREATE DATABASE ${POSTGRES_DB:-surfsense} OWNER ${POSTGRES_USER:-surfsense};\""
-    
-    # Enable pgvector extension
-    su - postgres -c "psql -d ${POSTGRES_DB:-surfsense} -c 'CREATE EXTENSION IF NOT EXISTS vector;'"
-    
-    # Create Electric SQL replication user (idempotent - uses IF NOT EXISTS)
-    echo "📡 Creating Electric SQL replication user..."
-    su - postgres -c "psql -d ${POSTGRES_DB:-surfsense} <<-EOSQL
-        DO \\\$\\\$
-        BEGIN
-            IF NOT EXISTS (SELECT FROM pg_user WHERE usename = '${ELECTRIC_DB_USER}') THEN
-                CREATE USER ${ELECTRIC_DB_USER} WITH REPLICATION PASSWORD '${ELECTRIC_DB_PASSWORD}';
-            END IF;
-        END
-        \\\$\\\$;
-
-        GRANT CONNECT ON DATABASE ${POSTGRES_DB:-surfsense} TO ${ELECTRIC_DB_USER};
-        GRANT USAGE ON SCHEMA public TO ${ELECTRIC_DB_USER};
-        GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${ELECTRIC_DB_USER};
-        GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO ${ELECTRIC_DB_USER};
-        ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ${ELECTRIC_DB_USER};
-        ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO ${ELECTRIC_DB_USER};
-
-        -- Create the publication for Electric SQL (if not exists)
-        DO \\\$\\\$
-        BEGIN
-            IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN
-                CREATE PUBLICATION electric_publication_default;
-            END IF;
-        END
-        \\\$\\\$;
-EOSQL"
-    echo "✅ Electric SQL user '${ELECTRIC_DB_USER}' created"
-    
-    # Stop temporary PostgreSQL
-    su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres stop"
-    
-    echo "✅ PostgreSQL initialized successfully"
-else
-    echo "✅ PostgreSQL data directory already exists"
-fi
-
-# ================================================
-# Initialize Redis data directory
-# ================================================
-mkdir -p /data/redis
-chmod 755 /data/redis
-echo "✅ Redis data directory ready"
-
-# ================================================
-# Copy frontend build to runtime location
-# ================================================
-if [ -d /app/frontend/.next/standalone ]; then
-    cp -r /app/frontend/.next/standalone/* /app/frontend/ 2>/dev/null || true
-    cp -r /app/frontend/.next/static /app/frontend/.next/static 2>/dev/null || true
-fi
-
-# ================================================
-# Runtime Environment Variable Replacement
-# ================================================
-# Next.js NEXT_PUBLIC_* vars are baked in at build time.
-# This replaces placeholder values with actual runtime env vars.
-echo "🔧 Applying runtime environment configuration..."
-
-# Set defaults if not provided
-NEXT_PUBLIC_FASTAPI_BACKEND_URL="${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:8000}"
-NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE="${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:-LOCAL}"
-NEXT_PUBLIC_ETL_SERVICE="${NEXT_PUBLIC_ETL_SERVICE:-DOCLING}"
-NEXT_PUBLIC_ELECTRIC_URL="${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}"
-NEXT_PUBLIC_ELECTRIC_AUTH_MODE="${NEXT_PUBLIC_ELECTRIC_AUTH_MODE:-insecure}"
-NEXT_PUBLIC_DEPLOYMENT_MODE="${NEXT_PUBLIC_DEPLOYMENT_MODE:-self-hosted}"
-
-# Replace placeholders in all JS files
-find /app/frontend -type f \( -name "*.js" -o -name "*.json" \) -exec sed -i \
-    -e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_URL__|${NEXT_PUBLIC_FASTAPI_BACKEND_URL}|g" \
-    -e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__|${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}|g" \
-    -e "s|__NEXT_PUBLIC_ETL_SERVICE__|${NEXT_PUBLIC_ETL_SERVICE}|g" \
-    -e "s|__NEXT_PUBLIC_ELECTRIC_URL__|${NEXT_PUBLIC_ELECTRIC_URL}|g" \
-    -e "s|__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__|${NEXT_PUBLIC_ELECTRIC_AUTH_MODE}|g" \
-    -e "s|__NEXT_PUBLIC_DEPLOYMENT_MODE__|${NEXT_PUBLIC_DEPLOYMENT_MODE}|g" \
-    {} +
-
-echo "✅ Environment configuration applied"
-echo "   Backend URL:      ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}"
-echo "   Auth Type:        ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}"
-echo "   ETL Service:      ${NEXT_PUBLIC_ETL_SERVICE}"
-echo "   Electric URL:     ${NEXT_PUBLIC_ELECTRIC_URL}"
-echo "   Deployment Mode:  ${NEXT_PUBLIC_DEPLOYMENT_MODE}"
-
-# ================================================
-# Run database migrations
-# ================================================
-run_migrations() {
-    echo "🔄 Running database migrations..."
-    
-    # Start PostgreSQL temporarily for migrations
-    su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres -l /tmp/postgres_migrate.log start"
-    sleep 5
-    
-    # Start Redis temporarily for migrations (some might need it)
-    redis-server --dir /data/redis --daemonize yes
-    sleep 2
-    
-    # Run alembic migrations
-    cd /app/backend
-    alembic upgrade head || echo "⚠️ Migrations may have already been applied"
-    
-    # Stop temporary services
-    redis-cli shutdown || true
-    su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres stop"
-    
-    echo "✅ Database migrations complete"
-}
-
-# Always run migrations on startup - alembic upgrade head is safe to run
-# every time. It only applies pending migrations (never re-runs applied ones,
-# never calls downgrade). This ensures updates are applied automatically.
-run_migrations
-
-# ================================================
-# Environment Variables Info
-# ================================================
-echo ""
-echo "==========================================="
-echo "  📋 Configuration"
-echo "==========================================="
-echo "  Frontend URL:    http://localhost:3000"
-echo "  Backend API:     ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}"
-echo "  API Docs:        ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}/docs"
-echo "  Electric URL:    ${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}"
-echo "  Auth Type:       ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}"
-echo "  ETL Service:     ${NEXT_PUBLIC_ETL_SERVICE}"
-echo "  TTS Service:     ${TTS_SERVICE}"
-echo "  STT Service:     ${STT_SERVICE}"
-echo "  Daytona Sandbox: ${DAYTONA_SANDBOX_ENABLED:-FALSE}"
-echo "==========================================="
-echo ""
-
-# ================================================
-# Start Supervisor (manages all services)
-# ================================================
-echo "🚀 Starting all services..."
-exec /usr/local/bin/supervisord -c /etc/supervisor/conf.d/surfsense.conf
-

scripts/docker/init-postgres.sh

@@ -1,77 +0,0 @@
-#!/bin/bash
-# PostgreSQL initialization script for SurfSense
-# This script is called during container startup if the database needs initialization
-
-set -e
-
-PGDATA=${PGDATA:-/data/postgres}
-POSTGRES_USER=${POSTGRES_USER:-surfsense}
-POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-surfsense}
-POSTGRES_DB=${POSTGRES_DB:-surfsense}
-
-# Electric SQL user credentials (configurable)
-ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric}
-ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
-
-echo "Initializing PostgreSQL..."
-
-# Check if PostgreSQL is already initialized
-if [ -f "$PGDATA/PG_VERSION" ]; then
-    echo "PostgreSQL data directory already exists. Skipping initialization."
-    exit 0
-fi
-
-# Initialize the database cluster
-/usr/lib/postgresql/14/bin/initdb -D "$PGDATA" --username=postgres
-
-# Configure PostgreSQL
-cat >> "$PGDATA/postgresql.conf" << EOF
-listen_addresses = '*'
-max_connections = 200
-shared_buffers = 256MB
-
-# Enable logical replication (required for Electric SQL)
-wal_level = logical
-max_replication_slots = 10
-max_wal_senders = 10
-
-# Performance settings
-checkpoint_timeout = 10min
-max_wal_size = 1GB
-min_wal_size = 80MB
-EOF
-
-cat >> "$PGDATA/pg_hba.conf" << EOF
-# Allow connections from anywhere with password
-host all all 0.0.0.0/0 md5
-host all all ::0/0 md5
-EOF
-
-# Start PostgreSQL temporarily
-/usr/lib/postgresql/14/bin/pg_ctl -D "$PGDATA" -l /tmp/postgres_init.log start
-
-# Wait for PostgreSQL to start
-sleep 3
-
-# Create user and database
-psql -U postgres << EOF
-CREATE USER $POSTGRES_USER WITH PASSWORD '$POSTGRES_PASSWORD' SUPERUSER;
-CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER;
-\c $POSTGRES_DB
-CREATE EXTENSION IF NOT EXISTS vector;
-
--- Create Electric SQL replication user
-CREATE USER $ELECTRIC_DB_USER WITH REPLICATION PASSWORD '$ELECTRIC_DB_PASSWORD';
-GRANT CONNECT ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER;
-GRANT USAGE ON SCHEMA public TO $ELECTRIC_DB_USER;
-GRANT SELECT ON ALL TABLES IN SCHEMA public TO $ELECTRIC_DB_USER;
-GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO $ELECTRIC_DB_USER;
-ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $ELECTRIC_DB_USER;
-ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO $ELECTRIC_DB_USER;
-EOF
-
-echo "PostgreSQL initialized successfully."
-
-# Stop PostgreSQL (supervisor will start it)
-/usr/lib/postgresql/14/bin/pg_ctl -D "$PGDATA" stop
-

scripts/docker/supervisor-allinone.conf

@@ -1,121 +0,0 @@
-[supervisord]
-nodaemon=true
-logfile=/dev/stdout
-logfile_maxbytes=0
-pidfile=/var/run/supervisord.pid
-loglevel=info
-user=root
-
-[unix_http_server]
-file=/var/run/supervisor.sock
-chmod=0700
-
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[supervisorctl]
-serverurl=unix:///var/run/supervisor.sock
-
-# PostgreSQL
-[program:postgresql]
-command=/usr/lib/postgresql/14/bin/postgres -D /data/postgres
-user=postgres
-autostart=true
-autorestart=true
-priority=10
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-environment=PGDATA="/data/postgres"
-
-# Redis
-[program:redis]
-command=/usr/bin/redis-server --dir /data/redis --appendonly yes
-autostart=true
-autorestart=true
-priority=20
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-
-# Backend API
-[program:backend]
-command=python main.py
-directory=/app/backend
-autostart=true
-autorestart=true
-priority=30
-startsecs=10
-startretries=3
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-environment=PYTHONPATH="/app/backend",UVICORN_LOOP="asyncio",UNSTRUCTURED_HAS_PATCHED_LOOP="1"
-
-# Celery Worker
-[program:celery-worker]
-command=celery -A app.celery_app worker --loglevel=info --concurrency=2 --pool=solo --queues=surfsense,surfsense.connectors
-directory=/app/backend
-autostart=true
-autorestart=true
-priority=40
-startsecs=15
-startretries=3
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-environment=PYTHONPATH="/app/backend"
-
-# Celery Beat (scheduler)
-[program:celery-beat]
-command=celery -A app.celery_app beat --loglevel=info
-directory=/app/backend
-autostart=true
-autorestart=true
-priority=50
-startsecs=20
-startretries=3
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-environment=PYTHONPATH="/app/backend"
-
-# Electric SQL (real-time sync)
-[program:electric]
-command=/app/electric-release/bin/entrypoint start
-autostart=true
-autorestart=true
-priority=25
-startsecs=10
-startretries=3
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-environment=DATABASE_URL="%(ENV_ELECTRIC_DATABASE_URL)s",ELECTRIC_INSECURE="%(ENV_ELECTRIC_INSECURE)s",ELECTRIC_WRITE_TO_PG_MODE="%(ENV_ELECTRIC_WRITE_TO_PG_MODE)s",RELEASE_COOKIE="surfsense_electric_cookie",PORT="%(ENV_ELECTRIC_PORT)s"
-
-# Frontend
-[program:frontend]
-command=node server.js
-directory=/app/frontend
-autostart=true
-autorestart=true
-priority=60
-startsecs=5
-startretries=3
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-environment=NODE_ENV="production",PORT="3000",HOSTNAME="0.0.0.0"
-
-# Process Groups
-[group:surfsense]
-programs=postgresql,redis,electric,backend,celery-worker,celery-beat,frontend
-priority=999
-

surfsense_backend/alembic/versions/66_add_notifications_table_and_electric_replication.py

@@ -8,7 +8,7 @@ Creates notifications table and sets up Electric SQL replication
 search_source_connectors, and documents tables.
 
 NOTE: Electric SQL user creation is idempotent (uses IF NOT EXISTS).
-- Docker deployments: user is pre-created by scripts/docker/init-electric-user.sh
+- Docker deployments: user is pre-created by docker/scripts/init-electric-user.sh
 - Local PostgreSQL: user is created here during migration
 Both approaches are safe to run together without conflicts as this migraiton is idempotent
 """

surfsense_backend/scripts/docker/entrypoint.sh

@@ -53,7 +53,7 @@ run_migrations() {
         sleep 1
     done
 
-    if timeout 60 alembic upgrade head 2>&1; then
+    if timeout 300 alembic upgrade head 2>&1; then
         echo "Migrations completed successfully."
     else
         echo "WARNING: Migration failed or timed out. Continuing anyway..."

surfsense_web/Dockerfile

@@ -29,15 +29,22 @@ WORKDIR /app
 # Enable pnpm
 RUN corepack enable pnpm
 
-# Accept build arguments for Next.js public env vars
-ARG NEXT_PUBLIC_FASTAPI_BACKEND_URL
-ARG NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE
-ARG NEXT_PUBLIC_ETL_SERVICE
+# Build with placeholder values for NEXT_PUBLIC_* variables.
+# These are replaced at container startup by docker-entrypoint.js
+# with real values from the container's environment variables.
+ARG NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__
+ARG NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__
+ARG NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__
+ARG NEXT_PUBLIC_ELECTRIC_URL=__NEXT_PUBLIC_ELECTRIC_URL__
+ARG NEXT_PUBLIC_ELECTRIC_AUTH_MODE=__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__
+ARG NEXT_PUBLIC_DEPLOYMENT_MODE=__NEXT_PUBLIC_DEPLOYMENT_MODE__
 
-# Set them as environment variables for the build
 ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=$NEXT_PUBLIC_FASTAPI_BACKEND_URL
 ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=$NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE
 ENV NEXT_PUBLIC_ETL_SERVICE=$NEXT_PUBLIC_ETL_SERVICE
+ENV NEXT_PUBLIC_ELECTRIC_URL=$NEXT_PUBLIC_ELECTRIC_URL
+ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=$NEXT_PUBLIC_ELECTRIC_AUTH_MODE
+ENV NEXT_PUBLIC_DEPLOYMENT_MODE=$NEXT_PUBLIC_DEPLOYMENT_MODE
 
 COPY --from=deps /app/node_modules ./node_modules
 COPY . .
@@ -67,6 +74,10 @@ COPY --from=builder /app/public ./public
 COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
 COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
 
+# Entrypoint scripts for runtime env var substitution
+COPY --chown=nextjs:nodejs docker-entrypoint.js ./docker-entrypoint.js
+COPY --chown=nextjs:nodejs --chmod=755 docker-entrypoint.sh ./docker-entrypoint.sh
+
 USER nextjs
 
 EXPOSE 3000
@@ -76,4 +87,4 @@ ENV PORT=3000
 # server.js is created by next build from the standalone output
 # https://nextjs.org/docs/pages/api-reference/config/next-config-js/output
 ENV HOSTNAME="0.0.0.0"
-CMD ["node", "server.js"] 
+ENTRYPOINT ["/bin/sh", "./docker-entrypoint.sh"]

surfsense_web/content/docs/connectors/airtable.mdx

@@ -88,16 +88,16 @@ After saving, you'll find your OAuth credentials on the integration page:
 
 ## Running SurfSense with Airtable Connector
 
-Add the Airtable environment variables to your Docker run command:
+Add the Airtable credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 \
-  -v surfsense-data:/data \
-  # Airtable Connector
-  -e AIRTABLE_CLIENT_ID=your_airtable_client_id \
-  -e AIRTABLE_CLIENT_SECRET=your_airtable_client_secret \
-  -e AIRTABLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/airtable/connector/callback \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+AIRTABLE_CLIENT_ID=your_airtable_client_id
+AIRTABLE_CLIENT_SECRET=your_airtable_client_secret
+AIRTABLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/airtable/connector/callback
+```
+
+Then restart the services:
+
+```bash
+docker compose up -d
 ```

surfsense_web/content/docs/connectors/clickup.mdx

@@ -44,16 +44,16 @@ After creating the app, you'll see your credentials:
 
 ## Running SurfSense with ClickUp Connector
 
-Add the ClickUp environment variables to your Docker run command:
+Add the ClickUp credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 \
-  -v surfsense-data:/data \
-  # ClickUp Connector
-  -e CLICKUP_CLIENT_ID=your_clickup_client_id \
-  -e CLICKUP_CLIENT_SECRET=your_clickup_client_secret \
-  -e CLICKUP_REDIRECT_URI=http://localhost:8000/api/v1/auth/clickup/connector/callback \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+CLICKUP_CLIENT_ID=your_clickup_client_id
+CLICKUP_CLIENT_SECRET=your_clickup_client_secret
+CLICKUP_REDIRECT_URI=http://localhost:8000/api/v1/auth/clickup/connector/callback
+```
+
+Then restart the services:
+
+```bash
+docker compose up -d
 ```

surfsense_web/content/docs/connectors/confluence.mdx

@@ -97,16 +97,16 @@ Select the **"Granular scopes"** tab and enable:
 
 ## Running SurfSense with Confluence Connector
 
-Add the Atlassian environment variables to your Docker run command:
+Add the Atlassian credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 \
-  -v surfsense-data:/data \
-  # Confluence Connector
-  -e ATLASSIAN_CLIENT_ID=your_atlassian_client_id \
-  -e ATLASSIAN_CLIENT_SECRET=your_atlassian_client_secret \
-  -e CONFLUENCE_REDIRECT_URI=http://localhost:8000/api/v1/auth/confluence/connector/callback \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+ATLASSIAN_CLIENT_ID=your_atlassian_client_id
+ATLASSIAN_CLIENT_SECRET=your_atlassian_client_secret
+CONFLUENCE_REDIRECT_URI=http://localhost:8000/api/v1/auth/confluence/connector/callback
+```
+
+Then restart the services:
+
+```bash
+docker compose up -d
 ```

surfsense_web/content/docs/connectors/discord.mdx

@@ -64,17 +64,17 @@ You'll also see your **Application ID** and **Public Key** on this page.
 
 ## Running SurfSense with Discord Connector
 
-Add the Discord environment variables to your Docker run command:
+Add the Discord credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 \
-  -v surfsense-data:/data \
-  # Discord Connector
-  -e DISCORD_CLIENT_ID=your_discord_client_id \
-  -e DISCORD_CLIENT_SECRET=your_discord_client_secret \
-  -e DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callback \
-  -e DISCORD_BOT_TOKEN=http://localhost:8000/api/v1/auth/discord/connector/callback \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+DISCORD_CLIENT_ID=your_discord_client_id
+DISCORD_CLIENT_SECRET=your_discord_client_secret
+DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callback
+DISCORD_BOT_TOKEN=your_discord_bot_token
+```
+
+Then restart the services:
+
+```bash
+docker compose up -d
 ```

surfsense_web/content/docs/connectors/gmail.mdx

@@ -70,16 +70,16 @@ This guide walks you through setting up a Google OAuth 2.0 integration for SurfS
 
 ## Running SurfSense with Gmail Connector
 
-Add the Google OAuth environment variables to your Docker run command:
+Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 \
-  -v surfsense-data:/data \
-  # Gmail Connector
-  -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \
-  -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \
-  -e GOOGLE_GMAIL_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/gmail/connector/callback \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+GOOGLE_OAUTH_CLIENT_ID=your_google_client_id
+GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret
+GOOGLE_GMAIL_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/gmail/connector/callback
+```
+
+Then restart the services:
+
+```bash
+docker compose up -d
 ```

surfsense_web/content/docs/connectors/google-calendar.mdx

@@ -69,16 +69,16 @@ This guide walks you through setting up a Google OAuth 2.0 integration for SurfS
 
 ## Running SurfSense with Google Calendar Connector
 
-Add the Google OAuth environment variables to your Docker run command:
+Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 \
-  -v surfsense-data:/data \
-  # Google Calendar Connector
-  -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \
-  -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \
-  -e GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+GOOGLE_OAUTH_CLIENT_ID=your_google_client_id
+GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret
+GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback
+```
+
+Then restart the services:
+
+```bash
+docker compose up -d
 ```

surfsense_web/content/docs/connectors/google-drive.mdx

@@ -70,16 +70,16 @@ This guide walks you through setting up a Google OAuth 2.0 integration for SurfS
 
 ## Running SurfSense with Google Drive Connector
 
-Add the Google OAuth environment variables to your Docker run command:
+Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 \
-  -v surfsense-data:/data \
-  # Google Drive Connector
-  -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \
-  -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \
-  -e GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/drive/connector/callback \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+GOOGLE_OAUTH_CLIENT_ID=your_google_client_id
+GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret
+GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/drive/connector/callback
+```
+
+Then restart the services:
+
+```bash
+docker compose up -d
 ```

surfsense_web/content/docs/connectors/jira.mdx

@@ -84,16 +84,16 @@ This guide walks you through setting up an Atlassian OAuth 2.0 (3LO) integration
 
 ## Running SurfSense with Jira Connector
 
-Add the Atlassian environment variables to your Docker run command:
+Add the Atlassian credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 \
-  -v surfsense-data:/data \
-  # Jira Connector
-  -e ATLASSIAN_CLIENT_ID=your_atlassian_client_id \
-  -e ATLASSIAN_CLIENT_SECRET=your_atlassian_client_secret \
-  -e JIRA_REDIRECT_URI=http://localhost:8000/api/v1/auth/jira/connector/callback \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+ATLASSIAN_CLIENT_ID=your_atlassian_client_id
+ATLASSIAN_CLIENT_SECRET=your_atlassian_client_secret
+JIRA_REDIRECT_URI=http://localhost:8000/api/v1/auth/jira/connector/callback
+```
+
+Then restart the services:
+
+```bash
+docker compose up -d
 ```

surfsense_web/content/docs/connectors/linear.mdx

@@ -53,17 +53,17 @@ After creating the application, you'll see your OAuth credentials:
 
 ## Running SurfSense with Linear Connector
 
-Add the Linear environment variables to your Docker run command:
+Add the Linear credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 \
-  -v surfsense-data:/data \
-  # Linear Connector
-  -e LINEAR_CLIENT_ID=your_linear_client_id \
-  -e LINEAR_CLIENT_SECRET=your_linear_client_secret \
-  -e LINEAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/linear/connector/callback \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+LINEAR_CLIENT_ID=your_linear_client_id
+LINEAR_CLIENT_SECRET=your_linear_client_secret
+LINEAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/linear/connector/callback
+```
+
+Then restart the services:
+
+```bash
+docker compose up -d
 ```
 

surfsense_web/content/docs/connectors/microsoft-teams.mdx

@@ -90,16 +90,16 @@ After registration, you'll be taken to the app's **Overview** page. Here you'll
 
 ## Running SurfSense with Microsoft Teams Connector
 
-Add the Microsoft Teams environment variables to your Docker run command:
+Add the Microsoft Teams credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 \
-  -v surfsense-data:/data \
-  # Microsoft Teams Connector
-  -e TEAMS_CLIENT_ID=your_microsoft_client_id \
-  -e TEAMS_CLIENT_SECRET=your_microsoft_client_secret \
-  -e TEAMS_REDIRECT_URI=http://localhost:8000/api/v1/auth/teams/connector/callback \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+TEAMS_CLIENT_ID=your_microsoft_client_id
+TEAMS_CLIENT_SECRET=your_microsoft_client_secret
+TEAMS_REDIRECT_URI=http://localhost:8000/api/v1/auth/teams/connector/callback
+```
+
+Then restart the services:
+
+```bash
+docker compose up -d
 ```

surfsense_web/content/docs/connectors/notion.mdx

@@ -91,16 +91,16 @@ For additional information:
 
 ## Running SurfSense with Notion Connector
 
-Add the Notion environment variables to your Docker run command:
+Add the Notion credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 \
-  -v surfsense-data:/data \
-  # Notion Connector
-  -e NOTION_OAUTH_CLIENT_ID=your_notion_client_id \
-  -e NOTION_OAUTH_CLIENT_SECRET=your_notion_client_secret \
-  -e NOTION_REDIRECT_URI=http://localhost:8000/api/v1/auth/notion/connector/callback \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+NOTION_OAUTH_CLIENT_ID=your_notion_client_id
+NOTION_OAUTH_CLIENT_SECRET=your_notion_client_secret
+NOTION_REDIRECT_URI=http://localhost:8000/api/v1/auth/notion/connector/callback
+```
+
+Then restart the services:
+
+```bash
+docker compose up -d
 ```

surfsense_web/content/docs/connectors/slack.mdx

@@ -80,16 +80,16 @@ Click **"Add an OAuth Scope"** to add each scope.
 
 ## Running SurfSense with Slack Connector
 
-Add the Slack environment variables to your Docker run command:
+Add the Slack credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 \
-  -v surfsense-data:/data \
-  # Slack Connector
-  -e SLACK_CLIENT_ID=your_slack_client_id \
-  -e SLACK_CLIENT_SECRET=your_slack_client_secret \
-  -e SLACK_REDIRECT_URI=https://localhost:8000/api/v1/auth/slack/connector/callback \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+SLACK_CLIENT_ID=your_slack_client_id
+SLACK_CLIENT_SECRET=your_slack_client_secret
+SLACK_REDIRECT_URI=http://localhost:8000/api/v1/auth/slack/connector/callback
+```
+
+Then restart the services:
+
+```bash
+docker compose up -d
 ```

surfsense_web/content/docs/docker-installation.mdx

@@ -4,511 +4,292 @@ description: Setting up SurfSense using Docker
 icon: Container
 ---
 
+This guide explains how to run SurfSense using Docker, with options ranging from a single-command install to a fully manual setup.
 
-This guide explains how to run SurfSense using Docker, with options ranging from quick single-command deployment to full production setups.
+## Quick Start
 
-## Quick Start with Docker 🐳
+### Option 1 — Install Script (recommended)
 
-Get SurfSense running in seconds with a single command:
+Downloads the compose files, generates a `SECRET_KEY`, starts all services, and sets up [Watchtower](https://github.com/nicholas-fedor/watchtower) for automatic daily updates.
 
 <Callout type="info">
-The all-in-one Docker image bundles PostgreSQL (with pgvector), Redis, and all SurfSense services. Perfect for quick evaluation and development.
+Windows users: install [WSL](https://learn.microsoft.com/en-us/windows/wsl/install) first and run the command below in the Ubuntu terminal.
 </Callout>
 
-<Callout type="warn">
-Make sure to include the `-v surfsense-data:/data` in your Docker command. This ensures your database and files are properly persisted.
-</Callout>
-
-### One-Line Installation
-
-**Linux/macOS:**
-
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \
-  -v surfsense-data:/data \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
 ```
 
-**Windows (PowerShell):**
+This creates a `./surfsense/` directory with `docker-compose.yml` and `.env`, then runs `docker compose up -d`.
 
-```powershell
-docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 `
-  -v surfsense-data:/data `
-  --name surfsense `
-  --restart unless-stopped `
-  ghcr.io/modsetter/surfsense:latest
-```
-
-> **Note:** A secure `SECRET_KEY` is automatically generated and persisted in the data volume on first run.
-
-### With Custom Configuration
-
-You can pass any [environment variable](/docs/manual-installation#backend-environment-variables) using `-e` flags:
+To skip Watchtower (e.g. in production where you manage updates yourself):
 
 ```bash
-docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \
-  -v surfsense-data:/data \
-  -e EMBEDDING_MODEL=openai://text-embedding-ada-002 \
-  -e OPENAI_API_KEY=your_openai_api_key \
-  -e AUTH_TYPE=GOOGLE \
-  -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \
-  -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \
-  -e ETL_SERVICE=LLAMACLOUD \
-  -e LLAMA_CLOUD_API_KEY=your_llama_cloud_key \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --no-watchtower
 ```
 
-<Callout type="info">
-- For Google OAuth, create credentials in the [Google Cloud Console](https://console.cloud.google.com/apis/credentials)
-- For Airtable connector, create an OAuth integration in the [Airtable Developer Hub](https://airtable.com/create/oauth)
-- If deploying behind a reverse proxy with HTTPS, add `-e BACKEND_URL=https://api.yourdomain.com`
-</Callout>
+To customise the check interval (default 24h), use `--watchtower-interval=SECONDS`.
 
-### Quick Start with Docker Compose
-
-For easier management with environment files:
+### Option 2 — Manual Docker Compose
 
 ```bash
-# Download the quick start compose file
-curl -o docker-compose.yml https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker-compose.quickstart.yml
-
-# Create .env file (optional - for custom configuration)
-cat > .env << EOF
-# EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
-# ETL_SERVICE=DOCLING
-# SECRET_KEY=your_custom_secret_key  # Auto-generated if not set
-EOF
-
-# Start SurfSense
+git clone https://github.com/MODSetter/SurfSense.git
+cd SurfSense/docker
+cp .env.example .env
+# Edit .env — at minimum set SECRET_KEY
 docker compose up -d
 ```
 
 After starting, access SurfSense at:
+
 - **Frontend**: [http://localhost:3000](http://localhost:3000)
 - **Backend API**: [http://localhost:8000](http://localhost:8000)
 - **API Docs**: [http://localhost:8000/docs](http://localhost:8000/docs)
-- **Electric-SQL**: [http://localhost:5133](http://localhost:5133)
+- **Electric SQL**: [http://localhost:5133](http://localhost:5133)
 
-### Quick Start Environment Variables
+---
 
-| Variable | Description | Default |
-|----------|-------------|---------|
-| SECRET_KEY | JWT secret key (auto-generated if not set) | Auto-generated |
-| AUTH_TYPE | Authentication: `LOCAL` or `GOOGLE` | LOCAL |
-| EMBEDDING_MODEL | Model for embeddings | sentence-transformers/all-MiniLM-L6-v2 |
-| ETL_SERVICE | Document parser: `DOCLING`, `UNSTRUCTURED`, `LLAMACLOUD` | DOCLING |
-| TTS_SERVICE | Text-to-speech for podcasts | local/kokoro |
-| STT_SERVICE | Speech-to-text for audio (model size: tiny, base, small, medium, large) | local/base |
-| REGISTRATION_ENABLED | Allow new user registration | TRUE |
+## Updating
 
-### Useful Commands
+**Option 1 — Watchtower daemon (recommended, auto-updates every 24 h):**
+
+If you used the install script (Option 1 above), Watchtower is already running. No extra setup needed.
+
+For manual Docker Compose installs (Option 2), start Watchtower separately:
 
 ```bash
-# View logs
-docker logs -f surfsense
-
-# Stop SurfSense
-docker stop surfsense
-
-# Start SurfSense
-docker start surfsense
-
-# Remove container (data preserved in volume)
-docker rm surfsense
-
-# Remove container AND data
-docker rm surfsense && docker volume rm surfsense-data
-```
-
-### Updating
-
-To update SurfSense to the latest version, you can use either of the following methods:
-
-<Callout type="info">
-Your data is safe! The `surfsense-data` volume persists across updates, and database migrations are applied automatically on every startup.
-</Callout>
-
-**Option 1: Using Watchtower (one-time auto-update)**
-
-[Watchtower](https://github.com/nicholas-fedor/watchtower) can automatically pull the latest image, stop the old container, and restart it with the same options:
-
-```bash
-docker run --rm \
+docker run -d --name watchtower \
+  --restart unless-stopped \
   -v /var/run/docker.sock:/var/run/docker.sock \
   nickfedor/watchtower \
-  --run-once surfsense
+  --label-enable \
+  --interval 86400
+```
+
+**Option 2 — Watchtower one-time update:**
+
+```bash
+docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \
+  nickfedor/watchtower --run-once \
+  --label-filter "com.docker.compose.project=surfsense"
 ```
 
 <Callout type="warn">
-Use the `nickfedor/watchtower` fork. The original `containrrr/watchtower` is no longer maintained and may fail with newer Docker versions.
+Use `nickfedor/watchtower`. The original `containrrr/watchtower` is no longer maintained and may fail with newer Docker versions.
 </Callout>
 
-**Option 2: Manual Update**
+**Option 3 — Manual:**
 
 ```bash
-# Stop and remove the current container
-docker rm -f surfsense
-
-# Pull the latest image
-docker pull ghcr.io/modsetter/surfsense:latest
-
-# Start with the new image
-docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \
-  -v surfsense-data:/data \
-  --name surfsense \
-  --restart unless-stopped \
-  ghcr.io/modsetter/surfsense:latest
+cd surfsense   # or SurfSense/docker if you cloned manually
+docker compose pull && docker compose up -d
 ```
 
-If you used Docker Compose for the quick start, updating is simpler:
+Database migrations are applied automatically on every startup.
+
+---
+
+## Configuration
+
+All configuration lives in a single `docker/.env` file (or `surfsense/.env` if you used the install script). Copy `.env.example` to `.env` and edit the values you need.
+
+### Required
+
+| Variable | Description |
+|----------|-------------|
+| `SECRET_KEY` | JWT secret key. Generate with: `openssl rand -base64 32`. Auto-generated by the install script. |
+
+### Core Settings
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `SURFSENSE_VERSION` | Image tag to deploy. Use `latest`, a clean version (e.g. `0.0.14`), or a specific build (e.g. `0.0.14.1`) | `latest` |
+| `AUTH_TYPE` | Authentication method: `LOCAL` (email/password) or `GOOGLE` (OAuth) | `LOCAL` |
+| `ETL_SERVICE` | Document parsing: `DOCLING` (local), `UNSTRUCTURED`, or `LLAMACLOUD` | `DOCLING` |
+| `EMBEDDING_MODEL` | Embedding model for vector search | `sentence-transformers/all-MiniLM-L6-v2` |
+| `TTS_SERVICE` | Text-to-speech provider for podcasts | `local/kokoro` |
+| `STT_SERVICE` | Speech-to-text provider for audio files | `local/base` |
+| `REGISTRATION_ENABLED` | Allow new user registrations | `TRUE` |
+
+### Ports
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `FRONTEND_PORT` | Frontend service port | `3000` |
+| `BACKEND_PORT` | Backend API service port | `8000` |
+| `ELECTRIC_PORT` | Electric SQL service port | `5133` |
+
+### Custom Domain / Reverse Proxy
+
+Only set these if serving SurfSense on a real domain via a reverse proxy (Caddy, Nginx, Cloudflare Tunnel, etc.). Leave commented out for standard localhost deployments.
+
+| Variable | Description |
+|----------|-------------|
+| `NEXT_FRONTEND_URL` | Public frontend URL (e.g. `https://app.yourdomain.com`) |
+| `BACKEND_URL` | Public backend URL for OAuth callbacks (e.g. `https://api.yourdomain.com`) |
+| `NEXT_PUBLIC_FASTAPI_BACKEND_URL` | Backend URL used by the frontend (e.g. `https://api.yourdomain.com`) |
+| `NEXT_PUBLIC_ELECTRIC_URL` | Electric SQL URL used by the frontend (e.g. `https://electric.yourdomain.com`) |
+
+### Database
+
+Defaults work out of the box. Change for security in production.
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `DB_USER` | PostgreSQL username | `surfsense` |
+| `DB_PASSWORD` | PostgreSQL password | `surfsense` |
+| `DB_NAME` | PostgreSQL database name | `surfsense` |
+| `DB_HOST` | PostgreSQL host | `db` |
+| `DB_PORT` | PostgreSQL port | `5432` |
+| `DB_SSLMODE` | SSL mode: `disable`, `require`, `verify-ca`, `verify-full` | `disable` |
+| `DATABASE_URL` | Full connection URL override. Use for managed databases (RDS, Supabase, etc.) | *(built from above)* |
+
+### Electric SQL
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `ELECTRIC_DB_USER` | Replication user for Electric SQL | `electric` |
+| `ELECTRIC_DB_PASSWORD` | Replication password for Electric SQL | `electric_password` |
+| `ELECTRIC_DATABASE_URL` | Full connection URL override for Electric. Set to `host.docker.internal` when pointing at a local Postgres instance | *(built from above)* |
+
+### Authentication
+
+| Variable | Description |
+|----------|-------------|
+| `GOOGLE_OAUTH_CLIENT_ID` | Google OAuth client ID (required if `AUTH_TYPE=GOOGLE`) |
+| `GOOGLE_OAUTH_CLIENT_SECRET` | Google OAuth client secret (required if `AUTH_TYPE=GOOGLE`) |
+
+Create credentials at the [Google Cloud Console](https://console.cloud.google.com/apis/credentials).
+
+### External API Keys
+
+| Variable | Description |
+|----------|-------------|
+| `FIRECRAWL_API_KEY` | Firecrawl API key for web crawling |
+| `UNSTRUCTURED_API_KEY` | Unstructured.io API key (required if `ETL_SERVICE=UNSTRUCTURED`) |
+| `LLAMA_CLOUD_API_KEY` | LlamaCloud API key (required if `ETL_SERVICE=LLAMACLOUD`) |
+
+### Connector OAuth Keys
+
+Uncomment the connectors you want to use. Redirect URIs follow the pattern `http://localhost:8000/api/v1/auth/<connector>/connector/callback`.
+
+| Connector | Variables |
+|-----------|-----------|
+| Google Drive / Gmail / Calendar | `GOOGLE_DRIVE_REDIRECT_URI`, `GOOGLE_GMAIL_REDIRECT_URI`, `GOOGLE_CALENDAR_REDIRECT_URI` |
+| Notion | `NOTION_CLIENT_ID`, `NOTION_CLIENT_SECRET`, `NOTION_REDIRECT_URI` |
+| Slack | `SLACK_CLIENT_ID`, `SLACK_CLIENT_SECRET`, `SLACK_REDIRECT_URI` |
+| Discord | `DISCORD_CLIENT_ID`, `DISCORD_CLIENT_SECRET`, `DISCORD_BOT_TOKEN`, `DISCORD_REDIRECT_URI` |
+| Jira & Confluence | `ATLASSIAN_CLIENT_ID`, `ATLASSIAN_CLIENT_SECRET`, `JIRA_REDIRECT_URI`, `CONFLUENCE_REDIRECT_URI` |
+| Linear | `LINEAR_CLIENT_ID`, `LINEAR_CLIENT_SECRET`, `LINEAR_REDIRECT_URI` |
+| ClickUp | `CLICKUP_CLIENT_ID`, `CLICKUP_CLIENT_SECRET`, `CLICKUP_REDIRECT_URI` |
+| Airtable | `AIRTABLE_CLIENT_ID`, `AIRTABLE_CLIENT_SECRET`, `AIRTABLE_REDIRECT_URI` |
+| Microsoft Teams | `TEAMS_CLIENT_ID`, `TEAMS_CLIENT_SECRET`, `TEAMS_REDIRECT_URI` |
+
+For Airtable, create an OAuth integration at the [Airtable Developer Hub](https://airtable.com/create/oauth).
+
+### Observability (optional)
+
+| Variable | Description |
+|----------|-------------|
+| `LANGSMITH_TRACING` | Enable LangSmith tracing (`true` / `false`) |
+| `LANGSMITH_ENDPOINT` | LangSmith API endpoint |
+| `LANGSMITH_API_KEY` | LangSmith API key |
+| `LANGSMITH_PROJECT` | LangSmith project name |
+
+### Advanced (optional)
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `SCHEDULE_CHECKER_INTERVAL` | How often to check for scheduled connector tasks (e.g. `5m`, `1h`) | `5m` |
+| `RERANKERS_ENABLED` | Enable document reranking for improved search | `FALSE` |
+| `RERANKERS_MODEL_NAME` | Reranker model name (e.g. `ms-marco-MiniLM-L-12-v2`) | |
+| `RERANKERS_MODEL_TYPE` | Reranker model type (e.g. `flashrank`) | |
+| `PAGES_LIMIT` | Max pages per user for ETL services | unlimited |
+
+---
+
+## Docker Services
+
+| Service | Description |
+|---------|-------------|
+| `db` | PostgreSQL with pgvector extension |
+| `redis` | Message broker for Celery |
+| `backend` | FastAPI application server |
+| `celery_worker` | Background task processing (document indexing, etc.) |
+| `celery_beat` | Periodic task scheduler (connector sync) |
+| `electric` | Electric SQL — real-time sync for the frontend |
+| `frontend` | Next.js web application |
+
+All services start automatically with `docker compose up -d`.
+
+The backend includes a health check — dependent services (workers, frontend) wait until the API is fully ready before starting. You can monitor startup progress with `docker compose ps` (look for `(health: starting)` → `(healthy)`).
+
+---
+
+## Development Compose File
+
+If you're contributing to SurfSense and want to build from source, use `docker-compose.dev.yml` instead:
 
 ```bash
-docker compose -f docker-compose.quickstart.yml pull
-docker compose -f docker-compose.quickstart.yml up -d
+cd SurfSense/docker
+docker compose -f docker-compose.dev.yml up --build
+```
+
+This file builds the backend and frontend from your local source code (instead of pulling prebuilt images) and includes pgAdmin for database inspection at [http://localhost:5050](http://localhost:5050). Use the production `docker-compose.yml` for all other cases.
+
+The following `.env` variables are **only used by the dev compose file** (they have no effect on the production `docker-compose.yml`):
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `PGADMIN_PORT` | pgAdmin web UI port | `5050` |
+| `PGADMIN_DEFAULT_EMAIL` | pgAdmin login email | `admin@surfsense.com` |
+| `PGADMIN_DEFAULT_PASSWORD` | pgAdmin login password | `surfsense` |
+| `REDIS_PORT` | Exposed Redis port (internal-only in prod) | `6379` |
+| `NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE` | Frontend build arg for auth type | `LOCAL` |
+| `NEXT_PUBLIC_ETL_SERVICE` | Frontend build arg for ETL service | `DOCLING` |
+| `NEXT_PUBLIC_DEPLOYMENT_MODE` | Frontend build arg for deployment mode | `self-hosted` |
+| `NEXT_PUBLIC_ELECTRIC_AUTH_MODE` | Frontend build arg for Electric auth | `insecure` |
+
+In the production compose file, the `NEXT_PUBLIC_*` frontend variables are automatically derived from `AUTH_TYPE`, `ETL_SERVICE`, and the port settings. In the dev compose file, they are passed as build args since the frontend is built from source.
+
+---
+
+## Migrating from the All-in-One Container
+
+<Callout type="warn">
+If you were previously using `docker-compose.quickstart.yml` (the legacy all-in-one `surfsense` container), your data lives in a `surfsense-data` volume and requires a **one-time migration** before switching to the current setup. PostgreSQL has been upgraded from version 14 to 17, so a simple volume swap will not work.
+
+See the full step-by-step guide: [Migrate from the All-in-One Container](/docs/how-to/migrate-from-allinone).
+</Callout>
+
+---
+
+## Useful Commands
+
+```bash
+# View logs (all services)
+docker compose logs -f
+
+# View logs for a specific service
+docker compose logs -f backend
+docker compose logs -f electric
+
+# Stop all services
+docker compose down
+
+# Restart a specific service
+docker compose restart backend
+
+# Stop and remove all containers + volumes (destructive!)
+docker compose down -v
 ```
 
 ---
 
-## Full Docker Compose Setup (Production)
-
-For production deployments with separate services and more control, use the full Docker Compose setup below.
-
-## Prerequisites
-
-Before you begin, ensure you have:
-
-- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) installed on your machine
-- [Git](https://git-scm.com/downloads) (to clone the repository)
-- Completed all the [prerequisite setup steps](/docs) including:
-  - Auth setup
-  - **File Processing ETL Service** (choose one):
-    - Unstructured.io API key (Supports 34+ formats)
-    - LlamaIndex API key (enhanced parsing, supports 50+ formats)
-    - Docling (local processing, no API key required, supports PDF, Office docs, images, HTML, CSV)
-  - Other required API keys
-
-## Installation Steps
-
-1. **Configure Environment Variables**
-    Set up the necessary environment variables:
-
-    **Linux/macOS:**
-
-    ```bash
-    # Copy example environment files
-    cp surfsense_backend/.env.example surfsense_backend/.env
-    cp surfsense_web/.env.example surfsense_web/.env
-    cp .env.example .env  # For Docker-specific settings
-    ```
-
-    **Windows (Command Prompt):**
-
-    ```cmd
-    copy surfsense_backend\.env.example surfsense_backend\.env
-    copy surfsense_web\.env.example surfsense_web\.env
-    copy .env.example .env
-    ```
-
-    **Windows (PowerShell):**
-
-    ```powershell
-    Copy-Item -Path surfsense_backend\.env.example -Destination surfsense_backend\.env
-    Copy-Item -Path surfsense_web\.env.example -Destination surfsense_web\.env
-    Copy-Item -Path .env.example -Destination .env
-    ```
-
-    Edit all `.env` files and fill in the required values:
-
-### Docker-Specific Environment Variables (Optional)
-
-| ENV VARIABLE               | DESCRIPTION                                                                 | DEFAULT VALUE       |
-|----------------------------|-----------------------------------------------------------------------------|---------------------|
-| FRONTEND_PORT              | Port for the frontend service                                               | 3000                |
-| BACKEND_PORT               | Port for the backend API service                                            | 8000                |
-| POSTGRES_PORT              | Port for the PostgreSQL database                                            | 5432                |
-| PGADMIN_PORT               | Port for pgAdmin web interface                                              | 5050                |
-| REDIS_PORT                 | Port for Redis (used by Celery)                                             | 6379                |
-| FLOWER_PORT                | Port for Flower (Celery monitoring tool)                                    | 5555                |
-| POSTGRES_USER              | PostgreSQL username                                                         | postgres            |
-| POSTGRES_PASSWORD          | PostgreSQL password                                                         | postgres            |
-| POSTGRES_DB                | PostgreSQL database name                                                    | surfsense           |
-| PGADMIN_DEFAULT_EMAIL      | Email for pgAdmin login                                                     | admin@surfsense.com |
-| PGADMIN_DEFAULT_PASSWORD   | Password for pgAdmin login                                                  | surfsense           |
-| NEXT_PUBLIC_FASTAPI_BACKEND_URL | URL of the backend API (used by frontend during build and runtime)     | http://localhost:8000 |
-| NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE | Authentication method for frontend: `LOCAL` or `GOOGLE`           | LOCAL               |
-| NEXT_PUBLIC_ETL_SERVICE    | Document parsing service for frontend UI: `UNSTRUCTURED`, `LLAMACLOUD`, or `DOCLING` | DOCLING |
-| ELECTRIC_PORT              | Port for Electric-SQL service                                               | 5133                |
-| POSTGRES_HOST              | PostgreSQL host for Electric connection (`db` for Docker PostgreSQL, `host.docker.internal` for local PostgreSQL) | db |
-| ELECTRIC_DB_USER           | PostgreSQL username for Electric connection                                 | electric            |
-| ELECTRIC_DB_PASSWORD       | PostgreSQL password for Electric connection                                 | electric_password   |
-| NEXT_PUBLIC_ELECTRIC_URL   | URL for Electric-SQL service (used by frontend)                             | http://localhost:5133 |
-
-**Note:** Frontend environment variables with the `NEXT_PUBLIC_` prefix are embedded into the Next.js production build at build time. Since the frontend now runs as a production build in Docker, these variables must be set in the root `.env` file (Docker-specific configuration) and will be passed as build arguments during the Docker build process.
-
-**Backend Environment Variables:**
-
-| ENV VARIABLE               | DESCRIPTION                                                                                                                                                                               |
-| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| DATABASE_URL               | PostgreSQL connection string (e.g., `postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense`)                                                                                    |
-| SECRET_KEY                 | JWT Secret key for authentication (should be a secure random string)                                                                                                                      |
-| NEXT_FRONTEND_URL          | URL where your frontend application is hosted (e.g., `http://localhost:3000`)                                                                                                             |
-| BACKEND_URL                | (Optional) Public URL of the backend for OAuth callbacks (e.g., `https://api.yourdomain.com`). Required when running behind a reverse proxy with HTTPS. Used to set correct OAuth redirect URLs and secure cookies. |
-| AUTH_TYPE                  | Authentication method: `GOOGLE` for OAuth with Google, `LOCAL` for email/password authentication                                                                                          |
-| GOOGLE_OAUTH_CLIENT_ID     | (Optional) Client ID from Google Cloud Console (required if AUTH_TYPE=GOOGLE)                                                                                                                        |
-| GOOGLE_OAUTH_CLIENT_SECRET | (Optional) Client secret from Google Cloud Console (required if AUTH_TYPE=GOOGLE)                                                                                                                    |
-| ELECTRIC_DB_USER           | (Optional) PostgreSQL username for Electric-SQL connection (default: `electric`)                                                                                                          |
-| ELECTRIC_DB_PASSWORD       | (Optional) PostgreSQL password for Electric-SQL connection (default: `electric_password`)                                                                                                 |
-| EMBEDDING_MODEL            | Name of the embedding model (e.g., `sentence-transformers/all-MiniLM-L6-v2`, `openai://text-embedding-ada-002`)                                                                                                                 |
-| RERANKERS_ENABLED          | (Optional) Enable or disable document reranking for improved search results (e.g., `TRUE` or `FALSE`, default: `FALSE`)                                                                  |
-| RERANKERS_MODEL_NAME       | Name of the reranker model (e.g., `ms-marco-MiniLM-L-12-v2`) (required if RERANKERS_ENABLED=TRUE)                                                                                        |
-| RERANKERS_MODEL_TYPE       | Type of reranker model (e.g., `flashrank`) (required if RERANKERS_ENABLED=TRUE)                                                                                                          |
-| TTS_SERVICE                | Text-to-Speech API provider for Podcasts (e.g., `local/kokoro`, `openai/tts-1`). See [supported providers](https://docs.litellm.ai/docs/text_to_speech#supported-providers)                            |
-| TTS_SERVICE_API_KEY        | (Optional if local) API key for the Text-to-Speech service                                                                                                                                                    |
-| TTS_SERVICE_API_BASE       | (Optional) Custom API base URL for the Text-to-Speech service                                                                                                                           |
-| STT_SERVICE                | Speech-to-Text API provider for Audio Files (e.g., `local/base`, `openai/whisper-1`). See [supported providers](https://docs.litellm.ai/docs/audio_transcription#supported-providers)                   |
-| STT_SERVICE_API_KEY        | (Optional if local) API key for the Speech-to-Text service                                                                                                                                                    |
-| STT_SERVICE_API_BASE       | (Optional) Custom API base URL for the Speech-to-Text service                                                                                                                      |
-| FIRECRAWL_API_KEY          | API key for Firecrawl service for web crawling                                                                                                                                          |
-| ETL_SERVICE                | Document parsing service: `UNSTRUCTURED` (supports 34+ formats), `LLAMACLOUD` (supports 50+ formats including legacy document types), or `DOCLING` (local processing, supports PDF, Office docs, images, HTML, CSV)                                                  |
-| UNSTRUCTURED_API_KEY       | API key for Unstructured.io service for document parsing (required if ETL_SERVICE=UNSTRUCTURED)                                                                                           |
-| LLAMA_CLOUD_API_KEY        | API key for LlamaCloud service for document parsing (required if ETL_SERVICE=LLAMACLOUD)                                                                                                  |
-| CELERY_BROKER_URL          | Redis connection URL for Celery broker (e.g., `redis://localhost:6379/0`)                                                                                                                |
-| CELERY_RESULT_BACKEND      | Redis connection URL for Celery result backend (e.g., `redis://localhost:6379/0`)                                                                                                        |
-| SCHEDULE_CHECKER_INTERVAL  | (Optional) How often to check for scheduled connector tasks. Format: `<number><unit>` where unit is `m` (minutes) or `h` (hours). Examples: `1m`, `5m`, `1h`, `2h` (default: `1m`)                  |
-| REGISTRATION_ENABLED       | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`)                                                                                                       |
-| PAGES_LIMIT                | (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version)                                                                                        |
-
-**Google Connector OAuth Configuration:**
-| ENV VARIABLE               | DESCRIPTION                                                                                                                                                                               |
-| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| GOOGLE_CALENDAR_REDIRECT_URI | (Optional) Redirect URI for Google Calendar connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/calendar/connector/callback`)                                     |
-| GOOGLE_GMAIL_REDIRECT_URI  | (Optional) Redirect URI for Gmail connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/gmail/connector/callback`)                                                   |
-| GOOGLE_DRIVE_REDIRECT_URI  | (Optional) Redirect URI for Google Drive connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/drive/connector/callback`)                                            |
-
-**Connector OAuth Configurations (Optional):**
-
-| ENV VARIABLE               | DESCRIPTION                                                                                                                                                                               |
-| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| AIRTABLE_CLIENT_ID         | (Optional) Airtable OAuth client ID from [Airtable Developer Hub](https://airtable.com/create/oauth)                                                                                      |
-| AIRTABLE_CLIENT_SECRET     | (Optional) Airtable OAuth client secret                                                                                                                                                   |
-| AIRTABLE_REDIRECT_URI      | (Optional) Redirect URI for Airtable connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/airtable/connector/callback`)                                                    |
-| CLICKUP_CLIENT_ID          | (Optional) ClickUp OAuth client ID                                                                                                                                                        |
-| CLICKUP_CLIENT_SECRET      | (Optional) ClickUp OAuth client secret                                                                                                                                                    |
-| CLICKUP_REDIRECT_URI       | (Optional) Redirect URI for ClickUp connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/clickup/connector/callback`)                                                      |
-| DISCORD_CLIENT_ID          | (Optional) Discord OAuth client ID                                                                                                                                                        |
-| DISCORD_CLIENT_SECRET      | (Optional) Discord OAuth client secret                                                                                                                                                    |
-| DISCORD_REDIRECT_URI       | (Optional) Redirect URI for Discord connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/discord/connector/callback`)                                                      |
-| DISCORD_BOT_TOKEN          | (Optional) Discord bot token from Developer Portal                                                                                                                                        |
-| ATLASSIAN_CLIENT_ID        | (Optional) Atlassian OAuth client ID (for Jira and Confluence)                                                                                                                            |
-| ATLASSIAN_CLIENT_SECRET    | (Optional) Atlassian OAuth client secret                                                                                                                                                  |
-| JIRA_REDIRECT_URI          | (Optional) Redirect URI for Jira connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/jira/connector/callback`)                                                            |
-| CONFLUENCE_REDIRECT_URI    | (Optional) Redirect URI for Confluence connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/confluence/connector/callback`)                                                |
-| LINEAR_CLIENT_ID           | (Optional) Linear OAuth client ID                                                                                                                                                         |
-| LINEAR_CLIENT_SECRET       | (Optional) Linear OAuth client secret                                                                                                                                                     |
-| LINEAR_REDIRECT_URI        | (Optional) Redirect URI for Linear connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/linear/connector/callback`)                                                        |
-| NOTION_CLIENT_ID           | (Optional) Notion OAuth client ID                                                                                                                                                         |
-| NOTION_CLIENT_SECRET       | (Optional) Notion OAuth client secret                                                                                                                                                     |
-| NOTION_REDIRECT_URI        | (Optional) Redirect URI for Notion connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/notion/connector/callback`)                                                        |
-| SLACK_CLIENT_ID            | (Optional) Slack OAuth client ID                                                                                                                                                          |
-| SLACK_CLIENT_SECRET        | (Optional) Slack OAuth client secret                                                                                                                                                      |
-| SLACK_REDIRECT_URI         | (Optional) Redirect URI for Slack connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/slack/connector/callback`)                                                          |
-| TEAMS_CLIENT_ID            | (Optional) Microsoft Teams OAuth client ID                                                                                                                                                |
-| TEAMS_CLIENT_SECRET        | (Optional) Microsoft Teams OAuth client secret                                                                                                                                            |
-| TEAMS_REDIRECT_URI         | (Optional) Redirect URI for Teams connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/teams/connector/callback`)                                                          |
-
-
-**Optional Backend LangSmith Observability:**
-| ENV VARIABLE | DESCRIPTION |
-|--------------|-------------|
-| LANGSMITH_TRACING | Enable LangSmith tracing (e.g., `true`) |
-| LANGSMITH_ENDPOINT | LangSmith API endpoint (e.g., `https://api.smith.langchain.com`) |
-| LANGSMITH_API_KEY | Your LangSmith API key |
-| LANGSMITH_PROJECT | LangSmith project name (e.g., `surfsense`) |
-
-**Backend Uvicorn Server Configuration:**
-| ENV VARIABLE | DESCRIPTION | DEFAULT VALUE |
-|------------------------------|---------------------------------------------|---------------|
-| UVICORN_HOST                 | Host address to bind the server             | 0.0.0.0       |
-| UVICORN_PORT                 | Port to run the backend API                 | 8000          |
-| UVICORN_LOG_LEVEL            | Logging level (e.g., info, debug, warning)  | info          |
-| UVICORN_PROXY_HEADERS        | Enable/disable proxy headers                | false         |
-| UVICORN_FORWARDED_ALLOW_IPS  | Comma-separated list of allowed IPs         | 127.0.0.1     |
-| UVICORN_WORKERS              | Number of worker processes                  | 1             |
-| UVICORN_ACCESS_LOG           | Enable/disable access log (true/false)      | true          |
-| UVICORN_LOOP                 | Event loop implementation                   | auto          |
-| UVICORN_HTTP                 | HTTP protocol implementation                | auto          |
-| UVICORN_WS                   | WebSocket protocol implementation           | auto          |
-| UVICORN_LIFESPAN             | Lifespan implementation                     | auto          |
-| UVICORN_LOG_CONFIG           | Path to logging config file or empty string |               |
-| UVICORN_SERVER_HEADER        | Enable/disable Server header                | true          |
-| UVICORN_DATE_HEADER          | Enable/disable Date header                  | true          |
-| UVICORN_LIMIT_CONCURRENCY    | Max concurrent connections                  |               |
-| UVICORN_LIMIT_MAX_REQUESTS   | Max requests before worker restart          |               |
-| UVICORN_TIMEOUT_KEEP_ALIVE   | Keep-alive timeout (seconds)                | 5             |
-| UVICORN_TIMEOUT_NOTIFY       | Worker shutdown notification timeout (sec)  | 30            |
-| UVICORN_SSL_KEYFILE          | Path to SSL key file                        |               |
-| UVICORN_SSL_CERTFILE         | Path to SSL certificate file                |               |
-| UVICORN_SSL_KEYFILE_PASSWORD | Password for SSL key file                   |               |
-| UVICORN_SSL_VERSION          | SSL version                                 |               |
-| UVICORN_SSL_CERT_REQS        | SSL certificate requirements                |               |
-| UVICORN_SSL_CA_CERTS         | Path to CA certificates file                |               |
-| UVICORN_SSL_CIPHERS          | SSL ciphers                                 |               |
-| UVICORN_HEADERS              | Comma-separated list of headers             |               |
-| UVICORN_USE_COLORS           | Enable/disable colored logs                 | true          |
-| UVICORN_UDS                  | Unix domain socket path                     |               |
-| UVICORN_FD                   | File descriptor to bind to                  |               |
-| UVICORN_ROOT_PATH            | Root path for the application               |               |
-
-For more details, see the [Uvicorn documentation](https://www.uvicorn.org/#command-line-options).
-
-### Frontend Environment Variables
-
-**Important:** Frontend environment variables are now configured in the **Docker-Specific Environment Variables** section above since the Next.js application runs as a production build in Docker. The following `NEXT_PUBLIC_*` variables should be set in your root `.env` file:
-
-- `NEXT_PUBLIC_FASTAPI_BACKEND_URL` - URL of the backend service
-- `NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE` - Authentication method (`LOCAL` or `GOOGLE`)
-- `NEXT_PUBLIC_ETL_SERVICE` - Document parsing service (should match backend `ETL_SERVICE`)
-- `NEXT_PUBLIC_ELECTRIC_URL` - URL for Electric-SQL service (default: `http://localhost:5133`)
-- `NEXT_PUBLIC_ELECTRIC_AUTH_MODE` - Electric-SQL authentication mode (default: `insecure`)
-
-These variables are embedded into the application during the Docker build process and affect the frontend's behavior and available features.
-
-2. **Build and Start Containers**
-
-   Start the Docker containers:
-
-   **Linux/macOS/Windows:**
-
-   ```bash
-   docker compose up --build
-   ```
-
-   To run in detached mode (in the background):
-
-   **Linux/macOS/Windows:**
-
-   ```bash
-   docker compose up -d
-   ```
-
-   **Note for Windows users:** If you're using older Docker Desktop versions, you might need to use `docker compose` (with a space) instead of `docker compose`.
-
-3. **Access the Applications**
-
-   Once the containers are running, you can access:
-
-   - Frontend: [http://localhost:3000](http://localhost:3000)
-   - Backend API: [http://localhost:8000](http://localhost:8000)
-   - API Documentation: [http://localhost:8000/docs](http://localhost:8000/docs)
-   - Electric-SQL: [http://localhost:5133](http://localhost:5133)
-   - pgAdmin: [http://localhost:5050](http://localhost:5050)
-
-## Docker Services Overview
-
-The Docker setup includes several services that work together:
-
-- **Backend**: FastAPI application server
-- **Frontend**: Next.js web application
-- **PostgreSQL (db)**: Database with pgvector extension
-- **Redis**: Message broker for Celery
-- **Electric-SQL**: Real-time sync service for database operations
-- **Celery Worker**: Handles background tasks (document processing, indexing, etc.)
-- **Celery Beat**: Scheduler for periodic tasks (enables scheduled connector indexing)
-  - The schedule interval can be configured using the `SCHEDULE_CHECKER_INTERVAL` environment variable in your backend `.env` file
-  - Default: checks every minute for connectors that need indexing
-- **pgAdmin**: Database management interface
-
-All services start automatically with `docker compose up`. The Celery Beat service ensures that periodic indexing functionality works out of the box.
-
-## Using pgAdmin
-
-pgAdmin is included in the Docker setup to help manage your PostgreSQL database. To connect:
-
-1. Open pgAdmin at [http://localhost:5050](http://localhost:5050)
-2. Login with the credentials from your `.env` file (default: admin@surfsense.com / surfsense)
-3. Right-click "Servers" > "Create" > "Server"
-4. In the "General" tab, name your connection (e.g., "SurfSense DB")
-5. In the "Connection" tab:
-   - Host: `db`
-   - Port: `5432`
-   - Maintenance database: `surfsense`
-   - Username: `postgres` (or your custom POSTGRES_USER)
-   - Password: `postgres` (or your custom POSTGRES_PASSWORD)
-6. Click "Save" to connect
-
-## Updating (Full Docker Compose)
-
-To update the full Docker Compose production setup to the latest version:
-
-```bash
-# Pull latest changes
-git pull
-
-# Rebuild and restart containers
-docker compose up --build -d
-```
-
-Database migrations are applied automatically on startup.
-
-## Useful Docker Commands
-
-### Container Management
-
-- **Stop containers:**
-
-  **Linux/macOS/Windows:**
-
-  ```bash
-  docker compose down
-  ```
-
-- **View logs:**
-
-  **Linux/macOS/Windows:**
-
-  ```bash
-  # All services
-  docker compose logs -f
-
-  # Specific service
-  docker compose logs -f backend
-  docker compose logs -f frontend
-  docker compose logs -f db
-  ```
-
-- **Restart a specific service:**
-
-  **Linux/macOS/Windows:**
-
-  ```bash
-  docker compose restart backend
-  ```
-
-- **Execute commands in a running container:**
-
-  **Linux/macOS/Windows:**
-
-  ```bash
-  # Backend
-  docker compose exec backend python -m pytest
-
-  # Frontend
-  docker compose exec frontend pnpm lint
-  ```
-
 ## Troubleshooting
 
-- **Linux/macOS:** If you encounter permission errors, you may need to run the docker commands with `sudo`.
-- **Windows:** If you see access denied errors, make sure you're running Command Prompt or PowerShell as Administrator.
-- If ports are already in use, modify the port mappings in the `docker-compose.yml` file.
-- For backend dependency issues, check the `Dockerfile` in the backend directory.
-- For frontend dependency issues, check the `Dockerfile` in the frontend directory.
-- **Windows-specific:** If you encounter line ending issues (CRLF vs LF), configure Git to handle line endings properly with `git config --global core.autocrlf true` before cloning the repository.
-
-## Next Steps
-
-Once your installation is complete, you can start using SurfSense! Navigate to the frontend URL and log in using your Google account.
+- **Ports already in use** — Change the relevant `*_PORT` variable in `.env` and restart.
+- **Permission errors on Linux** — You may need to prefix `docker` commands with `sudo`.
+- **Electric SQL not connecting** — Check `docker compose logs electric`. If it shows `domain does not exist: db`, ensure `ELECTRIC_DATABASE_URL` is not set to a stale value in `.env`.
+- **Real-time updates not working in browser** — Open DevTools → Console and look for `[Electric]` errors. Check that `NEXT_PUBLIC_ELECTRIC_URL` matches the running Electric SQL address.
+- **Line ending issues on Windows** — Run `git config --global core.autocrlf true` before cloning.

surfsense_web/content/docs/how-to/electric-sql.mdx

@@ -3,8 +3,6 @@ title: Electric SQL
 description: Setting up Electric SQL for real-time data synchronization in SurfSense
 ---
 
-# Electric SQL
-
 [Electric SQL](https://electric-sql.com/) enables real-time data synchronization in SurfSense, providing instant updates for inbox items, document indexing status, and connector sync progress without manual refresh. The frontend uses [PGlite](https://pglite.dev/) (a lightweight PostgreSQL in the browser) to maintain a local database that syncs with the backend via Electric SQL.
 
 ## What Does Electric SQL Do?
@@ -25,74 +23,29 @@ This means:
 
 ## Docker Setup
 
-### All-in-One Quickstart
-
-The simplest way to run SurfSense with Electric SQL is using the all-in-one Docker image. This bundles everything into a single container:
-
-- PostgreSQL + pgvector (vector database)
-- Redis (task queue)
-- Electric SQL (real-time sync)
-- Backend API
-- Frontend
+The `docker-compose.yml` includes the Electric SQL service. It is pre-configured to connect to the Docker-managed `db` container out of the box.
 
 ```bash
-docker run -d \
-  -p 3000:3000 \
-  -p 8000:8000 \
-  -p 5133:5133 \
-  -v surfsense-data:/data \
-  --name surfsense \
-  ghcr.io/modsetter/surfsense:latest
+docker compose up -d
 ```
 
-**With custom Electric SQL credentials:**
-
-```bash
-docker run -d \
-  -p 3000:3000 \
-  -p 8000:8000 \
-  -p 5133:5133 \
-  -v surfsense-data:/data \
-  -e ELECTRIC_DB_USER=your_electric_user \
-  -e ELECTRIC_DB_PASSWORD=your_electric_password \
-  --name surfsense \
-  ghcr.io/modsetter/surfsense:latest
-```
-
-Access SurfSense at `http://localhost:3000`. Electric SQL is automatically configured and running on port 5133.
-
-### Docker Compose
-
-For more control over individual services, use Docker Compose.
-
-**Quickstart (all-in-one image):**
-
-```bash
-docker compose -f docker-compose.quickstart.yml up -d
-```
-
-**Standard setup (separate services):**
-
-The `docker-compose.yml` includes the Electric SQL service configuration:
+The Electric SQL service configuration in `docker-compose.yml`:
 
 ```yaml
 electric:
-  image: electricsql/electric:latest
+  image: electricsql/electric:1.4.6
   ports:
     - "${ELECTRIC_PORT:-5133}:3000"
   environment:
-    - DATABASE_URL=${ELECTRIC_DATABASE_URL:-postgresql://${ELECTRIC_DB_USER:-electric}:${ELECTRIC_DB_PASSWORD:-electric_password}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-surfsense}?sslmode=disable}
-    - ELECTRIC_INSECURE=true
-    - ELECTRIC_WRITE_TO_PG_MODE=direct
-  restart: unless-stopped
-  healthcheck:
-    test: ["CMD", "curl", "-f", "http://localhost:3000/v1/health"]
-    interval: 10s
-    timeout: 5s
-    retries: 5
+    DATABASE_URL: ${ELECTRIC_DATABASE_URL:-postgresql://${ELECTRIC_DB_USER:-electric}:${ELECTRIC_DB_PASSWORD:-electric_password}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}}
+    ELECTRIC_INSECURE: "true"
+    ELECTRIC_WRITE_TO_PG_MODE: direct
+  depends_on:
+    db:
+      condition: service_healthy
 ```
 
-No additional configuration is required - Electric SQL is pre-configured to work with the Docker PostgreSQL instance.
+No additional configuration is required — Electric SQL is pre-configured to work with the Docker PostgreSQL instance.
 
 ## Manual Setup
 
@@ -102,19 +55,16 @@ Follow the steps below based on your PostgreSQL setup.
 
 Ensure your environment files are configured. If you haven't set up SurfSense yet, follow the [Manual Installation Guide](/docs/manual-installation) first.
 
-For Electric SQL, verify these variables are set:
-
-**Root `.env`:**
+For Electric SQL, verify these variables are set in `docker/.env`:
 
 ```bash
 ELECTRIC_PORT=5133
-POSTGRES_HOST=host.docker.internal  # Use 'db' for Docker PostgreSQL instance
 ELECTRIC_DB_USER=electric
 ELECTRIC_DB_PASSWORD=electric_password
 NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133
 ```
 
-**Frontend `.env` (`surfsense_web/.env`):**
+**Frontend (`surfsense_web/.env`):**
 
 ```bash
 NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133
@@ -125,32 +75,17 @@ NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure
 
 ### Option A: Using Docker PostgreSQL
 
-If you're using the Docker-managed PostgreSQL instance, follow these steps:
-
-**1. Update environment variable:**
-
-In your root `.env` file, set:
+If you're using the Docker-managed PostgreSQL instance, no extra configuration is needed. Just start the services:
 
 ```bash
-POSTGRES_HOST=db
+docker compose up -d db electric
 ```
 
-**2. Start PostgreSQL and Electric SQL:**
-
-```bash
-docker-compose up -d db electric
-```
-
-**3. Run database migration:**
+Then run the database migration and start the backend:
 
 ```bash
 cd surfsense_backend
 uv run alembic upgrade head
-```
-
-**4. Start the backend:**
-
-```bash
 uv run main.py
 ```
 
@@ -160,17 +95,17 @@ Electric SQL is now configured and connected to your Docker PostgreSQL database.
 
 ### Option B: Using Local PostgreSQL
 
-If you're using a local PostgreSQL installation, follow these steps:
+If you're using a local PostgreSQL installation (e.g. Postgres.app on macOS), follow these steps:
 
 **1. Enable logical replication in PostgreSQL:**
 
-Open your `postgresql.conf` file using vim (or your preferred editor):
+Open your `postgresql.conf` file:
 
 ```bash
 # Common locations:
-# macOS (Homebrew): /opt/homebrew/var/postgresql@15/postgresql.conf
-# Linux: /etc/postgresql/15/main/postgresql.conf
-# Windows: C:\Program Files\PostgreSQL\15\data\postgresql.conf
+# macOS (Postgres.app): ~/Library/Application Support/Postgres/var-17/postgresql.conf
+# macOS (Homebrew):     /opt/homebrew/var/postgresql@17/postgresql.conf
+# Linux:                /etc/postgresql/17/main/postgresql.conf
 
 sudo vim /path/to/postgresql.conf
 ```
@@ -178,38 +113,51 @@ sudo vim /path/to/postgresql.conf
 Add the following settings:
 
 ```ini
-# Enable logical replication (required for Electric SQL)
+# Required for Electric SQL
 wal_level = logical
 max_replication_slots = 10
 max_wal_senders = 10
 ```
 
-After saving the changes (`:wq` in vim), restart your PostgreSQL server for the configuration to take effect.
+After saving, restart PostgreSQL for the settings to take effect.
 
-**2. Update environment variable:**
+**2. Create the Electric replication user:**
 
-In your root `.env` file, set:
+Connect to your local database as a superuser and run:
 
-```bash
-POSTGRES_HOST=host.docker.internal
+```sql
+CREATE USER electric WITH REPLICATION PASSWORD 'electric_password';
+GRANT CONNECT ON DATABASE surfsense TO electric;
+GRANT CREATE ON DATABASE surfsense TO electric;
+GRANT USAGE ON SCHEMA public TO electric;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO electric;
+GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO electric;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO electric;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO electric;
+CREATE PUBLICATION electric_publication_default;
 ```
 
-**3. Start Electric SQL:**
+**3. Set `ELECTRIC_DATABASE_URL` in `docker/.env`:**
+
+Uncomment and update this line to point Electric at your local Postgres via `host.docker.internal` (the hostname Docker containers use to reach the host machine):
 
 ```bash
-docker-compose up -d electric
+ELECTRIC_DATABASE_URL=postgresql://electric:electric_password@host.docker.internal:5432/surfsense?sslmode=disable
 ```
 
-**4. Run database migration:**
+**4. Start Electric SQL only (skip the Docker `db` container):**
+
+```bash
+docker compose up -d --no-deps electric
+```
+
+The `--no-deps` flag starts only the `electric` service without starting the Docker-managed `db` container.
+
+**5. Run database migration and start the backend:**
 
 ```bash
 cd surfsense_backend
 uv run alembic upgrade head
-```
-
-**5. Start the backend:**
-
-```bash
 uv run main.py
 ```
 
@@ -219,12 +167,13 @@ Electric SQL is now configured and connected to your local PostgreSQL database.
 
 | Variable | Location | Description | Default |
 |----------|----------|-------------|---------|
-| `ELECTRIC_PORT` | Root `.env` | Port to expose Electric SQL | `5133` |
-| `POSTGRES_HOST` | Root `.env` | PostgreSQL host (`db` for Docker, `host.docker.internal` for local) | `host.docker.internal` |
-| `ELECTRIC_DB_USER` | Root `.env` | Database user for Electric | `electric` |
-| `ELECTRIC_DB_PASSWORD` | Root `.env` | Database password for Electric | `electric_password` |
+| `ELECTRIC_PORT` | `docker/.env` | Port to expose Electric SQL | `5133` |
+| `ELECTRIC_DB_USER` | `docker/.env` | Database user for Electric replication | `electric` |
+| `ELECTRIC_DB_PASSWORD` | `docker/.env` | Database password for Electric replication | `electric_password` |
+| `ELECTRIC_DATABASE_URL` | `docker/.env` | Full connection URL override for Electric. Set to use `host.docker.internal` when pointing at a local Postgres instance | *(built from above defaults)* |
 | `NEXT_PUBLIC_ELECTRIC_URL` | Frontend `.env` | Electric SQL server URL (PGlite connects to this) | `http://localhost:5133` |
 | `NEXT_PUBLIC_ELECTRIC_AUTH_MODE` | Frontend `.env` | Authentication mode (`insecure` for dev, `secure` for production) | `insecure` |
+
 ## Verify Setup
 
 To verify Electric SQL is running correctly:
@@ -262,7 +211,7 @@ You should receive:
 
 ### Data Not Syncing
 
-- Check Electric SQL logs: `docker logs electric`
+- Check Electric SQL logs: `docker compose logs electric`
 - Verify PostgreSQL replication is working
 - Ensure the Electric user has proper table permissions
 

surfsense_web/content/docs/how-to/meta.json

@@ -1,6 +1,6 @@
 {
 	"title": "How to",
+	"pages": ["electric-sql", "realtime-collaboration", "migrate-from-allinone"],
 	"icon": "BookOpen",
-	"pages": ["electric-sql", "realtime-collaboration"],
 	"defaultOpen": false
 }

surfsense_web/content/docs/how-to/migrate-from-allinone.mdx

@@ -0,0 +1,195 @@
+---
+title: Migrate from the All-in-One Container
+description: How to migrate your data from the legacy surfsense all-in-one Docker image to the current multi-container setup
+---
+
+The original SurfSense all-in-one image (`ghcr.io/modsetter/surfsense:latest`, run via `docker-compose.quickstart.yml`) stored all data — PostgreSQL, Redis, and configuration — in a single Docker volume named `surfsense-data`. The current setup uses separate named volumes and has upgraded PostgreSQL from **version 14 to 17**.
+
+Because PostgreSQL data files are not compatible between major versions, a **logical dump and restore** is required. This is a one-time migration.
+
+<Callout type="warn">
+This guide only applies to users who ran the legacy `docker-compose.quickstart.yml` (the all-in-one `surfsense` container). If you were already using `docker/docker-compose.yml`, you do not need to migrate.
+</Callout>
+
+---
+
+## Option A — One command (recommended)
+
+`install.sh` detects the legacy `surfsense-data` volume and handles the full migration automatically — no separate migration script needed. Just run the same install command you would use for a fresh install:
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
+```
+
+**What it does automatically:**
+
+1. Downloads all SurfSense files (including `migrate-database.sh`) into `./surfsense/`
+2. Detects the `surfsense-data` volume and enters migration mode
+3. Stops the old all-in-one container if it is still running
+4. Starts a temporary PostgreSQL 14 container and dumps your database
+5. Recovers your `SECRET_KEY` from the old volume
+6. Starts PostgreSQL 17, restores the dump, runs a smoke test
+7. Starts all services
+
+Your original `surfsense-data` volume is **never deleted** — you remove it manually after verifying.
+
+### After it completes
+
+1. Open [http://localhost:3000](http://localhost:3000) and confirm your data is intact.
+2. Once satisfied, remove the old volume (irreversible):
+   ```bash
+   docker volume rm surfsense-data
+   ```
+3. Delete the dump file once you no longer need it as a backup:
+   ```bash
+   rm ./surfsense_migration_backup.sql
+   ```
+
+### If the migration fails mid-way
+
+The dump file is saved to `./surfsense_migration_backup.sql` as a checkpoint. Simply re-run `install.sh` — it will detect the existing dump and skip straight to the restore step without re-extracting.
+
+---
+
+## Option B — Manual migration script (custom credentials)
+
+If you launched the old all-in-one container with custom database credentials (`POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` environment variables), the automatic path will use wrong credentials. Run `migrate-database.sh` manually first:
+
+```bash
+# 1. Extract data with your custom credentials
+bash ./surfsense/scripts/migrate-database.sh --db-user myuser --db-password mypass --db-name mydb
+
+# 2. Install and restore (detects the dump automatically)
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
+```
+
+Or download and run if you haven't run `install.sh` yet:
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/migrate-database.sh -o migrate-database.sh
+bash migrate-database.sh --db-user myuser --db-password mypass --db-name mydb
+```
+
+### Migration script options
+
+| Flag | Description | Default |
+|------|-------------|---------|
+| `--db-user USER` | Old PostgreSQL username | `surfsense` |
+| `--db-password PASS` | Old PostgreSQL password | `surfsense` |
+| `--db-name NAME` | Old PostgreSQL database | `surfsense` |
+| `--yes` / `-y` | Skip confirmation prompts (used automatically by `install.sh`) | — |
+
+---
+
+## Option C — Manual steps
+
+For users who prefer full control or whose platform doesn't support bash scripts (e.g. Windows without WSL2).
+
+### Step 1 — Stop the old all-in-one container
+
+Before mounting the `surfsense-data` volume into a new container, stop the existing one to prevent two PostgreSQL processes from writing to the same data directory:
+
+```bash
+docker stop surfsense 2>/dev/null || true
+```
+
+### Step 2 — Start a temporary PostgreSQL 14 container
+
+```bash
+docker run -d --name surfsense-pg14-temp \
+  -v surfsense-data:/data \
+  -e PGDATA=/data/postgres \
+  -e POSTGRES_USER=surfsense \
+  -e POSTGRES_PASSWORD=surfsense \
+  -e POSTGRES_DB=surfsense \
+  pgvector/pgvector:pg14
+```
+
+Wait ~10 seconds, then confirm it is healthy:
+
+```bash
+docker exec surfsense-pg14-temp pg_isready -U surfsense
+```
+
+### Step 3 — Dump the database
+
+```bash
+docker exec -e PGPASSWORD=surfsense surfsense-pg14-temp \
+  pg_dump -U surfsense surfsense > surfsense_backup.sql
+```
+
+### Step 4 — Recover your SECRET\_KEY
+
+```bash
+docker run --rm -v surfsense-data:/data alpine cat /data/.secret_key
+```
+
+### Step 5 — Set up the new stack
+
+```bash
+mkdir -p surfsense/scripts
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/docker-compose.yml -o surfsense/docker-compose.yml
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/.env.example -o surfsense/.env.example
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/postgresql.conf -o surfsense/postgresql.conf
+curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/init-electric-user.sh -o surfsense/scripts/init-electric-user.sh
+chmod +x surfsense/scripts/init-electric-user.sh
+cp surfsense/.env.example surfsense/.env
+```
+
+Set `SECRET_KEY` in `surfsense/.env` to the value from Step 4.
+
+### Step 6 — Start PostgreSQL 17 and restore
+
+```bash
+cd surfsense
+docker compose up -d db
+docker compose exec db pg_isready -U surfsense   # wait until ready
+docker compose exec -T db psql -U surfsense -d surfsense < ../surfsense_backup.sql
+```
+
+### Step 7 — Start all services
+
+```bash
+docker compose up -d
+```
+
+### Step 8 — Clean up
+
+```bash
+docker stop surfsense-pg14-temp && docker rm surfsense-pg14-temp
+docker volume rm surfsense-data   # only after verifying migration succeeded
+```
+
+---
+
+## Troubleshooting
+
+### `install.sh` runs normally with a blank database (no migration happened)
+
+The legacy volume was not detected. Confirm it exists:
+
+```bash
+docker volume ls | grep surfsense-data
+```
+
+If it doesn't appear, the old container may have used a different volume name. Check with:
+
+```bash
+docker volume ls | grep -i surfsense
+```
+
+### Extraction fails with permission errors
+
+The script detects the UID of the data files and runs the temporary PG14 container as that user. If you see permission errors in `./surfsense-migration.log`, run `migrate-database.sh` manually and check the log for details.
+
+### Cannot find `/data/.secret_key`
+
+The all-in-one entrypoint always writes the key to `/data/.secret_key` unless you explicitly set `SECRET_KEY=` as an environment variable. If the key is missing, the migration script auto-generates a new one (with a warning). You can update it manually in `./surfsense/.env` afterwards. Note that a new key invalidates all existing browser sessions — users will need to log in again.
+
+### Restore errors after re-running `install.sh`
+
+If `surfsense-postgres` volume already exists from a previous partial run, remove it before retrying:
+
+```bash
+docker volume rm surfsense-postgres
+```

surfsense_web/docker-entrypoint.js

@@ -0,0 +1,100 @@
+/**
+ * Runtime environment variable substitution for Next.js Docker images.
+ *
+ * Next.js inlines NEXT_PUBLIC_* values at build time. The Docker image is built
+ * with unique placeholder strings (e.g. __NEXT_PUBLIC_FASTAPI_BACKEND_URL__).
+ * This script replaces those placeholders with real values from the container's
+ * environment variables before the server starts.
+ *
+ * Runs once at container startup via docker-entrypoint.sh.
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const replacements = [
+  [
+    "__NEXT_PUBLIC_FASTAPI_BACKEND_URL__",
+    process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "http://localhost:8000",
+  ],
+  [
+    "__NEXT_PUBLIC_ELECTRIC_URL__",
+    process.env.NEXT_PUBLIC_ELECTRIC_URL || "http://localhost:5133",
+  ],
+  [
+    "__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__",
+    process.env.NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE || "LOCAL",
+  ],
+  [
+    "__NEXT_PUBLIC_ETL_SERVICE__",
+    process.env.NEXT_PUBLIC_ETL_SERVICE || "DOCLING",
+  ],
+  [
+    "__NEXT_PUBLIC_DEPLOYMENT_MODE__",
+    process.env.NEXT_PUBLIC_DEPLOYMENT_MODE || "self-hosted",
+  ],
+  [
+    "__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__",
+    process.env.NEXT_PUBLIC_ELECTRIC_AUTH_MODE || "insecure",
+  ],
+];
+
+let filesProcessed = 0;
+let filesModified = 0;
+
+function walk(dir) {
+  let entries;
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      walk(full);
+    } else if (entry.name.endsWith(".js")) {
+      filesProcessed++;
+      let content = fs.readFileSync(full, "utf8");
+      let changed = false;
+      for (const [placeholder, value] of replacements) {
+        if (content.includes(placeholder)) {
+          content = content.replaceAll(placeholder, value);
+          changed = true;
+        }
+      }
+      if (changed) {
+        fs.writeFileSync(full, content);
+        filesModified++;
+      }
+    }
+  }
+}
+
+console.log("[entrypoint] Replacing environment variable placeholders...");
+for (const [placeholder, value] of replacements) {
+  console.log(`  ${placeholder} -> ${value}`);
+}
+
+walk(path.join(__dirname, ".next"));
+
+const serverJs = path.join(__dirname, "server.js");
+if (fs.existsSync(serverJs)) {
+  let content = fs.readFileSync(serverJs, "utf8");
+  let changed = false;
+  filesProcessed++;
+  for (const [placeholder, value] of replacements) {
+    if (content.includes(placeholder)) {
+      content = content.replaceAll(placeholder, value);
+      changed = true;
+    }
+  }
+  if (changed) {
+    fs.writeFileSync(serverJs, content);
+    filesModified++;
+  }
+}
+
+console.log(
+  `[entrypoint] Done. Scanned ${filesProcessed} files, modified ${filesModified}.`
+);

surfsense_web/docker-entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -e
+
+node /app/docker-entrypoint.js
+
+exec node server.js