diff --git a/Dockerfile.allinone b/Dockerfile.allinone deleted file mode 100644 index e96618adc..000000000 --- a/Dockerfile.allinone +++ /dev/null @@ -1,285 +0,0 @@ -# SurfSense All-in-One Docker Image -# This image bundles PostgreSQL+pgvector, Redis, Electric SQL, Backend, and Frontend -# Usage: docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 -v surfsense-data:/data --name surfsense ghcr.io/modsetter/surfsense:latest -# -# Included Services (all run locally by default): -# - PostgreSQL 14 + pgvector (vector database) -# - Redis (task queue) -# - Electric SQL (real-time sync) -# - Docling (document processing, CPU-only, OCR disabled) -# - Kokoro TTS (local text-to-speech for podcasts) -# - Faster-Whisper (local speech-to-text for audio files) -# - Playwright Chromium (web scraping) -# -# Note: This is the CPU-only version. A :cuda tagged image with GPU support -# will be available in the future for faster AI inference. - -# ==================== -# Stage 1: Get Electric SQL Binary -# ==================== -FROM electricsql/electric:latest AS electric-builder - -# ==================== -# Stage 2: Build Frontend -# ==================== -FROM node:20-alpine AS frontend-builder - -WORKDIR /app - -# Install pnpm -RUN corepack enable pnpm - -# Copy package files -COPY surfsense_web/package.json surfsense_web/pnpm-lock.yaml* ./ -COPY surfsense_web/source.config.ts ./ -COPY surfsense_web/content ./content - -# Install dependencies (skip postinstall which requires all source files) -RUN pnpm install --frozen-lockfile --ignore-scripts - -# Copy source -COPY surfsense_web/ ./ - -# Run fumadocs-mdx postinstall now that source files are available -RUN pnpm fumadocs-mdx - -# Build with placeholder values that will be replaced at runtime -# These unique strings allow runtime substitution via entrypoint script -ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__ -ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__ -ENV NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__ -ENV NEXT_PUBLIC_ELECTRIC_URL=__NEXT_PUBLIC_ELECTRIC_URL__ -ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__ -ENV NEXT_PUBLIC_DEPLOYMENT_MODE=__NEXT_PUBLIC_DEPLOYMENT_MODE__ - -# Build -RUN pnpm run build - -# ==================== -# Stage 3: Runtime Image -# ==================== -FROM ubuntu:22.04 AS runtime - -# Prevent interactive prompts -ENV DEBIAN_FRONTEND=noninteractive - -# Install system dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - # PostgreSQL - postgresql-14 \ - postgresql-contrib-14 \ - # Build tools for pgvector - build-essential \ - postgresql-server-dev-14 \ - git \ - # Redis - redis-server \ - # Node.js prerequisites - curl \ - ca-certificates \ - gnupg \ - # Backend dependencies - gcc \ - wget \ - unzip \ - dos2unix \ - # For PPAs - software-properties-common \ - # ============================ - # Local TTS (Kokoro) dependencies - # ============================ - espeak-ng \ - libespeak-ng1 \ - # ============================ - # Local STT (Faster-Whisper) dependencies - # ============================ - ffmpeg \ - # ============================ - # Audio processing (soundfile) - # ============================ - libsndfile1 \ - # ============================ - # Image/OpenCV dependencies (for Docling) - # ============================ - libgl1 \ - libglib2.0-0 \ - libsm6 \ - libxext6 \ - libxrender1 \ - # ============================ - # Playwright browser dependencies - # ============================ - libnspr4 \ - libnss3 \ - libatk1.0-0 \ - libatk-bridge2.0-0 \ - libcups2 \ - libxkbcommon0 \ - libatspi2.0-0 \ - libxcomposite1 \ - libxdamage1 \ - libxrandr2 \ - libgbm1 \ - libcairo2 \ - libpango-1.0-0 \ - && rm -rf /var/lib/apt/lists/* - -# Install Pandoc 3.x from GitHub (apt ships 2.9 which has broken table rendering). -RUN ARCH=$(dpkg --print-architecture) && \ - wget -qO /tmp/pandoc.deb "https://github.com/jgm/pandoc/releases/download/3.9/pandoc-3.9-1-${ARCH}.deb" && \ - dpkg -i /tmp/pandoc.deb && \ - rm /tmp/pandoc.deb - - -# Install Node.js 20.x (for running frontend) -RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ - && apt-get install -y nodejs \ - && rm -rf /var/lib/apt/lists/* - -# Install Python 3.12 from deadsnakes PPA -RUN add-apt-repository ppa:deadsnakes/ppa -y \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - python3.12 \ - python3.12-venv \ - python3.12-dev \ - && rm -rf /var/lib/apt/lists/* - -# Set Python 3.12 as default -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.12 1 \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 - -# Install pip for Python 3.12 -RUN python3.12 -m ensurepip --upgrade \ - && python3.12 -m pip install --upgrade pip - -# Install supervisor via pip (system package incompatible with Python 3.12) -RUN pip install --no-cache-dir supervisor - -# Build and install pgvector -RUN cd /tmp \ - && git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git \ - && cd pgvector \ - && make \ - && make install \ - && rm -rf /tmp/pgvector - -# Update certificates -RUN update-ca-certificates - -# Create data directories -RUN mkdir -p /data/postgres /data/redis /data/surfsense \ - && chown -R postgres:postgres /data/postgres - -# ==================== -# Copy Frontend Build -# ==================== -WORKDIR /app/frontend - -# Copy only the standalone build (not node_modules) -COPY --from=frontend-builder /app/.next/standalone ./ -COPY --from=frontend-builder /app/.next/static ./.next/static -COPY --from=frontend-builder /app/public ./public - -COPY surfsense_web/content/docs /app/surfsense_web/content/docs - -# ==================== -# Copy Electric SQL Release -# ==================== -COPY --from=electric-builder /app /app/electric-release - -# ==================== -# Setup Backend -# ==================== -WORKDIR /app/backend - -# Copy backend dependency files -COPY surfsense_backend/pyproject.toml surfsense_backend/uv.lock ./ - -# Install PyTorch CPU-only (Docling needs it but OCR is disabled, no GPU needed) -RUN pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu - -# Install python dependencies -RUN pip install --no-cache-dir certifi pip-system-certs uv \ - && uv pip install --system --no-cache-dir -e . - -# Set SSL environment variables -RUN CERTIFI_PATH=$(python -c "import certifi; print(certifi.where())") \ - && echo "export SSL_CERT_FILE=$CERTIFI_PATH" >> /etc/profile.d/ssl.sh \ - && echo "export REQUESTS_CA_BUNDLE=$CERTIFI_PATH" >> /etc/profile.d/ssl.sh - -# Note: EasyOCR models NOT downloaded - OCR is disabled in docling_service.py -# GPU support will be added in a future :cuda tagged image - -# Install Playwright browsers -RUN pip install --no-cache-dir playwright \ - && playwright install chromium \ - && rm -rf /root/.cache/ms-playwright/ffmpeg* - -# Copy backend source -COPY surfsense_backend/ ./ - -# ==================== -# Configuration -# ==================== -WORKDIR /app - -# Copy supervisor configuration -COPY scripts/docker/supervisor-allinone.conf /etc/supervisor/conf.d/surfsense.conf - -# Copy entrypoint script -COPY scripts/docker/entrypoint-allinone.sh /app/entrypoint.sh -RUN dos2unix /app/entrypoint.sh && chmod +x /app/entrypoint.sh - -# PostgreSQL initialization script -COPY scripts/docker/init-postgres.sh /app/init-postgres.sh -RUN dos2unix /app/init-postgres.sh && chmod +x /app/init-postgres.sh - -# Clean up build dependencies to reduce image size -RUN apt-get purge -y build-essential postgresql-server-dev-14 \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Environment variables with defaults -ENV POSTGRES_USER=surfsense -ENV POSTGRES_PASSWORD=surfsense -ENV POSTGRES_DB=surfsense -ENV DATABASE_URL=postgresql+asyncpg://surfsense:surfsense@localhost:5432/surfsense -ENV CELERY_BROKER_URL=redis://localhost:6379/0 -ENV CELERY_RESULT_BACKEND=redis://localhost:6379/0 -ENV CELERY_TASK_DEFAULT_QUEUE=surfsense -ENV PYTHONPATH=/app/backend -ENV NEXT_FRONTEND_URL=http://localhost:3000 -ENV AUTH_TYPE=LOCAL -ENV ETL_SERVICE=DOCLING -ENV EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 - -# Frontend configuration (can be overridden at runtime) -# These are injected into the Next.js build at container startup -ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000 -ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL -ENV NEXT_PUBLIC_ETL_SERVICE=DOCLING - -# Electric SQL configuration (ELECTRIC_DATABASE_URL is built dynamically by entrypoint from these values) -ENV ELECTRIC_DB_USER=electric -ENV ELECTRIC_DB_PASSWORD=electric_password -# Note: ELECTRIC_DATABASE_URL is NOT set here - entrypoint builds it dynamically from ELECTRIC_DB_USER/PASSWORD -ENV ELECTRIC_INSECURE=true -ENV ELECTRIC_WRITE_TO_PG_MODE=direct -ENV ELECTRIC_PORT=5133 -ENV PORT=5133 -ENV NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133 -ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure - -# Data volume -VOLUME ["/data"] - -# Expose ports (Frontend: 3000, Backend: 8000, Electric: 5133) -EXPOSE 3000 8000 5133 - -# Health check -HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \ - CMD curl -f http://localhost:3000 || exit 1 - -# Run entrypoint -CMD ["/app/entrypoint.sh"] diff --git a/docker-compose.quickstart.yml b/docker-compose.quickstart.yml deleted file mode 100644 index ff72618b7..000000000 --- a/docker-compose.quickstart.yml +++ /dev/null @@ -1,80 +0,0 @@ -# SurfSense Quick Start Docker Compose -# -# This is a simplified docker-compose for quick local deployment using pre-built images. -# For production or customized deployments, use the main docker-compose.yml -# -# Usage: -# 1. (Optional) Create a .env file with your configuration -# 2. Run: docker compose -f docker-compose.quickstart.yml up -d -# 3. Access SurfSense at http://localhost:3000 -# -# All Environment Variables are Optional: -# - SECRET_KEY: JWT secret key (auto-generated and persisted if not set) -# - EMBEDDING_MODEL: Embedding model to use (default: sentence-transformers/all-MiniLM-L6-v2) -# - ETL_SERVICE: Document parsing service - DOCLING, UNSTRUCTURED, or LLAMACLOUD (default: DOCLING) -# - TTS_SERVICE: Text-to-speech service for podcasts (default: local/kokoro) -# - STT_SERVICE: Speech-to-text service with model size (default: local/base) -# - FIRECRAWL_API_KEY: For web crawling features - -version: "3.8" - -services: - # All-in-one SurfSense container - surfsense: - image: ghcr.io/modsetter/surfsense:latest - container_name: surfsense - ports: - - "${FRONTEND_PORT:-3000}:3000" - - "${BACKEND_PORT:-8000}:8000" - volumes: - - surfsense-data:/data - environment: - # Authentication (auto-generated if not set) - - SECRET_KEY=${SECRET_KEY:-} - - # Auth Configuration - - AUTH_TYPE=${AUTH_TYPE:-LOCAL} - - GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID:-} - - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-} - - # AI/ML Configuration - - EMBEDDING_MODEL=${EMBEDDING_MODEL:-sentence-transformers/all-MiniLM-L6-v2} - - RERANKERS_ENABLED=${RERANKERS_ENABLED:-FALSE} - - RERANKERS_MODEL_NAME=${RERANKERS_MODEL_NAME:-} - - RERANKERS_MODEL_TYPE=${RERANKERS_MODEL_TYPE:-} - - # Document Processing - - ETL_SERVICE=${ETL_SERVICE:-DOCLING} - - UNSTRUCTURED_API_KEY=${UNSTRUCTURED_API_KEY:-} - - LLAMA_CLOUD_API_KEY=${LLAMA_CLOUD_API_KEY:-} - - # Audio Services - - TTS_SERVICE=${TTS_SERVICE:-local/kokoro} - - TTS_SERVICE_API_KEY=${TTS_SERVICE_API_KEY:-} - - STT_SERVICE=${STT_SERVICE:-local/base} - - STT_SERVICE_API_KEY=${STT_SERVICE_API_KEY:-} - - # Web Crawling - - FIRECRAWL_API_KEY=${FIRECRAWL_API_KEY:-} - - # Optional Features - - REGISTRATION_ENABLED=${REGISTRATION_ENABLED:-TRUE} - - SCHEDULE_CHECKER_INTERVAL=${SCHEDULE_CHECKER_INTERVAL:-1m} - - # LangSmith Observability (optional) - - LANGSMITH_TRACING=${LANGSMITH_TRACING:-false} - - LANGSMITH_ENDPOINT=${LANGSMITH_ENDPOINT:-} - - LANGSMITH_API_KEY=${LANGSMITH_API_KEY:-} - - LANGSMITH_PROJECT=${LANGSMITH_PROJECT:-} - restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3000", "&&", "curl", "-f", "http://localhost:8000/docs"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 120s - -volumes: - surfsense-data: - name: surfsense-data - diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 2bf62b883..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,165 +0,0 @@ -services: - db: - image: ankane/pgvector:latest - ports: - - "${POSTGRES_PORT:-5432}:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - - ./scripts/docker/postgresql.conf:/etc/postgresql/postgresql.conf:ro - - ./scripts/docker/init-electric-user.sh:/docker-entrypoint-initdb.d/init-electric-user.sh:ro - environment: - - POSTGRES_USER=${POSTGRES_USER:-postgres} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} - - POSTGRES_DB=${POSTGRES_DB:-surfsense} - - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric} - - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password} - command: postgres -c config_file=/etc/postgresql/postgresql.conf - - pgadmin: - image: dpage/pgadmin4 - ports: - - "${PGADMIN_PORT:-5050}:80" - environment: - - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL:-admin@surfsense.com} - - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD:-surfsense} - volumes: - - pgadmin_data:/var/lib/pgadmin - depends_on: - - db - - redis: - image: redis:7-alpine - ports: - - "${REDIS_PORT:-6379}:6379" - volumes: - - redis_data:/data - command: redis-server --appendonly yes - - backend: - build: ./surfsense_backend - # image: ghcr.io/modsetter/surfsense_backend:latest - ports: - - "${BACKEND_PORT:-8000}:8000" - volumes: - - ./surfsense_backend/app:/app/app - - shared_temp:/tmp - # Uncomment and edit the line below to enable Obsidian vault indexing - # - /path/to/your/obsidian/vault:/obsidian-vault:ro - env_file: - - ./surfsense_backend/.env - environment: - - DATABASE_URL=postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-surfsense} - - CELERY_BROKER_URL=redis://redis:${REDIS_PORT:-6379}/0 - - CELERY_RESULT_BACKEND=redis://redis:${REDIS_PORT:-6379}/0 - - REDIS_APP_URL=redis://redis:${REDIS_PORT:-6379}/0 - # Queue name isolation - prevents task collision if Redis is shared with other apps - - CELERY_TASK_DEFAULT_QUEUE=surfsense - - PYTHONPATH=/app - - UVICORN_LOOP=asyncio - - UNSTRUCTURED_HAS_PATCHED_LOOP=1 - - LANGCHAIN_TRACING_V2=false - - LANGSMITH_TRACING=false - - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric} - - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password} - - AUTH_TYPE=${AUTH_TYPE:-LOCAL} - - NEXT_FRONTEND_URL=${NEXT_FRONTEND_URL:-http://localhost:3000} - depends_on: - - db - - redis - - # Run these services separately in production - # celery_worker: - # build: ./surfsense_backend - # # image: ghcr.io/modsetter/surfsense_backend:latest - # command: celery -A app.celery_app worker --loglevel=info --concurrency=1 --pool=solo - # volumes: - # - ./surfsense_backend:/app - # - shared_temp:/tmp - # env_file: - # - ./surfsense_backend/.env - # environment: - # - DATABASE_URL=postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-surfsense} - # - CELERY_BROKER_URL=redis://redis:${REDIS_PORT:-6379}/0 - # - CELERY_RESULT_BACKEND=redis://redis:${REDIS_PORT:-6379}/0 - # - PYTHONPATH=/app - # depends_on: - # - db - # - redis - # - backend - - # celery_beat: - # build: ./surfsense_backend - # # image: ghcr.io/modsetter/surfsense_backend:latest - # command: celery -A app.celery_app beat --loglevel=info - # volumes: - # - ./surfsense_backend:/app - # - shared_temp:/tmp - # env_file: - # - ./surfsense_backend/.env - # environment: - # - DATABASE_URL=postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-surfsense} - # - CELERY_BROKER_URL=redis://redis:${REDIS_PORT:-6379}/0 - # - CELERY_RESULT_BACKEND=redis://redis:${REDIS_PORT:-6379}/0 - # - PYTHONPATH=/app - # depends_on: - # - db - # - redis - # - celery_worker - - # flower: - # build: ./surfsense_backend - # # image: ghcr.io/modsetter/surfsense_backend:latest - # command: celery -A app.celery_app flower --port=5555 - # ports: - # - "${FLOWER_PORT:-5555}:5555" - # env_file: - # - ./surfsense_backend/.env - # environment: - # - CELERY_BROKER_URL=redis://redis:${REDIS_PORT:-6379}/0 - # - CELERY_RESULT_BACKEND=redis://redis:${REDIS_PORT:-6379}/0 - # - PYTHONPATH=/app - # depends_on: - # - redis - # - celery_worker - - electric: - image: electricsql/electric:latest - ports: - - "${ELECTRIC_PORT:-5133}:3000" - environment: - - DATABASE_URL=${ELECTRIC_DATABASE_URL:-postgresql://${ELECTRIC_DB_USER:-electric}:${ELECTRIC_DB_PASSWORD:-electric_password}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-surfsense}?sslmode=disable} - - ELECTRIC_INSECURE=true - - ELECTRIC_WRITE_TO_PG_MODE=direct - restart: unless-stopped - # depends_on: - # - db - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3000/v1/health"] - interval: 10s - timeout: 5s - retries: 5 - - frontend: - build: - context: ./surfsense_web - # image: ghcr.io/modsetter/surfsense_ui:latest - args: - NEXT_PUBLIC_FASTAPI_BACKEND_URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:8000} - NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:-LOCAL} - NEXT_PUBLIC_ETL_SERVICE: ${NEXT_PUBLIC_ETL_SERVICE:-DOCLING} - ports: - - "${FRONTEND_PORT:-3000}:3000" - env_file: - - ./surfsense_web/.env - environment: - - NEXT_PUBLIC_ELECTRIC_URL=${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133} - - NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure - depends_on: - - backend - - electric - -volumes: - postgres_data: - pgadmin_data: - redis_data: - shared_temp: diff --git a/scripts/docker/entrypoint-allinone.sh b/scripts/docker/entrypoint-allinone.sh deleted file mode 100644 index 4f88b3382..000000000 --- a/scripts/docker/entrypoint-allinone.sh +++ /dev/null @@ -1,243 +0,0 @@ -#!/bin/bash -set -e - -echo "===========================================" -echo " 🏄 SurfSense All-in-One Container" -echo "===========================================" - -# Create log directory -mkdir -p /var/log/supervisor - -# ================================================ -# Ensure data directory exists -# ================================================ -mkdir -p /data - -# ================================================ -# Generate SECRET_KEY if not provided -# ================================================ -if [ -z "$SECRET_KEY" ]; then - # Generate a random secret key and persist it - if [ -f /data/.secret_key ]; then - export SECRET_KEY=$(cat /data/.secret_key) - echo "✅ Using existing SECRET_KEY from persistent storage" - else - export SECRET_KEY=$(python3 -c "import secrets; print(secrets.token_urlsafe(32))") - echo "$SECRET_KEY" > /data/.secret_key - chmod 600 /data/.secret_key - echo "✅ Generated new SECRET_KEY (saved for persistence)" - fi -fi - -# ================================================ -# Set default TTS/STT services if not provided -# ================================================ -if [ -z "$TTS_SERVICE" ]; then - export TTS_SERVICE="local/kokoro" - echo "✅ Using default TTS_SERVICE: local/kokoro" -fi - -if [ -z "$STT_SERVICE" ]; then - export STT_SERVICE="local/base" - echo "✅ Using default STT_SERVICE: local/base" -fi - -# ================================================ -# Set Electric SQL configuration -# ================================================ -export ELECTRIC_DB_USER="${ELECTRIC_DB_USER:-electric}" -export ELECTRIC_DB_PASSWORD="${ELECTRIC_DB_PASSWORD:-electric_password}" -if [ -z "$ELECTRIC_DATABASE_URL" ]; then - export ELECTRIC_DATABASE_URL="postgresql://${ELECTRIC_DB_USER}:${ELECTRIC_DB_PASSWORD}@localhost:5432/${POSTGRES_DB:-surfsense}?sslmode=disable" - echo "✅ Electric SQL URL configured dynamically" -else - # Ensure sslmode=disable is in the URL if not already present - if [[ "$ELECTRIC_DATABASE_URL" != *"sslmode="* ]]; then - # Add sslmode=disable (handle both cases: with or without existing query params) - if [[ "$ELECTRIC_DATABASE_URL" == *"?"* ]]; then - export ELECTRIC_DATABASE_URL="${ELECTRIC_DATABASE_URL}&sslmode=disable" - else - export ELECTRIC_DATABASE_URL="${ELECTRIC_DATABASE_URL}?sslmode=disable" - fi - fi - echo "✅ Electric SQL URL configured from environment" -fi - -# Set Electric SQL port -export ELECTRIC_PORT="${ELECTRIC_PORT:-5133}" -export PORT="${ELECTRIC_PORT}" - -# ================================================ -# Initialize PostgreSQL if needed -# ================================================ -if [ ! -f /data/postgres/PG_VERSION ]; then - echo "📦 Initializing PostgreSQL database..." - - # Initialize PostgreSQL data directory - chown -R postgres:postgres /data/postgres - chmod 700 /data/postgres - - # Initialize with UTF8 encoding (required for proper text handling) - su - postgres -c "/usr/lib/postgresql/14/bin/initdb -D /data/postgres --encoding=UTF8 --locale=C.UTF-8" - - # Configure PostgreSQL for connections - echo "host all all 0.0.0.0/0 md5" >> /data/postgres/pg_hba.conf - echo "local all all trust" >> /data/postgres/pg_hba.conf - echo "listen_addresses='*'" >> /data/postgres/postgresql.conf - - # Enable logical replication for Electric SQL - echo "wal_level = logical" >> /data/postgres/postgresql.conf - echo "max_replication_slots = 10" >> /data/postgres/postgresql.conf - echo "max_wal_senders = 10" >> /data/postgres/postgresql.conf - - # Start PostgreSQL temporarily to create database and user - su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres -l /tmp/postgres_init.log start" - - # Wait for PostgreSQL to be ready - sleep 5 - - # Create user and database - su - postgres -c "psql -c \"CREATE USER ${POSTGRES_USER:-surfsense} WITH PASSWORD '${POSTGRES_PASSWORD:-surfsense}' SUPERUSER;\"" - su - postgres -c "psql -c \"CREATE DATABASE ${POSTGRES_DB:-surfsense} OWNER ${POSTGRES_USER:-surfsense};\"" - - # Enable pgvector extension - su - postgres -c "psql -d ${POSTGRES_DB:-surfsense} -c 'CREATE EXTENSION IF NOT EXISTS vector;'" - - # Create Electric SQL replication user (idempotent - uses IF NOT EXISTS) - echo "📡 Creating Electric SQL replication user..." - su - postgres -c "psql -d ${POSTGRES_DB:-surfsense} <<-EOSQL - DO \\\$\\\$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_user WHERE usename = '${ELECTRIC_DB_USER}') THEN - CREATE USER ${ELECTRIC_DB_USER} WITH REPLICATION PASSWORD '${ELECTRIC_DB_PASSWORD}'; - END IF; - END - \\\$\\\$; - - GRANT CONNECT ON DATABASE ${POSTGRES_DB:-surfsense} TO ${ELECTRIC_DB_USER}; - GRANT USAGE ON SCHEMA public TO ${ELECTRIC_DB_USER}; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${ELECTRIC_DB_USER}; - GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO ${ELECTRIC_DB_USER}; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ${ELECTRIC_DB_USER}; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO ${ELECTRIC_DB_USER}; - - -- Create the publication for Electric SQL (if not exists) - DO \\\$\\\$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN - CREATE PUBLICATION electric_publication_default; - END IF; - END - \\\$\\\$; -EOSQL" - echo "✅ Electric SQL user '${ELECTRIC_DB_USER}' created" - - # Stop temporary PostgreSQL - su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres stop" - - echo "✅ PostgreSQL initialized successfully" -else - echo "✅ PostgreSQL data directory already exists" -fi - -# ================================================ -# Initialize Redis data directory -# ================================================ -mkdir -p /data/redis -chmod 755 /data/redis -echo "✅ Redis data directory ready" - -# ================================================ -# Copy frontend build to runtime location -# ================================================ -if [ -d /app/frontend/.next/standalone ]; then - cp -r /app/frontend/.next/standalone/* /app/frontend/ 2>/dev/null || true - cp -r /app/frontend/.next/static /app/frontend/.next/static 2>/dev/null || true -fi - -# ================================================ -# Runtime Environment Variable Replacement -# ================================================ -# Next.js NEXT_PUBLIC_* vars are baked in at build time. -# This replaces placeholder values with actual runtime env vars. -echo "🔧 Applying runtime environment configuration..." - -# Set defaults if not provided -NEXT_PUBLIC_FASTAPI_BACKEND_URL="${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:8000}" -NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE="${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:-LOCAL}" -NEXT_PUBLIC_ETL_SERVICE="${NEXT_PUBLIC_ETL_SERVICE:-DOCLING}" -NEXT_PUBLIC_ELECTRIC_URL="${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}" -NEXT_PUBLIC_ELECTRIC_AUTH_MODE="${NEXT_PUBLIC_ELECTRIC_AUTH_MODE:-insecure}" -NEXT_PUBLIC_DEPLOYMENT_MODE="${NEXT_PUBLIC_DEPLOYMENT_MODE:-self-hosted}" - -# Replace placeholders in all JS files -find /app/frontend -type f \( -name "*.js" -o -name "*.json" \) -exec sed -i \ - -e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_URL__|${NEXT_PUBLIC_FASTAPI_BACKEND_URL}|g" \ - -e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__|${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}|g" \ - -e "s|__NEXT_PUBLIC_ETL_SERVICE__|${NEXT_PUBLIC_ETL_SERVICE}|g" \ - -e "s|__NEXT_PUBLIC_ELECTRIC_URL__|${NEXT_PUBLIC_ELECTRIC_URL}|g" \ - -e "s|__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__|${NEXT_PUBLIC_ELECTRIC_AUTH_MODE}|g" \ - -e "s|__NEXT_PUBLIC_DEPLOYMENT_MODE__|${NEXT_PUBLIC_DEPLOYMENT_MODE}|g" \ - {} + - -echo "✅ Environment configuration applied" -echo " Backend URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}" -echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}" -echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}" -echo " Electric URL: ${NEXT_PUBLIC_ELECTRIC_URL}" -echo " Deployment Mode: ${NEXT_PUBLIC_DEPLOYMENT_MODE}" - -# ================================================ -# Run database migrations -# ================================================ -run_migrations() { - echo "🔄 Running database migrations..." - - # Start PostgreSQL temporarily for migrations - su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres -l /tmp/postgres_migrate.log start" - sleep 5 - - # Start Redis temporarily for migrations (some might need it) - redis-server --dir /data/redis --daemonize yes - sleep 2 - - # Run alembic migrations - cd /app/backend - alembic upgrade head || echo "⚠️ Migrations may have already been applied" - - # Stop temporary services - redis-cli shutdown || true - su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres stop" - - echo "✅ Database migrations complete" -} - -# Always run migrations on startup - alembic upgrade head is safe to run -# every time. It only applies pending migrations (never re-runs applied ones, -# never calls downgrade). This ensures updates are applied automatically. -run_migrations - -# ================================================ -# Environment Variables Info -# ================================================ -echo "" -echo "===========================================" -echo " 📋 Configuration" -echo "===========================================" -echo " Frontend URL: http://localhost:3000" -echo " Backend API: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}" -echo " API Docs: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}/docs" -echo " Electric URL: ${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}" -echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}" -echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}" -echo " TTS Service: ${TTS_SERVICE}" -echo " STT Service: ${STT_SERVICE}" -echo "===========================================" -echo "" - -# ================================================ -# Start Supervisor (manages all services) -# ================================================ -echo "🚀 Starting all services..." -exec /usr/local/bin/supervisord -c /etc/supervisor/conf.d/surfsense.conf - diff --git a/scripts/docker/init-electric-user.sh b/scripts/docker/init-electric-user.sh deleted file mode 100755 index b3856c573..000000000 --- a/scripts/docker/init-electric-user.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -# ============================================================================ -# Electric SQL User Initialization Script (docker-compose only) -# ============================================================================ -# This script is ONLY used when running via docker-compose. -# -# How it works: -# - docker-compose.yml mounts this script into the PostgreSQL container's -# /docker-entrypoint-initdb.d/ directory -# - PostgreSQL automatically executes scripts in that directory on first -# container initialization -# -# For local PostgreSQL users (non-Docker), this script is NOT used. -# Instead, the Electric user is created by Alembic migration 66 -# (66_add_notifications_table_and_electric_replication.py). -# -# Both approaches are idempotent (use IF NOT EXISTS), so running both -# will not cause conflicts. -# ============================================================================ - -set -e - -# Use environment variables with defaults -ELECTRIC_DB_USER="${ELECTRIC_DB_USER:-electric}" -ELECTRIC_DB_PASSWORD="${ELECTRIC_DB_PASSWORD:-electric_password}" - -echo "Creating Electric SQL replication user: $ELECTRIC_DB_USER" - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - DO \$\$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_user WHERE usename = '$ELECTRIC_DB_USER') THEN - CREATE USER $ELECTRIC_DB_USER WITH REPLICATION PASSWORD '$ELECTRIC_DB_PASSWORD'; - END IF; - END - \$\$; - - GRANT CONNECT ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER; - GRANT CREATE ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER; - GRANT USAGE ON SCHEMA public TO $ELECTRIC_DB_USER; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO $ELECTRIC_DB_USER; - GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO $ELECTRIC_DB_USER; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $ELECTRIC_DB_USER; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO $ELECTRIC_DB_USER; - - -- Create the publication for Electric SQL (if not exists) - DO \$\$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN - CREATE PUBLICATION electric_publication_default; - END IF; - END - \$\$; -EOSQL - -echo "Electric SQL user '$ELECTRIC_DB_USER' and publication created successfully" diff --git a/scripts/docker/init-postgres.sh b/scripts/docker/init-postgres.sh deleted file mode 100644 index b6ddb6a50..000000000 --- a/scripts/docker/init-postgres.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -# PostgreSQL initialization script for SurfSense -# This script is called during container startup if the database needs initialization - -set -e - -PGDATA=${PGDATA:-/data/postgres} -POSTGRES_USER=${POSTGRES_USER:-surfsense} -POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-surfsense} -POSTGRES_DB=${POSTGRES_DB:-surfsense} - -# Electric SQL user credentials (configurable) -ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric} -ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password} - -echo "Initializing PostgreSQL..." - -# Check if PostgreSQL is already initialized -if [ -f "$PGDATA/PG_VERSION" ]; then - echo "PostgreSQL data directory already exists. Skipping initialization." - exit 0 -fi - -# Initialize the database cluster -/usr/lib/postgresql/14/bin/initdb -D "$PGDATA" --username=postgres - -# Configure PostgreSQL -cat >> "$PGDATA/postgresql.conf" << EOF -listen_addresses = '*' -max_connections = 200 -shared_buffers = 256MB - -# Enable logical replication (required for Electric SQL) -wal_level = logical -max_replication_slots = 10 -max_wal_senders = 10 - -# Performance settings -checkpoint_timeout = 10min -max_wal_size = 1GB -min_wal_size = 80MB -EOF - -cat >> "$PGDATA/pg_hba.conf" << EOF -# Allow connections from anywhere with password -host all all 0.0.0.0/0 md5 -host all all ::0/0 md5 -EOF - -# Start PostgreSQL temporarily -/usr/lib/postgresql/14/bin/pg_ctl -D "$PGDATA" -l /tmp/postgres_init.log start - -# Wait for PostgreSQL to start -sleep 3 - -# Create user and database -psql -U postgres << EOF -CREATE USER $POSTGRES_USER WITH PASSWORD '$POSTGRES_PASSWORD' SUPERUSER; -CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER; -\c $POSTGRES_DB -CREATE EXTENSION IF NOT EXISTS vector; - --- Create Electric SQL replication user -CREATE USER $ELECTRIC_DB_USER WITH REPLICATION PASSWORD '$ELECTRIC_DB_PASSWORD'; -GRANT CONNECT ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER; -GRANT USAGE ON SCHEMA public TO $ELECTRIC_DB_USER; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO $ELECTRIC_DB_USER; -GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO $ELECTRIC_DB_USER; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $ELECTRIC_DB_USER; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO $ELECTRIC_DB_USER; -EOF - -echo "PostgreSQL initialized successfully." - -# Stop PostgreSQL (supervisor will start it) -/usr/lib/postgresql/14/bin/pg_ctl -D "$PGDATA" stop - diff --git a/scripts/docker/postgresql.conf b/scripts/docker/postgresql.conf deleted file mode 100644 index 99b29ba30..000000000 --- a/scripts/docker/postgresql.conf +++ /dev/null @@ -1,20 +0,0 @@ -# PostgreSQL configuration for Electric SQL -# This file is mounted into the PostgreSQL container - -listen_addresses = '*' -max_connections = 200 -shared_buffers = 256MB - -# Enable logical replication (required for Electric SQL) -wal_level = logical -max_replication_slots = 10 -max_wal_senders = 10 - -# Performance settings -checkpoint_timeout = 10min -max_wal_size = 1GB -min_wal_size = 80MB - -# Logging (optional, for debugging) -# log_statement = 'all' -# log_replication_commands = on diff --git a/scripts/docker/supervisor-allinone.conf b/scripts/docker/supervisor-allinone.conf deleted file mode 100644 index 1a21fcc04..000000000 --- a/scripts/docker/supervisor-allinone.conf +++ /dev/null @@ -1,121 +0,0 @@ -[supervisord] -nodaemon=true -logfile=/dev/stdout -logfile_maxbytes=0 -pidfile=/var/run/supervisord.pid -loglevel=info -user=root - -[unix_http_server] -file=/var/run/supervisor.sock -chmod=0700 - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl=unix:///var/run/supervisor.sock - -# PostgreSQL -[program:postgresql] -command=/usr/lib/postgresql/14/bin/postgres -D /data/postgres -user=postgres -autostart=true -autorestart=true -priority=10 -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -environment=PGDATA="/data/postgres" - -# Redis -[program:redis] -command=/usr/bin/redis-server --dir /data/redis --appendonly yes -autostart=true -autorestart=true -priority=20 -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 - -# Backend API -[program:backend] -command=python main.py -directory=/app/backend -autostart=true -autorestart=true -priority=30 -startsecs=10 -startretries=3 -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -environment=PYTHONPATH="/app/backend",UVICORN_LOOP="asyncio",UNSTRUCTURED_HAS_PATCHED_LOOP="1" - -# Celery Worker -[program:celery-worker] -command=celery -A app.celery_app worker --loglevel=info --concurrency=2 --pool=solo --queues=surfsense,surfsense.connectors -directory=/app/backend -autostart=true -autorestart=true -priority=40 -startsecs=15 -startretries=3 -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -environment=PYTHONPATH="/app/backend" - -# Celery Beat (scheduler) -[program:celery-beat] -command=celery -A app.celery_app beat --loglevel=info -directory=/app/backend -autostart=true -autorestart=true -priority=50 -startsecs=20 -startretries=3 -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -environment=PYTHONPATH="/app/backend" - -# Electric SQL (real-time sync) -[program:electric] -command=/app/electric-release/bin/entrypoint start -autostart=true -autorestart=true -priority=25 -startsecs=10 -startretries=3 -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -environment=DATABASE_URL="%(ENV_ELECTRIC_DATABASE_URL)s",ELECTRIC_INSECURE="%(ENV_ELECTRIC_INSECURE)s",ELECTRIC_WRITE_TO_PG_MODE="%(ENV_ELECTRIC_WRITE_TO_PG_MODE)s",RELEASE_COOKIE="surfsense_electric_cookie",PORT="%(ENV_ELECTRIC_PORT)s" - -# Frontend -[program:frontend] -command=node server.js -directory=/app/frontend -autostart=true -autorestart=true -priority=60 -startsecs=5 -startretries=3 -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -environment=NODE_ENV="production",PORT="3000",HOSTNAME="0.0.0.0" - -# Process Groups -[group:surfsense] -programs=postgresql,redis,electric,backend,celery-worker,celery-beat,frontend -priority=999 -