diff --git a/Dockerfile.allinone b/Dockerfile.allinone index 95893c0b5..2e160d3dc 100644 --- a/Dockerfile.allinone +++ b/Dockerfile.allinone @@ -1,10 +1,11 @@ # SurfSense All-in-One Docker Image -# This image bundles PostgreSQL+pgvector, Redis, Backend, and Frontend -# Usage: docker run -d -p 3000:3000 -p 8000:8000 -v surfsense-data:/data --name surfsense ghcr.io/modsetter/surfsense:latest +# This image bundles PostgreSQL+pgvector, Redis, Electric SQL, Backend, and Frontend +# Usage: docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 -v surfsense-data:/data --name surfsense ghcr.io/modsetter/surfsense:latest # # Included Services (all run locally by default): # - PostgreSQL 14 + pgvector (vector database) # - Redis (task queue) +# - Electric SQL (real-time sync) # - Docling (document processing, CPU-only, OCR disabled) # - Kokoro TTS (local text-to-speech for podcasts) # - Faster-Whisper (local speech-to-text for audio files) @@ -14,7 +15,12 @@ # will be available in the future for faster AI inference. # ==================== -# Stage 1: Build Frontend +# Stage 1: Get Electric SQL Binary +# ==================== +FROM electricsql/electric:latest AS electric-builder + +# ==================== +# Stage 2: Build Frontend # ==================== FROM node:20-alpine AS frontend-builder @@ -42,12 +48,14 @@ RUN pnpm fumadocs-mdx ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__ ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__ ENV NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__ +ENV NEXT_PUBLIC_ELECTRIC_URL=__NEXT_PUBLIC_ELECTRIC_URL__ +ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__ # Build RUN pnpm run build # ==================== -# Stage 2: Runtime Image +# Stage 3: Runtime Image # ==================== FROM ubuntu:22.04 AS runtime @@ -167,6 +175,11 @@ COPY --from=frontend-builder /app/public ./public COPY surfsense_web/content/docs /app/surfsense_web/content/docs +# ==================== +# Copy Electric SQL Release +# ==================== +COPY --from=electric-builder /app /app/electric-release + # ==================== # Setup Backend # ==================== @@ -238,11 +251,22 @@ ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000 ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL ENV NEXT_PUBLIC_ETL_SERVICE=DOCLING +# Electric SQL configuration (ELECTRIC_DATABASE_URL is built dynamically by entrypoint from these values) +ENV ELECTRIC_DB_USER=electric +ENV ELECTRIC_DB_PASSWORD=electric_password +# Note: ELECTRIC_DATABASE_URL is NOT set here - entrypoint builds it dynamically from ELECTRIC_DB_USER/PASSWORD +ENV ELECTRIC_INSECURE=true +ENV ELECTRIC_WRITE_TO_PG_MODE=direct +ENV ELECTRIC_PORT=5133 +ENV PORT=5133 +ENV NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133 +ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure + # Data volume VOLUME ["/data"] -# Expose ports -EXPOSE 3000 8000 +# Expose ports (Frontend: 3000, Backend: 8000, Electric: 5133) +EXPOSE 3000 8000 5133 # Health check HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \ diff --git a/scripts/docker/entrypoint-allinone.sh b/scripts/docker/entrypoint-allinone.sh index 8248968ab..4cbb34761 100644 --- a/scripts/docker/entrypoint-allinone.sh +++ b/scripts/docker/entrypoint-allinone.sh @@ -42,6 +42,31 @@ if [ -z "$STT_SERVICE" ]; then echo "✅ Using default STT_SERVICE: local/base" fi +# ================================================ +# Set Electric SQL configuration +# ================================================ +export ELECTRIC_DB_USER="${ELECTRIC_DB_USER:-electric}" +export ELECTRIC_DB_PASSWORD="${ELECTRIC_DB_PASSWORD:-electric_password}" +if [ -z "$ELECTRIC_DATABASE_URL" ]; then + export ELECTRIC_DATABASE_URL="postgresql://${ELECTRIC_DB_USER}:${ELECTRIC_DB_PASSWORD}@localhost:5432/${POSTGRES_DB:-surfsense}?sslmode=disable" + echo "✅ Electric SQL URL configured dynamically" +else + # Ensure sslmode=disable is in the URL if not already present + if [[ "$ELECTRIC_DATABASE_URL" != *"sslmode="* ]]; then + # Add sslmode=disable (handle both cases: with or without existing query params) + if [[ "$ELECTRIC_DATABASE_URL" == *"?"* ]]; then + export ELECTRIC_DATABASE_URL="${ELECTRIC_DATABASE_URL}&sslmode=disable" + else + export ELECTRIC_DATABASE_URL="${ELECTRIC_DATABASE_URL}?sslmode=disable" + fi + fi + echo "✅ Electric SQL URL configured from environment" +fi + +# Set Electric SQL port +export ELECTRIC_PORT="${ELECTRIC_PORT:-5133}" +export PORT="${ELECTRIC_PORT}" + # ================================================ # Initialize PostgreSQL if needed # ================================================ @@ -60,6 +85,11 @@ if [ ! -f /data/postgres/PG_VERSION ]; then echo "local all all trust" >> /data/postgres/pg_hba.conf echo "listen_addresses='*'" >> /data/postgres/postgresql.conf + # Enable logical replication for Electric SQL + echo "wal_level = logical" >> /data/postgres/postgresql.conf + echo "max_replication_slots = 10" >> /data/postgres/postgresql.conf + echo "max_wal_senders = 10" >> /data/postgres/postgresql.conf + # Start PostgreSQL temporarily to create database and user su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres -l /tmp/postgres_init.log start" @@ -73,6 +103,35 @@ if [ ! -f /data/postgres/PG_VERSION ]; then # Enable pgvector extension su - postgres -c "psql -d ${POSTGRES_DB:-surfsense} -c 'CREATE EXTENSION IF NOT EXISTS vector;'" + # Create Electric SQL replication user (idempotent - uses IF NOT EXISTS) + echo "📡 Creating Electric SQL replication user..." + su - postgres -c "psql -d ${POSTGRES_DB:-surfsense} <<-EOSQL + DO \\\$\\\$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_user WHERE usename = '${ELECTRIC_DB_USER}') THEN + CREATE USER ${ELECTRIC_DB_USER} WITH REPLICATION PASSWORD '${ELECTRIC_DB_PASSWORD}'; + END IF; + END + \\\$\\\$; + + GRANT CONNECT ON DATABASE ${POSTGRES_DB:-surfsense} TO ${ELECTRIC_DB_USER}; + GRANT USAGE ON SCHEMA public TO ${ELECTRIC_DB_USER}; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${ELECTRIC_DB_USER}; + GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO ${ELECTRIC_DB_USER}; + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ${ELECTRIC_DB_USER}; + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO ${ELECTRIC_DB_USER}; + + -- Create the publication for Electric SQL (if not exists) + DO \\\$\\\$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN + CREATE PUBLICATION electric_publication_default; + END IF; + END + \\\$\\\$; +EOSQL" + echo "✅ Electric SQL user '${ELECTRIC_DB_USER}' created" + # Stop temporary PostgreSQL su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres stop" @@ -107,18 +166,23 @@ echo "🔧 Applying runtime environment configuration..." NEXT_PUBLIC_FASTAPI_BACKEND_URL="${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:8000}" NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE="${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:-LOCAL}" NEXT_PUBLIC_ETL_SERVICE="${NEXT_PUBLIC_ETL_SERVICE:-DOCLING}" +NEXT_PUBLIC_ELECTRIC_URL="${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}" +NEXT_PUBLIC_ELECTRIC_AUTH_MODE="${NEXT_PUBLIC_ELECTRIC_AUTH_MODE:-insecure}" # Replace placeholders in all JS files find /app/frontend -type f \( -name "*.js" -o -name "*.json" \) -exec sed -i \ -e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_URL__|${NEXT_PUBLIC_FASTAPI_BACKEND_URL}|g" \ -e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__|${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}|g" \ -e "s|__NEXT_PUBLIC_ETL_SERVICE__|${NEXT_PUBLIC_ETL_SERVICE}|g" \ + -e "s|__NEXT_PUBLIC_ELECTRIC_URL__|${NEXT_PUBLIC_ELECTRIC_URL}|g" \ + -e "s|__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__|${NEXT_PUBLIC_ELECTRIC_AUTH_MODE}|g" \ {} + echo "✅ Environment configuration applied" -echo " Backend URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}" -echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}" -echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}" +echo " Backend URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}" +echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}" +echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}" +echo " Electric URL: ${NEXT_PUBLIC_ELECTRIC_URL}" # ================================================ # Run database migrations @@ -161,6 +225,7 @@ echo "===========================================" echo " Frontend URL: http://localhost:3000" echo " Backend API: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}" echo " API Docs: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}/docs" +echo " Electric URL: ${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}" echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}" echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}" echo " TTS Service: ${TTS_SERVICE}" diff --git a/scripts/docker/init-electric-user.sh b/scripts/docker/init-electric-user.sh index fcd31b2e2..6ce9a42cd 100755 --- a/scripts/docker/init-electric-user.sh +++ b/scripts/docker/init-electric-user.sh @@ -1,11 +1,18 @@ #!/bin/sh # ============================================================================ -# Electric SQL User Initialization Script (Docker deployments) +# Electric SQL User Initialization Script (docker-compose only) # ============================================================================ -# Creates the Electric SQL replication user for Docker deployments. -# -# For local PostgreSQL users (non-Docker), this is handled by Alembic -# migration 66 (66_add_notifications_table_and_electric_replication.py). +# This script is ONLY used when running via docker-compose. +# +# How it works: +# - docker-compose.yml mounts this script into the PostgreSQL container's +# /docker-entrypoint-initdb.d/ directory +# - PostgreSQL automatically executes scripts in that directory on first +# container initialization +# +# For local PostgreSQL users (non-Docker), this script is NOT used. +# Instead, the Electric user is created by Alembic migration 66 +# (66_add_notifications_table_and_electric_replication.py). # # Both approaches are idempotent (use IF NOT EXISTS), so running both # will not cause conflicts. diff --git a/scripts/docker/init-postgres.sh b/scripts/docker/init-postgres.sh index a184e87b6..b6ddb6a50 100644 --- a/scripts/docker/init-postgres.sh +++ b/scripts/docker/init-postgres.sh @@ -9,6 +9,10 @@ POSTGRES_USER=${POSTGRES_USER:-surfsense} POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-surfsense} POSTGRES_DB=${POSTGRES_DB:-surfsense} +# Electric SQL user credentials (configurable) +ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric} +ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password} + echo "Initializing PostgreSQL..." # Check if PostgreSQL is already initialized @@ -57,13 +61,13 @@ CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER; CREATE EXTENSION IF NOT EXISTS vector; -- Create Electric SQL replication user -CREATE USER electric WITH REPLICATION PASSWORD 'electric_password'; -GRANT CONNECT ON DATABASE $POSTGRES_DB TO electric; -GRANT USAGE ON SCHEMA public TO electric; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO electric; -GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO electric; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO electric; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO electric; +CREATE USER $ELECTRIC_DB_USER WITH REPLICATION PASSWORD '$ELECTRIC_DB_PASSWORD'; +GRANT CONNECT ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER; +GRANT USAGE ON SCHEMA public TO $ELECTRIC_DB_USER; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO $ELECTRIC_DB_USER; +GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO $ELECTRIC_DB_USER; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $ELECTRIC_DB_USER; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO $ELECTRIC_DB_USER; EOF echo "PostgreSQL initialized successfully." diff --git a/scripts/docker/supervisor-allinone.conf b/scripts/docker/supervisor-allinone.conf index 6cada0dc2..eb2404b3c 100644 --- a/scripts/docker/supervisor-allinone.conf +++ b/scripts/docker/supervisor-allinone.conf @@ -85,6 +85,20 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 environment=PYTHONPATH="/app/backend" +# Electric SQL (real-time sync) +[program:electric] +command=/app/electric-release/bin/entrypoint start +autostart=true +autorestart=true +priority=25 +startsecs=10 +startretries=3 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +environment=DATABASE_URL="%(ENV_ELECTRIC_DATABASE_URL)s",ELECTRIC_INSECURE="%(ENV_ELECTRIC_INSECURE)s",ELECTRIC_WRITE_TO_PG_MODE="%(ENV_ELECTRIC_WRITE_TO_PG_MODE)s",RELEASE_COOKIE="surfsense_electric_cookie",PORT="%(ENV_ELECTRIC_PORT)s" + # Frontend [program:frontend] command=node server.js @@ -102,6 +116,6 @@ environment=NODE_ENV="production",PORT="3000",HOSTNAME="0.0.0.0" # Process Groups [group:surfsense] -programs=postgresql,redis,backend,celery-worker,celery-beat,frontend +programs=postgresql,redis,electric,backend,celery-worker,celery-beat,frontend priority=999