mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-04-25 00:36:31 +02:00
feat: docker all in one setup for electric-sql and edited comments in init-electric-user.sh
This commit is contained in:
Anish Sarkar
parent
35392c8e62
commit
eb1ddf0c92
5 changed files with 136 additions and 22 deletions
|
|
@ -1,10 +1,11 @@
|
|||
# SurfSense All-in-One Docker Image
|
||||
# This image bundles PostgreSQL+pgvector, Redis, Backend, and Frontend
|
||||
# Usage: docker run -d -p 3000:3000 -p 8000:8000 -v surfsense-data:/data --name surfsense ghcr.io/modsetter/surfsense:latest
|
||||
# This image bundles PostgreSQL+pgvector, Redis, Electric SQL, Backend, and Frontend
|
||||
# Usage: docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 -v surfsense-data:/data --name surfsense ghcr.io/modsetter/surfsense:latest
|
||||
#
|
||||
# Included Services (all run locally by default):
|
||||
# - PostgreSQL 14 + pgvector (vector database)
|
||||
# - Redis (task queue)
|
||||
# - Electric SQL (real-time sync)
|
||||
# - Docling (document processing, CPU-only, OCR disabled)
|
||||
# - Kokoro TTS (local text-to-speech for podcasts)
|
||||
# - Faster-Whisper (local speech-to-text for audio files)
|
||||
|
|
@ -14,7 +15,12 @@
|
|||
# will be available in the future for faster AI inference.
|
||||
|
||||
# ====================
|
||||
# Stage 1: Build Frontend
|
||||
# Stage 1: Get Electric SQL Binary
|
||||
# ====================
|
||||
FROM electricsql/electric:latest AS electric-builder
|
||||
|
||||
# ====================
|
||||
# Stage 2: Build Frontend
|
||||
# ====================
|
||||
FROM node:20-alpine AS frontend-builder
|
||||
|
||||
|
|
@ -42,12 +48,14 @@ RUN pnpm fumadocs-mdx
|
|||
ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__
|
||||
ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__
|
||||
ENV NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__
|
||||
ENV NEXT_PUBLIC_ELECTRIC_URL=__NEXT_PUBLIC_ELECTRIC_URL__
|
||||
ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__
|
||||
|
||||
# Build
|
||||
RUN pnpm run build
|
||||
|
||||
# ====================
|
||||
# Stage 2: Runtime Image
|
||||
# Stage 3: Runtime Image
|
||||
# ====================
|
||||
FROM ubuntu:22.04 AS runtime
|
||||
|
||||
|
|
@ -167,6 +175,11 @@ COPY --from=frontend-builder /app/public ./public
|
|||
|
||||
COPY surfsense_web/content/docs /app/surfsense_web/content/docs
|
||||
|
||||
# ====================
|
||||
# Copy Electric SQL Release
|
||||
# ====================
|
||||
COPY --from=electric-builder /app /app/electric-release
|
||||
|
||||
# ====================
|
||||
# Setup Backend
|
||||
# ====================
|
||||
|
|
@ -238,11 +251,22 @@ ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000
|
|||
ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL
|
||||
ENV NEXT_PUBLIC_ETL_SERVICE=DOCLING
|
||||
|
||||
# Electric SQL configuration (ELECTRIC_DATABASE_URL is built dynamically by entrypoint from these values)
|
||||
ENV ELECTRIC_DB_USER=electric
|
||||
ENV ELECTRIC_DB_PASSWORD=electric_password
|
||||
# Note: ELECTRIC_DATABASE_URL is NOT set here - entrypoint builds it dynamically from ELECTRIC_DB_USER/PASSWORD
|
||||
ENV ELECTRIC_INSECURE=true
|
||||
ENV ELECTRIC_WRITE_TO_PG_MODE=direct
|
||||
ENV ELECTRIC_PORT=5133
|
||||
ENV PORT=5133
|
||||
ENV NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133
|
||||
ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure
|
||||
|
||||
# Data volume
|
||||
VOLUME ["/data"]
|
||||
|
||||
# Expose ports
|
||||
EXPOSE 3000 8000
|
||||
# Expose ports (Frontend: 3000, Backend: 8000, Electric: 5133)
|
||||
EXPOSE 3000 8000 5133
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \
|
||||
|
|
|
|||
|
|
@ -42,6 +42,31 @@ if [ -z "$STT_SERVICE" ]; then
|
|||
echo "✅ Using default STT_SERVICE: local/base"
|
||||
fi
|
||||
|
||||
# ================================================
|
||||
# Set Electric SQL configuration
|
||||
# ================================================
|
||||
export ELECTRIC_DB_USER="${ELECTRIC_DB_USER:-electric}"
|
||||
export ELECTRIC_DB_PASSWORD="${ELECTRIC_DB_PASSWORD:-electric_password}"
|
||||
if [ -z "$ELECTRIC_DATABASE_URL" ]; then
|
||||
export ELECTRIC_DATABASE_URL="postgresql://${ELECTRIC_DB_USER}:${ELECTRIC_DB_PASSWORD}@localhost:5432/${POSTGRES_DB:-surfsense}?sslmode=disable"
|
||||
echo "✅ Electric SQL URL configured dynamically"
|
||||
else
|
||||
# Ensure sslmode=disable is in the URL if not already present
|
||||
if [[ "$ELECTRIC_DATABASE_URL" != *"sslmode="* ]]; then
|
||||
# Add sslmode=disable (handle both cases: with or without existing query params)
|
||||
if [[ "$ELECTRIC_DATABASE_URL" == *"?"* ]]; then
|
||||
export ELECTRIC_DATABASE_URL="${ELECTRIC_DATABASE_URL}&sslmode=disable"
|
||||
else
|
||||
export ELECTRIC_DATABASE_URL="${ELECTRIC_DATABASE_URL}?sslmode=disable"
|
||||
fi
|
||||
fi
|
||||
echo "✅ Electric SQL URL configured from environment"
|
||||
fi
|
||||
|
||||
# Set Electric SQL port
|
||||
export ELECTRIC_PORT="${ELECTRIC_PORT:-5133}"
|
||||
export PORT="${ELECTRIC_PORT}"
|
||||
|
||||
# ================================================
|
||||
# Initialize PostgreSQL if needed
|
||||
# ================================================
|
||||
|
|
@ -60,6 +85,11 @@ if [ ! -f /data/postgres/PG_VERSION ]; then
|
|||
echo "local all all trust" >> /data/postgres/pg_hba.conf
|
||||
echo "listen_addresses='*'" >> /data/postgres/postgresql.conf
|
||||
|
||||
# Enable logical replication for Electric SQL
|
||||
echo "wal_level = logical" >> /data/postgres/postgresql.conf
|
||||
echo "max_replication_slots = 10" >> /data/postgres/postgresql.conf
|
||||
echo "max_wal_senders = 10" >> /data/postgres/postgresql.conf
|
||||
|
||||
# Start PostgreSQL temporarily to create database and user
|
||||
su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres -l /tmp/postgres_init.log start"
|
||||
|
||||
|
|
@ -73,6 +103,35 @@ if [ ! -f /data/postgres/PG_VERSION ]; then
|
|||
# Enable pgvector extension
|
||||
su - postgres -c "psql -d ${POSTGRES_DB:-surfsense} -c 'CREATE EXTENSION IF NOT EXISTS vector;'"
|
||||
|
||||
# Create Electric SQL replication user (idempotent - uses IF NOT EXISTS)
|
||||
echo "📡 Creating Electric SQL replication user..."
|
||||
su - postgres -c "psql -d ${POSTGRES_DB:-surfsense} <<-EOSQL
|
||||
DO \\\$\\\$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_user WHERE usename = '${ELECTRIC_DB_USER}') THEN
|
||||
CREATE USER ${ELECTRIC_DB_USER} WITH REPLICATION PASSWORD '${ELECTRIC_DB_PASSWORD}';
|
||||
END IF;
|
||||
END
|
||||
\\\$\\\$;
|
||||
|
||||
GRANT CONNECT ON DATABASE ${POSTGRES_DB:-surfsense} TO ${ELECTRIC_DB_USER};
|
||||
GRANT USAGE ON SCHEMA public TO ${ELECTRIC_DB_USER};
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${ELECTRIC_DB_USER};
|
||||
GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO ${ELECTRIC_DB_USER};
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ${ELECTRIC_DB_USER};
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO ${ELECTRIC_DB_USER};
|
||||
|
||||
-- Create the publication for Electric SQL (if not exists)
|
||||
DO \\\$\\\$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN
|
||||
CREATE PUBLICATION electric_publication_default;
|
||||
END IF;
|
||||
END
|
||||
\\\$\\\$;
|
||||
EOSQL"
|
||||
echo "✅ Electric SQL user '${ELECTRIC_DB_USER}' created"
|
||||
|
||||
# Stop temporary PostgreSQL
|
||||
su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres stop"
|
||||
|
||||
|
|
@ -107,18 +166,23 @@ echo "🔧 Applying runtime environment configuration..."
|
|||
NEXT_PUBLIC_FASTAPI_BACKEND_URL="${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:8000}"
|
||||
NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE="${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:-LOCAL}"
|
||||
NEXT_PUBLIC_ETL_SERVICE="${NEXT_PUBLIC_ETL_SERVICE:-DOCLING}"
|
||||
NEXT_PUBLIC_ELECTRIC_URL="${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}"
|
||||
NEXT_PUBLIC_ELECTRIC_AUTH_MODE="${NEXT_PUBLIC_ELECTRIC_AUTH_MODE:-insecure}"
|
||||
|
||||
# Replace placeholders in all JS files
|
||||
find /app/frontend -type f \( -name "*.js" -o -name "*.json" \) -exec sed -i \
|
||||
-e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_URL__|${NEXT_PUBLIC_FASTAPI_BACKEND_URL}|g" \
|
||||
-e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__|${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}|g" \
|
||||
-e "s|__NEXT_PUBLIC_ETL_SERVICE__|${NEXT_PUBLIC_ETL_SERVICE}|g" \
|
||||
-e "s|__NEXT_PUBLIC_ELECTRIC_URL__|${NEXT_PUBLIC_ELECTRIC_URL}|g" \
|
||||
-e "s|__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__|${NEXT_PUBLIC_ELECTRIC_AUTH_MODE}|g" \
|
||||
{} +
|
||||
|
||||
echo "✅ Environment configuration applied"
|
||||
echo " Backend URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}"
|
||||
echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}"
|
||||
echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}"
|
||||
echo " Backend URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}"
|
||||
echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}"
|
||||
echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}"
|
||||
echo " Electric URL: ${NEXT_PUBLIC_ELECTRIC_URL}"
|
||||
|
||||
# ================================================
|
||||
# Run database migrations
|
||||
|
|
@ -161,6 +225,7 @@ echo "==========================================="
|
|||
echo " Frontend URL: http://localhost:3000"
|
||||
echo " Backend API: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}"
|
||||
echo " API Docs: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}/docs"
|
||||
echo " Electric URL: ${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}"
|
||||
echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}"
|
||||
echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}"
|
||||
echo " TTS Service: ${TTS_SERVICE}"
|
||||
|
|
|
|||
|
|
@ -1,11 +1,18 @@
|
|||
#!/bin/sh
|
||||
# ============================================================================
|
||||
# Electric SQL User Initialization Script (Docker deployments)
|
||||
# Electric SQL User Initialization Script (docker-compose only)
|
||||
# ============================================================================
|
||||
# Creates the Electric SQL replication user for Docker deployments.
|
||||
#
|
||||
# For local PostgreSQL users (non-Docker), this is handled by Alembic
|
||||
# migration 66 (66_add_notifications_table_and_electric_replication.py).
|
||||
# This script is ONLY used when running via docker-compose.
|
||||
#
|
||||
# How it works:
|
||||
# - docker-compose.yml mounts this script into the PostgreSQL container's
|
||||
# /docker-entrypoint-initdb.d/ directory
|
||||
# - PostgreSQL automatically executes scripts in that directory on first
|
||||
# container initialization
|
||||
#
|
||||
# For local PostgreSQL users (non-Docker), this script is NOT used.
|
||||
# Instead, the Electric user is created by Alembic migration 66
|
||||
# (66_add_notifications_table_and_electric_replication.py).
|
||||
#
|
||||
# Both approaches are idempotent (use IF NOT EXISTS), so running both
|
||||
# will not cause conflicts.
|
||||
|
|
|
|||
|
|
@ -9,6 +9,10 @@ POSTGRES_USER=${POSTGRES_USER:-surfsense}
|
|||
POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-surfsense}
|
||||
POSTGRES_DB=${POSTGRES_DB:-surfsense}
|
||||
|
||||
# Electric SQL user credentials (configurable)
|
||||
ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric}
|
||||
ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
|
||||
|
||||
echo "Initializing PostgreSQL..."
|
||||
|
||||
# Check if PostgreSQL is already initialized
|
||||
|
|
@ -57,13 +61,13 @@ CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER;
|
|||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
|
||||
-- Create Electric SQL replication user
|
||||
CREATE USER electric WITH REPLICATION PASSWORD 'electric_password';
|
||||
GRANT CONNECT ON DATABASE $POSTGRES_DB TO electric;
|
||||
GRANT USAGE ON SCHEMA public TO electric;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA public TO electric;
|
||||
GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO electric;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO electric;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO electric;
|
||||
CREATE USER $ELECTRIC_DB_USER WITH REPLICATION PASSWORD '$ELECTRIC_DB_PASSWORD';
|
||||
GRANT CONNECT ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER;
|
||||
GRANT USAGE ON SCHEMA public TO $ELECTRIC_DB_USER;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA public TO $ELECTRIC_DB_USER;
|
||||
GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO $ELECTRIC_DB_USER;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $ELECTRIC_DB_USER;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO $ELECTRIC_DB_USER;
|
||||
EOF
|
||||
|
||||
echo "PostgreSQL initialized successfully."
|
||||
|
|
|
|||
|
|
@ -85,6 +85,20 @@ stderr_logfile=/dev/stderr
|
|||
stderr_logfile_maxbytes=0
|
||||
environment=PYTHONPATH="/app/backend"
|
||||
|
||||
# Electric SQL (real-time sync)
|
||||
[program:electric]
|
||||
command=/app/electric-release/bin/entrypoint start
|
||||
autostart=true
|
||||
autorestart=true
|
||||
priority=25
|
||||
startsecs=10
|
||||
startretries=3
|
||||
stdout_logfile=/dev/stdout
|
||||
stdout_logfile_maxbytes=0
|
||||
stderr_logfile=/dev/stderr
|
||||
stderr_logfile_maxbytes=0
|
||||
environment=DATABASE_URL="%(ENV_ELECTRIC_DATABASE_URL)s",ELECTRIC_INSECURE="%(ENV_ELECTRIC_INSECURE)s",ELECTRIC_WRITE_TO_PG_MODE="%(ENV_ELECTRIC_WRITE_TO_PG_MODE)s",RELEASE_COOKIE="surfsense_electric_cookie",PORT="%(ENV_ELECTRIC_PORT)s"
|
||||
|
||||
# Frontend
|
||||
[program:frontend]
|
||||
command=node server.js
|
||||
|
|
@ -102,6 +116,6 @@ environment=NODE_ENV="production",PORT="3000",HOSTNAME="0.0.0.0"
|
|||
|
||||
# Process Groups
|
||||
[group:surfsense]
|
||||
programs=postgresql,redis,backend,celery-worker,celery-beat,frontend
|
||||
programs=postgresql,redis,electric,backend,celery-worker,celery-beat,frontend
|
||||
priority=999
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue