chore: remove legacy Docker configuration files and initialization scripts for SurfSense

This commit is contained in:
Anish Sarkar 2026-02-24 23:01:38 +05:30
parent ce1f8c872f
commit 68be0d8675
8 changed files with 0 additions and 1047 deletions

View file

@ -1,243 +0,0 @@
#!/bin/bash
set -e
echo "==========================================="
echo " 🏄 SurfSense All-in-One Container"
echo "==========================================="
# Create log directory
mkdir -p /var/log/supervisor
# ================================================
# Ensure data directory exists
# ================================================
mkdir -p /data
# ================================================
# Generate SECRET_KEY if not provided
# ================================================
if [ -z "$SECRET_KEY" ]; then
# Generate a random secret key and persist it
if [ -f /data/.secret_key ]; then
export SECRET_KEY=$(cat /data/.secret_key)
echo "✅ Using existing SECRET_KEY from persistent storage"
else
export SECRET_KEY=$(python3 -c "import secrets; print(secrets.token_urlsafe(32))")
echo "$SECRET_KEY" > /data/.secret_key
chmod 600 /data/.secret_key
echo "✅ Generated new SECRET_KEY (saved for persistence)"
fi
fi
# ================================================
# Set default TTS/STT services if not provided
# ================================================
if [ -z "$TTS_SERVICE" ]; then
export TTS_SERVICE="local/kokoro"
echo "✅ Using default TTS_SERVICE: local/kokoro"
fi
if [ -z "$STT_SERVICE" ]; then
export STT_SERVICE="local/base"
echo "✅ Using default STT_SERVICE: local/base"
fi
# ================================================
# Set Electric SQL configuration
# ================================================
export ELECTRIC_DB_USER="${ELECTRIC_DB_USER:-electric}"
export ELECTRIC_DB_PASSWORD="${ELECTRIC_DB_PASSWORD:-electric_password}"
if [ -z "$ELECTRIC_DATABASE_URL" ]; then
export ELECTRIC_DATABASE_URL="postgresql://${ELECTRIC_DB_USER}:${ELECTRIC_DB_PASSWORD}@localhost:5432/${POSTGRES_DB:-surfsense}?sslmode=disable"
echo "✅ Electric SQL URL configured dynamically"
else
# Ensure sslmode=disable is in the URL if not already present
if [[ "$ELECTRIC_DATABASE_URL" != *"sslmode="* ]]; then
# Add sslmode=disable (handle both cases: with or without existing query params)
if [[ "$ELECTRIC_DATABASE_URL" == *"?"* ]]; then
export ELECTRIC_DATABASE_URL="${ELECTRIC_DATABASE_URL}&sslmode=disable"
else
export ELECTRIC_DATABASE_URL="${ELECTRIC_DATABASE_URL}?sslmode=disable"
fi
fi
echo "✅ Electric SQL URL configured from environment"
fi
# Set Electric SQL port
export ELECTRIC_PORT="${ELECTRIC_PORT:-5133}"
export PORT="${ELECTRIC_PORT}"
# ================================================
# Initialize PostgreSQL if needed
# ================================================
if [ ! -f /data/postgres/PG_VERSION ]; then
echo "📦 Initializing PostgreSQL database..."
# Initialize PostgreSQL data directory
chown -R postgres:postgres /data/postgres
chmod 700 /data/postgres
# Initialize with UTF8 encoding (required for proper text handling)
su - postgres -c "/usr/lib/postgresql/14/bin/initdb -D /data/postgres --encoding=UTF8 --locale=C.UTF-8"
# Configure PostgreSQL for connections
echo "host all all 0.0.0.0/0 md5" >> /data/postgres/pg_hba.conf
echo "local all all trust" >> /data/postgres/pg_hba.conf
echo "listen_addresses='*'" >> /data/postgres/postgresql.conf
# Enable logical replication for Electric SQL
echo "wal_level = logical" >> /data/postgres/postgresql.conf
echo "max_replication_slots = 10" >> /data/postgres/postgresql.conf
echo "max_wal_senders = 10" >> /data/postgres/postgresql.conf
# Start PostgreSQL temporarily to create database and user
su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres -l /tmp/postgres_init.log start"
# Wait for PostgreSQL to be ready
sleep 5
# Create user and database
su - postgres -c "psql -c \"CREATE USER ${POSTGRES_USER:-surfsense} WITH PASSWORD '${POSTGRES_PASSWORD:-surfsense}' SUPERUSER;\""
su - postgres -c "psql -c \"CREATE DATABASE ${POSTGRES_DB:-surfsense} OWNER ${POSTGRES_USER:-surfsense};\""
# Enable pgvector extension
su - postgres -c "psql -d ${POSTGRES_DB:-surfsense} -c 'CREATE EXTENSION IF NOT EXISTS vector;'"
# Create Electric SQL replication user (idempotent - uses IF NOT EXISTS)
echo "📡 Creating Electric SQL replication user..."
su - postgres -c "psql -d ${POSTGRES_DB:-surfsense} <<-EOSQL
DO \\\$\\\$
BEGIN
IF NOT EXISTS (SELECT FROM pg_user WHERE usename = '${ELECTRIC_DB_USER}') THEN
CREATE USER ${ELECTRIC_DB_USER} WITH REPLICATION PASSWORD '${ELECTRIC_DB_PASSWORD}';
END IF;
END
\\\$\\\$;
GRANT CONNECT ON DATABASE ${POSTGRES_DB:-surfsense} TO ${ELECTRIC_DB_USER};
GRANT USAGE ON SCHEMA public TO ${ELECTRIC_DB_USER};
GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${ELECTRIC_DB_USER};
GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO ${ELECTRIC_DB_USER};
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ${ELECTRIC_DB_USER};
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO ${ELECTRIC_DB_USER};
-- Create the publication for Electric SQL (if not exists)
DO \\\$\\\$
BEGIN
IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN
CREATE PUBLICATION electric_publication_default;
END IF;
END
\\\$\\\$;
EOSQL"
echo "✅ Electric SQL user '${ELECTRIC_DB_USER}' created"
# Stop temporary PostgreSQL
su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres stop"
echo "✅ PostgreSQL initialized successfully"
else
echo "✅ PostgreSQL data directory already exists"
fi
# ================================================
# Initialize Redis data directory
# ================================================
mkdir -p /data/redis
chmod 755 /data/redis
echo "✅ Redis data directory ready"
# ================================================
# Copy frontend build to runtime location
# ================================================
if [ -d /app/frontend/.next/standalone ]; then
cp -r /app/frontend/.next/standalone/* /app/frontend/ 2>/dev/null || true
cp -r /app/frontend/.next/static /app/frontend/.next/static 2>/dev/null || true
fi
# ================================================
# Runtime Environment Variable Replacement
# ================================================
# Next.js NEXT_PUBLIC_* vars are baked in at build time.
# This replaces placeholder values with actual runtime env vars.
echo "🔧 Applying runtime environment configuration..."
# Set defaults if not provided
NEXT_PUBLIC_FASTAPI_BACKEND_URL="${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:8000}"
NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE="${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:-LOCAL}"
NEXT_PUBLIC_ETL_SERVICE="${NEXT_PUBLIC_ETL_SERVICE:-DOCLING}"
NEXT_PUBLIC_ELECTRIC_URL="${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}"
NEXT_PUBLIC_ELECTRIC_AUTH_MODE="${NEXT_PUBLIC_ELECTRIC_AUTH_MODE:-insecure}"
NEXT_PUBLIC_DEPLOYMENT_MODE="${NEXT_PUBLIC_DEPLOYMENT_MODE:-self-hosted}"
# Replace placeholders in all JS files
find /app/frontend -type f \( -name "*.js" -o -name "*.json" \) -exec sed -i \
-e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_URL__|${NEXT_PUBLIC_FASTAPI_BACKEND_URL}|g" \
-e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__|${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}|g" \
-e "s|__NEXT_PUBLIC_ETL_SERVICE__|${NEXT_PUBLIC_ETL_SERVICE}|g" \
-e "s|__NEXT_PUBLIC_ELECTRIC_URL__|${NEXT_PUBLIC_ELECTRIC_URL}|g" \
-e "s|__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__|${NEXT_PUBLIC_ELECTRIC_AUTH_MODE}|g" \
-e "s|__NEXT_PUBLIC_DEPLOYMENT_MODE__|${NEXT_PUBLIC_DEPLOYMENT_MODE}|g" \
{} +
echo "✅ Environment configuration applied"
echo " Backend URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}"
echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}"
echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}"
echo " Electric URL: ${NEXT_PUBLIC_ELECTRIC_URL}"
echo " Deployment Mode: ${NEXT_PUBLIC_DEPLOYMENT_MODE}"
# ================================================
# Run database migrations
# ================================================
run_migrations() {
echo "🔄 Running database migrations..."
# Start PostgreSQL temporarily for migrations
su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres -l /tmp/postgres_migrate.log start"
sleep 5
# Start Redis temporarily for migrations (some might need it)
redis-server --dir /data/redis --daemonize yes
sleep 2
# Run alembic migrations
cd /app/backend
alembic upgrade head || echo "⚠️ Migrations may have already been applied"
# Stop temporary services
redis-cli shutdown || true
su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres stop"
echo "✅ Database migrations complete"
}
# Always run migrations on startup - alembic upgrade head is safe to run
# every time. It only applies pending migrations (never re-runs applied ones,
# never calls downgrade). This ensures updates are applied automatically.
run_migrations
# ================================================
# Environment Variables Info
# ================================================
echo ""
echo "==========================================="
echo " 📋 Configuration"
echo "==========================================="
echo " Frontend URL: http://localhost:3000"
echo " Backend API: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}"
echo " API Docs: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}/docs"
echo " Electric URL: ${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}"
echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}"
echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}"
echo " TTS Service: ${TTS_SERVICE}"
echo " STT Service: ${STT_SERVICE}"
echo "==========================================="
echo ""
# ================================================
# Start Supervisor (manages all services)
# ================================================
echo "🚀 Starting all services..."
exec /usr/local/bin/supervisord -c /etc/supervisor/conf.d/surfsense.conf

View file

@ -1,56 +0,0 @@
#!/bin/sh
# ============================================================================
# Electric SQL User Initialization Script (docker-compose only)
# ============================================================================
# This script is ONLY used when running via docker-compose.
#
# How it works:
# - docker-compose.yml mounts this script into the PostgreSQL container's
# /docker-entrypoint-initdb.d/ directory
# - PostgreSQL automatically executes scripts in that directory on first
# container initialization
#
# For local PostgreSQL users (non-Docker), this script is NOT used.
# Instead, the Electric user is created by Alembic migration 66
# (66_add_notifications_table_and_electric_replication.py).
#
# Both approaches are idempotent (use IF NOT EXISTS), so running both
# will not cause conflicts.
# ============================================================================
set -e
# Use environment variables with defaults
ELECTRIC_DB_USER="${ELECTRIC_DB_USER:-electric}"
ELECTRIC_DB_PASSWORD="${ELECTRIC_DB_PASSWORD:-electric_password}"
echo "Creating Electric SQL replication user: $ELECTRIC_DB_USER"
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
DO \$\$
BEGIN
IF NOT EXISTS (SELECT FROM pg_user WHERE usename = '$ELECTRIC_DB_USER') THEN
CREATE USER $ELECTRIC_DB_USER WITH REPLICATION PASSWORD '$ELECTRIC_DB_PASSWORD';
END IF;
END
\$\$;
GRANT CONNECT ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER;
GRANT CREATE ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER;
GRANT USAGE ON SCHEMA public TO $ELECTRIC_DB_USER;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO $ELECTRIC_DB_USER;
GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO $ELECTRIC_DB_USER;
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $ELECTRIC_DB_USER;
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO $ELECTRIC_DB_USER;
-- Create the publication for Electric SQL (if not exists)
DO \$\$
BEGIN
IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN
CREATE PUBLICATION electric_publication_default;
END IF;
END
\$\$;
EOSQL
echo "Electric SQL user '$ELECTRIC_DB_USER' and publication created successfully"

View file

@ -1,77 +0,0 @@
#!/bin/bash
# PostgreSQL initialization script for SurfSense
# This script is called during container startup if the database needs initialization
set -e
PGDATA=${PGDATA:-/data/postgres}
POSTGRES_USER=${POSTGRES_USER:-surfsense}
POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-surfsense}
POSTGRES_DB=${POSTGRES_DB:-surfsense}
# Electric SQL user credentials (configurable)
ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric}
ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
echo "Initializing PostgreSQL..."
# Check if PostgreSQL is already initialized
if [ -f "$PGDATA/PG_VERSION" ]; then
echo "PostgreSQL data directory already exists. Skipping initialization."
exit 0
fi
# Initialize the database cluster
/usr/lib/postgresql/14/bin/initdb -D "$PGDATA" --username=postgres
# Configure PostgreSQL
cat >> "$PGDATA/postgresql.conf" << EOF
listen_addresses = '*'
max_connections = 200
shared_buffers = 256MB
# Enable logical replication (required for Electric SQL)
wal_level = logical
max_replication_slots = 10
max_wal_senders = 10
# Performance settings
checkpoint_timeout = 10min
max_wal_size = 1GB
min_wal_size = 80MB
EOF
cat >> "$PGDATA/pg_hba.conf" << EOF
# Allow connections from anywhere with password
host all all 0.0.0.0/0 md5
host all all ::0/0 md5
EOF
# Start PostgreSQL temporarily
/usr/lib/postgresql/14/bin/pg_ctl -D "$PGDATA" -l /tmp/postgres_init.log start
# Wait for PostgreSQL to start
sleep 3
# Create user and database
psql -U postgres << EOF
CREATE USER $POSTGRES_USER WITH PASSWORD '$POSTGRES_PASSWORD' SUPERUSER;
CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER;
\c $POSTGRES_DB
CREATE EXTENSION IF NOT EXISTS vector;
-- Create Electric SQL replication user
CREATE USER $ELECTRIC_DB_USER WITH REPLICATION PASSWORD '$ELECTRIC_DB_PASSWORD';
GRANT CONNECT ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER;
GRANT USAGE ON SCHEMA public TO $ELECTRIC_DB_USER;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO $ELECTRIC_DB_USER;
GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO $ELECTRIC_DB_USER;
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $ELECTRIC_DB_USER;
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO $ELECTRIC_DB_USER;
EOF
echo "PostgreSQL initialized successfully."
# Stop PostgreSQL (supervisor will start it)
/usr/lib/postgresql/14/bin/pg_ctl -D "$PGDATA" stop

View file

@ -1,20 +0,0 @@
# PostgreSQL configuration for Electric SQL
# This file is mounted into the PostgreSQL container
listen_addresses = '*'
max_connections = 200
shared_buffers = 256MB
# Enable logical replication (required for Electric SQL)
wal_level = logical
max_replication_slots = 10
max_wal_senders = 10
# Performance settings
checkpoint_timeout = 10min
max_wal_size = 1GB
min_wal_size = 80MB
# Logging (optional, for debugging)
# log_statement = 'all'
# log_replication_commands = on

View file

@ -1,121 +0,0 @@
[supervisord]
nodaemon=true
logfile=/dev/stdout
logfile_maxbytes=0
pidfile=/var/run/supervisord.pid
loglevel=info
user=root
[unix_http_server]
file=/var/run/supervisor.sock
chmod=0700
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///var/run/supervisor.sock
# PostgreSQL
[program:postgresql]
command=/usr/lib/postgresql/14/bin/postgres -D /data/postgres
user=postgres
autostart=true
autorestart=true
priority=10
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
environment=PGDATA="/data/postgres"
# Redis
[program:redis]
command=/usr/bin/redis-server --dir /data/redis --appendonly yes
autostart=true
autorestart=true
priority=20
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
# Backend API
[program:backend]
command=python main.py
directory=/app/backend
autostart=true
autorestart=true
priority=30
startsecs=10
startretries=3
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
environment=PYTHONPATH="/app/backend",UVICORN_LOOP="asyncio",UNSTRUCTURED_HAS_PATCHED_LOOP="1"
# Celery Worker
[program:celery-worker]
command=celery -A app.celery_app worker --loglevel=info --concurrency=2 --pool=solo --queues=surfsense,surfsense.connectors
directory=/app/backend
autostart=true
autorestart=true
priority=40
startsecs=15
startretries=3
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
environment=PYTHONPATH="/app/backend"
# Celery Beat (scheduler)
[program:celery-beat]
command=celery -A app.celery_app beat --loglevel=info
directory=/app/backend
autostart=true
autorestart=true
priority=50
startsecs=20
startretries=3
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
environment=PYTHONPATH="/app/backend"
# Electric SQL (real-time sync)
[program:electric]
command=/app/electric-release/bin/entrypoint start
autostart=true
autorestart=true
priority=25
startsecs=10
startretries=3
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
environment=DATABASE_URL="%(ENV_ELECTRIC_DATABASE_URL)s",ELECTRIC_INSECURE="%(ENV_ELECTRIC_INSECURE)s",ELECTRIC_WRITE_TO_PG_MODE="%(ENV_ELECTRIC_WRITE_TO_PG_MODE)s",RELEASE_COOKIE="surfsense_electric_cookie",PORT="%(ENV_ELECTRIC_PORT)s"
# Frontend
[program:frontend]
command=node server.js
directory=/app/frontend
autostart=true
autorestart=true
priority=60
startsecs=5
startretries=3
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
environment=NODE_ENV="production",PORT="3000",HOSTNAME="0.0.0.0"
# Process Groups
[group:surfsense]
programs=postgresql,redis,electric,backend,celery-worker,celery-beat,frontend
priority=999