mirror of
https://github.com/dograh-hq/dograh.git
synced 2026-06-07 07:55:16 +02:00
Add alembic and start services scripts
This commit is contained in:
Abhishek Kumar
parent
44232b37f8
commit
443490b2dd
9 changed files with 761 additions and 1 deletions
20
scripts/dograh-services.service
Normal file
20
scripts/dograh-services.service
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
[Unit]
|
||||
Description=Dograh Services
|
||||
After=network.target postgresql.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=forking
|
||||
User=a
|
||||
Group=a
|
||||
WorkingDirectory=/home/a/dograh
|
||||
Environment="PATH=/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
ExecStart=/home/a/dograh/scripts/start_services.sh
|
||||
ExecStop=/home/a/dograh/scripts/stop_services.sh
|
||||
TimeoutStopSec=30
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
5
scripts/format.sh
Executable file
5
scripts/format.sh
Executable file
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/sh -e
|
||||
set -euo pipefail
|
||||
|
||||
ruff check api --select I --select F401 --fix
|
||||
ruff format api
|
||||
8
scripts/lint.sh
Normal file
8
scripts/lint.sh
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
mypy api
|
||||
ruff check api --check
|
||||
ruff format api --check
|
||||
33
scripts/makemigrate.sh
Normal file
33
scripts/makemigrate.sh
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -e # Exit immediately if a command exits with a non-zero status
|
||||
|
||||
# Set PYTHONPATH to the parent directory of the script's location
|
||||
export PYTHONPATH="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")"
|
||||
|
||||
# Define color codes
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
env_file="api/.env"
|
||||
|
||||
# Check if environment file exists
|
||||
if [ ! -f "$env_file" ]; then
|
||||
echo -e "${RED}Error: Environment file $env_file not found.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Load environment variables
|
||||
export $(grep -v '^#' "$env_file" | xargs)
|
||||
|
||||
# Prompt for the name of the migration
|
||||
read -p "Enter the migration name (minimum 5 characters): " migration_name
|
||||
|
||||
# Check if the migration name is empty or less than 5 characters
|
||||
if [[ -z "$migration_name" || ${#migration_name} -lt 5 ]]; then
|
||||
echo -e "${RED}Error: Migration name must be at least 5 characters long.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Generate the Alembic revision with the provided migration name
|
||||
alembic -c api/alembic.ini revision --autogenerate -m "$migration_name"
|
||||
27
scripts/migrate.sh
Normal file
27
scripts/migrate.sh
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -e # Exit immediately if a command exits with a non-zero status
|
||||
|
||||
# Set PYTHONPATH to the parent directory of the script's location
|
||||
export PYTHONPATH="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")"
|
||||
|
||||
# Define color codes
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
env_file="api/.env"
|
||||
|
||||
# Check if environment file exists
|
||||
if [ ! -f "$env_file" ]; then
|
||||
echo -e "${RED}Error: Environment file $env_file not found.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Load environment variables
|
||||
export $(grep -v '^#' "$env_file" | xargs)
|
||||
|
||||
# Run migrations
|
||||
alembic -c api/alembic.ini upgrade head
|
||||
|
||||
# Create initial data in DB
|
||||
# python api/initial_data.py
|
||||
31
scripts/pre_commit.sh
Executable file
31
scripts/pre_commit.sh
Executable file
|
|
@ -0,0 +1,31 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
###############################################################################
|
||||
# Ensure Ruff is installed (first try pipx, fall back to pip --user).
|
||||
###############################################################################
|
||||
if ! command -v ruff >/dev/null 2>&1; then
|
||||
echo "⇢ Ruff not found on PATH – installing…"
|
||||
if command -v pipx >/dev/null 2>&1; then
|
||||
# install into an isolated environment if pipx is present
|
||||
pipx install --quiet ruff
|
||||
else
|
||||
# otherwise install into the current (or user-level) Python environment
|
||||
pip install --quiet --upgrade --user ruff
|
||||
fi
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# 1 – Python formatting (calls Ruff + Black, etc.)
|
||||
###############################################################################
|
||||
sh scripts/format.sh
|
||||
|
||||
###############################################################################
|
||||
# 2 – ESLint autofix inside the Next.js app
|
||||
###############################################################################
|
||||
(cd ui && npm run fix-lint)
|
||||
|
||||
###############################################################################
|
||||
# 3 – Restage any files changed by the fixers so the commit includes them
|
||||
###############################################################################
|
||||
git add -u
|
||||
450
scripts/rolling_update_uvicorn.sh
Executable file
450
scripts/rolling_update_uvicorn.sh
Executable file
|
|
@ -0,0 +1,450 @@
|
|||
#!/usr/bin/env bash
|
||||
# rolling_update_uvicorn.sh — Zero-downtime rolling update for uvicorn workers
|
||||
#
|
||||
# Usage: ./rolling_update_uvicorn.sh <NEW_PORT>
|
||||
# Example: ./rolling_update_uvicorn.sh 8001
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Check if running as root or with sudo
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "This script must be run as root or with sudo"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
### CONFIGURATION #############################################################
|
||||
ENV_FILE="api/.env"
|
||||
RUN_DIR="run"
|
||||
LOG_ROOT="logs"
|
||||
HEALTH_CHECK_ENDPOINT="/api/v1/health" # Adjust as needed
|
||||
MAX_WAIT_SECONDS=310 # Max wait for graceful shutdown (5 minutes + 10 seconds grace)
|
||||
|
||||
# Load environment to get ENVIRONMENT variable
|
||||
set -a && . "$ENV_FILE" && set +a
|
||||
ENVIRONMENT="${ENVIRONMENT:-staging}"
|
||||
|
||||
# Set nginx upstream config based on environment
|
||||
if [[ "$ENVIRONMENT" == "production" ]]; then
|
||||
NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_production_upstream.conf"
|
||||
UPSTREAM_NAME="dograh_production_backend"
|
||||
echo "Rolling update for PRODUCTION environment"
|
||||
else
|
||||
NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_staging_upstream.conf"
|
||||
UPSTREAM_NAME="dograh_staging_backend"
|
||||
echo "Rolling update for STAGING environment"
|
||||
fi
|
||||
|
||||
### FUNCTIONS ##################################################################
|
||||
|
||||
log_info() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO: $*"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARN: $*"
|
||||
}
|
||||
|
||||
check_port_availability() {
|
||||
local port=$1
|
||||
if lsof -Pi :$port -sTCP:LISTEN -t >/dev/null 2>&1; then
|
||||
return 1 # Port is in use
|
||||
fi
|
||||
return 0 # Port is available
|
||||
}
|
||||
|
||||
wait_for_health_check() {
|
||||
local port=$1
|
||||
local max_attempts=30
|
||||
local attempt=0
|
||||
|
||||
log_info "Waiting for new uvicorn workers to be healthy on port $port..."
|
||||
|
||||
while [ $attempt -lt $max_attempts ]; do
|
||||
if curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:${port}${HEALTH_CHECK_ENDPOINT}" | grep -q "200"; then
|
||||
log_info "Health check passed on port $port"
|
||||
return 0
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
log_info "Health check attempt $attempt/$max_attempts..."
|
||||
sleep 1
|
||||
done
|
||||
|
||||
log_error "Health check failed after $max_attempts attempts"
|
||||
return 1
|
||||
}
|
||||
|
||||
get_old_uvicorn_pids() {
|
||||
local pidfile="$RUN_DIR/uvicorn.pid"
|
||||
local pids=""
|
||||
|
||||
if [[ -f "$pidfile" ]]; then
|
||||
# Read the main PID
|
||||
local main_pid=$(<"$pidfile")
|
||||
if kill -0 "$main_pid" 2>/dev/null; then
|
||||
# Get all PIDs in the process group
|
||||
pids=$(ps -o pid= -g $(ps -o pgid= -p "$main_pid" | tr -d ' ') 2>/dev/null || echo "$main_pid")
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "$pids"
|
||||
}
|
||||
|
||||
graceful_shutdown_old_workers() {
|
||||
local old_pids="$1"
|
||||
|
||||
if [[ -z "$old_pids" ]]; then
|
||||
log_warning "No old uvicorn workers found to shut down"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "Starting graceful shutdown of old uvicorn workers (PIDs: $(echo $old_pids | tr '\n' ' '))"
|
||||
|
||||
# Send SIGTERM to trigger graceful shutdown
|
||||
for pid in $old_pids; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
log_info "Sending SIGTERM to PID $pid"
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
|
||||
# Wait for processes to exit gracefully
|
||||
local start_time=$(date +%s)
|
||||
local all_dead=false
|
||||
|
||||
while [[ $(($(date +%s) - start_time)) -lt $MAX_WAIT_SECONDS ]]; do
|
||||
all_dead=true
|
||||
for pid in $old_pids; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
all_dead=false
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if $all_dead; then
|
||||
log_info "All old workers shut down gracefully"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "Waiting for workers to complete active requests... ($(( $(date +%s) - start_time ))s elapsed)"
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# Force kill if still running after timeout
|
||||
log_warning "Timeout reached, force killing remaining workers"
|
||||
for pid in $old_pids; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
log_warning "Force killing PID $pid"
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
|
||||
sleep 1
|
||||
return 0
|
||||
}
|
||||
|
||||
update_nginx_upstream() {
|
||||
local new_port=$1
|
||||
local old_port=$2
|
||||
|
||||
log_info "Updating nginx upstream configuration for $ENVIRONMENT..."
|
||||
|
||||
# Create or update the upstream configuration
|
||||
cat > "${NGINX_UPSTREAM_CONF}.tmp" <<EOF
|
||||
# Auto-generated by rolling_update_uvicorn.sh for $ENVIRONMENT
|
||||
# Last updated: $(date)
|
||||
upstream ${UPSTREAM_NAME} {
|
||||
server 127.0.0.1:${new_port} max_fails=3 fail_timeout=30s;
|
||||
}
|
||||
EOF
|
||||
|
||||
# Atomic move (with sudo if needed)
|
||||
if [[ $EUID -eq 0 ]]; then
|
||||
mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}"
|
||||
else
|
||||
sudo mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}" 2>/dev/null || {
|
||||
log_error "Could not update nginx config (need sudo). Run: sudo $0 $NEW_PORT"
|
||||
return 1
|
||||
}
|
||||
fi
|
||||
|
||||
# Test nginx configuration (with sudo if needed)
|
||||
if nginx -t 2>/dev/null || sudo nginx -t 2>/dev/null; then
|
||||
log_info "Nginx configuration test passed"
|
||||
# Reload nginx to pick up new configuration (with sudo if needed)
|
||||
if nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null; then
|
||||
log_info "Nginx reloaded successfully"
|
||||
else
|
||||
log_error "Could not reload nginx"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log_error "Nginx configuration test failed, rolling back"
|
||||
# Restore old configuration if possible
|
||||
if [[ -n "$old_port" ]]; then
|
||||
cat > "${NGINX_UPSTREAM_CONF}.tmp" <<EOF
|
||||
upstream ${UPSTREAM_NAME} {
|
||||
server 127.0.0.1:${old_port} max_fails=3 fail_timeout=30s;
|
||||
}
|
||||
EOF
|
||||
if [[ $EUID -eq 0 ]]; then
|
||||
mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}"
|
||||
else
|
||||
sudo mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}" 2>/dev/null || true
|
||||
fi
|
||||
nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null || true
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
start_new_uvicorn_workers() {
|
||||
local new_port=$1
|
||||
|
||||
log_info "Starting new uvicorn workers on port $new_port..."
|
||||
|
||||
# Get configuration from environment
|
||||
set -a && . "$ENV_FILE" && set +a
|
||||
|
||||
if [[ -z "${FASTAPI_WORKERS:-}" ]]; then
|
||||
log_error "FASTAPI_WORKERS environment variable is not set"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -z "${CONDA_ENV_NAME:-}" ]]; then
|
||||
log_error "CONDA_ENV_NAME environment variable is not set"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Source conda if not already available
|
||||
if ! command -v conda &>/dev/null; then
|
||||
source /opt/conda/etc/profile.d/conda.sh
|
||||
fi
|
||||
eval "$(conda shell.bash hook)"
|
||||
conda activate "$CONDA_ENV_NAME"
|
||||
|
||||
# Use the latest log directory (where start_services.sh put logs)
|
||||
# Resolve the symlink to get the actual directory
|
||||
local log_dir="$LOG_ROOT/latest"
|
||||
if [[ -L "$log_dir" ]]; then
|
||||
# It's a symlink, resolve it
|
||||
log_dir=$(readlink -f "$log_dir")
|
||||
fi
|
||||
|
||||
if [[ ! -d "$log_dir" ]]; then
|
||||
log_error "No latest log directory found. Run start_services.sh first."
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Export rotation settings
|
||||
export LOG_ROTATION_SIZE="${LOG_ROTATION_SIZE:-100 MB}"
|
||||
export LOG_RETENTION="${LOG_RETENTION:-7 days}"
|
||||
export LOG_COMPRESSION="${LOG_COMPRESSION:-gz}"
|
||||
|
||||
# Create unique log filename using timestamp and script PID to avoid conflicts
|
||||
local script_pid=$$ # PID of this rolling_update script (for uniqueness)
|
||||
local timestamp=$(date '+%H%M%S')
|
||||
export LOG_FILE_PATH="$log_dir/uvicorn-rollover-${timestamp}-${script_pid}.log"
|
||||
|
||||
log_info "Starting uvicorn with $FASTAPI_WORKERS workers on port $new_port"
|
||||
log_info "Logs: $LOG_FILE_PATH"
|
||||
|
||||
# If running as root, switch to original user for uvicorn process
|
||||
if [[ $EUID -eq 0 ]] && [[ -n "${SUDO_USER:-}" ]]; then
|
||||
log_info "Starting uvicorn as user: $SUDO_USER (not root)"
|
||||
|
||||
# Run uvicorn as the original user, similar to start_services.sh
|
||||
# Using setsid and passing LOG_FILE_PATH for loguru to pick up
|
||||
sudo -u "$SUDO_USER" bash -c "
|
||||
cd '$PWD'
|
||||
export HOME='$(getent passwd $SUDO_USER | cut -d: -f6)'
|
||||
export LOG_FILE_PATH='$LOG_FILE_PATH'
|
||||
export LOG_ROTATION_SIZE='$LOG_ROTATION_SIZE'
|
||||
export LOG_RETENTION='$LOG_RETENTION'
|
||||
export LOG_COMPRESSION='$LOG_COMPRESSION'
|
||||
set -a && source '$ENV_FILE' && set +a
|
||||
source /opt/conda/etc/profile.d/conda.sh
|
||||
conda activate '$CONDA_ENV_NAME'
|
||||
setsid nohup bash -c \"LOG_FILE_PATH='$LOG_FILE_PATH' uvicorn api.app:app --host 0.0.0.0 --port $new_port --workers $FASTAPI_WORKERS\" >/dev/null 2>&1 &
|
||||
echo \$! > '$RUN_DIR/uvicorn_new.pid'
|
||||
"
|
||||
# Read the PID that was written
|
||||
local new_pid=$(<"$RUN_DIR/uvicorn_new.pid")
|
||||
else
|
||||
# Start in new process group with setsid (same as start_services.sh)
|
||||
# Each service gets its own LOG_FILE_PATH environment variable
|
||||
setsid nohup bash -c "LOG_FILE_PATH='$LOG_FILE_PATH' uvicorn api.app:app --host 0.0.0.0 --port $new_port --workers $FASTAPI_WORKERS" >/dev/null 2>&1 &
|
||||
|
||||
local new_pid=$!
|
||||
echo "$new_pid" > "$RUN_DIR/uvicorn_new.pid"
|
||||
fi
|
||||
|
||||
# Save port information
|
||||
echo "$new_port" > "$RUN_DIR/uvicorn_new.port"
|
||||
|
||||
log_info "New uvicorn started with PID $new_pid"
|
||||
|
||||
# Wait a bit for startup
|
||||
sleep 5
|
||||
|
||||
# Check if process is still running
|
||||
if ! kill -0 "$new_pid" 2>/dev/null; then
|
||||
log_error "New uvicorn process died immediately"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
finalize_rollover() {
|
||||
log_info "Finalizing rollover..."
|
||||
|
||||
# Move new PID file to main PID file
|
||||
if [[ -f "$RUN_DIR/uvicorn_new.pid" ]]; then
|
||||
mv "$RUN_DIR/uvicorn_new.pid" "$RUN_DIR/uvicorn.pid"
|
||||
fi
|
||||
|
||||
# Store the new port for reference
|
||||
if [[ -f "$RUN_DIR/uvicorn_new.port" ]]; then
|
||||
mv "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn.port"
|
||||
fi
|
||||
|
||||
# Clean up old PID file if it exists
|
||||
rm -f "$RUN_DIR/uvicorn_old.pid"
|
||||
|
||||
log_info "Rollover completed successfully"
|
||||
}
|
||||
|
||||
rollback() {
|
||||
local old_port=$1
|
||||
local new_pid=$2
|
||||
|
||||
log_error "Rolling back due to failure..."
|
||||
|
||||
# Kill new workers if they exist
|
||||
if [[ -n "$new_pid" ]] && kill -0 "$new_pid" 2>/dev/null; then
|
||||
log_info "Killing new uvicorn workers (PID: $new_pid)"
|
||||
kill -KILL -"$new_pid" 2>/dev/null || kill -KILL "$new_pid" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Clean up temporary files
|
||||
rm -f "$RUN_DIR/uvicorn_new.pid" "$RUN_DIR/uvicorn_new.port"
|
||||
|
||||
# Restore nginx configuration if old port is known
|
||||
if [[ -n "$old_port" ]]; then
|
||||
update_nginx_upstream "$old_port" ""
|
||||
fi
|
||||
|
||||
log_error "Rollback completed"
|
||||
}
|
||||
|
||||
### MAIN LOGIC ################################################################
|
||||
|
||||
# Check arguments
|
||||
if [[ $# -ne 1 ]]; then
|
||||
echo "Usage: $0 <NEW_PORT>"
|
||||
echo "Example: $0 8001"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NEW_PORT=$1
|
||||
|
||||
# Check nginx permissions early and exit if we can't update nginx
|
||||
if [[ ! -w $(dirname "$NGINX_UPSTREAM_CONF") ]] && [[ $EUID -ne 0 ]]; then
|
||||
if ! sudo -n true 2>/dev/null; then
|
||||
log_error "This script needs sudo access to update nginx configuration"
|
||||
log_error "Cannot proceed without nginx update permissions"
|
||||
echo ""
|
||||
echo "Please run with sudo:"
|
||||
echo " sudo $0 $NEW_PORT"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Validate port number
|
||||
if ! [[ "$NEW_PORT" =~ ^[0-9]+$ ]] || [ "$NEW_PORT" -lt 1 ] || [ "$NEW_PORT" -gt 65535 ]; then
|
||||
log_error "Invalid port number: $NEW_PORT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if port is available
|
||||
if ! check_port_availability "$NEW_PORT"; then
|
||||
log_error "Port $NEW_PORT is already in use"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get old port from file or environment
|
||||
OLD_PORT=""
|
||||
if [[ -f "$RUN_DIR/uvicorn.port" ]]; then
|
||||
OLD_PORT=$(<"$RUN_DIR/uvicorn.port")
|
||||
elif [[ -f "$ENV_FILE" ]]; then
|
||||
set -a && . "$ENV_FILE" && set +a
|
||||
OLD_PORT="${FASTAPI_PORT:-}"
|
||||
fi
|
||||
|
||||
if [[ "$NEW_PORT" == "$OLD_PORT" ]]; then
|
||||
log_error "New port is the same as old port ($NEW_PORT)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Starting rolling update from port ${OLD_PORT:-unknown} to port $NEW_PORT"
|
||||
|
||||
# Create run directory if it doesn't exist
|
||||
mkdir -p "$RUN_DIR"
|
||||
|
||||
# Get old uvicorn PIDs before starting new ones
|
||||
OLD_PIDS=$(get_old_uvicorn_pids)
|
||||
if [[ -n "$OLD_PIDS" ]]; then
|
||||
# Save old PIDs for potential rollback
|
||||
echo "$OLD_PIDS" > "$RUN_DIR/uvicorn_old.pid"
|
||||
log_info "Found old uvicorn workers: $(echo $OLD_PIDS | tr '\n' ' ')"
|
||||
else
|
||||
log_warning "No existing uvicorn workers found"
|
||||
fi
|
||||
|
||||
# Start new uvicorn workers
|
||||
if ! start_new_uvicorn_workers "$NEW_PORT"; then
|
||||
log_error "Failed to start new uvicorn workers"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NEW_PID=$(<"$RUN_DIR/uvicorn_new.pid")
|
||||
|
||||
# Wait for new workers to be healthy
|
||||
if ! wait_for_health_check "$NEW_PORT"; then
|
||||
log_error "New workers failed health check"
|
||||
rollback "$OLD_PORT" "$NEW_PID"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Update nginx to point to new workers
|
||||
if ! update_nginx_upstream "$NEW_PORT" "$OLD_PORT"; then
|
||||
log_error "Failed to update nginx configuration"
|
||||
rollback "$OLD_PORT" "$NEW_PID"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Give nginx some time to start routing to new workers
|
||||
log_info "Waiting for nginx to stabilize..."
|
||||
sleep 5
|
||||
|
||||
# Gracefully shutdown old workers
|
||||
if [[ -n "$OLD_PIDS" ]]; then
|
||||
graceful_shutdown_old_workers "$OLD_PIDS"
|
||||
fi
|
||||
|
||||
# Finalize the rollover
|
||||
finalize_rollover
|
||||
|
||||
# Summary
|
||||
echo "──────────────────────────────────────────────────"
|
||||
echo "✓ Rolling update completed successfully"
|
||||
echo " Old port: ${OLD_PORT:-none}"
|
||||
echo " New port: $NEW_PORT"
|
||||
echo " New PID: $NEW_PID"
|
||||
echo " Logs: $LOG_ROOT/latest/"
|
||||
echo "──────────────────────────────────────────────────"
|
||||
187
scripts/start_services.sh
Executable file
187
scripts/start_services.sh
Executable file
|
|
@ -0,0 +1,187 @@
|
|||
#!/usr/bin/env bash
|
||||
# restart_services.sh — safer, simplified
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
### CONFIGURATION #############################################################
|
||||
ENV_FILE="api/.env"
|
||||
RUN_DIR="run" # where we keep *.pid
|
||||
LOG_ROOT="logs"
|
||||
|
||||
### 1) Load environment vars so that configurations like FASTAPI_WORKERS are loaded #
|
||||
set -a && . "$ENV_FILE" && set +a
|
||||
|
||||
# Get ENVIRONMENT for nginx config selection
|
||||
ENVIRONMENT="${ENVIRONMENT:-staging}"
|
||||
|
||||
if [[ -z "${FASTAPI_PORT:-}" ]]; then
|
||||
echo "Error: FASTAPI_PORT environment variable is not set."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${FASTAPI_WORKERS:-}" ]]; then
|
||||
echo "Error: FASTAPI_WORKERS environment variable is not set."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${CONDA_ENV_NAME:-}" ]]; then
|
||||
echo "Error: CONDA_ENV_NAME environment variable is not set."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Default ARQ_WORKERS to 1 if not set
|
||||
ARQ_WORKERS=${ARQ_WORKERS:-1}
|
||||
|
||||
# map "service name" → "command to run"
|
||||
declare -A SERVICES=(
|
||||
[ari_manager]="python -m api.services.telephony.ari_manager"
|
||||
[campaign_orchestrator]="python -m api.services.campaign.campaign_orchestrator"
|
||||
[uvicorn]="uvicorn api.app:app --host 0.0.0.0 --port $FASTAPI_PORT --workers $FASTAPI_WORKERS"
|
||||
)
|
||||
|
||||
# Add ARQ workers dynamically based on ARQ_WORKERS environment variable
|
||||
for ((i=1; i<=ARQ_WORKERS; i++)); do
|
||||
SERVICES[arq$i]="python -m arq api.tasks.arq.WorkerSettings --custom-log-dict api.tasks.arq.LOG_CONFIG"
|
||||
done
|
||||
|
||||
### 2) Activate conda #########################################################
|
||||
# Source conda if not already available (needed when running from systemd)
|
||||
if ! command -v conda &>/dev/null; then
|
||||
source /opt/conda/etc/profile.d/conda.sh
|
||||
fi
|
||||
eval "$(conda shell.bash hook)"
|
||||
conda activate "$CONDA_ENV_NAME"
|
||||
|
||||
### 3) Stop old services (only via PID files) #################################
|
||||
mkdir -p "$RUN_DIR"
|
||||
for name in "${!SERVICES[@]}"; do
|
||||
pidfile="$RUN_DIR/$name.pid"
|
||||
if [[ -f $pidfile ]]; then
|
||||
oldpid=$(<"$pidfile")
|
||||
if kill -0 "$oldpid" 2>/dev/null; then
|
||||
echo "Stopping $name (PID $oldpid and its process group)…"
|
||||
# Kill the entire process group (negative PID)
|
||||
# First try SIGTERM
|
||||
kill -TERM -"$oldpid" 2>/dev/null || kill -TERM "$oldpid" 2>/dev/null || true
|
||||
sleep 4
|
||||
# If still running, use SIGKILL
|
||||
if kill -0 "$oldpid" 2>/dev/null; then
|
||||
echo "⚠️ $name did not exit cleanly, forcing stop..."
|
||||
kill -KILL -"$oldpid" 2>/dev/null || kill -KILL "$oldpid" 2>/dev/null || true
|
||||
sleep 1
|
||||
fi
|
||||
fi
|
||||
rm -f "$pidfile"
|
||||
else
|
||||
echo "No PID file for $name, skipping stop."
|
||||
fi
|
||||
done
|
||||
|
||||
# Clean up any port tracking files for uvicorn
|
||||
rm -f "$RUN_DIR/uvicorn.port" "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn_old.pid"
|
||||
|
||||
|
||||
### 4) Run migrations #########################################################
|
||||
alembic -c api/alembic.ini upgrade head
|
||||
|
||||
### 5) Prepare logs ###########################################################
|
||||
timestamp=$(date '+%Y-%m-%d_%H-%M-%S')
|
||||
LOG_DIR="$LOG_ROOT/$timestamp"
|
||||
mkdir -p "$LOG_DIR"
|
||||
# Create relative symlink
|
||||
cd "$LOG_ROOT" && ln -sfn "$timestamp" latest && cd - >/dev/null
|
||||
|
||||
### 6) (Optional) Free FastAPI port ###########################################
|
||||
FASTAPI_PORT=$FASTAPI_PORT
|
||||
if command -v lsof &>/dev/null; then
|
||||
lsof -ti tcp:"$FASTAPI_PORT" | xargs -r kill -9 || true
|
||||
fi
|
||||
|
||||
### 7) Start services #########################################################
|
||||
# Export rotation settings for loguru (if using file logging)
|
||||
export LOG_ROTATION_SIZE="${LOG_ROTATION_SIZE:-100 MB}"
|
||||
export LOG_RETENTION="${LOG_RETENTION:-7 days}"
|
||||
export LOG_COMPRESSION="${LOG_COMPRESSION:-gz}"
|
||||
|
||||
for name in "${!SERVICES[@]}"; do
|
||||
cmd=${SERVICES[$name]}
|
||||
echo "→ Starting $name with loguru rotation…"
|
||||
|
||||
# Export LOG_FILE_PATH for this specific service
|
||||
export LOG_FILE_PATH="$LOG_DIR/$name.log"
|
||||
|
||||
# Start in new process group with setsid
|
||||
# Each service gets its own LOG_FILE_PATH environment variable
|
||||
setsid nohup bash -c "LOG_FILE_PATH='$LOG_DIR/$name.log' $cmd" >/dev/null 2>&1 &
|
||||
|
||||
# Get the PID of the setsid process
|
||||
pid=$!
|
||||
echo $pid >"$RUN_DIR/$name.pid"
|
||||
|
||||
# For uvicorn, also save the port for rolling updates and update nginx
|
||||
if [[ "$name" == "uvicorn" ]]; then
|
||||
echo "$FASTAPI_PORT" >"$RUN_DIR/uvicorn.port"
|
||||
|
||||
# Update nginx upstream configuration if nginx is installed
|
||||
if command -v nginx &>/dev/null && [[ -d /etc/nginx ]]; then
|
||||
# Determine which upstream config to update based on ENVIRONMENT
|
||||
if [[ "${ENVIRONMENT:-}" == "production" ]]; then
|
||||
NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_production_upstream.conf"
|
||||
UPSTREAM_NAME="dograh_production_backend"
|
||||
echo "→ Updating PRODUCTION nginx upstream to port $FASTAPI_PORT…"
|
||||
else
|
||||
# Default to staging for any non-production environment
|
||||
NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_staging_upstream.conf"
|
||||
UPSTREAM_NAME="dograh_staging_backend"
|
||||
echo "→ Updating STAGING nginx upstream to port $FASTAPI_PORT…"
|
||||
fi
|
||||
|
||||
if [[ -w $(dirname "$NGINX_UPSTREAM_CONF") ]] || [[ $EUID -eq 0 ]]; then
|
||||
cat > "${NGINX_UPSTREAM_CONF}.tmp" <<EOF
|
||||
# Auto-generated by start_services.sh for ${ENVIRONMENT:-staging}
|
||||
# Last updated: $(date)
|
||||
upstream ${UPSTREAM_NAME} {
|
||||
server 127.0.0.1:${FASTAPI_PORT} max_fails=3 fail_timeout=30s;
|
||||
}
|
||||
EOF
|
||||
# Atomic move (may need sudo)
|
||||
if [[ $EUID -eq 0 ]]; then
|
||||
mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}"
|
||||
else
|
||||
sudo mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}" 2>/dev/null || \
|
||||
echo "⚠️ Could not update nginx config (need sudo). Run: sudo $0"
|
||||
fi
|
||||
|
||||
# Test and reload nginx if config was updated
|
||||
if [[ -f "$NGINX_UPSTREAM_CONF" ]]; then
|
||||
if nginx -t 2>/dev/null || sudo nginx -t 2>/dev/null; then
|
||||
echo "→ Reloading nginx…"
|
||||
nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null || \
|
||||
echo "⚠️ Could not reload nginx (may need sudo)"
|
||||
else
|
||||
echo "⚠️ Nginx configuration test failed"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "⚠️ Cannot write to nginx config directory (need sudo privileges)"
|
||||
echo " Run: sudo $0 to update nginx configuration"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
disown -a
|
||||
|
||||
### 8) Summary #################################################################
|
||||
echo
|
||||
echo "──────────────────────────────────────────────────"
|
||||
for name in "${!SERVICES[@]}"; do
|
||||
pid=$(<"$RUN_DIR/$name.pid")
|
||||
echo "✓ $name (PID $pid) → $LOG_DIR/$name.log"
|
||||
done
|
||||
echo " Rotation: ${LOG_ROTATION_SIZE:-100 MB}"
|
||||
echo " Retention: ${LOG_RETENTION:-7 days}"
|
||||
echo " Compression: ${LOG_COMPRESSION:-gz}"
|
||||
echo "Logs: tail -f $LOG_DIR/*.log"
|
||||
echo "Rotated logs: ls $LOG_DIR/*.log.*"
|
||||
echo "To stop: run this script again or kill -TERM -<PID> for process groups"
|
||||
echo "──────────────────────────────────────────────────"
|
||||
Loading…
Add table
Add a link
Reference in a new issue