mirror of
https://github.com/dograh-hq/dograh.git
synced 2026-06-07 07:55:16 +02:00
feat: add rolling updates for production deployment (#175)
* feat: rolling update uvicorn workers * script fixes
This commit is contained in:
Abhishek
GitHub
parent
ad6261333d
commit
aed5a782fb
10 changed files with 785 additions and 497 deletions
219
scripts/start_services_dev.sh
Executable file
219
scripts/start_services_dev.sh
Executable file
|
|
@ -0,0 +1,219 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e # Exit on error
|
||||
|
||||
###############################################################################
|
||||
### CONFIGURATION
|
||||
###############################################################################
|
||||
|
||||
# Determine BASE_DIR as parent of the scripts directory
|
||||
BASE_DIR="$(cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" && pwd)"
|
||||
|
||||
ENV_FILE="$BASE_DIR/api/.env"
|
||||
RUN_DIR="$BASE_DIR/run" # Where we keep *.pid
|
||||
BASE_LOG_DIR="$BASE_DIR/logs" # Base logs directory
|
||||
|
||||
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
||||
LOG_DIR="$BASE_LOG_DIR/$TIMESTAMP" # Timestamped log directory
|
||||
LATEST_LINK="$BASE_LOG_DIR/latest" # Symlink to latest logs
|
||||
VENV_PATH="$BASE_DIR/venv"
|
||||
|
||||
ARQ_WORKERS=${ARQ_WORKERS:-1}
|
||||
LOG_TO_FILE=${LOG_TO_FILE:-true}
|
||||
|
||||
cd "$BASE_DIR"
|
||||
echo "Starting Dograh Services (DEV MODE) at $(date) in BASE_DIR: ${BASE_DIR}"
|
||||
echo "Auto-reload enabled for api/ directory changes"
|
||||
|
||||
###############################################################################
|
||||
### 1) Load environment variables
|
||||
###############################################################################
|
||||
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a && . "$ENV_FILE" && set +a
|
||||
fi
|
||||
|
||||
UVICORN_BASE_PORT=${UVICORN_BASE_PORT:-8000}
|
||||
|
||||
###############################################################################
|
||||
### 2) Define services
|
||||
###############################################################################
|
||||
|
||||
SERVICE_NAMES=(
|
||||
"ari_manager"
|
||||
"campaign_orchestrator"
|
||||
"uvicorn"
|
||||
)
|
||||
|
||||
SERVICE_COMMANDS=(
|
||||
"python -m api.services.telephony.ari_manager"
|
||||
"python -m api.services.campaign.campaign_orchestrator"
|
||||
"uvicorn api.app:app --host 0.0.0.0 --port $UVICORN_BASE_PORT --reload --reload-dir api"
|
||||
)
|
||||
|
||||
# Add ARQ workers dynamically
|
||||
for ((i=1; i<=ARQ_WORKERS; i++)); do
|
||||
SERVICE_NAMES+=("arq$i")
|
||||
SERVICE_COMMANDS+=("python -m arq api.tasks.arq.WorkerSettings --custom-log-dict api.tasks.arq.LOG_CONFIG")
|
||||
done
|
||||
|
||||
###############################################################################
|
||||
### 3) Activate virtual environment
|
||||
###############################################################################
|
||||
|
||||
if [[ -d "$VENV_PATH" && -f "$VENV_PATH/bin/activate" ]]; then
|
||||
source "$VENV_PATH/bin/activate"
|
||||
echo "Virtual environment activated: $VENV_PATH"
|
||||
else
|
||||
echo "Warning: Virtual environment not found at $VENV_PATH"
|
||||
echo "Continuing without virtual environment activation..."
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
### 4) Stop old services
|
||||
###############################################################################
|
||||
|
||||
mkdir -p "$RUN_DIR"
|
||||
|
||||
# Function to get all descendant PIDs of a process (children, grandchildren, etc.)
|
||||
get_descendants() {
|
||||
local parent_pid=$1
|
||||
local descendants=""
|
||||
local children
|
||||
|
||||
# Get direct children
|
||||
children=$(pgrep -P "$parent_pid" 2>/dev/null || true)
|
||||
|
||||
for child in $children; do
|
||||
# Recursively get descendants of each child
|
||||
descendants="$descendants $child $(get_descendants "$child")"
|
||||
done
|
||||
|
||||
echo "$descendants"
|
||||
}
|
||||
|
||||
# Function to kill a process and all its descendants
|
||||
kill_process_tree() {
|
||||
local pid=$1
|
||||
local signal=$2
|
||||
local descendants
|
||||
|
||||
descendants=$(get_descendants "$pid")
|
||||
|
||||
# Kill children first (bottom-up), then parent
|
||||
for desc_pid in $descendants; do
|
||||
if kill -0 "$desc_pid" 2>/dev/null; then
|
||||
kill "$signal" "$desc_pid" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
|
||||
# Kill the parent
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill "$signal" "$pid" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
for name in "${SERVICE_NAMES[@]}"; do
|
||||
pidfile="$RUN_DIR/$name.pid"
|
||||
|
||||
if [[ -f $pidfile ]]; then
|
||||
oldpid=$(<"$pidfile")
|
||||
|
||||
if kill -0 "$oldpid" 2>/dev/null; then
|
||||
echo "Stopping $name (PID $oldpid and all descendants)…"
|
||||
|
||||
kill_process_tree "$oldpid" "-TERM"
|
||||
sleep 4
|
||||
|
||||
still_alive=false
|
||||
if kill -0 "$oldpid" 2>/dev/null; then
|
||||
still_alive=true
|
||||
else
|
||||
for desc_pid in $(get_descendants "$oldpid"); do
|
||||
if kill -0 "$desc_pid" 2>/dev/null; then
|
||||
still_alive=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if $still_alive; then
|
||||
echo "⚠️ $name did not exit cleanly, forcing stop..."
|
||||
kill_process_tree "$oldpid" "-KILL"
|
||||
sleep 1
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -f "$pidfile"
|
||||
else
|
||||
echo "No PID file for $name, skipping stop."
|
||||
fi
|
||||
done
|
||||
|
||||
# Clean up legacy port tracking files
|
||||
rm -f "$RUN_DIR/uvicorn.port" "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn_old.pid" "$RUN_DIR/active_band"
|
||||
|
||||
###############################################################################
|
||||
### 5) Run migrations
|
||||
###############################################################################
|
||||
|
||||
alembic -c "$BASE_DIR/api/alembic.ini" upgrade head
|
||||
|
||||
###############################################################################
|
||||
### 6) Prepare logs
|
||||
###############################################################################
|
||||
|
||||
mkdir -p "$BASE_LOG_DIR" "$LOG_DIR"
|
||||
|
||||
if [[ -L "$LATEST_LINK" ]]; then
|
||||
rm "$LATEST_LINK"
|
||||
fi
|
||||
ln -s "$TIMESTAMP" "$LATEST_LINK"
|
||||
|
||||
echo "Log directory: $LOG_DIR"
|
||||
echo "Latest symlink: $LATEST_LINK -> $TIMESTAMP"
|
||||
|
||||
###############################################################################
|
||||
### 7) Start services
|
||||
###############################################################################
|
||||
|
||||
for i in "${!SERVICE_NAMES[@]}"; do
|
||||
name="${SERVICE_NAMES[$i]}"
|
||||
cmd="${SERVICE_COMMANDS[$i]}"
|
||||
echo "→ Starting $name"
|
||||
|
||||
(
|
||||
cd "$BASE_DIR"
|
||||
if [[ "$LOG_TO_FILE" == "true" ]]; then
|
||||
export LOG_FILE_PATH="$LOG_DIR/$name.log"
|
||||
exec $cmd >>"$LOG_DIR/$name.log" 2>&1
|
||||
else
|
||||
exec $cmd
|
||||
fi
|
||||
) &
|
||||
|
||||
pid=$!
|
||||
echo $pid >"$RUN_DIR/$name.pid"
|
||||
echo " Started with PID $pid"
|
||||
|
||||
done
|
||||
|
||||
###############################################################################
|
||||
### 8) Summary
|
||||
###############################################################################
|
||||
|
||||
echo
|
||||
echo "──────────────────────────────────────────────────"
|
||||
echo "Mode: DEVELOPMENT (auto-reload enabled)"
|
||||
echo ""
|
||||
for name in "${SERVICE_NAMES[@]}"; do
|
||||
pid=$(<"$RUN_DIR/$name.pid")
|
||||
echo "✓ $name (PID $pid) → $LOG_DIR/$name.log"
|
||||
done
|
||||
echo ""
|
||||
echo " Rotation: ${LOG_ROTATION_SIZE:-100 MB}"
|
||||
echo " Retention: ${LOG_RETENTION:-7 days}"
|
||||
echo " Compression: ${LOG_COMPRESSION:-gz}"
|
||||
echo "Logs: tail -f $LOG_DIR/*.log"
|
||||
echo "Rotated logs: ls $LOG_DIR/*.log.*"
|
||||
echo "To stop: ./scripts/stop_services.sh"
|
||||
echo "──────────────────────────────────────────────────"
|
||||
Loading…
Add table
Add a link
Reference in a new issue