#!/usr/bin/env bash # restart_services.sh — safer, simplified set -euo pipefail ### CONFIGURATION ############################################################# ENV_FILE="api/.env" RUN_DIR="run" # where we keep *.pid LOG_ROOT="logs" ### 1) Load environment vars so that configurations like FASTAPI_WORKERS are loaded # set -a && . "$ENV_FILE" && set +a # Get ENVIRONMENT for nginx config selection ENVIRONMENT="${ENVIRONMENT:-staging}" if [[ -z "${FASTAPI_PORT:-}" ]]; then echo "Error: FASTAPI_PORT environment variable is not set." exit 1 fi if [[ -z "${FASTAPI_WORKERS:-}" ]]; then echo "Error: FASTAPI_WORKERS environment variable is not set." exit 1 fi if [[ -z "${CONDA_ENV_NAME:-}" ]]; then echo "Error: CONDA_ENV_NAME environment variable is not set." exit 1 fi # Default ARQ_WORKERS to 1 if not set ARQ_WORKERS=${ARQ_WORKERS:-1} # map "service name" → "command to run" declare -A SERVICES=( [ari_manager]="python -m api.services.telephony.ari_manager" [campaign_orchestrator]="python -m api.services.campaign.campaign_orchestrator" [uvicorn]="uvicorn api.app:app --host 0.0.0.0 --port $FASTAPI_PORT --workers $FASTAPI_WORKERS" ) # Add ARQ workers dynamically based on ARQ_WORKERS environment variable for ((i=1; i<=ARQ_WORKERS; i++)); do SERVICES[arq$i]="python -m arq api.tasks.arq.WorkerSettings --custom-log-dict api.tasks.arq.LOG_CONFIG" done ### 2) Activate conda ######################################################### # Source conda if not already available (needed when running from systemd) if ! command -v conda &>/dev/null; then source /opt/conda/etc/profile.d/conda.sh fi eval "$(conda shell.bash hook)" conda activate "$CONDA_ENV_NAME" ### 3) Stop old services (only via PID files) ################################# mkdir -p "$RUN_DIR" for name in "${!SERVICES[@]}"; do pidfile="$RUN_DIR/$name.pid" if [[ -f $pidfile ]]; then oldpid=$(<"$pidfile") if kill -0 "$oldpid" 2>/dev/null; then echo "Stopping $name (PID $oldpid and its process group)…" # Kill the entire process group (negative PID) # First try SIGTERM kill -TERM -"$oldpid" 2>/dev/null || kill -TERM "$oldpid" 2>/dev/null || true sleep 4 # If still running, use SIGKILL if kill -0 "$oldpid" 2>/dev/null; then echo "⚠️ $name did not exit cleanly, forcing stop..." kill -KILL -"$oldpid" 2>/dev/null || kill -KILL "$oldpid" 2>/dev/null || true sleep 1 fi fi rm -f "$pidfile" else echo "No PID file for $name, skipping stop." fi done # Clean up any port tracking files for uvicorn rm -f "$RUN_DIR/uvicorn.port" "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn_old.pid" ### 4) Run migrations ######################################################### alembic -c api/alembic.ini upgrade head ### 5) Prepare logs ########################################################### timestamp=$(date '+%Y-%m-%d_%H-%M-%S') LOG_DIR="$LOG_ROOT/$timestamp" mkdir -p "$LOG_DIR" # Create relative symlink cd "$LOG_ROOT" && ln -sfn "$timestamp" latest && cd - >/dev/null ### 6) (Optional) Free FastAPI port ########################################### FASTAPI_PORT=$FASTAPI_PORT if command -v lsof &>/dev/null; then lsof -ti tcp:"$FASTAPI_PORT" | xargs -r kill -9 || true fi ### 7) Start services ######################################################### # Export rotation settings for loguru (if using file logging) export LOG_ROTATION_SIZE="${LOG_ROTATION_SIZE:-100 MB}" export LOG_RETENTION="${LOG_RETENTION:-7 days}" export LOG_COMPRESSION="${LOG_COMPRESSION:-gz}" for name in "${!SERVICES[@]}"; do cmd=${SERVICES[$name]} echo "→ Starting $name with loguru rotation…" # Export LOG_FILE_PATH for this specific service export LOG_FILE_PATH="$LOG_DIR/$name.log" # Start in new process group with setsid # Each service gets its own LOG_FILE_PATH environment variable setsid nohup bash -c "LOG_FILE_PATH='$LOG_DIR/$name.log' $cmd" >/dev/null 2>&1 & # Get the PID of the setsid process pid=$! echo $pid >"$RUN_DIR/$name.pid" # For uvicorn, also save the port for rolling updates and update nginx if [[ "$name" == "uvicorn" ]]; then echo "$FASTAPI_PORT" >"$RUN_DIR/uvicorn.port" # Update nginx upstream configuration if nginx is installed if command -v nginx &>/dev/null && [[ -d /etc/nginx ]]; then # Determine which upstream config to update based on ENVIRONMENT if [[ "${ENVIRONMENT:-}" == "production" ]]; then NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_production_upstream.conf" UPSTREAM_NAME="dograh_production_backend" echo "→ Updating PRODUCTION nginx upstream to port $FASTAPI_PORT…" else # Default to staging for any non-production environment NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_staging_upstream.conf" UPSTREAM_NAME="dograh_staging_backend" echo "→ Updating STAGING nginx upstream to port $FASTAPI_PORT…" fi if [[ -w $(dirname "$NGINX_UPSTREAM_CONF") ]] || [[ $EUID -eq 0 ]]; then cat > "${NGINX_UPSTREAM_CONF}.tmp" </dev/null || \ echo "⚠️ Could not update nginx config (need sudo). Run: sudo $0" fi # Test and reload nginx if config was updated if [[ -f "$NGINX_UPSTREAM_CONF" ]]; then if nginx -t 2>/dev/null || sudo nginx -t 2>/dev/null; then echo "→ Reloading nginx…" nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null || \ echo "⚠️ Could not reload nginx (may need sudo)" else echo "⚠️ Nginx configuration test failed" fi fi else echo "⚠️ Cannot write to nginx config directory (need sudo privileges)" echo " Run: sudo $0 to update nginx configuration" fi fi fi done disown -a ### 8) Summary ################################################################# echo echo "──────────────────────────────────────────────────" for name in "${!SERVICES[@]}"; do pid=$(<"$RUN_DIR/$name.pid") echo "✓ $name (PID $pid) → $LOG_DIR/$name.log" done echo " Rotation: ${LOG_ROTATION_SIZE:-100 MB}" echo " Retention: ${LOG_RETENTION:-7 days}" echo " Compression: ${LOG_COMPRESSION:-gz}" echo "Logs: tail -f $LOG_DIR/*.log" echo "Rotated logs: ls $LOG_DIR/*.log.*" echo "To stop: run this script again or kill -TERM - for process groups" echo "──────────────────────────────────────────────────"