#!/usr/bin/env bash set -e # Exit on error ############################################################################### ### ARGUMENT PARSING ############################################################################### DEV_MODE=false show_help() { echo "Usage: $0 [OPTIONS]" echo "" echo "Options:" echo " --dev Enable development mode with auto-reload for API changes" echo " --help Show this help message" echo "" echo "Examples:" echo " $0 # Start in production mode" echo " $0 --dev # Start in development mode with auto-reload" } while [[ $# -gt 0 ]]; do case $1 in --dev) DEV_MODE=true shift ;; --help|-h) show_help exit 0 ;; *) echo "Unknown option: $1" show_help exit 1 ;; esac done ############################################################################### ### CONFIGURATION ############################################################################### # Determine BASE_DIR as parent of the scripts directory BASE_DIR="$(cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" && pwd)" ENV_FILE="$BASE_DIR/api/.env" RUN_DIR="$BASE_DIR/run" # Where we keep *.pid BASE_LOG_DIR="$BASE_DIR/logs" # Base logs directory TIMESTAMP=$(date +"%Y%m%d_%H%M%S") LOG_DIR="$BASE_LOG_DIR/$TIMESTAMP" # Timestamped log directory LATEST_LINK="$BASE_LOG_DIR/latest" # Symlink to latest logs VENV_PATH="$BASE_DIR/venv" ARQ_WORKERS=${ARQ_WORKERS:-1} LOG_TO_FILE=${LOG_TO_FILE:-true} # Set to false in Docker to use stdout WAIT_FOR_PROCESSES=${WAIT_FOR_PROCESSES:-false} # Set to true in Docker to keep container alive # Log startup cd "$BASE_DIR" if $DEV_MODE; then echo "Starting Dograh Services (DEV MODE) at $(date) in BASE_DIR: ${BASE_DIR}" echo "Auto-reload enabled for api/ directory changes" else echo "Starting Dograh Services at $(date) in BASE_DIR: ${BASE_DIR}" fi ############################################################################### ### 1) Load environment variables ############################################################################### # Load environment from a file if it exists if [[ -f "$ENV_FILE" ]]; then set -a && . "$ENV_FILE" && set +a fi FASTAPI_PORT=${FASTAPI_PORT:-8000} CPU_CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1) FASTAPI_WORKERS=${FASTAPI_WORKERS:-$CPU_CORES} ############################################################################### ### 2) Define services ############################################################################### # Map "service name" → "command to run" # Using arrays for bash 3.2 compatibility SERVICE_NAMES=( "ari_manager" "campaign_orchestrator" "uvicorn" ) # Build uvicorn command based on mode if $DEV_MODE; then # Dev mode: single worker with auto-reload (--reload is incompatible with --workers > 1) UVICORN_CMD="uvicorn api.app:app --host 0.0.0.0 --port $FASTAPI_PORT --reload --reload-dir api" else # Production mode: multiple workers, no reload UVICORN_CMD="uvicorn api.app:app --host 0.0.0.0 --port $FASTAPI_PORT --workers $FASTAPI_WORKERS" fi SERVICE_COMMANDS=( "python -m api.services.telephony.ari_manager" "python -m api.services.campaign.campaign_orchestrator" "$UVICORN_CMD" ) # Add ARQ workers dynamically for ((i=1; i<=ARQ_WORKERS; i++)); do SERVICE_NAMES+=("arq$i") SERVICE_COMMANDS+=("python -m arq api.tasks.arq.WorkerSettings --custom-log-dict api.tasks.arq.LOG_CONFIG") done ############################################################################### ### 3) Activate virtual environment ############################################################################### if [[ -d "$VENV_PATH" && -f "$VENV_PATH/bin/activate" ]]; then source "$VENV_PATH/bin/activate" echo "Virtual environment activated: $VENV_PATH" else echo "Warning: Virtual environment not found at $VENV_PATH" echo "Continuing without virtual environment activation..." fi ############################################################################### ### 4) Stop old services ############################################################################### mkdir -p "$RUN_DIR" # Function to get all descendant PIDs of a process (children, grandchildren, etc.) get_descendants() { local parent_pid=$1 local descendants="" local children # Get direct children children=$(pgrep -P "$parent_pid" 2>/dev/null || true) for child in $children; do # Recursively get descendants of each child descendants="$descendants $child $(get_descendants "$child")" done echo "$descendants" } # Function to kill a process and all its descendants kill_process_tree() { local pid=$1 local signal=$2 local descendants descendants=$(get_descendants "$pid") # Kill children first (bottom-up), then parent for desc_pid in $descendants; do if kill -0 "$desc_pid" 2>/dev/null; then kill "$signal" "$desc_pid" 2>/dev/null || true fi done # Kill the parent if kill -0 "$pid" 2>/dev/null; then kill "$signal" "$pid" 2>/dev/null || true fi } for name in "${SERVICE_NAMES[@]}"; do pidfile="$RUN_DIR/$name.pid" if [[ -f $pidfile ]]; then oldpid=$(<"$pidfile") if kill -0 "$oldpid" 2>/dev/null; then echo "Stopping $name (PID $oldpid and all descendants)…" # Kill the entire process tree (parent + all descendants) kill_process_tree "$oldpid" "-TERM" sleep 4 # Check if parent or any descendants are still alive still_alive=false if kill -0 "$oldpid" 2>/dev/null; then still_alive=true else for desc_pid in $(get_descendants "$oldpid"); do if kill -0 "$desc_pid" 2>/dev/null; then still_alive=true break fi done fi if $still_alive; then echo "⚠️ $name did not exit cleanly, forcing stop..." kill_process_tree "$oldpid" "-KILL" sleep 1 fi fi rm -f "$pidfile" else echo "No PID file for $name, skipping stop." fi done # Clean up any port tracking files for uvicorn rm -f "$RUN_DIR/uvicorn.port" "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn_old.pid" ############################################################################### ### 5) Run migrations ############################################################################### alembic -c "$BASE_DIR/api/alembic.ini" upgrade head ############################################################################### ### 6) Prepare logs ############################################################################### mkdir -p "$BASE_LOG_DIR" "$LOG_DIR" # Remove old symlink and create a new one if [[ -L "$LATEST_LINK" ]]; then rm "$LATEST_LINK" fi ln -s "$TIMESTAMP" "$LATEST_LINK" echo "Log directory: $LOG_DIR" echo "Latest symlink: $LATEST_LINK -> $TIMESTAMP" ############################################################################### ### 7) Start services ############################################################################### for i in "${!SERVICE_NAMES[@]}"; do name="${SERVICE_NAMES[$i]}" cmd="${SERVICE_COMMANDS[$i]}" echo "→ Starting $name" ( cd "$BASE_DIR" if [[ "$LOG_TO_FILE" == "true" ]]; then export LOG_FILE_PATH="$LOG_DIR/$name.log" exec $cmd >>"$LOG_DIR/$name.log" 2>&1 else # Log to stdout/stderr for Docker exec $cmd fi ) & pid=$! echo $pid >"$RUN_DIR/$name.pid" echo " Started with PID $pid" if [[ "$name" == "uvicorn" ]]; then echo "$FASTAPI_PORT" >"$RUN_DIR/uvicorn.port" fi done ############################################################################### ### 8) Summary ############################################################################### echo echo "──────────────────────────────────────────────────" if $DEV_MODE; then echo "Mode: DEVELOPMENT (auto-reload enabled)" else echo "Mode: PRODUCTION" fi echo "" for name in "${SERVICE_NAMES[@]}"; do pid=$(<"$RUN_DIR/$name.pid") echo "✓ $name (PID $pid) → $LOG_DIR/$name.log" done echo "" echo " Rotation: ${LOG_ROTATION_SIZE:-100 MB}" echo " Retention: ${LOG_RETENTION:-7 days}" echo " Compression: ${LOG_COMPRESSION:-gz}" echo "Logs: tail -f $LOG_DIR/*.log" echo "Rotated logs: ls $LOG_DIR/*.log.*" echo "To stop: ./scripts/stop_services.sh" echo "──────────────────────────────────────────────────" # In Docker mode, wait for all background processes to keep container alive if [[ "$WAIT_FOR_PROCESSES" == "true" ]]; then wait fi