dograh/scripts/start_services.sh

#!/usr/bin/env bash
set -e  # Exit on error

###############################################################################
### ARGUMENT PARSING
###############################################################################

DEV_MODE=false

show_help() {
  echo "Usage: $0 [OPTIONS]"
  echo ""
  echo "Options:"
  echo "  --dev     Enable development mode with auto-reload for API changes"
  echo "  --help    Show this help message"
  echo ""
  echo "Examples:"
  echo "  $0           # Start in production mode"
  echo "  $0 --dev     # Start in development mode with auto-reload"
}

while [[ $# -gt 0 ]]; do
  case $1 in
    --dev)
      DEV_MODE=true
      shift
      ;;
    --help|-h)
      show_help
      exit 0
      ;;
    *)
      echo "Unknown option: $1"
      show_help
      exit 1
      ;;
  esac
done

###############################################################################
### CONFIGURATION
###############################################################################

# Determine BASE_DIR as parent of the scripts directory
BASE_DIR="$(cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" && pwd)"

ENV_FILE="$BASE_DIR/api/.env"
RUN_DIR="$BASE_DIR/run"                 # Where we keep *.pid
BASE_LOG_DIR="$BASE_DIR/logs"           # Base logs directory

TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
LOG_DIR="$BASE_LOG_DIR/$TIMESTAMP"      # Timestamped log directory
LATEST_LINK="$BASE_LOG_DIR/latest"      # Symlink to latest logs
VENV_PATH="$BASE_DIR/venv"

ARQ_WORKERS=${ARQ_WORKERS:-1}
LOG_TO_FILE=${LOG_TO_FILE:-true}    # Set to false in Docker to use stdout
WAIT_FOR_PROCESSES=${WAIT_FOR_PROCESSES:-false}  # Set to true in Docker to keep container alive

# Log startup
cd "$BASE_DIR"
if $DEV_MODE; then
  echo "Starting Dograh Services (DEV MODE) at $(date) in BASE_DIR: ${BASE_DIR}"
  echo "Auto-reload enabled for api/ directory changes"
else
  echo "Starting Dograh Services at $(date) in BASE_DIR: ${BASE_DIR}"
fi

###############################################################################
### 1) Load environment variables
###############################################################################

# Load environment from a file if it exists
if [[ -f "$ENV_FILE" ]]; then
  set -a && . "$ENV_FILE" && set +a
fi

FASTAPI_PORT=${FASTAPI_PORT:-8000}
CPU_CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
FASTAPI_WORKERS=${FASTAPI_WORKERS:-$CPU_CORES}

###############################################################################
### 2) Define services
###############################################################################

# Map "service name" → "command to run"
# Using arrays for bash 3.2 compatibility
SERVICE_NAMES=(
  "ari_manager"
  "campaign_orchestrator"
  "uvicorn"
)

# Build uvicorn command based on mode
if $DEV_MODE; then
  # Dev mode: single worker with auto-reload (--reload is incompatible with --workers > 1)
  UVICORN_CMD="uvicorn api.app:app --host 0.0.0.0 --port $FASTAPI_PORT --reload --reload-dir api"
else
  # Production mode: multiple workers, no reload
  UVICORN_CMD="uvicorn api.app:app --host 0.0.0.0 --port $FASTAPI_PORT --workers $FASTAPI_WORKERS"
fi

SERVICE_COMMANDS=(
  "python -m api.services.telephony.ari_manager"
  "python -m api.services.campaign.campaign_orchestrator"
  "$UVICORN_CMD"
)

# Add ARQ workers dynamically
for ((i=1; i<=ARQ_WORKERS; i++)); do
  SERVICE_NAMES+=("arq$i")
  SERVICE_COMMANDS+=("python -m arq api.tasks.arq.WorkerSettings --custom-log-dict api.tasks.arq.LOG_CONFIG")
done

###############################################################################
### 3) Activate virtual environment
###############################################################################

if [[ -d "$VENV_PATH" && -f "$VENV_PATH/bin/activate" ]]; then
  source "$VENV_PATH/bin/activate"
  echo "Virtual environment activated: $VENV_PATH"
else
  echo "Warning: Virtual environment not found at $VENV_PATH"
  echo "Continuing without virtual environment activation..."
fi

###############################################################################
### 4) Stop old services
###############################################################################

mkdir -p "$RUN_DIR"

# Function to get all descendant PIDs of a process (children, grandchildren, etc.)
get_descendants() {
  local parent_pid=$1
  local descendants=""
  local children

  # Get direct children
  children=$(pgrep -P "$parent_pid" 2>/dev/null || true)

  for child in $children; do
    # Recursively get descendants of each child
    descendants="$descendants $child $(get_descendants "$child")"
  done

  echo "$descendants"
}

# Function to kill a process and all its descendants
kill_process_tree() {
  local pid=$1
  local signal=$2
  local descendants

  descendants=$(get_descendants "$pid")

  # Kill children first (bottom-up), then parent
  for desc_pid in $descendants; do
    if kill -0 "$desc_pid" 2>/dev/null; then
      kill "$signal" "$desc_pid" 2>/dev/null || true
    fi
  done

  # Kill the parent
  if kill -0 "$pid" 2>/dev/null; then
    kill "$signal" "$pid" 2>/dev/null || true
  fi
}

for name in "${SERVICE_NAMES[@]}"; do
  pidfile="$RUN_DIR/$name.pid"

  if [[ -f $pidfile ]]; then
    oldpid=$(<"$pidfile")

    if kill -0 "$oldpid" 2>/dev/null; then
      echo "Stopping $name (PID $oldpid and all descendants)…"

      # Kill the entire process tree (parent + all descendants)
      kill_process_tree "$oldpid" "-TERM"
      sleep 4

      # Check if parent or any descendants are still alive
      still_alive=false
      if kill -0 "$oldpid" 2>/dev/null; then
        still_alive=true
      else
        for desc_pid in $(get_descendants "$oldpid"); do
          if kill -0 "$desc_pid" 2>/dev/null; then
            still_alive=true
            break
          fi
        done
      fi

      if $still_alive; then
        echo "⚠️  $name did not exit cleanly, forcing stop..."
        kill_process_tree "$oldpid" "-KILL"
        sleep 1
      fi
    fi

    rm -f "$pidfile"
  else
    echo "No PID file for $name, skipping stop."
  fi
done

# Clean up any port tracking files for uvicorn
rm -f "$RUN_DIR/uvicorn.port" "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn_old.pid"

###############################################################################
### 5) Run migrations
###############################################################################

alembic -c "$BASE_DIR/api/alembic.ini" upgrade head

###############################################################################
### 6) Prepare logs
###############################################################################

mkdir -p "$BASE_LOG_DIR" "$LOG_DIR"

# Remove old symlink and create a new one
if [[ -L "$LATEST_LINK" ]]; then
  rm "$LATEST_LINK"
fi
ln -s "$TIMESTAMP" "$LATEST_LINK"

echo "Log directory: $LOG_DIR"
echo "Latest symlink: $LATEST_LINK -> $TIMESTAMP"

###############################################################################
### 7) Start services
###############################################################################

for i in "${!SERVICE_NAMES[@]}"; do
  name="${SERVICE_NAMES[$i]}"
  cmd="${SERVICE_COMMANDS[$i]}"
  echo "→ Starting $name"

  (
    cd "$BASE_DIR"
    if [[ "$LOG_TO_FILE" == "true" ]]; then
      export LOG_FILE_PATH="$LOG_DIR/$name.log"
      exec $cmd >>"$LOG_DIR/$name.log" 2>&1
    else
      # Log to stdout/stderr for Docker
      exec $cmd
    fi
  ) &

  pid=$!
  echo $pid >"$RUN_DIR/$name.pid"
  echo "  Started with PID $pid"

  if [[ "$name" == "uvicorn" ]]; then
    echo "$FASTAPI_PORT" >"$RUN_DIR/uvicorn.port"
  fi
done

###############################################################################
### 8) Summary
###############################################################################

echo
echo "──────────────────────────────────────────────────"
if $DEV_MODE; then
  echo "Mode: DEVELOPMENT (auto-reload enabled)"
else
  echo "Mode: PRODUCTION"
fi
echo ""
for name in "${SERVICE_NAMES[@]}"; do
  pid=$(<"$RUN_DIR/$name.pid")
  echo "✓ $name (PID $pid) → $LOG_DIR/$name.log"
done
echo ""
echo "  Rotation: ${LOG_ROTATION_SIZE:-100 MB}"
echo "  Retention: ${LOG_RETENTION:-7 days}"
echo "  Compression: ${LOG_COMPRESSION:-gz}"
echo "Logs: tail -f $LOG_DIR/*.log"
echo "Rotated logs: ls $LOG_DIR/*.log.*"
echo "To stop: ./scripts/stop_services.sh"
echo "──────────────────────────────────────────────────"

# In Docker mode, wait for all background processes to keep container alive
if [[ "$WAIT_FOR_PROCESSES" == "true" ]]; then
  wait
fi