diff --git a/.gitignore b/.gitignore
index f0ffdd1..410cc1c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,4 +8,3 @@ __pycache__
docs/
infrastructure/
nginx/
-scripts/
diff --git a/api/logging_config.py b/api/logging_config.py
index de68fb3..a8ea420 100644
--- a/api/logging_config.py
+++ b/api/logging_config.py
@@ -1,73 +1,21 @@
-import atexit
-import logging
import os
-import queue
import sys
-from logging.handlers import QueueHandler, QueueListener
import loguru
-from axiom_py import Client
-from axiom_py.logging import AxiomHandler
from pipecat.utils.context import run_id_var, turn_var
from api.enums import Environment
from api.utils.worker import get_worker_id, is_worker_process
-# ----- NEW CODE START -----
-# Helper to map string log level to Python logging level, adding support for "TRACE"
-TRACE_LEVEL_NUM = 5 # Below DEBUG (10)
-
-
-def _get_logging_level(level_name: str) -> int:
- """Return numeric logging level for a given level name.
-
- Supports the standard logging levels as well as the custom ``TRACE`` level
- used by *loguru*. If ``TRACE`` is requested and not yet defined in the
- ``logging`` module, it will be registered dynamically.
- """
- level_name = level_name.upper()
-
- # Standard levels are present on the ``logging`` module.
- if hasattr(logging, level_name):
- return getattr(logging, level_name)
-
- # Add support for TRACE (finer-grained than DEBUG)
- if level_name == "TRACE":
- if not hasattr(logging, "TRACE"):
- logging.addLevelName(TRACE_LEVEL_NUM, "TRACE")
-
- def trace(self, message, *args, **kwargs): # type: ignore[override]
- if self.isEnabledFor(TRACE_LEVEL_NUM):
- self._log(TRACE_LEVEL_NUM, message, args, **kwargs)
-
- logging.Logger.trace = trace # type: ignore[attr-defined]
- return TRACE_LEVEL_NUM
-
- # Fallback to DEBUG if an unknown level is provided
- return logging.DEBUG
-
-
-# ----- NEW CODE END -----
-
ENVIRONMENT = os.getenv("ENVIRONMENT", Environment.LOCAL.value)
ENABLE_TURN_LOGGING = os.getenv("ENABLE_TURN_LOGGING", "false").lower() == "true"
-# Log rotation settings from environment
-LOG_ROTATION_SIZE = os.getenv("LOG_ROTATION_SIZE", "100 MB") # e.g., "100 MB", "1 GB"
-LOG_ROTATION_TIME = os.getenv("LOG_ROTATION_TIME", None) # e.g., "00:00", "12:00"
-LOG_RETENTION = os.getenv(
- "LOG_RETENTION", "7 days"
-) # e.g., "7 days", "1 week", "10 files"
-LOG_COMPRESSION = os.getenv(
- "LOG_COMPRESSION", "gz"
-) # "gz", "bz2", "xz", "tar", "tar.gz", "tar.bz2", "tar.xz", "zip"
-LOG_FILE_PATH = os.getenv(
- "LOG_FILE_PATH", None
-) # If set, write to file instead of stdout
+# We write different uvicorn forked worker log to a different
+# file which is then synced to cloudwatch logs
+LOG_FILE_PATH = os.getenv("LOG_FILE_PATH", None)
# Track if logging has been initialized
_logging_initialized = False
-_axiom_listener = None
def inject_run_id(record):
@@ -97,11 +45,11 @@ def inject_run_id(record):
def setup_logging():
"""Set up logging for the main application"""
- global _logging_initialized, _axiom_listener
+ global _logging_initialized
# Return early if already initialized
if _logging_initialized:
- return _axiom_listener
+ return
log_level = os.getenv("LOG_LEVEL", "DEBUG").upper()
@@ -125,31 +73,8 @@ def setup_logging():
else:
log_format = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level} | [run_id={extra[run_id]}] | {file.name}:{line} | {message}"
- # Add handler - either file with rotation or console
+ # Add handler - either file or console
if LOG_FILE_PATH:
- # File handler with rotation
- rotation_config = {}
-
- # Size-based rotation (e.g., "100 MB", "1 GB")
- if LOG_ROTATION_SIZE:
- rotation_config["rotation"] = LOG_ROTATION_SIZE
-
- # Time-based rotation (e.g., "00:00" for daily at midnight)
- if LOG_ROTATION_TIME:
- rotation_config["rotation"] = LOG_ROTATION_TIME
-
- # If no rotation specified, default to 100 MB
- if not rotation_config:
- rotation_config["rotation"] = "100 MB"
-
- # Retention policy (e.g., "7 days", "10 files")
- if LOG_RETENTION:
- rotation_config["retention"] = LOG_RETENTION
-
- # Compression format
- if LOG_COMPRESSION and LOG_COMPRESSION.lower() != "none":
- rotation_config["compression"] = LOG_COMPRESSION
-
# Determine the actual log file path
actual_log_path = LOG_FILE_PATH
@@ -159,7 +84,6 @@ def setup_logging():
# Split the path to insert worker ID before extension
base_path, ext = os.path.splitext(LOG_FILE_PATH)
actual_log_path = f"{base_path}-worker-{worker_id}{ext}"
- loguru.logger.info(f"Worker {worker_id} will log to: {actual_log_path}")
patched.add(
actual_log_path,
@@ -167,7 +91,6 @@ def setup_logging():
level=log_level,
colorize=False, # No colors in file logs
enqueue=True, # Thread-safe writing
- **rotation_config,
)
else:
# Console handler (existing behavior)
@@ -178,40 +101,5 @@ def setup_logging():
colorize=True,
)
- # Set up queue-based logging for Axiom
- log_q = queue.Queue(-1) # infinite size (tweak if needed)
- queue_handler = QueueHandler(log_q) # puts LogRecord on the queue
- queue_handler.setLevel(_get_logging_level(log_level))
-
- # Set up Axiom handler if credentials are available
- axiom_token = os.environ.get("AXIOM_TOKEN")
- axiom_org = os.environ.get("AXIOM_ORG")
- axiom_dataset = os.getenv("AXIOM_LOG_DATASET")
-
- if axiom_token and axiom_org and axiom_dataset:
- client = Client(token=axiom_token, org_id=axiom_org)
- axiom_handler = AxiomHandler(client, axiom_dataset)
- axiom_handler.setLevel(_get_logging_level(log_level))
-
- listener = QueueListener(
- log_q,
- axiom_handler,
- respect_handler_level=True,
- )
- listener.start()
-
- patched.add(queue_handler, level=log_level, enqueue=False)
-
- # Register cleanup
- atexit.register(listener.stop)
-
- # Return the listener for manual cleanup if needed
- loguru.logger = patched
- _logging_initialized = True
- _axiom_listener = listener
- return listener
- else:
- # No Axiom logging available
- loguru.logger = patched
- _logging_initialized = True
- return None
+ loguru.logger = patched
+ _logging_initialized = True
diff --git a/api/requirements.txt b/api/requirements.txt
index 4e3644c..d92d3e1 100644
--- a/api/requirements.txt
+++ b/api/requirements.txt
@@ -7,7 +7,6 @@ redis==5.3.1
uvicorn==0.35.0
aioboto3==15.1.0
arq==0.26.3
-axiom-py==0.9.0
twilio==9.8.0
minio==7.2.16
alembic-postgresql-enum==1.8.0
diff --git a/scripts/format.sh b/scripts/format.sh
new file mode 100755
index 0000000..3423179
--- /dev/null
+++ b/scripts/format.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -e
+set -euo pipefail
+
+ruff check api --select I --select F401 --fix
+ruff format api
diff --git a/scripts/lint.sh b/scripts/lint.sh
new file mode 100644
index 0000000..2b87d68
--- /dev/null
+++ b/scripts/lint.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+mypy api
+ruff check api --check
+ruff format api --check
\ No newline at end of file
diff --git a/scripts/makemigrate.sh b/scripts/makemigrate.sh
new file mode 100644
index 0000000..ba81621
--- /dev/null
+++ b/scripts/makemigrate.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -e # Exit immediately if a command exits with a non-zero status
+
+# Set PYTHONPATH to the parent directory of the script's location
+export PYTHONPATH="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")"
+
+# Define color codes
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+env_file="api/.env"
+
+# Check if environment file exists
+if [ ! -f "$env_file" ]; then
+ echo -e "${RED}Error: Environment file $env_file not found.${NC}"
+ exit 1
+fi
+
+# Load environment variables
+export $(grep -v '^#' "$env_file" | xargs)
+
+# Prompt for the name of the migration
+read -p "Enter the migration name (minimum 5 characters): " migration_name
+
+# Check if the migration name is empty or less than 5 characters
+if [[ -z "$migration_name" || ${#migration_name} -lt 5 ]]; then
+ echo -e "${RED}Error: Migration name must be at least 5 characters long.${NC}"
+ exit 1
+fi
+
+# Generate the Alembic revision with the provided migration name
+alembic -c api/alembic.ini revision --autogenerate -m "$migration_name"
diff --git a/scripts/migrate.sh b/scripts/migrate.sh
new file mode 100644
index 0000000..6c83c74
--- /dev/null
+++ b/scripts/migrate.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e # Exit immediately if a command exits with a non-zero status
+
+# Set PYTHONPATH to the parent directory of the script's location
+export PYTHONPATH="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")"
+
+# Define color codes
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+env_file="api/.env"
+
+# Check if environment file exists
+if [ ! -f "$env_file" ]; then
+ echo -e "${RED}Error: Environment file $env_file not found.${NC}"
+ exit 1
+fi
+
+# Load environment variables
+export $(grep -v '^#' "$env_file" | xargs)
+
+# Run migrations
+alembic -c api/alembic.ini upgrade head
+
+# Create initial data in DB
+# python api/initial_data.py
\ No newline at end of file
diff --git a/scripts/pre_commit.sh b/scripts/pre_commit.sh
new file mode 100755
index 0000000..1e4301b
--- /dev/null
+++ b/scripts/pre_commit.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+###############################################################################
+# Ensure Ruff is installed (first try pipx, fall back to pip --user).
+###############################################################################
+if ! command -v ruff >/dev/null 2>&1; then
+ echo "⇢ Ruff not found on PATH – installing…"
+ if command -v pipx >/dev/null 2>&1; then
+ # install into an isolated environment if pipx is present
+ pipx install --quiet ruff
+ else
+ # otherwise install into the current (or user-level) Python environment
+ pip install --quiet --upgrade --user ruff
+ fi
+fi
+
+###############################################################################
+# 1 – Python formatting (calls Ruff + Black, etc.)
+###############################################################################
+sh scripts/format.sh
+
+###############################################################################
+# 2 – ESLint autofix inside the Next.js app
+###############################################################################
+(cd ui && npm run fix-lint)
+
+###############################################################################
+# 3 – Restage any files changed by the fixers so the commit includes them
+###############################################################################
+git add -u
diff --git a/scripts/rolling_update_uvicorn.sh b/scripts/rolling_update_uvicorn.sh
new file mode 100755
index 0000000..7cba8c1
--- /dev/null
+++ b/scripts/rolling_update_uvicorn.sh
@@ -0,0 +1,310 @@
+#!/usr/bin/env bash
+# rolling_update_uvicorn.sh — Zero-downtime rolling update for uvicorn workers
+
+set -e # Exit on error
+
+### CONFIGURATION #############################################################
+ENV_FILE="api/.env"
+RUN_DIR="run"
+LOG_DIR="logs"
+VENV_PATH="/home/ubuntu/dograh_venv"
+HEALTH_CHECK_ENDPOINT="/api/v1/health" # Adjust as needed
+MAX_WAIT_SECONDS=310 # Max wait for graceful shutdown (5 minutes + 10 seconds grace)
+
+# Load environment
+set -a && . "$ENV_FILE" && set +a
+
+cd /home/ubuntu/app
+
+### FUNCTIONS ##################################################################
+
+log_info() {
+ echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO: $*"
+}
+
+log_error() {
+ echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2
+}
+
+log_warning() {
+ echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARN: $*"
+}
+
+check_port_availability() {
+ local port=$1
+ if lsof -Pi :$port -sTCP:LISTEN -t >/dev/null 2>&1; then
+ return 1 # Port is in use
+ fi
+ return 0 # Port is available
+}
+
+wait_for_health_check() {
+ local port=$1
+ local max_attempts=30
+ local attempt=0
+
+ log_info "Waiting for new uvicorn workers to be healthy on port $port..."
+
+ while [ $attempt -lt $max_attempts ]; do
+ if curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:${port}${HEALTH_CHECK_ENDPOINT}" | grep -q "200"; then
+ log_info "Health check passed on port $port"
+ return 0
+ fi
+ attempt=$((attempt + 1))
+ log_info "Health check attempt $attempt/$max_attempts..."
+ sleep 1
+ done
+
+ log_error "Health check failed after $max_attempts attempts"
+ return 1
+}
+
+get_old_uvicorn_pids() {
+ local pidfile="$RUN_DIR/uvicorn.pid"
+ local pids=""
+
+ if [[ -f "$pidfile" ]]; then
+ # Read the main PID
+ local main_pid=$(<"$pidfile")
+ if kill -0 "$main_pid" 2>/dev/null; then
+ # Get all PIDs in the process group
+ pids=$(ps -o pid= -g $(ps -o pgid= -p "$main_pid" | tr -d ' ') 2>/dev/null || echo "$main_pid")
+ fi
+ fi
+
+ echo "$pids"
+}
+
+graceful_shutdown_old_workers() {
+ local old_pids="$1"
+
+ if [[ -z "$old_pids" ]]; then
+ log_warning "No old uvicorn workers found to shut down"
+ return 0
+ fi
+
+ log_info "Starting graceful shutdown of old uvicorn workers (PIDs: $(echo $old_pids | tr '\n' ' '))"
+
+ # Send SIGTERM to trigger graceful shutdown
+ for pid in $old_pids; do
+ if kill -0 "$pid" 2>/dev/null; then
+ log_info "Sending SIGTERM to PID $pid"
+ kill -TERM "$pid" 2>/dev/null || true
+ fi
+ done
+
+ # Wait for processes to exit gracefully
+ local start_time=$(date +%s)
+ local all_dead=false
+
+ while [[ $(($(date +%s) - start_time)) -lt $MAX_WAIT_SECONDS ]]; do
+ all_dead=true
+ for pid in $old_pids; do
+ if kill -0 "$pid" 2>/dev/null; then
+ all_dead=false
+ break
+ fi
+ done
+
+ if $all_dead; then
+ log_info "All old workers shut down gracefully"
+ return 0
+ fi
+
+ log_info "Waiting for workers to complete active requests... ($(( $(date +%s) - start_time ))s elapsed)"
+ sleep 5
+ done
+
+ # Force kill if still running after timeout
+ log_warning "Timeout reached, force killing remaining workers"
+ for pid in $old_pids; do
+ if kill -0 "$pid" 2>/dev/null; then
+ log_warning "Force killing PID $pid"
+ kill -KILL "$pid" 2>/dev/null || true
+ fi
+ done
+
+ sleep 1
+ return 0
+}
+
+start_new_uvicorn_workers() {
+ local new_port=$1
+
+ log_info "Starting new uvicorn workers on port $new_port..."
+
+ # Get configuration from environment
+ set -a && . "$ENV_FILE" && set +a
+
+ if [[ -z "${FASTAPI_WORKERS:-}" ]]; then
+ log_error "FASTAPI_WORKERS environment variable is not set"
+ return 1
+ fi
+
+ # Activate virtual environment
+ source ${VENV_PATH}/bin/activate
+
+ # Use the log directory (where start_services.sh put logs)
+ local log_dir="$LOG_DIR"
+
+ if [[ ! -d "$log_dir" ]]; then
+ log_error "No log directory found. Run start_services.sh first."
+ return 1
+ fi
+
+ # Create unique log filename using timestamp and script PID to avoid conflicts
+ local script_pid=$$ # PID of this rolling_update script (for uniqueness)
+ local timestamp=$(date '+%H%M%S')
+ export LOG_FILE_PATH="$log_dir/uvicorn-rollover-${timestamp}-${script_pid}.log"
+
+ log_info "Starting uvicorn with $FASTAPI_WORKERS workers on port $new_port"
+ log_info "Logs: $LOG_FILE_PATH"
+
+ # Start in new process group with setsid (same as start_services.sh)
+ # Each service gets its own LOG_FILE_PATH environment variable
+ setsid nohup bash -c "LOG_FILE_PATH='$LOG_FILE_PATH' uvicorn api.app:app --host 0.0.0.0 --port $new_port --workers $FASTAPI_WORKERS" >/dev/null 2>&1 &
+
+ local new_pid=$!
+ echo "$new_pid" > "$RUN_DIR/uvicorn_new.pid"
+
+ # Save port information
+ echo "$new_port" > "$RUN_DIR/uvicorn_new.port"
+
+ log_info "New uvicorn started with PID $new_pid"
+
+ # Wait a bit for startup
+ sleep 5
+
+ # Check if process is still running
+ if ! kill -0 "$new_pid" 2>/dev/null; then
+ log_error "New uvicorn process died immediately"
+ return 1
+ fi
+
+ return 0
+}
+
+finalize_rollover() {
+ log_info "Finalizing rollover..."
+
+ # Move new PID file to main PID file
+ if [[ -f "$RUN_DIR/uvicorn_new.pid" ]]; then
+ mv "$RUN_DIR/uvicorn_new.pid" "$RUN_DIR/uvicorn.pid"
+ fi
+
+ # Store the new port for reference
+ if [[ -f "$RUN_DIR/uvicorn_new.port" ]]; then
+ mv "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn.port"
+ fi
+
+ # Clean up old PID file if it exists
+ rm -f "$RUN_DIR/uvicorn_old.pid"
+
+ log_info "Rollover completed successfully"
+}
+
+rollback() {
+ local old_port=$1
+ local new_pid=$2
+
+ log_error "Rolling back due to failure..."
+
+ # Kill new workers if they exist
+ if [[ -n "$new_pid" ]] && kill -0 "$new_pid" 2>/dev/null; then
+ log_info "Killing new uvicorn workers (PID: $new_pid)"
+ kill -KILL -"$new_pid" 2>/dev/null || kill -KILL "$new_pid" 2>/dev/null || true
+ fi
+
+ # Clean up temporary files
+ rm -f "$RUN_DIR/uvicorn_new.pid" "$RUN_DIR/uvicorn_new.port"
+
+ log_error "Rollback completed"
+}
+
+### MAIN LOGIC ################################################################
+
+# Check arguments
+if [[ $# -ne 1 ]]; then
+ echo "Usage: $0 "
+ echo "Example: $0 8001"
+ exit 1
+fi
+
+NEW_PORT=$1
+
+# Validate port number
+if ! [[ "$NEW_PORT" =~ ^[0-9]+$ ]] || [ "$NEW_PORT" -lt 1 ] || [ "$NEW_PORT" -gt 65535 ]; then
+ log_error "Invalid port number: $NEW_PORT"
+ exit 1
+fi
+
+# Check if port is available
+if ! check_port_availability "$NEW_PORT"; then
+ log_error "Port $NEW_PORT is already in use"
+ exit 1
+fi
+
+# Get old port from file or environment
+OLD_PORT=""
+if [[ -f "$RUN_DIR/uvicorn.port" ]]; then
+ OLD_PORT=$(<"$RUN_DIR/uvicorn.port")
+elif [[ -f "$ENV_FILE" ]]; then
+ set -a && . "$ENV_FILE" && set +a
+ OLD_PORT="${FASTAPI_PORT:-}"
+fi
+
+if [[ "$NEW_PORT" == "$OLD_PORT" ]]; then
+ log_error "New port is the same as old port ($NEW_PORT)"
+ exit 1
+fi
+
+log_info "Starting rolling update from port ${OLD_PORT:-unknown} to port $NEW_PORT"
+
+# Create run directory if it doesn't exist
+mkdir -p "$RUN_DIR"
+
+# Get old uvicorn PIDs before starting new ones
+OLD_PIDS=$(get_old_uvicorn_pids)
+if [[ -n "$OLD_PIDS" ]]; then
+ # Save old PIDs for potential rollback
+ echo "$OLD_PIDS" > "$RUN_DIR/uvicorn_old.pid"
+ log_info "Found old uvicorn workers: $(echo $OLD_PIDS | tr '\n' ' ')"
+else
+ log_warning "No existing uvicorn workers found"
+fi
+
+# Start new uvicorn workers
+if ! start_new_uvicorn_workers "$NEW_PORT"; then
+ log_error "Failed to start new uvicorn workers"
+ exit 1
+fi
+
+NEW_PID=$(<"$RUN_DIR/uvicorn_new.pid")
+
+# Wait for new workers to be healthy
+if ! wait_for_health_check "$NEW_PORT"; then
+ log_error "New workers failed health check"
+ rollback "$OLD_PORT" "$NEW_PID"
+ exit 1
+fi
+
+# Give the system some time to stabilize before shutting down old workers
+log_info "Waiting for system to stabilize..."
+sleep 5
+
+# Gracefully shutdown old workers
+if [[ -n "$OLD_PIDS" ]]; then
+ graceful_shutdown_old_workers "$OLD_PIDS"
+fi
+
+# Finalize the rollover
+finalize_rollover
+
+# Summary
+echo "──────────────────────────────────────────────────"
+echo "✓ Rolling update completed successfully"
+echo " Old port: ${OLD_PORT:-none}"
+echo " New port: $NEW_PORT"
+echo " New PID: $NEW_PID"
+echo " Logs: $LOG_DIR/"
+echo "──────────────────────────────────────────────────"
\ No newline at end of file
diff --git a/scripts/start_services.sh b/scripts/start_services.sh
new file mode 100755
index 0000000..3f054f8
--- /dev/null
+++ b/scripts/start_services.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+# start_services.sh
+
+set -e # Exit on error
+
+### CONFIGURATION #############################################################
+ENV_FILE="api/.env"
+RUN_DIR="run" # where we keep *.pid
+LOG_DIR="logs"
+VENV_PATH="/home/ubuntu/dograh_venv"
+ARQ_WORKERS=${ARQ_WORKERS:-1}
+
+# Log startup
+echo "Starting Dograh Services at $(date)"
+
+### 1) Load environment vars so that configurations like FASTAPI_WORKERS are loaded
+set -a && . "$ENV_FILE" && set +a
+
+cd /home/ubuntu/app
+
+if [[ -z "${FASTAPI_PORT:-}" ]]; then
+ echo "Error: FASTAPI_PORT environment variable is not set."
+ exit 1
+fi
+
+if [[ -z "${FASTAPI_WORKERS:-}" ]]; then
+ echo "Error: FASTAPI_WORKERS environment variable is not set."
+ exit 1
+fi
+
+# map "service name" → "command to run"
+declare -A SERVICES=(
+ [ari_manager]="python -m api.services.telephony.ari_manager"
+ [campaign_orchestrator]="python -m api.services.campaign.campaign_orchestrator"
+ [uvicorn]="uvicorn api.app:app --host 0.0.0.0 --port $FASTAPI_PORT --workers $FASTAPI_WORKERS"
+)
+
+# Add ARQ workers dynamically based on ARQ_WORKERS environment variable
+for ((i=1; i<=ARQ_WORKERS; i++)); do
+ SERVICES[arq$i]="python -m arq api.tasks.arq.WorkerSettings --custom-log-dict api.tasks.arq.LOG_CONFIG"
+done
+
+### 2) Activate virtual environment #########################################
+source ${VENV_PATH}/bin/activate
+
+### 3) Stop old services (only via PID files) #################################
+mkdir -p "$RUN_DIR"
+for name in "${!SERVICES[@]}"; do
+ pidfile="$RUN_DIR/$name.pid"
+ if [[ -f $pidfile ]]; then
+ oldpid=$(<"$pidfile")
+ if kill -0 "$oldpid"; then
+ echo "Stopping $name (PID $oldpid and its process group)…"
+ # Kill the entire process group (negative PID)
+ # First try SIGTERM
+ kill -TERM -"$oldpid" || kill -TERM "$oldpid" || true
+ sleep 4
+ # If still running, use SIGKILL
+ if kill -0 "$oldpid"; then
+ echo "⚠️ $name did not exit cleanly, forcing stop..."
+ kill -KILL -"$oldpid" || kill -KILL "$oldpid" || true
+ sleep 1
+ fi
+ fi
+ rm -f "$pidfile"
+ else
+ echo "No PID file for $name, skipping stop."
+ fi
+done
+
+# Clean up any port tracking files for uvicorn
+rm -f "$RUN_DIR/uvicorn.port" "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn_old.pid"
+
+### 4) Run migrations #########################################################
+alembic -c api/alembic.ini upgrade head
+
+### 5) Prepare logs ###########################################################
+mkdir -p "$LOG_DIR"
+
+### 7) Start services #########################################################
+for name in "${!SERVICES[@]}"; do
+ cmd=${SERVICES[$name]}
+ echo "→ Starting $name"
+
+ # Export LOG_FILE_PATH for this specific service
+ export LOG_FILE_PATH="$LOG_DIR/$name.log"
+
+ # Start in new process group with setsid
+ # Each service gets its own LOG_FILE_PATH environment variable
+ setsid nohup bash -c "LOG_FILE_PATH='$LOG_DIR/$name.log' $cmd" >/dev/null 2>&1 &
+
+ # Get the PID of the setsid process
+ pid=$!
+ echo $pid >"$RUN_DIR/$name.pid"
+
+ # For uvicorn, also save the port for rolling updates
+ if [[ "$name" == "uvicorn" ]]; then
+ echo "$FASTAPI_PORT" >"$RUN_DIR/uvicorn.port"
+ fi
+done
+disown -a
+
+### 8) Summary #################################################################
+echo
+echo "──────────────────────────────────────────────────"
+for name in "${!SERVICES[@]}"; do
+ pid=$(<"$RUN_DIR/$name.pid")
+ echo "✓ $name (PID $pid) → $LOG_DIR/$name.log"
+done
+echo " Rotation: ${LOG_ROTATION_SIZE:-100 MB}"
+echo " Retention: ${LOG_RETENTION:-7 days}"
+echo " Compression: ${LOG_COMPRESSION:-gz}"
+echo "Logs: tail -f $LOG_DIR/*.log"
+echo "Rotated logs: ls $LOG_DIR/*.log.*"
+echo "To stop: run this script again or kill -TERM - for process groups"
+echo "──────────────────────────────────────────────────"