diff --git a/.gitignore b/.gitignore index f0ffdd1..410cc1c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ __pycache__ docs/ infrastructure/ nginx/ -scripts/ diff --git a/scripts/dograh-services.service b/scripts/dograh-services.service new file mode 100644 index 0000000..1c02ccc --- /dev/null +++ b/scripts/dograh-services.service @@ -0,0 +1,20 @@ +[Unit] +Description=Dograh Services +After=network.target postgresql.service +Wants=network-online.target + +[Service] +Type=forking +User=a +Group=a +WorkingDirectory=/home/a/dograh +Environment="PATH=/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +ExecStart=/home/a/dograh/scripts/start_services.sh +ExecStop=/home/a/dograh/scripts/stop_services.sh +TimeoutStopSec=30 +Restart=on-failure +RestartSec=10 +StandardOutput=journal +StandardError=journal + +[Install] \ No newline at end of file diff --git a/scripts/format.sh b/scripts/format.sh new file mode 100755 index 0000000..3423179 --- /dev/null +++ b/scripts/format.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e +set -euo pipefail + +ruff check api --select I --select F401 --fix +ruff format api diff --git a/scripts/lint.sh b/scripts/lint.sh new file mode 100644 index 0000000..2b87d68 --- /dev/null +++ b/scripts/lint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e +set -x + +mypy api +ruff check api --check +ruff format api --check \ No newline at end of file diff --git a/scripts/makemigrate.sh b/scripts/makemigrate.sh new file mode 100644 index 0000000..ba81621 --- /dev/null +++ b/scripts/makemigrate.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -e # Exit immediately if a command exits with a non-zero status + +# Set PYTHONPATH to the parent directory of the script's location +export PYTHONPATH="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")" + +# Define color codes +RED='\033[0;31m' +NC='\033[0m' # No Color + +env_file="api/.env" + +# Check if environment file exists +if [ ! -f "$env_file" ]; then + echo -e "${RED}Error: Environment file $env_file not found.${NC}" + exit 1 +fi + +# Load environment variables +export $(grep -v '^#' "$env_file" | xargs) + +# Prompt for the name of the migration +read -p "Enter the migration name (minimum 5 characters): " migration_name + +# Check if the migration name is empty or less than 5 characters +if [[ -z "$migration_name" || ${#migration_name} -lt 5 ]]; then + echo -e "${RED}Error: Migration name must be at least 5 characters long.${NC}" + exit 1 +fi + +# Generate the Alembic revision with the provided migration name +alembic -c api/alembic.ini revision --autogenerate -m "$migration_name" diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100644 index 0000000..6c83c74 --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e # Exit immediately if a command exits with a non-zero status + +# Set PYTHONPATH to the parent directory of the script's location +export PYTHONPATH="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")" + +# Define color codes +RED='\033[0;31m' +NC='\033[0m' # No Color + +env_file="api/.env" + +# Check if environment file exists +if [ ! -f "$env_file" ]; then + echo -e "${RED}Error: Environment file $env_file not found.${NC}" + exit 1 +fi + +# Load environment variables +export $(grep -v '^#' "$env_file" | xargs) + +# Run migrations +alembic -c api/alembic.ini upgrade head + +# Create initial data in DB +# python api/initial_data.py \ No newline at end of file diff --git a/scripts/pre_commit.sh b/scripts/pre_commit.sh new file mode 100755 index 0000000..1e4301b --- /dev/null +++ b/scripts/pre_commit.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +############################################################################### +# Ensure Ruff is installed (first try pipx, fall back to pip --user). +############################################################################### +if ! command -v ruff >/dev/null 2>&1; then + echo "⇢ Ruff not found on PATH – installing…" + if command -v pipx >/dev/null 2>&1; then + # install into an isolated environment if pipx is present + pipx install --quiet ruff + else + # otherwise install into the current (or user-level) Python environment + pip install --quiet --upgrade --user ruff + fi +fi + +############################################################################### +# 1 – Python formatting (calls Ruff + Black, etc.) +############################################################################### +sh scripts/format.sh + +############################################################################### +# 2 – ESLint autofix inside the Next.js app +############################################################################### +(cd ui && npm run fix-lint) + +############################################################################### +# 3 – Restage any files changed by the fixers so the commit includes them +############################################################################### +git add -u diff --git a/scripts/rolling_update_uvicorn.sh b/scripts/rolling_update_uvicorn.sh new file mode 100755 index 0000000..e6b0100 --- /dev/null +++ b/scripts/rolling_update_uvicorn.sh @@ -0,0 +1,450 @@ +#!/usr/bin/env bash +# rolling_update_uvicorn.sh — Zero-downtime rolling update for uvicorn workers +# +# Usage: ./rolling_update_uvicorn.sh +# Example: ./rolling_update_uvicorn.sh 8001 + +set -euo pipefail + +# Check if running as root or with sudo +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root or with sudo" + exit 1 +fi + +### CONFIGURATION ############################################################# +ENV_FILE="api/.env" +RUN_DIR="run" +LOG_ROOT="logs" +HEALTH_CHECK_ENDPOINT="/api/v1/health" # Adjust as needed +MAX_WAIT_SECONDS=310 # Max wait for graceful shutdown (5 minutes + 10 seconds grace) + +# Load environment to get ENVIRONMENT variable +set -a && . "$ENV_FILE" && set +a +ENVIRONMENT="${ENVIRONMENT:-staging}" + +# Set nginx upstream config based on environment +if [[ "$ENVIRONMENT" == "production" ]]; then + NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_production_upstream.conf" + UPSTREAM_NAME="dograh_production_backend" + echo "Rolling update for PRODUCTION environment" +else + NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_staging_upstream.conf" + UPSTREAM_NAME="dograh_staging_backend" + echo "Rolling update for STAGING environment" +fi + +### FUNCTIONS ################################################################## + +log_info() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO: $*" +} + +log_error() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2 +} + +log_warning() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARN: $*" +} + +check_port_availability() { + local port=$1 + if lsof -Pi :$port -sTCP:LISTEN -t >/dev/null 2>&1; then + return 1 # Port is in use + fi + return 0 # Port is available +} + +wait_for_health_check() { + local port=$1 + local max_attempts=30 + local attempt=0 + + log_info "Waiting for new uvicorn workers to be healthy on port $port..." + + while [ $attempt -lt $max_attempts ]; do + if curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:${port}${HEALTH_CHECK_ENDPOINT}" | grep -q "200"; then + log_info "Health check passed on port $port" + return 0 + fi + attempt=$((attempt + 1)) + log_info "Health check attempt $attempt/$max_attempts..." + sleep 1 + done + + log_error "Health check failed after $max_attempts attempts" + return 1 +} + +get_old_uvicorn_pids() { + local pidfile="$RUN_DIR/uvicorn.pid" + local pids="" + + if [[ -f "$pidfile" ]]; then + # Read the main PID + local main_pid=$(<"$pidfile") + if kill -0 "$main_pid" 2>/dev/null; then + # Get all PIDs in the process group + pids=$(ps -o pid= -g $(ps -o pgid= -p "$main_pid" | tr -d ' ') 2>/dev/null || echo "$main_pid") + fi + fi + + echo "$pids" +} + +graceful_shutdown_old_workers() { + local old_pids="$1" + + if [[ -z "$old_pids" ]]; then + log_warning "No old uvicorn workers found to shut down" + return 0 + fi + + log_info "Starting graceful shutdown of old uvicorn workers (PIDs: $(echo $old_pids | tr '\n' ' '))" + + # Send SIGTERM to trigger graceful shutdown + for pid in $old_pids; do + if kill -0 "$pid" 2>/dev/null; then + log_info "Sending SIGTERM to PID $pid" + kill -TERM "$pid" 2>/dev/null || true + fi + done + + # Wait for processes to exit gracefully + local start_time=$(date +%s) + local all_dead=false + + while [[ $(($(date +%s) - start_time)) -lt $MAX_WAIT_SECONDS ]]; do + all_dead=true + for pid in $old_pids; do + if kill -0 "$pid" 2>/dev/null; then + all_dead=false + break + fi + done + + if $all_dead; then + log_info "All old workers shut down gracefully" + return 0 + fi + + log_info "Waiting for workers to complete active requests... ($(( $(date +%s) - start_time ))s elapsed)" + sleep 5 + done + + # Force kill if still running after timeout + log_warning "Timeout reached, force killing remaining workers" + for pid in $old_pids; do + if kill -0 "$pid" 2>/dev/null; then + log_warning "Force killing PID $pid" + kill -KILL "$pid" 2>/dev/null || true + fi + done + + sleep 1 + return 0 +} + +update_nginx_upstream() { + local new_port=$1 + local old_port=$2 + + log_info "Updating nginx upstream configuration for $ENVIRONMENT..." + + # Create or update the upstream configuration + cat > "${NGINX_UPSTREAM_CONF}.tmp" </dev/null || { + log_error "Could not update nginx config (need sudo). Run: sudo $0 $NEW_PORT" + return 1 + } + fi + + # Test nginx configuration (with sudo if needed) + if nginx -t 2>/dev/null || sudo nginx -t 2>/dev/null; then + log_info "Nginx configuration test passed" + # Reload nginx to pick up new configuration (with sudo if needed) + if nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null; then + log_info "Nginx reloaded successfully" + else + log_error "Could not reload nginx" + return 1 + fi + else + log_error "Nginx configuration test failed, rolling back" + # Restore old configuration if possible + if [[ -n "$old_port" ]]; then + cat > "${NGINX_UPSTREAM_CONF}.tmp" </dev/null || true + fi + nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null || true + fi + return 1 + fi +} + +start_new_uvicorn_workers() { + local new_port=$1 + + log_info "Starting new uvicorn workers on port $new_port..." + + # Get configuration from environment + set -a && . "$ENV_FILE" && set +a + + if [[ -z "${FASTAPI_WORKERS:-}" ]]; then + log_error "FASTAPI_WORKERS environment variable is not set" + return 1 + fi + + if [[ -z "${CONDA_ENV_NAME:-}" ]]; then + log_error "CONDA_ENV_NAME environment variable is not set" + return 1 + fi + + # Source conda if not already available + if ! command -v conda &>/dev/null; then + source /opt/conda/etc/profile.d/conda.sh + fi + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV_NAME" + + # Use the latest log directory (where start_services.sh put logs) + # Resolve the symlink to get the actual directory + local log_dir="$LOG_ROOT/latest" + if [[ -L "$log_dir" ]]; then + # It's a symlink, resolve it + log_dir=$(readlink -f "$log_dir") + fi + + if [[ ! -d "$log_dir" ]]; then + log_error "No latest log directory found. Run start_services.sh first." + return 1 + fi + + # Export rotation settings + export LOG_ROTATION_SIZE="${LOG_ROTATION_SIZE:-100 MB}" + export LOG_RETENTION="${LOG_RETENTION:-7 days}" + export LOG_COMPRESSION="${LOG_COMPRESSION:-gz}" + + # Create unique log filename using timestamp and script PID to avoid conflicts + local script_pid=$$ # PID of this rolling_update script (for uniqueness) + local timestamp=$(date '+%H%M%S') + export LOG_FILE_PATH="$log_dir/uvicorn-rollover-${timestamp}-${script_pid}.log" + + log_info "Starting uvicorn with $FASTAPI_WORKERS workers on port $new_port" + log_info "Logs: $LOG_FILE_PATH" + + # If running as root, switch to original user for uvicorn process + if [[ $EUID -eq 0 ]] && [[ -n "${SUDO_USER:-}" ]]; then + log_info "Starting uvicorn as user: $SUDO_USER (not root)" + + # Run uvicorn as the original user, similar to start_services.sh + # Using setsid and passing LOG_FILE_PATH for loguru to pick up + sudo -u "$SUDO_USER" bash -c " + cd '$PWD' + export HOME='$(getent passwd $SUDO_USER | cut -d: -f6)' + export LOG_FILE_PATH='$LOG_FILE_PATH' + export LOG_ROTATION_SIZE='$LOG_ROTATION_SIZE' + export LOG_RETENTION='$LOG_RETENTION' + export LOG_COMPRESSION='$LOG_COMPRESSION' + set -a && source '$ENV_FILE' && set +a + source /opt/conda/etc/profile.d/conda.sh + conda activate '$CONDA_ENV_NAME' + setsid nohup bash -c \"LOG_FILE_PATH='$LOG_FILE_PATH' uvicorn api.app:app --host 0.0.0.0 --port $new_port --workers $FASTAPI_WORKERS\" >/dev/null 2>&1 & + echo \$! > '$RUN_DIR/uvicorn_new.pid' + " + # Read the PID that was written + local new_pid=$(<"$RUN_DIR/uvicorn_new.pid") + else + # Start in new process group with setsid (same as start_services.sh) + # Each service gets its own LOG_FILE_PATH environment variable + setsid nohup bash -c "LOG_FILE_PATH='$LOG_FILE_PATH' uvicorn api.app:app --host 0.0.0.0 --port $new_port --workers $FASTAPI_WORKERS" >/dev/null 2>&1 & + + local new_pid=$! + echo "$new_pid" > "$RUN_DIR/uvicorn_new.pid" + fi + + # Save port information + echo "$new_port" > "$RUN_DIR/uvicorn_new.port" + + log_info "New uvicorn started with PID $new_pid" + + # Wait a bit for startup + sleep 5 + + # Check if process is still running + if ! kill -0 "$new_pid" 2>/dev/null; then + log_error "New uvicorn process died immediately" + return 1 + fi + + return 0 +} + +finalize_rollover() { + log_info "Finalizing rollover..." + + # Move new PID file to main PID file + if [[ -f "$RUN_DIR/uvicorn_new.pid" ]]; then + mv "$RUN_DIR/uvicorn_new.pid" "$RUN_DIR/uvicorn.pid" + fi + + # Store the new port for reference + if [[ -f "$RUN_DIR/uvicorn_new.port" ]]; then + mv "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn.port" + fi + + # Clean up old PID file if it exists + rm -f "$RUN_DIR/uvicorn_old.pid" + + log_info "Rollover completed successfully" +} + +rollback() { + local old_port=$1 + local new_pid=$2 + + log_error "Rolling back due to failure..." + + # Kill new workers if they exist + if [[ -n "$new_pid" ]] && kill -0 "$new_pid" 2>/dev/null; then + log_info "Killing new uvicorn workers (PID: $new_pid)" + kill -KILL -"$new_pid" 2>/dev/null || kill -KILL "$new_pid" 2>/dev/null || true + fi + + # Clean up temporary files + rm -f "$RUN_DIR/uvicorn_new.pid" "$RUN_DIR/uvicorn_new.port" + + # Restore nginx configuration if old port is known + if [[ -n "$old_port" ]]; then + update_nginx_upstream "$old_port" "" + fi + + log_error "Rollback completed" +} + +### MAIN LOGIC ################################################################ + +# Check arguments +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + echo "Example: $0 8001" + exit 1 +fi + +NEW_PORT=$1 + +# Check nginx permissions early and exit if we can't update nginx +if [[ ! -w $(dirname "$NGINX_UPSTREAM_CONF") ]] && [[ $EUID -ne 0 ]]; then + if ! sudo -n true 2>/dev/null; then + log_error "This script needs sudo access to update nginx configuration" + log_error "Cannot proceed without nginx update permissions" + echo "" + echo "Please run with sudo:" + echo " sudo $0 $NEW_PORT" + echo "" + exit 1 + fi +fi + +# Validate port number +if ! [[ "$NEW_PORT" =~ ^[0-9]+$ ]] || [ "$NEW_PORT" -lt 1 ] || [ "$NEW_PORT" -gt 65535 ]; then + log_error "Invalid port number: $NEW_PORT" + exit 1 +fi + +# Check if port is available +if ! check_port_availability "$NEW_PORT"; then + log_error "Port $NEW_PORT is already in use" + exit 1 +fi + +# Get old port from file or environment +OLD_PORT="" +if [[ -f "$RUN_DIR/uvicorn.port" ]]; then + OLD_PORT=$(<"$RUN_DIR/uvicorn.port") +elif [[ -f "$ENV_FILE" ]]; then + set -a && . "$ENV_FILE" && set +a + OLD_PORT="${FASTAPI_PORT:-}" +fi + +if [[ "$NEW_PORT" == "$OLD_PORT" ]]; then + log_error "New port is the same as old port ($NEW_PORT)" + exit 1 +fi + +log_info "Starting rolling update from port ${OLD_PORT:-unknown} to port $NEW_PORT" + +# Create run directory if it doesn't exist +mkdir -p "$RUN_DIR" + +# Get old uvicorn PIDs before starting new ones +OLD_PIDS=$(get_old_uvicorn_pids) +if [[ -n "$OLD_PIDS" ]]; then + # Save old PIDs for potential rollback + echo "$OLD_PIDS" > "$RUN_DIR/uvicorn_old.pid" + log_info "Found old uvicorn workers: $(echo $OLD_PIDS | tr '\n' ' ')" +else + log_warning "No existing uvicorn workers found" +fi + +# Start new uvicorn workers +if ! start_new_uvicorn_workers "$NEW_PORT"; then + log_error "Failed to start new uvicorn workers" + exit 1 +fi + +NEW_PID=$(<"$RUN_DIR/uvicorn_new.pid") + +# Wait for new workers to be healthy +if ! wait_for_health_check "$NEW_PORT"; then + log_error "New workers failed health check" + rollback "$OLD_PORT" "$NEW_PID" + exit 1 +fi + +# Update nginx to point to new workers +if ! update_nginx_upstream "$NEW_PORT" "$OLD_PORT"; then + log_error "Failed to update nginx configuration" + rollback "$OLD_PORT" "$NEW_PID" + exit 1 +fi + +# Give nginx some time to start routing to new workers +log_info "Waiting for nginx to stabilize..." +sleep 5 + +# Gracefully shutdown old workers +if [[ -n "$OLD_PIDS" ]]; then + graceful_shutdown_old_workers "$OLD_PIDS" +fi + +# Finalize the rollover +finalize_rollover + +# Summary +echo "──────────────────────────────────────────────────" +echo "✓ Rolling update completed successfully" +echo " Old port: ${OLD_PORT:-none}" +echo " New port: $NEW_PORT" +echo " New PID: $NEW_PID" +echo " Logs: $LOG_ROOT/latest/" +echo "──────────────────────────────────────────────────" \ No newline at end of file diff --git a/scripts/start_services.sh b/scripts/start_services.sh new file mode 100755 index 0000000..ed36ad1 --- /dev/null +++ b/scripts/start_services.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# restart_services.sh — safer, simplified + +set -euo pipefail + +### CONFIGURATION ############################################################# +ENV_FILE="api/.env" +RUN_DIR="run" # where we keep *.pid +LOG_ROOT="logs" + +### 1) Load environment vars so that configurations like FASTAPI_WORKERS are loaded # +set -a && . "$ENV_FILE" && set +a + +# Get ENVIRONMENT for nginx config selection +ENVIRONMENT="${ENVIRONMENT:-staging}" + +if [[ -z "${FASTAPI_PORT:-}" ]]; then + echo "Error: FASTAPI_PORT environment variable is not set." + exit 1 +fi + +if [[ -z "${FASTAPI_WORKERS:-}" ]]; then + echo "Error: FASTAPI_WORKERS environment variable is not set." + exit 1 +fi + +if [[ -z "${CONDA_ENV_NAME:-}" ]]; then + echo "Error: CONDA_ENV_NAME environment variable is not set." + exit 1 +fi + +# Default ARQ_WORKERS to 1 if not set +ARQ_WORKERS=${ARQ_WORKERS:-1} + +# map "service name" → "command to run" +declare -A SERVICES=( + [ari_manager]="python -m api.services.telephony.ari_manager" + [campaign_orchestrator]="python -m api.services.campaign.campaign_orchestrator" + [uvicorn]="uvicorn api.app:app --host 0.0.0.0 --port $FASTAPI_PORT --workers $FASTAPI_WORKERS" +) + +# Add ARQ workers dynamically based on ARQ_WORKERS environment variable +for ((i=1; i<=ARQ_WORKERS; i++)); do + SERVICES[arq$i]="python -m arq api.tasks.arq.WorkerSettings --custom-log-dict api.tasks.arq.LOG_CONFIG" +done + +### 2) Activate conda ######################################################### +# Source conda if not already available (needed when running from systemd) +if ! command -v conda &>/dev/null; then + source /opt/conda/etc/profile.d/conda.sh +fi +eval "$(conda shell.bash hook)" +conda activate "$CONDA_ENV_NAME" + +### 3) Stop old services (only via PID files) ################################# +mkdir -p "$RUN_DIR" +for name in "${!SERVICES[@]}"; do + pidfile="$RUN_DIR/$name.pid" + if [[ -f $pidfile ]]; then + oldpid=$(<"$pidfile") + if kill -0 "$oldpid" 2>/dev/null; then + echo "Stopping $name (PID $oldpid and its process group)…" + # Kill the entire process group (negative PID) + # First try SIGTERM + kill -TERM -"$oldpid" 2>/dev/null || kill -TERM "$oldpid" 2>/dev/null || true + sleep 4 + # If still running, use SIGKILL + if kill -0 "$oldpid" 2>/dev/null; then + echo "⚠️ $name did not exit cleanly, forcing stop..." + kill -KILL -"$oldpid" 2>/dev/null || kill -KILL "$oldpid" 2>/dev/null || true + sleep 1 + fi + fi + rm -f "$pidfile" + else + echo "No PID file for $name, skipping stop." + fi +done + +# Clean up any port tracking files for uvicorn +rm -f "$RUN_DIR/uvicorn.port" "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn_old.pid" + + +### 4) Run migrations ######################################################### +alembic -c api/alembic.ini upgrade head + +### 5) Prepare logs ########################################################### +timestamp=$(date '+%Y-%m-%d_%H-%M-%S') +LOG_DIR="$LOG_ROOT/$timestamp" +mkdir -p "$LOG_DIR" +# Create relative symlink +cd "$LOG_ROOT" && ln -sfn "$timestamp" latest && cd - >/dev/null + +### 6) (Optional) Free FastAPI port ########################################### +FASTAPI_PORT=$FASTAPI_PORT +if command -v lsof &>/dev/null; then + lsof -ti tcp:"$FASTAPI_PORT" | xargs -r kill -9 || true +fi + +### 7) Start services ######################################################### +# Export rotation settings for loguru (if using file logging) +export LOG_ROTATION_SIZE="${LOG_ROTATION_SIZE:-100 MB}" +export LOG_RETENTION="${LOG_RETENTION:-7 days}" +export LOG_COMPRESSION="${LOG_COMPRESSION:-gz}" + +for name in "${!SERVICES[@]}"; do + cmd=${SERVICES[$name]} + echo "→ Starting $name with loguru rotation…" + + # Export LOG_FILE_PATH for this specific service + export LOG_FILE_PATH="$LOG_DIR/$name.log" + + # Start in new process group with setsid + # Each service gets its own LOG_FILE_PATH environment variable + setsid nohup bash -c "LOG_FILE_PATH='$LOG_DIR/$name.log' $cmd" >/dev/null 2>&1 & + + # Get the PID of the setsid process + pid=$! + echo $pid >"$RUN_DIR/$name.pid" + + # For uvicorn, also save the port for rolling updates and update nginx + if [[ "$name" == "uvicorn" ]]; then + echo "$FASTAPI_PORT" >"$RUN_DIR/uvicorn.port" + + # Update nginx upstream configuration if nginx is installed + if command -v nginx &>/dev/null && [[ -d /etc/nginx ]]; then + # Determine which upstream config to update based on ENVIRONMENT + if [[ "${ENVIRONMENT:-}" == "production" ]]; then + NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_production_upstream.conf" + UPSTREAM_NAME="dograh_production_backend" + echo "→ Updating PRODUCTION nginx upstream to port $FASTAPI_PORT…" + else + # Default to staging for any non-production environment + NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_staging_upstream.conf" + UPSTREAM_NAME="dograh_staging_backend" + echo "→ Updating STAGING nginx upstream to port $FASTAPI_PORT…" + fi + + if [[ -w $(dirname "$NGINX_UPSTREAM_CONF") ]] || [[ $EUID -eq 0 ]]; then + cat > "${NGINX_UPSTREAM_CONF}.tmp" </dev/null || \ + echo "⚠️ Could not update nginx config (need sudo). Run: sudo $0" + fi + + # Test and reload nginx if config was updated + if [[ -f "$NGINX_UPSTREAM_CONF" ]]; then + if nginx -t 2>/dev/null || sudo nginx -t 2>/dev/null; then + echo "→ Reloading nginx…" + nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null || \ + echo "⚠️ Could not reload nginx (may need sudo)" + else + echo "⚠️ Nginx configuration test failed" + fi + fi + else + echo "⚠️ Cannot write to nginx config directory (need sudo privileges)" + echo " Run: sudo $0 to update nginx configuration" + fi + fi + fi +done +disown -a + +### 8) Summary ################################################################# +echo +echo "──────────────────────────────────────────────────" +for name in "${!SERVICES[@]}"; do + pid=$(<"$RUN_DIR/$name.pid") + echo "✓ $name (PID $pid) → $LOG_DIR/$name.log" +done +echo " Rotation: ${LOG_ROTATION_SIZE:-100 MB}" +echo " Retention: ${LOG_RETENTION:-7 days}" +echo " Compression: ${LOG_COMPRESSION:-gz}" +echo "Logs: tail -f $LOG_DIR/*.log" +echo "Rotated logs: ls $LOG_DIR/*.log.*" +echo "To stop: run this script again or kill -TERM - for process groups" +echo "──────────────────────────────────────────────────"