From 443490b2dd796bdec783e70128a3ec87eeae6539 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek@a6k.me>
Date: Fri, 26 Sep 2025 09:09:06 +0530
Subject: [PATCH 1/4] Add alembic and start services scripts

---
 .gitignore                        |   1 -
 scripts/dograh-services.service   |  20 ++
 scripts/format.sh                 |   5 +
 scripts/lint.sh                   |   8 +
 scripts/makemigrate.sh            |  33 +++
 scripts/migrate.sh                |  27 ++
 scripts/pre_commit.sh             |  31 ++
 scripts/rolling_update_uvicorn.sh | 450 ++++++++++++++++++++++++++++++
 scripts/start_services.sh         | 187 +++++++++++++
 9 files changed, 761 insertions(+), 1 deletion(-)
 create mode 100644 scripts/dograh-services.service
 create mode 100755 scripts/format.sh
 create mode 100644 scripts/lint.sh
 create mode 100644 scripts/makemigrate.sh
 create mode 100644 scripts/migrate.sh
 create mode 100755 scripts/pre_commit.sh
 create mode 100755 scripts/rolling_update_uvicorn.sh
 create mode 100755 scripts/start_services.sh

diff --git a/.gitignore b/.gitignore
index f0ffdd1..410cc1c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,4 +8,3 @@ __pycache__
 docs/
 infrastructure/
 nginx/
-scripts/
diff --git a/scripts/dograh-services.service b/scripts/dograh-services.service
new file mode 100644
index 0000000..1c02ccc
--- /dev/null
+++ b/scripts/dograh-services.service
@@ -0,0 +1,20 @@
+[Unit]
+Description=Dograh Services
+After=network.target postgresql.service
+Wants=network-online.target
+
+[Service]
+Type=forking
+User=a
+Group=a
+WorkingDirectory=/home/a/dograh
+Environment="PATH=/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ExecStart=/home/a/dograh/scripts/start_services.sh
+ExecStop=/home/a/dograh/scripts/stop_services.sh
+TimeoutStopSec=30
+Restart=on-failure
+RestartSec=10
+StandardOutput=journal
+StandardError=journal
+
+[Install]
\ No newline at end of file
diff --git a/scripts/format.sh b/scripts/format.sh
new file mode 100755
index 0000000..3423179
--- /dev/null
+++ b/scripts/format.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -e
+set -euo pipefail
+
+ruff check api --select I --select F401 --fix
+ruff format api
diff --git a/scripts/lint.sh b/scripts/lint.sh
new file mode 100644
index 0000000..2b87d68
--- /dev/null
+++ b/scripts/lint.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+mypy api
+ruff check api --check
+ruff format api --check
\ No newline at end of file
diff --git a/scripts/makemigrate.sh b/scripts/makemigrate.sh
new file mode 100644
index 0000000..ba81621
--- /dev/null
+++ b/scripts/makemigrate.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -e  # Exit immediately if a command exits with a non-zero status
+
+# Set PYTHONPATH to the parent directory of the script's location
+export PYTHONPATH="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")"
+
+# Define color codes
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+env_file="api/.env"
+
+# Check if environment file exists
+if [ ! -f "$env_file" ]; then
+    echo -e "${RED}Error: Environment file $env_file not found.${NC}"
+    exit 1
+fi
+
+# Load environment variables
+export $(grep -v '^#' "$env_file" | xargs)
+
+# Prompt for the name of the migration
+read -p "Enter the migration name (minimum 5 characters): " migration_name
+
+# Check if the migration name is empty or less than 5 characters
+if [[ -z "$migration_name" || ${#migration_name} -lt 5 ]]; then
+  echo -e "${RED}Error: Migration name must be at least 5 characters long.${NC}"
+  exit 1
+fi
+
+# Generate the Alembic revision with the provided migration name
+alembic -c api/alembic.ini revision --autogenerate -m "$migration_name"
diff --git a/scripts/migrate.sh b/scripts/migrate.sh
new file mode 100644
index 0000000..6c83c74
--- /dev/null
+++ b/scripts/migrate.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e  # Exit immediately if a command exits with a non-zero status
+
+# Set PYTHONPATH to the parent directory of the script's location
+export PYTHONPATH="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")"
+
+# Define color codes
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+env_file="api/.env"
+
+# Check if environment file exists
+if [ ! -f "$env_file" ]; then
+    echo -e "${RED}Error: Environment file $env_file not found.${NC}"
+    exit 1
+fi
+
+# Load environment variables
+export $(grep -v '^#' "$env_file" | xargs)
+
+# Run migrations
+alembic -c api/alembic.ini upgrade head
+
+# Create initial data in DB
+# python api/initial_data.py
\ No newline at end of file
diff --git a/scripts/pre_commit.sh b/scripts/pre_commit.sh
new file mode 100755
index 0000000..1e4301b
--- /dev/null
+++ b/scripts/pre_commit.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+###############################################################################
+# Ensure Ruff is installed (first try pipx, fall back to pip --user).
+###############################################################################
+if ! command -v ruff >/dev/null 2>&1; then
+  echo "⇢ Ruff not found on PATH – installing…"
+  if command -v pipx >/dev/null 2>&1; then
+    # install into an isolated environment if pipx is present
+    pipx install --quiet ruff
+  else
+    # otherwise install into the current (or user-level) Python environment
+    pip install --quiet --upgrade --user ruff
+  fi
+fi
+
+###############################################################################
+# 1 – Python formatting (calls Ruff + Black, etc.)
+###############################################################################
+sh scripts/format.sh
+
+###############################################################################
+# 2 – ESLint autofix inside the Next.js app
+###############################################################################
+(cd ui && npm run fix-lint)
+
+###############################################################################
+# 3 – Restage any files changed by the fixers so the commit includes them
+###############################################################################
+git add -u
diff --git a/scripts/rolling_update_uvicorn.sh b/scripts/rolling_update_uvicorn.sh
new file mode 100755
index 0000000..e6b0100
--- /dev/null
+++ b/scripts/rolling_update_uvicorn.sh
@@ -0,0 +1,450 @@
+#!/usr/bin/env bash
+# rolling_update_uvicorn.sh — Zero-downtime rolling update for uvicorn workers
+#
+# Usage: ./rolling_update_uvicorn.sh <NEW_PORT>
+# Example: ./rolling_update_uvicorn.sh 8001
+
+set -euo pipefail
+
+# Check if running as root or with sudo
+if [[ $EUID -ne 0 ]]; then
+   echo "This script must be run as root or with sudo" 
+   exit 1
+fi
+
+### CONFIGURATION #############################################################
+ENV_FILE="api/.env"
+RUN_DIR="run"
+LOG_ROOT="logs"
+HEALTH_CHECK_ENDPOINT="/api/v1/health"  # Adjust as needed
+MAX_WAIT_SECONDS=310  # Max wait for graceful shutdown (5 minutes + 10 seconds grace)
+
+# Load environment to get ENVIRONMENT variable
+set -a && . "$ENV_FILE" && set +a
+ENVIRONMENT="${ENVIRONMENT:-staging}"
+
+# Set nginx upstream config based on environment
+if [[ "$ENVIRONMENT" == "production" ]]; then
+    NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_production_upstream.conf"
+    UPSTREAM_NAME="dograh_production_backend"
+    echo "Rolling update for PRODUCTION environment"
+else
+    NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_staging_upstream.conf"
+    UPSTREAM_NAME="dograh_staging_backend"
+    echo "Rolling update for STAGING environment"
+fi
+
+### FUNCTIONS ##################################################################
+
+log_info() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO: $*"
+}
+
+log_error() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2
+}
+
+log_warning() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARN: $*"
+}
+
+check_port_availability() {
+    local port=$1
+    if lsof -Pi :$port -sTCP:LISTEN -t >/dev/null 2>&1; then
+        return 1  # Port is in use
+    fi
+    return 0  # Port is available
+}
+
+wait_for_health_check() {
+    local port=$1
+    local max_attempts=30
+    local attempt=0
+    
+    log_info "Waiting for new uvicorn workers to be healthy on port $port..."
+    
+    while [ $attempt -lt $max_attempts ]; do
+        if curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:${port}${HEALTH_CHECK_ENDPOINT}" | grep -q "200"; then
+            log_info "Health check passed on port $port"
+            return 0
+        fi
+        attempt=$((attempt + 1))
+        log_info "Health check attempt $attempt/$max_attempts..."
+        sleep 1
+    done
+    
+    log_error "Health check failed after $max_attempts attempts"
+    return 1
+}
+
+get_old_uvicorn_pids() {
+    local pidfile="$RUN_DIR/uvicorn.pid"
+    local pids=""
+    
+    if [[ -f "$pidfile" ]]; then
+        # Read the main PID
+        local main_pid=$(<"$pidfile")
+        if kill -0 "$main_pid" 2>/dev/null; then
+            # Get all PIDs in the process group
+            pids=$(ps -o pid= -g $(ps -o pgid= -p "$main_pid" | tr -d ' ') 2>/dev/null || echo "$main_pid")
+        fi
+    fi
+    
+    echo "$pids"
+}
+
+graceful_shutdown_old_workers() {
+    local old_pids="$1"
+    
+    if [[ -z "$old_pids" ]]; then
+        log_warning "No old uvicorn workers found to shut down"
+        return 0
+    fi
+    
+    log_info "Starting graceful shutdown of old uvicorn workers (PIDs: $(echo $old_pids | tr '\n' ' '))"
+    
+    # Send SIGTERM to trigger graceful shutdown
+    for pid in $old_pids; do
+        if kill -0 "$pid" 2>/dev/null; then
+            log_info "Sending SIGTERM to PID $pid"
+            kill -TERM "$pid" 2>/dev/null || true
+        fi
+    done
+    
+    # Wait for processes to exit gracefully
+    local start_time=$(date +%s)
+    local all_dead=false
+    
+    while [[ $(($(date +%s) - start_time)) -lt $MAX_WAIT_SECONDS ]]; do
+        all_dead=true
+        for pid in $old_pids; do
+            if kill -0 "$pid" 2>/dev/null; then
+                all_dead=false
+                break
+            fi
+        done
+        
+        if $all_dead; then
+            log_info "All old workers shut down gracefully"
+            return 0
+        fi
+        
+        log_info "Waiting for workers to complete active requests... ($(( $(date +%s) - start_time ))s elapsed)"
+        sleep 5
+    done
+    
+    # Force kill if still running after timeout
+    log_warning "Timeout reached, force killing remaining workers"
+    for pid in $old_pids; do
+        if kill -0 "$pid" 2>/dev/null; then
+            log_warning "Force killing PID $pid"
+            kill -KILL "$pid" 2>/dev/null || true
+        fi
+    done
+    
+    sleep 1
+    return 0
+}
+
+update_nginx_upstream() {
+    local new_port=$1
+    local old_port=$2
+    
+    log_info "Updating nginx upstream configuration for $ENVIRONMENT..."
+    
+    # Create or update the upstream configuration
+    cat > "${NGINX_UPSTREAM_CONF}.tmp" <<EOF
+# Auto-generated by rolling_update_uvicorn.sh for $ENVIRONMENT
+# Last updated: $(date)
+upstream ${UPSTREAM_NAME} {
+    server 127.0.0.1:${new_port} max_fails=3 fail_timeout=30s;
+}
+EOF
+    
+    # Atomic move (with sudo if needed)
+    if [[ $EUID -eq 0 ]]; then
+        mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}"
+    else
+        sudo mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}" 2>/dev/null || {
+            log_error "Could not update nginx config (need sudo). Run: sudo $0 $NEW_PORT"
+            return 1
+        }
+    fi
+    
+    # Test nginx configuration (with sudo if needed)
+    if nginx -t 2>/dev/null || sudo nginx -t 2>/dev/null; then
+        log_info "Nginx configuration test passed"
+        # Reload nginx to pick up new configuration (with sudo if needed)
+        if nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null; then
+            log_info "Nginx reloaded successfully"
+        else
+            log_error "Could not reload nginx"
+            return 1
+        fi
+    else
+        log_error "Nginx configuration test failed, rolling back"
+        # Restore old configuration if possible
+        if [[ -n "$old_port" ]]; then
+            cat > "${NGINX_UPSTREAM_CONF}.tmp" <<EOF
+upstream ${UPSTREAM_NAME} {
+    server 127.0.0.1:${old_port} max_fails=3 fail_timeout=30s;
+}
+EOF
+            if [[ $EUID -eq 0 ]]; then
+                mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}"
+            else
+                sudo mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}" 2>/dev/null || true
+            fi
+            nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null || true
+        fi
+        return 1
+    fi
+}
+
+start_new_uvicorn_workers() {
+    local new_port=$1
+    
+    log_info "Starting new uvicorn workers on port $new_port..."
+    
+    # Get configuration from environment
+    set -a && . "$ENV_FILE" && set +a
+    
+    if [[ -z "${FASTAPI_WORKERS:-}" ]]; then
+        log_error "FASTAPI_WORKERS environment variable is not set"
+        return 1
+    fi
+    
+    if [[ -z "${CONDA_ENV_NAME:-}" ]]; then
+        log_error "CONDA_ENV_NAME environment variable is not set"
+        return 1
+    fi
+    
+    # Source conda if not already available
+    if ! command -v conda &>/dev/null; then
+        source /opt/conda/etc/profile.d/conda.sh
+    fi
+    eval "$(conda shell.bash hook)"
+    conda activate "$CONDA_ENV_NAME"
+    
+    # Use the latest log directory (where start_services.sh put logs)
+    # Resolve the symlink to get the actual directory
+    local log_dir="$LOG_ROOT/latest"
+    if [[ -L "$log_dir" ]]; then
+        # It's a symlink, resolve it
+        log_dir=$(readlink -f "$log_dir")
+    fi
+    
+    if [[ ! -d "$log_dir" ]]; then
+        log_error "No latest log directory found. Run start_services.sh first."
+        return 1
+    fi
+    
+    # Export rotation settings
+    export LOG_ROTATION_SIZE="${LOG_ROTATION_SIZE:-100 MB}"
+    export LOG_RETENTION="${LOG_RETENTION:-7 days}"
+    export LOG_COMPRESSION="${LOG_COMPRESSION:-gz}"
+    
+    # Create unique log filename using timestamp and script PID to avoid conflicts
+    local script_pid=$$  # PID of this rolling_update script (for uniqueness)
+    local timestamp=$(date '+%H%M%S')
+    export LOG_FILE_PATH="$log_dir/uvicorn-rollover-${timestamp}-${script_pid}.log"
+    
+    log_info "Starting uvicorn with $FASTAPI_WORKERS workers on port $new_port"
+    log_info "Logs: $LOG_FILE_PATH"
+    
+    # If running as root, switch to original user for uvicorn process
+    if [[ $EUID -eq 0 ]] && [[ -n "${SUDO_USER:-}" ]]; then
+        log_info "Starting uvicorn as user: $SUDO_USER (not root)"
+        
+        # Run uvicorn as the original user, similar to start_services.sh
+        # Using setsid and passing LOG_FILE_PATH for loguru to pick up
+        sudo -u "$SUDO_USER" bash -c "
+            cd '$PWD'
+            export HOME='$(getent passwd $SUDO_USER | cut -d: -f6)'
+            export LOG_FILE_PATH='$LOG_FILE_PATH'
+            export LOG_ROTATION_SIZE='$LOG_ROTATION_SIZE'
+            export LOG_RETENTION='$LOG_RETENTION'
+            export LOG_COMPRESSION='$LOG_COMPRESSION'
+            set -a && source '$ENV_FILE' && set +a
+            source /opt/conda/etc/profile.d/conda.sh
+            conda activate '$CONDA_ENV_NAME'
+            setsid nohup bash -c \"LOG_FILE_PATH='$LOG_FILE_PATH' uvicorn api.app:app --host 0.0.0.0 --port $new_port --workers $FASTAPI_WORKERS\" >/dev/null 2>&1 &
+            echo \$! > '$RUN_DIR/uvicorn_new.pid'
+        "
+        # Read the PID that was written
+        local new_pid=$(<"$RUN_DIR/uvicorn_new.pid")
+    else
+        # Start in new process group with setsid (same as start_services.sh)
+        # Each service gets its own LOG_FILE_PATH environment variable
+        setsid nohup bash -c "LOG_FILE_PATH='$LOG_FILE_PATH' uvicorn api.app:app --host 0.0.0.0 --port $new_port --workers $FASTAPI_WORKERS" >/dev/null 2>&1 &
+        
+        local new_pid=$!
+        echo "$new_pid" > "$RUN_DIR/uvicorn_new.pid"
+    fi
+    
+    # Save port information
+    echo "$new_port" > "$RUN_DIR/uvicorn_new.port"
+    
+    log_info "New uvicorn started with PID $new_pid"
+    
+    # Wait a bit for startup
+    sleep 5
+    
+    # Check if process is still running
+    if ! kill -0 "$new_pid" 2>/dev/null; then
+        log_error "New uvicorn process died immediately"
+        return 1
+    fi
+    
+    return 0
+}
+
+finalize_rollover() {
+    log_info "Finalizing rollover..."
+    
+    # Move new PID file to main PID file
+    if [[ -f "$RUN_DIR/uvicorn_new.pid" ]]; then
+        mv "$RUN_DIR/uvicorn_new.pid" "$RUN_DIR/uvicorn.pid"
+    fi
+    
+    # Store the new port for reference
+    if [[ -f "$RUN_DIR/uvicorn_new.port" ]]; then
+        mv "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn.port"
+    fi
+    
+    # Clean up old PID file if it exists
+    rm -f "$RUN_DIR/uvicorn_old.pid"
+    
+    log_info "Rollover completed successfully"
+}
+
+rollback() {
+    local old_port=$1
+    local new_pid=$2
+    
+    log_error "Rolling back due to failure..."
+    
+    # Kill new workers if they exist
+    if [[ -n "$new_pid" ]] && kill -0 "$new_pid" 2>/dev/null; then
+        log_info "Killing new uvicorn workers (PID: $new_pid)"
+        kill -KILL -"$new_pid" 2>/dev/null || kill -KILL "$new_pid" 2>/dev/null || true
+    fi
+    
+    # Clean up temporary files
+    rm -f "$RUN_DIR/uvicorn_new.pid" "$RUN_DIR/uvicorn_new.port"
+    
+    # Restore nginx configuration if old port is known
+    if [[ -n "$old_port" ]]; then
+        update_nginx_upstream "$old_port" ""
+    fi
+    
+    log_error "Rollback completed"
+}
+
+### MAIN LOGIC ################################################################
+
+# Check arguments
+if [[ $# -ne 1 ]]; then
+    echo "Usage: $0 <NEW_PORT>"
+    echo "Example: $0 8001"
+    exit 1
+fi
+
+NEW_PORT=$1
+
+# Check nginx permissions early and exit if we can't update nginx
+if [[ ! -w $(dirname "$NGINX_UPSTREAM_CONF") ]] && [[ $EUID -ne 0 ]]; then
+    if ! sudo -n true 2>/dev/null; then
+        log_error "This script needs sudo access to update nginx configuration"
+        log_error "Cannot proceed without nginx update permissions"
+        echo ""
+        echo "Please run with sudo:"
+        echo "  sudo $0 $NEW_PORT"
+        echo ""
+        exit 1
+    fi
+fi
+
+# Validate port number
+if ! [[ "$NEW_PORT" =~ ^[0-9]+$ ]] || [ "$NEW_PORT" -lt 1 ] || [ "$NEW_PORT" -gt 65535 ]; then
+    log_error "Invalid port number: $NEW_PORT"
+    exit 1
+fi
+
+# Check if port is available
+if ! check_port_availability "$NEW_PORT"; then
+    log_error "Port $NEW_PORT is already in use"
+    exit 1
+fi
+
+# Get old port from file or environment
+OLD_PORT=""
+if [[ -f "$RUN_DIR/uvicorn.port" ]]; then
+    OLD_PORT=$(<"$RUN_DIR/uvicorn.port")
+elif [[ -f "$ENV_FILE" ]]; then
+    set -a && . "$ENV_FILE" && set +a
+    OLD_PORT="${FASTAPI_PORT:-}"
+fi
+
+if [[ "$NEW_PORT" == "$OLD_PORT" ]]; then
+    log_error "New port is the same as old port ($NEW_PORT)"
+    exit 1
+fi
+
+log_info "Starting rolling update from port ${OLD_PORT:-unknown} to port $NEW_PORT"
+
+# Create run directory if it doesn't exist
+mkdir -p "$RUN_DIR"
+
+# Get old uvicorn PIDs before starting new ones
+OLD_PIDS=$(get_old_uvicorn_pids)
+if [[ -n "$OLD_PIDS" ]]; then
+    # Save old PIDs for potential rollback
+    echo "$OLD_PIDS" > "$RUN_DIR/uvicorn_old.pid"
+    log_info "Found old uvicorn workers: $(echo $OLD_PIDS | tr '\n' ' ')"
+else
+    log_warning "No existing uvicorn workers found"
+fi
+
+# Start new uvicorn workers
+if ! start_new_uvicorn_workers "$NEW_PORT"; then
+    log_error "Failed to start new uvicorn workers"
+    exit 1
+fi
+
+NEW_PID=$(<"$RUN_DIR/uvicorn_new.pid")
+
+# Wait for new workers to be healthy
+if ! wait_for_health_check "$NEW_PORT"; then
+    log_error "New workers failed health check"
+    rollback "$OLD_PORT" "$NEW_PID"
+    exit 1
+fi
+
+# Update nginx to point to new workers
+if ! update_nginx_upstream "$NEW_PORT" "$OLD_PORT"; then
+    log_error "Failed to update nginx configuration"
+    rollback "$OLD_PORT" "$NEW_PID"
+    exit 1
+fi
+
+# Give nginx some time to start routing to new workers
+log_info "Waiting for nginx to stabilize..."
+sleep 5
+
+# Gracefully shutdown old workers
+if [[ -n "$OLD_PIDS" ]]; then
+    graceful_shutdown_old_workers "$OLD_PIDS"
+fi
+
+# Finalize the rollover
+finalize_rollover
+
+# Summary
+echo "──────────────────────────────────────────────────"
+echo "✓ Rolling update completed successfully"
+echo "  Old port: ${OLD_PORT:-none}"
+echo "  New port: $NEW_PORT"
+echo "  New PID: $NEW_PID"
+echo "  Logs: $LOG_ROOT/latest/"
+echo "──────────────────────────────────────────────────"
\ No newline at end of file
diff --git a/scripts/start_services.sh b/scripts/start_services.sh
new file mode 100755
index 0000000..ed36ad1
--- /dev/null
+++ b/scripts/start_services.sh
@@ -0,0 +1,187 @@
+#!/usr/bin/env bash
+# restart_services.sh — safer, simplified
+
+set -euo pipefail
+
+### CONFIGURATION #############################################################
+ENV_FILE="api/.env"
+RUN_DIR="run"                 # where we keep *.pid
+LOG_ROOT="logs"
+
+### 1) Load environment vars so that configurations like FASTAPI_WORKERS are loaded #
+set -a && . "$ENV_FILE" && set +a
+
+# Get ENVIRONMENT for nginx config selection
+ENVIRONMENT="${ENVIRONMENT:-staging}"
+
+if [[ -z "${FASTAPI_PORT:-}" ]]; then
+  echo "Error: FASTAPI_PORT environment variable is not set."
+  exit 1
+fi
+
+if [[ -z "${FASTAPI_WORKERS:-}" ]]; then
+  echo "Error: FASTAPI_WORKERS environment variable is not set."
+  exit 1
+fi
+
+if [[ -z "${CONDA_ENV_NAME:-}" ]]; then
+  echo "Error: CONDA_ENV_NAME environment variable is not set."
+  exit 1
+fi
+
+# Default ARQ_WORKERS to 1 if not set
+ARQ_WORKERS=${ARQ_WORKERS:-1}
+
+# map "service name" → "command to run"
+declare -A SERVICES=(
+  [ari_manager]="python -m api.services.telephony.ari_manager"
+  [campaign_orchestrator]="python -m api.services.campaign.campaign_orchestrator"
+  [uvicorn]="uvicorn api.app:app --host 0.0.0.0 --port $FASTAPI_PORT --workers $FASTAPI_WORKERS"
+)
+
+# Add ARQ workers dynamically based on ARQ_WORKERS environment variable
+for ((i=1; i<=ARQ_WORKERS; i++)); do
+  SERVICES[arq$i]="python -m arq api.tasks.arq.WorkerSettings --custom-log-dict api.tasks.arq.LOG_CONFIG"
+done
+
+### 2) Activate conda #########################################################
+# Source conda if not already available (needed when running from systemd)
+if ! command -v conda &>/dev/null; then
+  source /opt/conda/etc/profile.d/conda.sh
+fi
+eval "$(conda shell.bash hook)"
+conda activate "$CONDA_ENV_NAME"
+
+### 3) Stop old services (only via PID files) #################################
+mkdir -p "$RUN_DIR"
+for name in "${!SERVICES[@]}"; do
+  pidfile="$RUN_DIR/$name.pid"
+  if [[ -f $pidfile ]]; then
+    oldpid=$(<"$pidfile")
+    if kill -0 "$oldpid" 2>/dev/null; then
+      echo "Stopping $name (PID $oldpid and its process group)…"
+      # Kill the entire process group (negative PID)
+      # First try SIGTERM
+      kill -TERM -"$oldpid" 2>/dev/null || kill -TERM "$oldpid" 2>/dev/null || true
+      sleep 4
+      # If still running, use SIGKILL
+      if kill -0 "$oldpid" 2>/dev/null; then
+        echo "⚠️  $name did not exit cleanly, forcing stop..."
+        kill -KILL -"$oldpid" 2>/dev/null || kill -KILL "$oldpid" 2>/dev/null || true
+        sleep 1
+      fi
+    fi
+    rm -f "$pidfile"
+  else
+    echo "No PID file for $name, skipping stop."
+  fi
+done
+
+# Clean up any port tracking files for uvicorn
+rm -f "$RUN_DIR/uvicorn.port" "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn_old.pid"
+
+
+### 4) Run migrations #########################################################
+alembic -c api/alembic.ini upgrade head
+
+### 5) Prepare logs ###########################################################
+timestamp=$(date '+%Y-%m-%d_%H-%M-%S')
+LOG_DIR="$LOG_ROOT/$timestamp"
+mkdir -p "$LOG_DIR"
+# Create relative symlink
+cd "$LOG_ROOT" && ln -sfn "$timestamp" latest && cd - >/dev/null
+
+### 6) (Optional) Free FastAPI port ###########################################
+FASTAPI_PORT=$FASTAPI_PORT
+if command -v lsof &>/dev/null; then
+  lsof -ti tcp:"$FASTAPI_PORT" | xargs -r kill -9 || true
+fi
+
+### 7) Start services #########################################################
+# Export rotation settings for loguru (if using file logging)
+export LOG_ROTATION_SIZE="${LOG_ROTATION_SIZE:-100 MB}"
+export LOG_RETENTION="${LOG_RETENTION:-7 days}"
+export LOG_COMPRESSION="${LOG_COMPRESSION:-gz}"
+
+for name in "${!SERVICES[@]}"; do
+  cmd=${SERVICES[$name]}
+  echo "→ Starting $name with loguru rotation…"
+  
+  # Export LOG_FILE_PATH for this specific service
+  export LOG_FILE_PATH="$LOG_DIR/$name.log"
+  
+  # Start in new process group with setsid
+  # Each service gets its own LOG_FILE_PATH environment variable
+  setsid nohup bash -c "LOG_FILE_PATH='$LOG_DIR/$name.log' $cmd" >/dev/null 2>&1 &
+  
+  # Get the PID of the setsid process
+  pid=$!
+  echo $pid >"$RUN_DIR/$name.pid"
+  
+  # For uvicorn, also save the port for rolling updates and update nginx
+  if [[ "$name" == "uvicorn" ]]; then
+    echo "$FASTAPI_PORT" >"$RUN_DIR/uvicorn.port"
+    
+    # Update nginx upstream configuration if nginx is installed
+    if command -v nginx &>/dev/null && [[ -d /etc/nginx ]]; then
+      # Determine which upstream config to update based on ENVIRONMENT
+      if [[ "${ENVIRONMENT:-}" == "production" ]]; then
+        NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_production_upstream.conf"
+        UPSTREAM_NAME="dograh_production_backend"
+        echo "→ Updating PRODUCTION nginx upstream to port $FASTAPI_PORT…"
+      else
+        # Default to staging for any non-production environment
+        NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_staging_upstream.conf"
+        UPSTREAM_NAME="dograh_staging_backend"
+        echo "→ Updating STAGING nginx upstream to port $FASTAPI_PORT…"
+      fi
+      
+      if [[ -w $(dirname "$NGINX_UPSTREAM_CONF") ]] || [[ $EUID -eq 0 ]]; then
+        cat > "${NGINX_UPSTREAM_CONF}.tmp" <<EOF
+# Auto-generated by start_services.sh for ${ENVIRONMENT:-staging}
+# Last updated: $(date)
+upstream ${UPSTREAM_NAME} {
+    server 127.0.0.1:${FASTAPI_PORT} max_fails=3 fail_timeout=30s;
+}
+EOF
+        # Atomic move (may need sudo)
+        if [[ $EUID -eq 0 ]]; then
+          mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}"
+        else
+          sudo mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}" 2>/dev/null || \
+            echo "⚠️  Could not update nginx config (need sudo). Run: sudo $0"
+        fi
+        
+        # Test and reload nginx if config was updated
+        if [[ -f "$NGINX_UPSTREAM_CONF" ]]; then
+          if nginx -t 2>/dev/null || sudo nginx -t 2>/dev/null; then
+            echo "→ Reloading nginx…"
+            nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null || \
+              echo "⚠️  Could not reload nginx (may need sudo)"
+          else
+            echo "⚠️  Nginx configuration test failed"
+          fi
+        fi
+      else
+        echo "⚠️  Cannot write to nginx config directory (need sudo privileges)"
+        echo "   Run: sudo $0 to update nginx configuration"
+      fi
+    fi
+  fi
+done
+disown -a
+
+### 8) Summary #################################################################
+echo
+echo "──────────────────────────────────────────────────"
+for name in "${!SERVICES[@]}"; do
+  pid=$(<"$RUN_DIR/$name.pid")
+  echo "✓ $name (PID $pid) → $LOG_DIR/$name.log"
+done
+echo "  Rotation: ${LOG_ROTATION_SIZE:-100 MB}"
+echo "  Retention: ${LOG_RETENTION:-7 days}"
+echo "  Compression: ${LOG_COMPRESSION:-gz}"
+echo "Logs: tail -f $LOG_DIR/*.log"
+echo "Rotated logs: ls $LOG_DIR/*.log.*"
+echo "To stop: run this script again or kill -TERM -<PID> for process groups"
+echo "──────────────────────────────────────────────────"

From 39dade175508e03c14bcbb16ecd9e068b96a5303 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek@a6k.me>
Date: Fri, 26 Sep 2025 10:57:51 +0530
Subject: [PATCH 2/4] Remove axiom and fix scripts

---
 api/logging_config.py             | 128 ++--------------------
 api/requirements.txt              |   1 -
 scripts/rolling_update_uvicorn.sh | 176 +++---------------------------
 scripts/start_services.sh         | 106 +++---------------
 4 files changed, 43 insertions(+), 368 deletions(-)

diff --git a/api/logging_config.py b/api/logging_config.py
index de68fb3..a8ea420 100644
--- a/api/logging_config.py
+++ b/api/logging_config.py
@@ -1,73 +1,21 @@
-import atexit
-import logging
 import os
-import queue
 import sys
-from logging.handlers import QueueHandler, QueueListener
 
 import loguru
-from axiom_py import Client
-from axiom_py.logging import AxiomHandler
 from pipecat.utils.context import run_id_var, turn_var
 
 from api.enums import Environment
 from api.utils.worker import get_worker_id, is_worker_process
 
-# ----- NEW CODE START -----
-# Helper to map string log level to Python logging level, adding support for "TRACE"
-TRACE_LEVEL_NUM = 5  # Below DEBUG (10)
-
-
-def _get_logging_level(level_name: str) -> int:
-    """Return numeric logging level for a given level name.
-
-    Supports the standard logging levels as well as the custom ``TRACE`` level
-    used by *loguru*. If ``TRACE`` is requested and not yet defined in the
-    ``logging`` module, it will be registered dynamically.
-    """
-    level_name = level_name.upper()
-
-    # Standard levels are present on the ``logging`` module.
-    if hasattr(logging, level_name):
-        return getattr(logging, level_name)
-
-    # Add support for TRACE (finer-grained than DEBUG)
-    if level_name == "TRACE":
-        if not hasattr(logging, "TRACE"):
-            logging.addLevelName(TRACE_LEVEL_NUM, "TRACE")
-
-            def trace(self, message, *args, **kwargs):  # type: ignore[override]
-                if self.isEnabledFor(TRACE_LEVEL_NUM):
-                    self._log(TRACE_LEVEL_NUM, message, args, **kwargs)
-
-            logging.Logger.trace = trace  # type: ignore[attr-defined]
-        return TRACE_LEVEL_NUM
-
-    # Fallback to DEBUG if an unknown level is provided
-    return logging.DEBUG
-
-
-# ----- NEW CODE END -----
-
 ENVIRONMENT = os.getenv("ENVIRONMENT", Environment.LOCAL.value)
 ENABLE_TURN_LOGGING = os.getenv("ENABLE_TURN_LOGGING", "false").lower() == "true"
 
-# Log rotation settings from environment
-LOG_ROTATION_SIZE = os.getenv("LOG_ROTATION_SIZE", "100 MB")  # e.g., "100 MB", "1 GB"
-LOG_ROTATION_TIME = os.getenv("LOG_ROTATION_TIME", None)  # e.g., "00:00", "12:00"
-LOG_RETENTION = os.getenv(
-    "LOG_RETENTION", "7 days"
-)  # e.g., "7 days", "1 week", "10 files"
-LOG_COMPRESSION = os.getenv(
-    "LOG_COMPRESSION", "gz"
-)  # "gz", "bz2", "xz", "tar", "tar.gz", "tar.bz2", "tar.xz", "zip"
-LOG_FILE_PATH = os.getenv(
-    "LOG_FILE_PATH", None
-)  # If set, write to file instead of stdout
+# We write different uvicorn forked worker log to a different
+# file which is then synced to cloudwatch logs
+LOG_FILE_PATH = os.getenv("LOG_FILE_PATH", None)
 
 # Track if logging has been initialized
 _logging_initialized = False
-_axiom_listener = None
 
 
 def inject_run_id(record):
@@ -97,11 +45,11 @@ def inject_run_id(record):
 
 def setup_logging():
     """Set up logging for the main application"""
-    global _logging_initialized, _axiom_listener
+    global _logging_initialized
 
     # Return early if already initialized
     if _logging_initialized:
-        return _axiom_listener
+        return
 
     log_level = os.getenv("LOG_LEVEL", "DEBUG").upper()
 
@@ -125,31 +73,8 @@ def setup_logging():
     else:
         log_format = "{time:YYYY-MM-DD HH:mm:ss.SSS} | <level>{level}</level> | [run_id={extra[run_id]}] | {file.name}:{line} | {message}"
 
-    # Add handler - either file with rotation or console
+    # Add handler - either file or console
     if LOG_FILE_PATH:
-        # File handler with rotation
-        rotation_config = {}
-
-        # Size-based rotation (e.g., "100 MB", "1 GB")
-        if LOG_ROTATION_SIZE:
-            rotation_config["rotation"] = LOG_ROTATION_SIZE
-
-        # Time-based rotation (e.g., "00:00" for daily at midnight)
-        if LOG_ROTATION_TIME:
-            rotation_config["rotation"] = LOG_ROTATION_TIME
-
-        # If no rotation specified, default to 100 MB
-        if not rotation_config:
-            rotation_config["rotation"] = "100 MB"
-
-        # Retention policy (e.g., "7 days", "10 files")
-        if LOG_RETENTION:
-            rotation_config["retention"] = LOG_RETENTION
-
-        # Compression format
-        if LOG_COMPRESSION and LOG_COMPRESSION.lower() != "none":
-            rotation_config["compression"] = LOG_COMPRESSION
-
         # Determine the actual log file path
         actual_log_path = LOG_FILE_PATH
 
@@ -159,7 +84,6 @@ def setup_logging():
             # Split the path to insert worker ID before extension
             base_path, ext = os.path.splitext(LOG_FILE_PATH)
             actual_log_path = f"{base_path}-worker-{worker_id}{ext}"
-            loguru.logger.info(f"Worker {worker_id} will log to: {actual_log_path}")
 
         patched.add(
             actual_log_path,
@@ -167,7 +91,6 @@ def setup_logging():
             level=log_level,
             colorize=False,  # No colors in file logs
             enqueue=True,  # Thread-safe writing
-            **rotation_config,
         )
     else:
         # Console handler (existing behavior)
@@ -178,40 +101,5 @@ def setup_logging():
             colorize=True,
         )
 
-    # Set up queue-based logging for Axiom
-    log_q = queue.Queue(-1)  # infinite size (tweak if needed)
-    queue_handler = QueueHandler(log_q)  # puts LogRecord on the queue
-    queue_handler.setLevel(_get_logging_level(log_level))
-
-    # Set up Axiom handler if credentials are available
-    axiom_token = os.environ.get("AXIOM_TOKEN")
-    axiom_org = os.environ.get("AXIOM_ORG")
-    axiom_dataset = os.getenv("AXIOM_LOG_DATASET")
-
-    if axiom_token and axiom_org and axiom_dataset:
-        client = Client(token=axiom_token, org_id=axiom_org)
-        axiom_handler = AxiomHandler(client, axiom_dataset)
-        axiom_handler.setLevel(_get_logging_level(log_level))
-
-        listener = QueueListener(
-            log_q,
-            axiom_handler,
-            respect_handler_level=True,
-        )
-        listener.start()
-
-        patched.add(queue_handler, level=log_level, enqueue=False)
-
-        # Register cleanup
-        atexit.register(listener.stop)
-
-        # Return the listener for manual cleanup if needed
-        loguru.logger = patched
-        _logging_initialized = True
-        _axiom_listener = listener
-        return listener
-    else:
-        # No Axiom logging available
-        loguru.logger = patched
-        _logging_initialized = True
-        return None
+    loguru.logger = patched
+    _logging_initialized = True
diff --git a/api/requirements.txt b/api/requirements.txt
index 4e3644c..d92d3e1 100644
--- a/api/requirements.txt
+++ b/api/requirements.txt
@@ -7,7 +7,6 @@ redis==5.3.1
 uvicorn==0.35.0
 aioboto3==15.1.0
 arq==0.26.3
-axiom-py==0.9.0
 twilio==9.8.0
 minio==7.2.16
 alembic-postgresql-enum==1.8.0
diff --git a/scripts/rolling_update_uvicorn.sh b/scripts/rolling_update_uvicorn.sh
index e6b0100..8d0c85b 100755
--- a/scripts/rolling_update_uvicorn.sh
+++ b/scripts/rolling_update_uvicorn.sh
@@ -1,38 +1,18 @@
 #!/usr/bin/env bash
 # rolling_update_uvicorn.sh — Zero-downtime rolling update for uvicorn workers
-#
-# Usage: ./rolling_update_uvicorn.sh <NEW_PORT>
-# Example: ./rolling_update_uvicorn.sh 8001
 
-set -euo pipefail
-
-# Check if running as root or with sudo
-if [[ $EUID -ne 0 ]]; then
-   echo "This script must be run as root or with sudo" 
-   exit 1
-fi
+set -e  # Exit on error
 
 ### CONFIGURATION #############################################################
 ENV_FILE="api/.env"
 RUN_DIR="run"
 LOG_ROOT="logs"
+VENV_PATH="/home/ubuntu/dograh_venv"
 HEALTH_CHECK_ENDPOINT="/api/v1/health"  # Adjust as needed
 MAX_WAIT_SECONDS=310  # Max wait for graceful shutdown (5 minutes + 10 seconds grace)
 
-# Load environment to get ENVIRONMENT variable
+# Load environment
 set -a && . "$ENV_FILE" && set +a
-ENVIRONMENT="${ENVIRONMENT:-staging}"
-
-# Set nginx upstream config based on environment
-if [[ "$ENVIRONMENT" == "production" ]]; then
-    NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_production_upstream.conf"
-    UPSTREAM_NAME="dograh_production_backend"
-    echo "Rolling update for PRODUCTION environment"
-else
-    NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_staging_upstream.conf"
-    UPSTREAM_NAME="dograh_staging_backend"
-    echo "Rolling update for STAGING environment"
-fi
 
 ### FUNCTIONS ##################################################################
 
@@ -146,61 +126,6 @@ graceful_shutdown_old_workers() {
     return 0
 }
 
-update_nginx_upstream() {
-    local new_port=$1
-    local old_port=$2
-    
-    log_info "Updating nginx upstream configuration for $ENVIRONMENT..."
-    
-    # Create or update the upstream configuration
-    cat > "${NGINX_UPSTREAM_CONF}.tmp" <<EOF
-# Auto-generated by rolling_update_uvicorn.sh for $ENVIRONMENT
-# Last updated: $(date)
-upstream ${UPSTREAM_NAME} {
-    server 127.0.0.1:${new_port} max_fails=3 fail_timeout=30s;
-}
-EOF
-    
-    # Atomic move (with sudo if needed)
-    if [[ $EUID -eq 0 ]]; then
-        mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}"
-    else
-        sudo mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}" 2>/dev/null || {
-            log_error "Could not update nginx config (need sudo). Run: sudo $0 $NEW_PORT"
-            return 1
-        }
-    fi
-    
-    # Test nginx configuration (with sudo if needed)
-    if nginx -t 2>/dev/null || sudo nginx -t 2>/dev/null; then
-        log_info "Nginx configuration test passed"
-        # Reload nginx to pick up new configuration (with sudo if needed)
-        if nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null; then
-            log_info "Nginx reloaded successfully"
-        else
-            log_error "Could not reload nginx"
-            return 1
-        fi
-    else
-        log_error "Nginx configuration test failed, rolling back"
-        # Restore old configuration if possible
-        if [[ -n "$old_port" ]]; then
-            cat > "${NGINX_UPSTREAM_CONF}.tmp" <<EOF
-upstream ${UPSTREAM_NAME} {
-    server 127.0.0.1:${old_port} max_fails=3 fail_timeout=30s;
-}
-EOF
-            if [[ $EUID -eq 0 ]]; then
-                mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}"
-            else
-                sudo mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}" 2>/dev/null || true
-            fi
-            nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null || true
-        fi
-        return 1
-    fi
-}
-
 start_new_uvicorn_workers() {
     local new_port=$1
     
@@ -214,36 +139,17 @@ start_new_uvicorn_workers() {
         return 1
     fi
     
-    if [[ -z "${CONDA_ENV_NAME:-}" ]]; then
-        log_error "CONDA_ENV_NAME environment variable is not set"
-        return 1
-    fi
-    
-    # Source conda if not already available
-    if ! command -v conda &>/dev/null; then
-        source /opt/conda/etc/profile.d/conda.sh
-    fi
-    eval "$(conda shell.bash hook)"
-    conda activate "$CONDA_ENV_NAME"
+    # Activate virtual environment
+    source ${VENV_PATH}/bin/activate
     
     # Use the latest log directory (where start_services.sh put logs)
-    # Resolve the symlink to get the actual directory
     local log_dir="$LOG_ROOT/latest"
-    if [[ -L "$log_dir" ]]; then
-        # It's a symlink, resolve it
-        log_dir=$(readlink -f "$log_dir")
-    fi
     
     if [[ ! -d "$log_dir" ]]; then
         log_error "No latest log directory found. Run start_services.sh first."
         return 1
     fi
     
-    # Export rotation settings
-    export LOG_ROTATION_SIZE="${LOG_ROTATION_SIZE:-100 MB}"
-    export LOG_RETENTION="${LOG_RETENTION:-7 days}"
-    export LOG_COMPRESSION="${LOG_COMPRESSION:-gz}"
-    
     # Create unique log filename using timestamp and script PID to avoid conflicts
     local script_pid=$$  # PID of this rolling_update script (for uniqueness)
     local timestamp=$(date '+%H%M%S')
@@ -252,35 +158,12 @@ start_new_uvicorn_workers() {
     log_info "Starting uvicorn with $FASTAPI_WORKERS workers on port $new_port"
     log_info "Logs: $LOG_FILE_PATH"
     
-    # If running as root, switch to original user for uvicorn process
-    if [[ $EUID -eq 0 ]] && [[ -n "${SUDO_USER:-}" ]]; then
-        log_info "Starting uvicorn as user: $SUDO_USER (not root)"
-        
-        # Run uvicorn as the original user, similar to start_services.sh
-        # Using setsid and passing LOG_FILE_PATH for loguru to pick up
-        sudo -u "$SUDO_USER" bash -c "
-            cd '$PWD'
-            export HOME='$(getent passwd $SUDO_USER | cut -d: -f6)'
-            export LOG_FILE_PATH='$LOG_FILE_PATH'
-            export LOG_ROTATION_SIZE='$LOG_ROTATION_SIZE'
-            export LOG_RETENTION='$LOG_RETENTION'
-            export LOG_COMPRESSION='$LOG_COMPRESSION'
-            set -a && source '$ENV_FILE' && set +a
-            source /opt/conda/etc/profile.d/conda.sh
-            conda activate '$CONDA_ENV_NAME'
-            setsid nohup bash -c \"LOG_FILE_PATH='$LOG_FILE_PATH' uvicorn api.app:app --host 0.0.0.0 --port $new_port --workers $FASTAPI_WORKERS\" >/dev/null 2>&1 &
-            echo \$! > '$RUN_DIR/uvicorn_new.pid'
-        "
-        # Read the PID that was written
-        local new_pid=$(<"$RUN_DIR/uvicorn_new.pid")
-    else
-        # Start in new process group with setsid (same as start_services.sh)
-        # Each service gets its own LOG_FILE_PATH environment variable
-        setsid nohup bash -c "LOG_FILE_PATH='$LOG_FILE_PATH' uvicorn api.app:app --host 0.0.0.0 --port $new_port --workers $FASTAPI_WORKERS" >/dev/null 2>&1 &
-        
-        local new_pid=$!
-        echo "$new_pid" > "$RUN_DIR/uvicorn_new.pid"
-    fi
+    # Start in new process group with setsid (same as start_services.sh)
+    # Each service gets its own LOG_FILE_PATH environment variable
+    setsid nohup bash -c "LOG_FILE_PATH='$LOG_FILE_PATH' uvicorn api.app:app --host 0.0.0.0 --port $new_port --workers $FASTAPI_WORKERS" >/dev/null 2>&1 &
+
+    local new_pid=$!
+    echo "$new_pid" > "$RUN_DIR/uvicorn_new.pid"
     
     # Save port information
     echo "$new_port" > "$RUN_DIR/uvicorn_new.port"
@@ -321,23 +204,18 @@ finalize_rollover() {
 rollback() {
     local old_port=$1
     local new_pid=$2
-    
+
     log_error "Rolling back due to failure..."
-    
+
     # Kill new workers if they exist
     if [[ -n "$new_pid" ]] && kill -0 "$new_pid" 2>/dev/null; then
         log_info "Killing new uvicorn workers (PID: $new_pid)"
         kill -KILL -"$new_pid" 2>/dev/null || kill -KILL "$new_pid" 2>/dev/null || true
     fi
-    
+
     # Clean up temporary files
     rm -f "$RUN_DIR/uvicorn_new.pid" "$RUN_DIR/uvicorn_new.port"
-    
-    # Restore nginx configuration if old port is known
-    if [[ -n "$old_port" ]]; then
-        update_nginx_upstream "$old_port" ""
-    fi
-    
+
     log_error "Rollback completed"
 }
 
@@ -352,19 +230,6 @@ fi
 
 NEW_PORT=$1
 
-# Check nginx permissions early and exit if we can't update nginx
-if [[ ! -w $(dirname "$NGINX_UPSTREAM_CONF") ]] && [[ $EUID -ne 0 ]]; then
-    if ! sudo -n true 2>/dev/null; then
-        log_error "This script needs sudo access to update nginx configuration"
-        log_error "Cannot proceed without nginx update permissions"
-        echo ""
-        echo "Please run with sudo:"
-        echo "  sudo $0 $NEW_PORT"
-        echo ""
-        exit 1
-    fi
-fi
-
 # Validate port number
 if ! [[ "$NEW_PORT" =~ ^[0-9]+$ ]] || [ "$NEW_PORT" -lt 1 ] || [ "$NEW_PORT" -gt 65535 ]; then
     log_error "Invalid port number: $NEW_PORT"
@@ -421,15 +286,8 @@ if ! wait_for_health_check "$NEW_PORT"; then
     exit 1
 fi
 
-# Update nginx to point to new workers
-if ! update_nginx_upstream "$NEW_PORT" "$OLD_PORT"; then
-    log_error "Failed to update nginx configuration"
-    rollback "$OLD_PORT" "$NEW_PID"
-    exit 1
-fi
-
-# Give nginx some time to start routing to new workers
-log_info "Waiting for nginx to stabilize..."
+# Give the system some time to stabilize before shutting down old workers
+log_info "Waiting for system to stabilize..."
 sleep 5
 
 # Gracefully shutdown old workers
diff --git a/scripts/start_services.sh b/scripts/start_services.sh
index ed36ad1..e02b262 100755
--- a/scripts/start_services.sh
+++ b/scripts/start_services.sh
@@ -1,18 +1,22 @@
 #!/usr/bin/env bash
-# restart_services.sh — safer, simplified
+# start_services.sh
 
-set -euo pipefail
+set -e  # Exit on error
 
 ### CONFIGURATION #############################################################
 ENV_FILE="api/.env"
 RUN_DIR="run"                 # where we keep *.pid
 LOG_ROOT="logs"
+VENV_PATH="/home/ubuntu/dograh_venv"
+ARQ_WORKERS=${ARQ_WORKERS:-1}
 
-### 1) Load environment vars so that configurations like FASTAPI_WORKERS are loaded #
+# Log startup
+echo "Starting Dograh Services at $(date)"
+
+### 1) Load environment vars so that configurations like FASTAPI_WORKERS are loaded
 set -a && . "$ENV_FILE" && set +a
 
-# Get ENVIRONMENT for nginx config selection
-ENVIRONMENT="${ENVIRONMENT:-staging}"
+cd /home/ubuntu/app
 
 if [[ -z "${FASTAPI_PORT:-}" ]]; then
   echo "Error: FASTAPI_PORT environment variable is not set."
@@ -24,14 +28,6 @@ if [[ -z "${FASTAPI_WORKERS:-}" ]]; then
   exit 1
 fi
 
-if [[ -z "${CONDA_ENV_NAME:-}" ]]; then
-  echo "Error: CONDA_ENV_NAME environment variable is not set."
-  exit 1
-fi
-
-# Default ARQ_WORKERS to 1 if not set
-ARQ_WORKERS=${ARQ_WORKERS:-1}
-
 # map "service name" → "command to run"
 declare -A SERVICES=(
   [ari_manager]="python -m api.services.telephony.ari_manager"
@@ -44,13 +40,8 @@ for ((i=1; i<=ARQ_WORKERS; i++)); do
   SERVICES[arq$i]="python -m arq api.tasks.arq.WorkerSettings --custom-log-dict api.tasks.arq.LOG_CONFIG"
 done
 
-### 2) Activate conda #########################################################
-# Source conda if not already available (needed when running from systemd)
-if ! command -v conda &>/dev/null; then
-  source /opt/conda/etc/profile.d/conda.sh
-fi
-eval "$(conda shell.bash hook)"
-conda activate "$CONDA_ENV_NAME"
+### 2) Activate virtual environment #########################################
+source ${VENV_PATH}/bin/activate
 
 ### 3) Stop old services (only via PID files) #################################
 mkdir -p "$RUN_DIR"
@@ -58,16 +49,16 @@ for name in "${!SERVICES[@]}"; do
   pidfile="$RUN_DIR/$name.pid"
   if [[ -f $pidfile ]]; then
     oldpid=$(<"$pidfile")
-    if kill -0 "$oldpid" 2>/dev/null; then
+    if kill -0 "$oldpid"; then
       echo "Stopping $name (PID $oldpid and its process group)…"
       # Kill the entire process group (negative PID)
       # First try SIGTERM
-      kill -TERM -"$oldpid" 2>/dev/null || kill -TERM "$oldpid" 2>/dev/null || true
+      kill -TERM -"$oldpid" || kill -TERM "$oldpid" || true
       sleep 4
       # If still running, use SIGKILL
-      if kill -0 "$oldpid" 2>/dev/null; then
+      if kill -0 "$oldpid"; then
         echo "⚠️  $name did not exit cleanly, forcing stop..."
-        kill -KILL -"$oldpid" 2>/dev/null || kill -KILL "$oldpid" 2>/dev/null || true
+        kill -KILL -"$oldpid" || kill -KILL "$oldpid" || true
         sleep 1
       fi
     fi
@@ -80,32 +71,17 @@ done
 # Clean up any port tracking files for uvicorn
 rm -f "$RUN_DIR/uvicorn.port" "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn_old.pid"
 
-
 ### 4) Run migrations #########################################################
 alembic -c api/alembic.ini upgrade head
 
 ### 5) Prepare logs ###########################################################
-timestamp=$(date '+%Y-%m-%d_%H-%M-%S')
-LOG_DIR="$LOG_ROOT/$timestamp"
+LOG_DIR="$LOG_ROOT/latest"
 mkdir -p "$LOG_DIR"
-# Create relative symlink
-cd "$LOG_ROOT" && ln -sfn "$timestamp" latest && cd - >/dev/null
-
-### 6) (Optional) Free FastAPI port ###########################################
-FASTAPI_PORT=$FASTAPI_PORT
-if command -v lsof &>/dev/null; then
-  lsof -ti tcp:"$FASTAPI_PORT" | xargs -r kill -9 || true
-fi
 
 ### 7) Start services #########################################################
-# Export rotation settings for loguru (if using file logging)
-export LOG_ROTATION_SIZE="${LOG_ROTATION_SIZE:-100 MB}"
-export LOG_RETENTION="${LOG_RETENTION:-7 days}"
-export LOG_COMPRESSION="${LOG_COMPRESSION:-gz}"
-
 for name in "${!SERVICES[@]}"; do
   cmd=${SERVICES[$name]}
-  echo "→ Starting $name with loguru rotation…"
+  echo "→ Starting $name"
   
   # Export LOG_FILE_PATH for this specific service
   export LOG_FILE_PATH="$LOG_DIR/$name.log"
@@ -118,55 +94,9 @@ for name in "${!SERVICES[@]}"; do
   pid=$!
   echo $pid >"$RUN_DIR/$name.pid"
   
-  # For uvicorn, also save the port for rolling updates and update nginx
+  # For uvicorn, also save the port for rolling updates
   if [[ "$name" == "uvicorn" ]]; then
     echo "$FASTAPI_PORT" >"$RUN_DIR/uvicorn.port"
-    
-    # Update nginx upstream configuration if nginx is installed
-    if command -v nginx &>/dev/null && [[ -d /etc/nginx ]]; then
-      # Determine which upstream config to update based on ENVIRONMENT
-      if [[ "${ENVIRONMENT:-}" == "production" ]]; then
-        NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_production_upstream.conf"
-        UPSTREAM_NAME="dograh_production_backend"
-        echo "→ Updating PRODUCTION nginx upstream to port $FASTAPI_PORT…"
-      else
-        # Default to staging for any non-production environment
-        NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_staging_upstream.conf"
-        UPSTREAM_NAME="dograh_staging_backend"
-        echo "→ Updating STAGING nginx upstream to port $FASTAPI_PORT…"
-      fi
-      
-      if [[ -w $(dirname "$NGINX_UPSTREAM_CONF") ]] || [[ $EUID -eq 0 ]]; then
-        cat > "${NGINX_UPSTREAM_CONF}.tmp" <<EOF
-# Auto-generated by start_services.sh for ${ENVIRONMENT:-staging}
-# Last updated: $(date)
-upstream ${UPSTREAM_NAME} {
-    server 127.0.0.1:${FASTAPI_PORT} max_fails=3 fail_timeout=30s;
-}
-EOF
-        # Atomic move (may need sudo)
-        if [[ $EUID -eq 0 ]]; then
-          mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}"
-        else
-          sudo mv "${NGINX_UPSTREAM_CONF}.tmp" "${NGINX_UPSTREAM_CONF}" 2>/dev/null || \
-            echo "⚠️  Could not update nginx config (need sudo). Run: sudo $0"
-        fi
-        
-        # Test and reload nginx if config was updated
-        if [[ -f "$NGINX_UPSTREAM_CONF" ]]; then
-          if nginx -t 2>/dev/null || sudo nginx -t 2>/dev/null; then
-            echo "→ Reloading nginx…"
-            nginx -s reload 2>/dev/null || sudo nginx -s reload 2>/dev/null || \
-              echo "⚠️  Could not reload nginx (may need sudo)"
-          else
-            echo "⚠️  Nginx configuration test failed"
-          fi
-        fi
-      else
-        echo "⚠️  Cannot write to nginx config directory (need sudo privileges)"
-        echo "   Run: sudo $0 to update nginx configuration"
-      fi
-    fi
   fi
 done
 disown -a

From 1349887649ef7a1ff70a215d9c78f771a7b09d47 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek@a6k.me>
Date: Fri, 26 Sep 2025 12:28:05 +0530
Subject: [PATCH 3/4] Export logs to LOG_DIR

---
 scripts/rolling_update_uvicorn.sh | 12 +++++++-----
 scripts/start_services.sh         |  3 +--
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/scripts/rolling_update_uvicorn.sh b/scripts/rolling_update_uvicorn.sh
index 8d0c85b..7cba8c1 100755
--- a/scripts/rolling_update_uvicorn.sh
+++ b/scripts/rolling_update_uvicorn.sh
@@ -6,7 +6,7 @@ set -e  # Exit on error
 ### CONFIGURATION #############################################################
 ENV_FILE="api/.env"
 RUN_DIR="run"
-LOG_ROOT="logs"
+LOG_DIR="logs"
 VENV_PATH="/home/ubuntu/dograh_venv"
 HEALTH_CHECK_ENDPOINT="/api/v1/health"  # Adjust as needed
 MAX_WAIT_SECONDS=310  # Max wait for graceful shutdown (5 minutes + 10 seconds grace)
@@ -14,6 +14,8 @@ MAX_WAIT_SECONDS=310  # Max wait for graceful shutdown (5 minutes + 10 seconds g
 # Load environment
 set -a && . "$ENV_FILE" && set +a
 
+cd /home/ubuntu/app
+
 ### FUNCTIONS ##################################################################
 
 log_info() {
@@ -142,11 +144,11 @@ start_new_uvicorn_workers() {
     # Activate virtual environment
     source ${VENV_PATH}/bin/activate
     
-    # Use the latest log directory (where start_services.sh put logs)
-    local log_dir="$LOG_ROOT/latest"
+    # Use the log directory (where start_services.sh put logs)
+    local log_dir="$LOG_DIR"
     
     if [[ ! -d "$log_dir" ]]; then
-        log_error "No latest log directory found. Run start_services.sh first."
+        log_error "No log directory found. Run start_services.sh first."
         return 1
     fi
     
@@ -304,5 +306,5 @@ echo "✓ Rolling update completed successfully"
 echo "  Old port: ${OLD_PORT:-none}"
 echo "  New port: $NEW_PORT"
 echo "  New PID: $NEW_PID"
-echo "  Logs: $LOG_ROOT/latest/"
+echo "  Logs: $LOG_DIR/"
 echo "──────────────────────────────────────────────────"
\ No newline at end of file
diff --git a/scripts/start_services.sh b/scripts/start_services.sh
index e02b262..3f054f8 100755
--- a/scripts/start_services.sh
+++ b/scripts/start_services.sh
@@ -6,7 +6,7 @@ set -e  # Exit on error
 ### CONFIGURATION #############################################################
 ENV_FILE="api/.env"
 RUN_DIR="run"                 # where we keep *.pid
-LOG_ROOT="logs"
+LOG_DIR="logs"
 VENV_PATH="/home/ubuntu/dograh_venv"
 ARQ_WORKERS=${ARQ_WORKERS:-1}
 
@@ -75,7 +75,6 @@ rm -f "$RUN_DIR/uvicorn.port" "$RUN_DIR/uvicorn_new.port" "$RUN_DIR/uvicorn_old.
 alembic -c api/alembic.ini upgrade head
 
 ### 5) Prepare logs ###########################################################
-LOG_DIR="$LOG_ROOT/latest"
 mkdir -p "$LOG_DIR"
 
 ### 7) Start services #########################################################

From 778044e70b74c9858c2c34a0946c12dea4aa284c Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek@a6k.me>
Date: Fri, 26 Sep 2025 13:07:29 +0530
Subject: [PATCH 4/4] Remove .service declaration

---
 scripts/dograh-services.service | 20 --------------------
 1 file changed, 20 deletions(-)
 delete mode 100644 scripts/dograh-services.service

diff --git a/scripts/dograh-services.service b/scripts/dograh-services.service
deleted file mode 100644
index 1c02ccc..0000000
--- a/scripts/dograh-services.service
+++ /dev/null
@@ -1,20 +0,0 @@
-[Unit]
-Description=Dograh Services
-After=network.target postgresql.service
-Wants=network-online.target
-
-[Service]
-Type=forking
-User=a
-Group=a
-WorkingDirectory=/home/a/dograh
-Environment="PATH=/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-ExecStart=/home/a/dograh/scripts/start_services.sh
-ExecStop=/home/a/dograh/scripts/stop_services.sh
-TimeoutStopSec=30
-Restart=on-failure
-RestartSec=10
-StandardOutput=journal
-StandardError=journal
-
-[Install]
\ No newline at end of file