feat: add FASTAPI_WORKERS parameter

scripts/setup_remote.sh

@@ -116,6 +116,26 @@ fi
 # Telemetry opt-out (default: true)
 ENABLE_TELEMETRY="${ENABLE_TELEMETRY:-true}"
 
+# Number of uvicorn worker processes. Each runs as its own process on a
+# distinct port (8000, 8001, ...) and nginx balances across them with
+# least_conn. Better than uvicorn --workers for long-lived WebSocket
+# connections, which would otherwise stick to whichever worker accepted them.
+if [[ -z "$FASTAPI_WORKERS" ]]; then
+    if [[ -t 0 ]]; then
+        echo ""
+        echo -e "${YELLOW}Number of FastAPI workers (uvicorn processes nginx will load-balance):${NC}"
+        read -p "[4]: " FASTAPI_WORKERS
+        FASTAPI_WORKERS="${FASTAPI_WORKERS:-4}"
+    else
+        FASTAPI_WORKERS="4"
+    fi
+fi
+
+if ! [[ "$FASTAPI_WORKERS" =~ ^[1-9][0-9]*$ ]]; then
+    echo -e "${RED}Error: FASTAPI_WORKERS must be a positive integer (got: $FASTAPI_WORKERS)${NC}"
+    exit 1
+fi
+
 # Where setup artifacts (.env, certs, nginx.conf, etc.) will land. Build mode
 # with an existing repo writes them next to docker-compose.yaml in cwd;
 # everything else writes into a fresh dograh/ subdirectory.
@@ -165,9 +185,10 @@ fi
 
 echo ""
 echo -e "${GREEN}Configuration:${NC}"
-echo -e "  Server IP:     ${BLUE}$SERVER_IP${NC}"
-echo -e "  TURN Secret:   ${BLUE}********${NC}"
-echo -e "  Deploy mode:   ${BLUE}$DEPLOY_MODE${NC}"
+echo -e "  Server IP:        ${BLUE}$SERVER_IP${NC}"
+echo -e "  TURN Secret:      ${BLUE}********${NC}"
+echo -e "  Deploy mode:      ${BLUE}$DEPLOY_MODE${NC}"
+echo -e "  FastAPI workers:  ${BLUE}$FASTAPI_WORKERS${NC}  (ports 8000..$((8000 + FASTAPI_WORKERS - 1)))"
 if [[ "$DEPLOY_MODE" == "build" ]]; then
     if [[ "$REPO_SOURCE" == "clone" ]]; then
         echo -e "  Source:        ${BLUE}clone $FORK_REPO@$BRANCH${NC}"
@@ -209,7 +230,22 @@ else
 fi
 
 echo -e "${BLUE}[2/$TOTAL] Creating nginx.conf...${NC}"
-cat > nginx.conf << 'NGINX_EOF'
+# Build the upstream block first (needs shell interpolation for the server
+# lines), then append the static server blocks via a quoted heredoc. The
+# SERVER_IP_PLACEHOLDER gets replaced by sed below.
+{
+    echo "# Backend API workers — one uvicorn process per port, balanced by least_conn."
+    echo "# Generated by setup_remote.sh; regenerate to change worker count."
+    echo "upstream dograh_api {"
+    echo "    least_conn;"
+    for ((i=0; i<FASTAPI_WORKERS; i++)); do
+        port=$((8000 + i))
+        echo "    server api:$port max_fails=3 fail_timeout=10s;"
+    done
+    echo "    keepalive 32;"
+    echo "}"
+    echo ""
+    cat << 'NGINX_EOF'
 server {
     listen 80;
     server_name SERVER_IP_PLACEHOLDER;
@@ -229,11 +265,15 @@ server {
     ssl_protocols TLSv1.2 TLSv1.3;
     ssl_prefer_server_ciphers on;
 
-    # Backend API and WebSockets - bypass the UI, go straight to api:8000
+    # Backend API and WebSockets - bypass the UI, go straight to the
+    # api workers via the least_conn upstream defined above.
     location /api/v1/ {
-        proxy_pass http://api:8000;
+        proxy_pass http://dograh_api;
         proxy_http_version 1.1;
 
+        # Retry on a dead/restarting worker
+        proxy_next_upstream error timeout http_502 http_503 http_504;
+
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection "upgrade";
 
@@ -287,6 +327,7 @@ server {
     }
 }
 NGINX_EOF
+} > nginx.conf
 
 # Replace placeholder with actual IP
 sed -i.bak "s/SERVER_IP_PLACEHOLDER/$SERVER_IP/g" nginx.conf && rm -f nginx.conf.bak
@@ -364,6 +405,12 @@ OSS_JWT_SECRET=$OSS_JWT_SECRET
 
 # Telemetry (set to false to disable)
 ENABLE_TELEMETRY=$ENABLE_TELEMETRY
+
+# Number of uvicorn worker processes; nginx load-balances across them
+# (ports 8000..$((8000 + FASTAPI_WORKERS - 1))) with least_conn.
+# Must match the upstream block in nginx.conf — re-run setup_remote.sh
+# (with DOGRAH_FORCE_OVERWRITE=1) to change.
+FASTAPI_WORKERS=$FASTAPI_WORKERS
 ENV_EOF
 echo -e "${GREEN}✓ .env file created${NC}"
 
@@ -382,14 +429,14 @@ services:
     build:
       context: .
       dockerfile: api/Dockerfile
-    image: dograh-local/dograh-api:dev
+    image: dograh-local/dograh-api:local
     pull_policy: never
 
   ui:
     build:
       context: .
       dockerfile: ui/Dockerfile
-    image: dograh-local/dograh-ui:dev
+    image: dograh-local/dograh-ui:local
     pull_policy: never
 OVERRIDE_EOF
     echo -e "${GREEN}✓ docker-compose.override.yaml created${NC}"

scripts/start_services_docker.sh

@@ -61,7 +61,16 @@ start() {
 
 start ari_manager           python -m api.services.telephony.ari_manager
 start campaign_orchestrator python -m api.services.campaign.campaign_orchestrator
-start uvicorn               uvicorn api.app:app --host 0.0.0.0 --port "$UVICORN_BASE_PORT" --workers "$FASTAPI_WORKERS"
+
+# Spawn FASTAPI_WORKERS independent uvicorn processes on consecutive ports
+# starting at UVICORN_BASE_PORT. nginx upstream (configured in setup_remote.sh)
+# balances across them with least_conn — better than uvicorn --workers for
+# long-lived WebSocket connections, which would otherwise stick to whichever
+# worker accepted them first.
+for ((i=0; i<FASTAPI_WORKERS; i++)); do
+  port=$((UVICORN_BASE_PORT + i))
+  start "uvicorn$i" uvicorn api.app:app --host 0.0.0.0 --port "$port" --workers 1
+done
 
 for ((i=1; i<=ARQ_WORKERS; i++)); do
   start "arq$i" python -m arq api.tasks.arq.WorkerSettings --custom-log-dict api.tasks.arq.LOG_CONFIG