fix: add a devops secret header

api/.env.example

@@ -12,6 +12,11 @@ UI_APP_URL="http://localhost:3000"
 DATABASE_URL="postgresql+asyncpg://postgres:postgres@localhost:5432/postgres"
 REDIS_URL="redis://:redissecret@localhost:6379"
 
+# Internal devops secret for deployment scripts and lifecycle hooks.
+# scripts/rolling_update.sh sends this to protected operational endpoints via
+# X-Dograh-Devops-Secret. Use a unique random value in production.
+DOGRAH_DEVOPS_SECRET="change-me-dograh-devops-secret"
+
 # AWS S3 Configuration
 ENABLE_AWS_S3="false"
 # AWS_ACCESS_KEY_ID=""

api/.env.test.example

@@ -14,4 +14,6 @@ UI_APP_URL=http://localhost:3000
 DATABASE_URL="postgresql+asyncpg://postgres:postgres@localhost:5432/test_db"
 REDIS_URL="redis://:redissecret@localhost:6379/0"
 
+DOGRAH_DEVOPS_SECRET="test-dograh-devops-secret"
+
 MINIO_PUBLIC_ENDPOINT=http://localhost:9000

api/constants.py

@@ -54,6 +54,7 @@ STACK_AUTH_PROJECT_ID = os.getenv("STACK_AUTH_PROJECT_ID")
 STACK_PUBLISHABLE_CLIENT_KEY = os.getenv("STACK_PUBLISHABLE_CLIENT_KEY")
 DOGRAH_MPS_SECRET_KEY = os.getenv("DOGRAH_MPS_SECRET_KEY", None)
 MPS_API_URL = os.getenv("MPS_API_URL", "https://services.dograh.com")
+DOGRAH_DEVOPS_SECRET = os.getenv("DOGRAH_DEVOPS_SECRET") or None
 
 # Storage Configuration
 ENABLE_AWS_S3 = os.getenv("ENABLE_AWS_S3", "false").lower() == "true"

api/routes/main.py

@@ -1,4 +1,7 @@
-from fastapi import APIRouter
+import secrets
+from typing import Annotated
+
+from fastapi import APIRouter, Header, HTTPException, status
 from loguru import logger
 from pydantic import BaseModel
 
@@ -131,8 +134,35 @@ class ActiveCallsResponse(BaseModel):
     active_calls: int
 
 
+DOGRAH_DEVOPS_SECRET_HEADER = "X-Dograh-Devops-Secret"
+
+
+def _verify_devops_secret(
+    configured_secret: str | None,
+    provided_secret: str | None,
+) -> None:
+    if not configured_secret:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="Devops secret is not configured",
+        )
+    if not provided_secret or not secrets.compare_digest(
+        provided_secret,
+        configured_secret,
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Forbidden",
+        )
+
+
 @router.get("/health/active-calls", response_model=ActiveCallsResponse)
-async def active_calls() -> ActiveCallsResponse:
+async def active_calls(
+    x_dograh_devops_secret: Annotated[
+        str | None,
+        Header(alias=DOGRAH_DEVOPS_SECRET_HEADER),
+    ] = None,
+) -> ActiveCallsResponse:
     """In-flight call count for THIS worker — the drain signal for deploys.
 
     A deploy orchestrator polls this per worker and waits for zero before
@@ -141,6 +171,8 @@ async def active_calls() -> ActiveCallsResponse:
     count is per-process: one uvicorn per VM port (scripts/rolling_update.sh)
     or per Kubernetes pod (preStop hook). See api/services/pipecat/active_calls.py.
     """
+    from api.constants import DOGRAH_DEVOPS_SECRET
     from api.services.pipecat.active_calls import active_call_count
 
+    _verify_devops_secret(DOGRAH_DEVOPS_SECRET, x_dograh_devops_secret)
     return ActiveCallsResponse(active_calls=active_call_count())

api/tests/test_active_calls.py

@@ -6,6 +6,10 @@ worker. The guarantees that matter for draining: register/unregister are
 idempotent, and the count only reaches zero when every registered run is gone.
 """
 
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from api.routes import main as main_routes
 from api.services.pipecat import active_calls
 
 
@@ -14,6 +18,21 @@ def setup_function():
     active_calls._active_run_ids.clear()
 
 
+def _make_active_calls_client(
+    monkeypatch,
+    configured_secret: str | None = "test-dograh-devops-secret",
+) -> TestClient:
+    monkeypatch.setattr("api.constants.DOGRAH_DEVOPS_SECRET", configured_secret)
+    app = FastAPI()
+    app.add_api_route(
+        "/api/v1/health/active-calls",
+        main_routes.active_calls,
+        methods=["GET"],
+        response_model=main_routes.ActiveCallsResponse,
+    )
+    return TestClient(app)
+
+
 def test_starts_empty():
     assert active_calls.active_call_count() == 0
 
@@ -51,3 +70,46 @@ def test_full_lifecycle_drains_to_zero():
     assert active_calls.active_call_count() == 1
     active_calls.unregister_active_call(42)
     assert active_calls.active_call_count() == 0
+
+
+def test_active_calls_route_requires_configured_secret(monkeypatch):
+    client = _make_active_calls_client(monkeypatch, configured_secret=None)
+
+    response = client.get(
+        "/api/v1/health/active-calls",
+        headers={"X-Dograh-Devops-Secret": "test-dograh-devops-secret"},
+    )
+
+    assert response.status_code == 503
+
+
+def test_active_calls_route_rejects_missing_secret_header(monkeypatch):
+    client = _make_active_calls_client(monkeypatch)
+
+    response = client.get("/api/v1/health/active-calls")
+
+    assert response.status_code == 403
+
+
+def test_active_calls_route_rejects_wrong_secret(monkeypatch):
+    client = _make_active_calls_client(monkeypatch)
+
+    response = client.get(
+        "/api/v1/health/active-calls",
+        headers={"X-Dograh-Devops-Secret": "wrong"},
+    )
+
+    assert response.status_code == 403
+
+
+def test_active_calls_route_returns_count_with_secret(monkeypatch):
+    active_calls.register_active_call(42)
+    client = _make_active_calls_client(monkeypatch)
+
+    response = client.get(
+        "/api/v1/health/active-calls",
+        headers={"X-Dograh-Devops-Secret": "test-dograh-devops-secret"},
+    )
+
+    assert response.status_code == 200
+    assert response.json() == {"active_calls": 1}

scripts/rolling_update.sh

@@ -29,6 +29,7 @@ NGINX_UPSTREAM_CONF="/etc/nginx/conf.d/dograh_upstream.conf"
 
 HEALTH_CHECK_ENDPOINT="/api/v1/health"
 ACTIVE_CALLS_ENDPOINT="/api/v1/health/active-calls"
+DOGRAH_DEVOPS_SECRET_HEADER="X-Dograh-Devops-Secret"
 
 # Load environment
 if [[ -f "$ENV_FILE" ]]; then
@@ -57,6 +58,15 @@ log_info()  { echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO:  $*"; }
 log_warn()  { echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARN:  $*"; }
 log_error() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2; }
 
+if [[ -z "${DOGRAH_DEVOPS_SECRET:-}" ]]; then
+  log_error "DOGRAH_DEVOPS_SECRET is not set. Add it to $ENV_FILE before running rolling_update.sh."
+  exit 1
+fi
+if [[ "$DOGRAH_DEVOPS_SECRET" == "change-me-dograh-devops-secret" ]]; then
+  log_error "DOGRAH_DEVOPS_SECRET still has the example placeholder value. Replace it in $ENV_FILE."
+  exit 1
+fi
+
 # Band port calculation: band A = base, band B = base + 100
 band_base_port() {
   local band=$1
@@ -100,17 +110,38 @@ kill_process_tree() {
 }
 
 # Active in-progress call count for a single worker, via its health endpoint.
-# A worker that is unreachable (already exited) or returns no/garbage body
-# reports 0, so it never blocks the drain.
+# A worker that is unreachable (already exited) reports 0, so it never blocks the
+# drain. Non-200 responses or malformed bodies are hard failures: otherwise an
+# auth/configuration error could be mistaken for a fully drained worker.
 count_active_calls_on_port() {
   local port=$1
-  local body n
-  body=$(curl -s --max-time 3 \
+  local response http_code body n
+  response=$(curl -sS --max-time 3 \
+    -H "${DOGRAH_DEVOPS_SECRET_HEADER}: ${DOGRAH_DEVOPS_SECRET}" \
+    -w $'\n%{http_code}' \
     "http://127.0.0.1:${port}${ACTIVE_CALLS_ENDPOINT}" 2>/dev/null || true)
+  http_code="${response##*$'\n'}"
+  body="${response%$'\n'*}"
+
+  if [[ "$http_code" == "000" ]]; then
+    printf '0'
+    return 0
+  fi
+
+  if [[ "$http_code" != "200" ]]; then
+    log_error "uvicorn_${port} active-calls endpoint returned HTTP ${http_code}. Check DOGRAH_DEVOPS_SECRET in $ENV_FILE."
+    return 1
+  fi
+
   n=$(printf '%s' "$body" \
     | grep -o '"active_calls"[[:space:]]*:[[:space:]]*[0-9]\+' \
     | grep -o '[0-9]\+$' || true)
-  printf '%s' "${n:-0}"
+  if [[ -z "$n" ]]; then
+    log_error "uvicorn_${port} active-calls endpoint returned an invalid response body."
+    return 1
+  fi
+
+  printf '%s' "$n"
 }
 
 ###############################################################################
@@ -399,7 +430,10 @@ while true; do
     # Only poll workers still alive; an exited worker holds no calls.
     pidfile="$RUN_DIR/uvicorn_${port}.pid"
     if [[ -f "$pidfile" ]] && kill -0 "$(<"$pidfile")" 2>/dev/null; then
-      active=$((active + $(count_active_calls_on_port "$port")))
+      if ! call_count=$(count_active_calls_on_port "$port"); then
+        exit 1
+      fi
+      active=$((active + call_count))
     fi
   done
 

scripts/start_services.sh

@@ -33,6 +33,15 @@ if [[ -f "$ENV_FILE" ]]; then
   set -a && . "$ENV_FILE" && set +a
 fi
 
+if [[ -z "${DOGRAH_DEVOPS_SECRET:-}" ]]; then
+  echo "ERROR: DOGRAH_DEVOPS_SECRET is not set. Add it to $ENV_FILE before starting production services."
+  exit 1
+fi
+if [[ "$DOGRAH_DEVOPS_SECRET" == "change-me-dograh-devops-secret" ]]; then
+  echo "ERROR: DOGRAH_DEVOPS_SECRET still has the example placeholder value. Replace it in $ENV_FILE."
+  exit 1
+fi
+
 UVICORN_BASE_PORT=${UVICORN_BASE_PORT:-8000}
 CPU_CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
 FASTAPI_WORKERS=${FASTAPI_WORKERS:-$CPU_CORES}

ui/src/client/types.gen.ts

@@ -176,6 +176,16 @@ export type AwsBedrockLlmConfiguration = {
     aws_region?: string;
 };
 
+/**
+ * ActiveCallsResponse
+ */
+export type ActiveCallsResponse = {
+    /**
+     * Active Calls
+     */
+    active_calls: number;
+};
+
 /**
  * AmbientNoiseUploadRequest
  */
@@ -13844,3 +13854,38 @@ export type HealthApiV1HealthGetResponses = {
 };
 
 export type HealthApiV1HealthGetResponse = HealthApiV1HealthGetResponses[keyof HealthApiV1HealthGetResponses];
+
+export type ActiveCallsApiV1HealthActiveCallsGetData = {
+    body?: never;
+    headers?: {
+        /**
+         * X-Dograh-Devops-Secret
+         */
+        'X-Dograh-Devops-Secret'?: string | null;
+    };
+    path?: never;
+    query?: never;
+    url: '/api/v1/health/active-calls';
+};
+
+export type ActiveCallsApiV1HealthActiveCallsGetErrors = {
+    /**
+     * Not found
+     */
+    404: unknown;
+    /**
+     * Validation Error
+     */
+    422: HttpValidationError;
+};
+
+export type ActiveCallsApiV1HealthActiveCallsGetError = ActiveCallsApiV1HealthActiveCallsGetErrors[keyof ActiveCallsApiV1HealthActiveCallsGetErrors];
+
+export type ActiveCallsApiV1HealthActiveCallsGetResponses = {
+    /**
+     * Successful Response
+     */
+    200: ActiveCallsResponse;
+};
+
+export type ActiveCallsApiV1HealthActiveCallsGetResponse = ActiveCallsApiV1HealthActiveCallsGetResponses[keyof ActiveCallsApiV1HealthActiveCallsGetResponses];