diff --git a/deploy/helm/dograh/README.md b/deploy/helm/dograh/README.md index a0dba4aa..f81fc612 100644 --- a/deploy/helm/dograh/README.md +++ b/deploy/helm/dograh/README.md @@ -32,10 +32,10 @@ and `examples/values-aws.yaml` for topology-specific overrides. | Workload | Replicas | Strategy | Notes | |------------------------------|-------------|-----------------|-------| | `dograh-web` | 2 (HPA opt) | RollingUpdate | Long-lived WS, graceful drain | -| `dograh-arq-worker` | 1 (knob) | RollingUpdate | Stateless | +| `dograh-arq-worker` | 1 (HPA opt) | RollingUpdate | Stateless | | `dograh-ari-manager` | **1 fixed** | **Recreate** | Telephony singleton | | `dograh-campaign-orchestrator` | **1 fixed** | **Recreate** | Campaign singleton (in-memory locks) | -| `dograh-ui` | 2 | RollingUpdate | Next.js SSR | +| `dograh-ui` | 2 (HPA opt) | RollingUpdate | Next.js SSR | | `dograh-coturn` | 1 | Recreate | LoadBalancer Service, port-pinned | HTTP traffic: Gateway API (default) or Ingress (fallback). @@ -53,10 +53,13 @@ silent. Each is exposed in `values.yaml` for operator override. - **Liveness probes on singletons: `exec` (`pgrep`).** No HTTP endpoint exists on ari-manager / campaign-orchestrator; process-alive check is the simplest correct signal. -- **HPA on web: CPU/memory, disabled by default.** Plan recommends HPA - but CPU/memory is a poor signal for WS workloads. Default - `autoscaling.web.enabled=false`; flip on with a knowing eye and plan - to replace with a connection-count metric. +- **HPA on web / workers / ui: CPU(/memory), disabled by default.** + Templates exist for all three tiers (`web-hpa.yaml`, + `arq-worker-hpa.yaml`, `ui-hpa.yaml`) but every `autoscaling.` + block ships `enabled: false` — flip on per tier with a knowing eye. + CPU/memory is a poor signal for the WS-heavy web tier and a coarse + one for the IO-bound ARQ workers (plan: queue-depth / active-call + scaling, see TODOs); it is a reasonable signal for the Next.js UI. - **Singleton replica counts: hard-coded.** No `replicaCount` knob exposed on ari-manager / campaign-orchestrator. Prevents accidental `kubectl scale` corrupting in-memory dedup state. @@ -95,8 +98,10 @@ volume. The remaining `/tmp` uses (`audio_file_cache.py`, - **MinIO public route via separate hostname.** Make `/voice-audio/` path-prefix the default but allow operators to opt into a dedicated hostname. -- **KEDA for ARQ workers.** When a queue-depth metric is available, - switch ARQ from fixed replicas to KEDA-driven scaling. +- **KEDA for ARQ workers.** When a queue-depth or active-calls metric + is available, switch ARQ from CPU HPA to KEDA-driven scaling. Keep + `autoscaling.workers.enabled=false` when a KEDA ScaledObject owns the + Deployment so the chart doesn't render a competing HPA. ## Validation @@ -108,6 +113,7 @@ helm template test-release . > /tmp/render-default.yaml helm template test-release . -f examples/values-single-node.yaml > /tmp/render-single.yaml helm template test-release . -f examples/values-managed.yaml > /tmp/render-managed.yaml helm template test-release . -f examples/values-aws.yaml > /tmp/render-aws.yaml +helm template test-release . -f examples/values-k3s-prod.yaml > /tmp/render-k3s.yaml ``` Spot-check expectations: @@ -131,7 +137,8 @@ deploy/helm/dograh/ ├── examples/ │ ├── values-single-node.yaml │ ├── values-managed.yaml -│ └── values-aws.yaml +│ ├── values-aws.yaml +│ └── values-k3s-prod.yaml └── templates/ ├── _helpers.tpl ├── NOTES.txt @@ -144,10 +151,12 @@ deploy/helm/dograh/ ├── web-hpa.yaml ├── web-pdb.yaml ├── arq-worker-deployment.yaml + ├── arq-worker-hpa.yaml ├── ari-manager-deployment.yaml ├── campaign-orchestrator-deployment.yaml ├── ui-deployment.yaml ├── ui-service.yaml + ├── ui-hpa.yaml ├── ui-pdb.yaml ├── coturn-deployment.yaml ├── coturn-service.yaml diff --git a/deploy/helm/dograh/examples/values-k3s-prod.yaml b/deploy/helm/dograh/examples/values-k3s-prod.yaml new file mode 100644 index 00000000..1d743e27 --- /dev/null +++ b/deploy/helm/dograh/examples/values-k3s-prod.yaml @@ -0,0 +1,117 @@ +# Dograh — production values for k3s +# Hosted-AI only (no local models). All state on the node's local disk via +# k3s local-path StorageClass. TLS terminated at Cloudflare edge; re-encrypted +# to origin using the shared *.yourdomain.com Cloudflare Origin CA cert +# (secret `cloudflare-origin-tls`, copied from a neighboring namespace). +# +# HPA is enabled for web / workers / ui with min=1, max=5. Requires +# metrics-server in the cluster (k3s ships it by default). + +image: + tag: latest # pin to a released tag once stable + +# --- Bundled stateful deps (in-cluster, on local-path PVCs) ----------------- +database: { mode: internal } +redis: { mode: internal } +storage: { mode: internalMinio } + +# --- HTTP exposure ---------------------------------------------------------- +exposure: + mode: ingress + ingress: + className: traefik + host: aicalling.yourdomain.com + tls: + enabled: true + secretName: cloudflare-origin-tls + +# --- Runtime config (rendered into the ConfigMap) -------------------------- +config: + environment: production + logLevel: INFO + backendApiEndpoint: https://aicalling.yourdomain.com # kills the CF-tunnel fallback + minioPublicEndpoint: https://aicalling.yourdomain.com # browser fetches /voice-audio/ via Traefik + enableSignup: false # invite-only: 403 the public signup endpoint + +# --- Workloads not needed for hosted-AI web-only calls --------------------- +ariManager: { enabled: false } # Asterisk SIP singleton +campaignOrchestrator: { enabled: false } # scheduled outbound singleton +coturn: { enabled: false } # TURN relay + +# --- Web tier (FastAPI + WebSocket signaling) ------------------------------ +web: + replicaCount: 1 + resources: + requests: { cpu: 100m, memory: 384Mi } + limits: { cpu: "1", memory: 1Gi } + pdb: { enabled: false } + +# --- ARQ background workers ------------------------------------------------ +workers: + replicaCount: 1 + resources: + requests: { cpu: 50m, memory: 128Mi } + limits: { cpu: 500m, memory: 512Mi } + +# --- Next.js UI ------------------------------------------------------------ +ui: + replicaCount: 1 + resources: + requests: { cpu: 50m, memory: 256Mi } + limits: { cpu: 500m, memory: 512Mi } + pdb: { enabled: false } + +# --- Alembic migration Job (post-install / pre-upgrade hook) --------------- +# CPU request kept low so it schedules on a tight node; migrations are +# short-lived and don't need much headroom. +migrate: + resources: + requests: { cpu: 20m, memory: 256Mi } + limits: { cpu: 500m, memory: 512Mi } + +# --- HPA: 1 → 5 on CPU 70% only (memory HPA opt-in — see comments) --------- +# The chart ships all autoscaling blocks disabled; this file opts in for all +# three tiers. NOTE: CPU is a poor scale signal for the web tier (long-lived +# WebSockets don't move CPU much) and a coarse one for the IO-bound ARQ +# workers — see the autoscaling notes in values.yaml. Fine starting point for +# a single node; minReplicas=1 is why the web/ui PDBs are disabled above. +autoscaling: + web: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: null # idle FastAPI already close to request; CPU HPA is enough + workers: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: null # idle Python (ARQ) sits near the memory request; CPU HPA is enough + ui: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: null # idle Node.js (Next.js SSR) sits near the memory request; CPU HPA is enough + +# --- Internal Postgres (pgvector/pg17) ------------------------------------- +postgresql: + persistence: { size: 5Gi, storageClass: local-path } + resources: + requests: { cpu: 100m, memory: 256Mi } + limits: { cpu: "1", memory: 1Gi } + +# --- Internal Redis -------------------------------------------------------- +redisinternal: + persistence: { size: 1Gi, storageClass: local-path } + resources: + requests: { cpu: 50m, memory: 64Mi } + limits: { cpu: 300m, memory: 256Mi } + +# --- Internal MinIO (audio + artifacts on node disk) ----------------------- +minio: + persistence: { size: 10Gi, storageClass: local-path } + resources: + requests: { cpu: 50m, memory: 128Mi } + limits: { cpu: 500m, memory: 512Mi } diff --git a/deploy/helm/dograh/templates/arq-worker-deployment.yaml b/deploy/helm/dograh/templates/arq-worker-deployment.yaml index d63c78cc..99960255 100644 --- a/deploy/helm/dograh/templates/arq-worker-deployment.yaml +++ b/deploy/helm/dograh/templates/arq-worker-deployment.yaml @@ -7,7 +7,13 @@ metadata: {{- include "dograh.labels" . | nindent 4 }} app.kubernetes.io/component: arq-worker spec: + {{- /* Omit spec.replicas only when the HPA object will actually render — matches + the gate in templates/arq-worker-hpa.yaml. If HPA is enabled but both + metric targets are null the HPA is suppressed, so we must keep the + static replica count here to avoid a Deployment with no owner. */ -}} + {{- if not (and .Values.autoscaling.workers.enabled (or .Values.autoscaling.workers.targetCPUUtilizationPercentage .Values.autoscaling.workers.targetMemoryUtilizationPercentage)) }} replicas: {{ .Values.workers.replicaCount }} + {{- end }} selector: matchLabels: {{- include "dograh.selectorLabels" . | nindent 6 }} diff --git a/deploy/helm/dograh/templates/arq-worker-hpa.yaml b/deploy/helm/dograh/templates/arq-worker-hpa.yaml new file mode 100644 index 00000000..a4fc54d2 --- /dev/null +++ b/deploy/helm/dograh/templates/arq-worker-hpa.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.autoscaling.workers.enabled (or .Values.autoscaling.workers.targetCPUUtilizationPercentage .Values.autoscaling.workers.targetMemoryUtilizationPercentage) }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "dograh.arqWorker.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "dograh.labels" . | nindent 4 }} + app.kubernetes.io/component: arq-worker +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "dograh.arqWorker.fullname" . }} + minReplicas: {{ .Values.autoscaling.workers.minReplicas }} + maxReplicas: {{ .Values.autoscaling.workers.maxReplicas }} + metrics: + {{- if .Values.autoscaling.workers.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.workers.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.workers.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.workers.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/deploy/helm/dograh/templates/ui-deployment.yaml b/deploy/helm/dograh/templates/ui-deployment.yaml index 9282220d..f5110ff8 100644 --- a/deploy/helm/dograh/templates/ui-deployment.yaml +++ b/deploy/helm/dograh/templates/ui-deployment.yaml @@ -8,7 +8,13 @@ metadata: {{- include "dograh.labels" . | nindent 4 }} app.kubernetes.io/component: ui spec: + {{- /* Omit spec.replicas only when the HPA object will actually render — matches + the gate in templates/ui-hpa.yaml. If HPA is enabled but both metric + targets are null the HPA is suppressed, so we must keep the static + replica count here to avoid a Deployment with no owner. */ -}} + {{- if not (and .Values.autoscaling.ui.enabled (or .Values.autoscaling.ui.targetCPUUtilizationPercentage .Values.autoscaling.ui.targetMemoryUtilizationPercentage)) }} replicas: {{ .Values.ui.replicaCount }} + {{- end }} selector: matchLabels: {{- include "dograh.selectorLabels" . | nindent 6 }} diff --git a/deploy/helm/dograh/templates/ui-hpa.yaml b/deploy/helm/dograh/templates/ui-hpa.yaml new file mode 100644 index 00000000..080e0694 --- /dev/null +++ b/deploy/helm/dograh/templates/ui-hpa.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.ui.enabled .Values.autoscaling.ui.enabled (or .Values.autoscaling.ui.targetCPUUtilizationPercentage .Values.autoscaling.ui.targetMemoryUtilizationPercentage) }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "dograh.ui.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "dograh.labels" . | nindent 4 }} + app.kubernetes.io/component: ui +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "dograh.ui.fullname" . }} + minReplicas: {{ .Values.autoscaling.ui.minReplicas }} + maxReplicas: {{ .Values.autoscaling.ui.maxReplicas }} + metrics: + {{- if .Values.autoscaling.ui.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.ui.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.ui.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.ui.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/deploy/helm/dograh/templates/web-deployment.yaml b/deploy/helm/dograh/templates/web-deployment.yaml index 459baa3b..5dcaf18c 100644 --- a/deploy/helm/dograh/templates/web-deployment.yaml +++ b/deploy/helm/dograh/templates/web-deployment.yaml @@ -7,7 +7,11 @@ metadata: {{- include "dograh.labels" . | nindent 4 }} app.kubernetes.io/component: web spec: - {{- if not .Values.autoscaling.web.enabled }} + {{- /* Omit spec.replicas only when the HPA object will actually render — matches + the gate in templates/web-hpa.yaml. If HPA is enabled but both metric + targets are null the HPA is suppressed, so we must keep the static + replica count here to avoid a Deployment with no owner. */ -}} + {{- if not (and .Values.autoscaling.web.enabled (or .Values.autoscaling.web.targetCPUUtilizationPercentage .Values.autoscaling.web.targetMemoryUtilizationPercentage)) }} replicas: {{ .Values.web.replicaCount }} {{- end }} selector: diff --git a/deploy/helm/dograh/templates/web-hpa.yaml b/deploy/helm/dograh/templates/web-hpa.yaml index cdc04868..2054f011 100644 --- a/deploy/helm/dograh/templates/web-hpa.yaml +++ b/deploy/helm/dograh/templates/web-hpa.yaml @@ -1,4 +1,4 @@ -{{- if .Values.autoscaling.web.enabled }} +{{- if and .Values.autoscaling.web.enabled (or .Values.autoscaling.web.targetCPUUtilizationPercentage .Values.autoscaling.web.targetMemoryUtilizationPercentage) }} # WARNING: CPU/memory is a poor signal for WebRTC signaling. WebSockets # are long-lived and low-CPU; this HPA will not respond to connection # pressure. Replace with a custom metric (active connections, active diff --git a/deploy/helm/dograh/values.yaml b/deploy/helm/dograh/values.yaml index 9a66a7cb..47d08fb8 100644 --- a/deploy/helm/dograh/values.yaml +++ b/deploy/helm/dograh/values.yaml @@ -453,12 +453,30 @@ migrate: memory: 512Mi # ----------------------------------------------------------------------------- -# Horizontal Pod Autoscaling — web tier only. +# Horizontal Pod Autoscaling. # -# WARNING: CPU/memory is a poor signal for WebRTC signaling workloads. +# All tiers default to `enabled: false`; opt in per tier. Requires +# metrics-server in the cluster. HPA reads Deployment `replicas` on first sync +# and then owns it, so `web.replicaCount` etc. are ignored once the +# corresponding autoscaling block is enabled — treat `minReplicas` as the new +# floor. Enabling a tier's HPA also removes `replicas` from its Deployment, so +# on the first upgrade after enabling, the tier briefly resets to 1 pod until +# the HPA reconciles — enable during a quiet window. +# +# WARNING (web): CPU/memory is a poor signal for WebRTC signaling workloads. # WebSockets are long-lived, low-CPU, and steady-memory; CPU will look flat # while you saturate per-pod connection limits. Replace this with a custom # metric (active WS connections, active calls) once one is exposed. +# +# Workers (ARQ): jobs are mostly IO-bound (webhook delivery, embeddings, LLM +# calls), so CPU can stay flat while the queue backs up. CPU HPA is a coarse +# stopgap; the long-term plan is queue-depth / active-call driven scaling via +# KEDA or a custom metric (see README TODOs). If an external scaler (e.g. a +# KEDA ScaledObject) owns the worker Deployment, keep this block disabled so +# the chart does not render a competing HPA. +# +# UI (Next.js SSR) correlates reasonably with CPU; resource-metric HPA is a +# fine choice there. # ----------------------------------------------------------------------------- autoscaling: web: @@ -467,6 +485,26 @@ autoscaling: maxReplicas: 10 targetCPUUtilizationPercentage: 70 targetMemoryUtilizationPercentage: 80 + workers: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 70 + # NOTE: memory HPA disabled by default. Idle Python (ARQ worker) at the + # chart's default 256Mi request sits near the 80% target on cold start, + # so HPA would scale to maxReplicas with no real load. Enable only after + # sizing your workload's steady-state memory well below the target. + targetMemoryUtilizationPercentage: null + ui: + enabled: false + # Floor of 2 matches ui.replicaCount and keeps ui.pdb (minAvailable: 1) + # satisfiable during node drains. Single-node installs that drop this to 1 + # should also disable ui.pdb (see examples/values-k3s-prod.yaml). + minReplicas: 2 + maxReplicas: 5 + targetCPUUtilizationPercentage: 70 + # Same reason as workers — idle Next.js SSR sits close to the 256Mi request. + targetMemoryUtilizationPercentage: null # ----------------------------------------------------------------------------- # Bundled stateful deps for the internal/all-in-one modes. These are plain