Merge branch 'main' into feat/vici-dial

api/.env.example

@@ -2,6 +2,9 @@
 ENVIRONMENT="local"
 LOG_LEVEL="DEBUG"
 
+# Set to "false" to disable public signup (invite-only installs)
+ENABLE_SIGNUP="true"
+
 # Change these values if you deploy the backend and frontend
 # on any hosting provider with some DNS. Please ensure to 
 # provide the URL with scheme like http or https
@@ -12,12 +15,24 @@ UI_APP_URL="http://localhost:3000"
 DATABASE_URL="postgresql+asyncpg://postgres:postgres@localhost:5432/postgres"
 REDIS_URL="redis://:redissecret@localhost:6379"
 
+# Internal devops secret for deployment scripts and lifecycle hooks.
+# scripts/rolling_update.sh sends this to protected operational endpoints via
+# X-Dograh-Devops-Secret. Use a unique random value in production.
+DOGRAH_DEVOPS_SECRET="change-me-dograh-devops-secret"
+
 # AWS S3 Configuration
 ENABLE_AWS_S3="false"
 # AWS_ACCESS_KEY_ID=""
 # AWS_SECRET_ACCESS_KEY=""
 # S3_BUCKET=""
 # S3_REGION=""
+# --- S3-compatible servers (MinIO, rustfs, Ceph, ...) ---
+# Use the S3 backend (ENABLE_AWS_S3=true) against a non-AWS, S3-compatible
+# server by overriding the endpoint and signing. Unlike the MinIO backend, the
+# S3 backend emits real presigned URLs, so the bucket can stay private.
+# S3_ENDPOINT_URL=""        # e.g. https://s3.example.com (blank = AWS default)
+# S3_SIGNATURE_VERSION=""   # blank = botocore default; set "s3v4" if the server requires SigV4
+# S3_ADDRESSING_STYLE=""    # blank = auto; set "path" if the server / TLS cert requires path-style
 
 # MinIO Configuration if using containerised MinIO instead of
 # AWS S3

api/.env.test.example

@@ -14,4 +14,6 @@ UI_APP_URL=http://localhost:3000
 DATABASE_URL="postgresql+asyncpg://postgres:postgres@localhost:5432/test_db"
 REDIS_URL="redis://:redissecret@localhost:6379/0"
 
+DOGRAH_DEVOPS_SECRET="test-dograh-devops-secret"
+
 MINIO_PUBLIC_ENDPOINT=http://localhost:9000

api/Dockerfile

@@ -59,56 +59,53 @@ RUN npm ci --omit=dev && npm cache clean --force
 # Stage 3: Static ffmpeg binary (avoids apt ffmpeg pulling mesa/libllvm for
 # hardware acceleration we don't use server-side).
 #
-# Resilient download: johnvansickle.com is the primary source but it's a single
-# self-hosted host with no CDN and goes down intermittently. Use bounded-timeout
-# retries, then fall back to a pinned BtbN/FFmpeg-Builds autobuild. Every archive
-# is SHA256-verified before extraction. The two sources have different internal
-# layouts, so locate the binaries with `find` rather than a fixed strip path.
+# Source: BtbN/FFmpeg-Builds, served from GitHub's release-assets CDN (fast,
+# highly available, multi-arch). We pin a specific build for reproducibility,
+# but to a *month-end* autobuild tag — not a daily one. BtbN prunes daily
+# autobuilds after ~2 weeks (the previous pin was a daily tag and started
+# 404ing once GC'd), but keeps one month-end snapshot per month long-term
+# (~2 years back). A dated tag's assets are immutable, so the per-arch sha256
+# below never rots: builds stay reproducible AND integrity-verified.
+#
+# To upgrade ffmpeg: bump BTBN_TAG + BTBN_REV to a newer month-end autobuild
+# and refresh the two sha256s. No download needed — read tag, revision and
+# per-asset sha256 straight from the GitHub release-asset metadata:
+#   gh api repos/BtbN/FFmpeg-Builds/releases/tags/<tag> \
+#     --jq '.assets[] | select(.name|test("(linux64|linuxarm64)-gpl\\.tar\\.xz$")) | "\(.name) \(.digest)"'
+#
+# `--speed-limit/--speed-time` aborts a *stalled* transfer after 30s of <1KB/s
+# (the cause of "stuck" builds) without killing a slow-but-progressing
+# download; `--max-time` is a hard backstop; `--retry` rides out transient CDN
+# hiccups. The archive nests binaries under bin/, so locate them with `find`.
 FROM debian:trixie-slim AS ffmpeg-static
 ARG TARGETARCH
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        curl ca-certificates xz-utils \
-    && rm -rf /var/lib/apt/lists/* \
-    && case "${TARGETARCH}" in \
-         amd64) \
-           primary_url="https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz" ; \
-           primary_sha256="abda8d77ce8309141f83ab8edf0596834087c52467f6badf376a6a2a4c87cf67" ; \
-           fallback_url="https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2026-05-30-13-19/ffmpeg-N-124681-gb8c5376eb4-linux64-gpl.tar.xz" ; \
-           fallback_sha256="6cfd689ee95ff128e89080af10c93f16e48760eb2acc124c5c8258dc922cc13b" ; \
-           ;; \
-         arm64) \
-           primary_url="https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-arm64-static.tar.xz" ; \
-           primary_sha256="f4149bb2b0784e30e99bdda85471c9b5930d3402014e934a5098b41d0f7201b1" ; \
-           fallback_url="https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2026-05-30-13-19/ffmpeg-N-124681-gb8c5376eb4-linuxarm64-gpl.tar.xz" ; \
-           fallback_sha256="b90a31f1d0b030f5d8a3d11cfec736e369bd5a1371b19bf65421a07f72b1d547" ; \
-           ;; \
-         *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
-       esac \
-    && mkdir -p /tmp/ffmpeg \
-    && ok= \
-    && for source in \
-         "primary ${primary_sha256} ${primary_url}" \
-         "fallback ${fallback_sha256} ${fallback_url}" ; do \
-         source_name="${source%% *}" ; \
-         source_data="${source#* }" ; \
-         sha256="${source_data%% *}" ; \
-         url="${source_data#* }" ; \
-         echo "Downloading ffmpeg (${source_name}) from ${url}" ; \
-         if curl -fsSL --connect-timeout 20 --max-time 300 \
-                 --retry 3 --retry-delay 5 --retry-all-errors \
-                 -o /tmp/ffmpeg.tar.xz "${url}" \
-             && echo "${sha256}  /tmp/ffmpeg.tar.xz" | sha256sum -c - ; then ok=1 ; break ; fi ; \
-         rm -f /tmp/ffmpeg.tar.xz ; \
-         echo "ffmpeg source failed, trying next: ${url}" >&2 ; \
-       done \
-    && [ -n "${ok}" ] || { echo "all ffmpeg download sources failed" >&2 ; exit 1 ; } \
-    && tar -xJf /tmp/ffmpeg.tar.xz -C /tmp/ffmpeg \
-    && ffmpeg_bin="$(find /tmp/ffmpeg -type f -name ffmpeg | head -n1)" \
-    && ffprobe_bin="$(find /tmp/ffmpeg -type f -name ffprobe | head -n1)" \
-    && [ -n "${ffmpeg_bin}" ] && [ -n "${ffprobe_bin}" ] \
-    && mv "${ffmpeg_bin}" "${ffprobe_bin}" /usr/local/bin/ \
-    && chmod +x /usr/local/bin/ffmpeg /usr/local/bin/ffprobe \
-    && rm -rf /tmp/ffmpeg /tmp/ffmpeg.tar.xz
+ARG BTBN_TAG=autobuild-2026-05-31-13-22
+ARG BTBN_REV=N-124714-g49a77d37be
+RUN set -eu ; \
+    apt-get update && apt-get install -y --no-install-recommends \
+        curl ca-certificates xz-utils ; \
+    rm -rf /var/lib/apt/lists/* ; \
+    case "${TARGETARCH}" in \
+      amd64) btbn_arch=linux64 ; \
+             sha256=ee052121296e6479325e09c6097d48e72a4af472d18c2b94388b5405dcde6cce ;; \
+      arm64) btbn_arch=linuxarm64 ; \
+             sha256=e97545305043794cdf7b698d713e29291464e0c35bb8e0f3ff1f62e4c56eedd6 ;; \
+      *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2 ; exit 1 ;; \
+    esac ; \
+    url="https://github.com/BtbN/FFmpeg-Builds/releases/download/${BTBN_TAG}/ffmpeg-${BTBN_REV}-${btbn_arch}-gpl.tar.xz" ; \
+    mkdir -p /tmp/ffmpeg ; cd /tmp/ffmpeg ; \
+    echo "Downloading ffmpeg (${BTBN_TAG}) from ${url}" ; \
+    curl -fsSL --connect-timeout 20 --speed-limit 1024 --speed-time 30 \
+         --max-time 600 --retry 3 --retry-delay 5 --retry-all-errors \
+         -o ffmpeg.tar.xz "${url}" ; \
+    echo "${sha256}  ffmpeg.tar.xz" | sha256sum -c - ; \
+    tar -xJf ffmpeg.tar.xz ; \
+    ffmpeg_bin="$(find /tmp/ffmpeg -type f -name ffmpeg | head -n1)" ; \
+    ffprobe_bin="$(find /tmp/ffmpeg -type f -name ffprobe | head -n1)" ; \
+    [ -n "${ffmpeg_bin}" ] && [ -n "${ffprobe_bin}" ] ; \
+    mv "${ffmpeg_bin}" "${ffprobe_bin}" /usr/local/bin/ ; \
+    chmod +x /usr/local/bin/ffmpeg /usr/local/bin/ffprobe ; \
+    rm -rf /tmp/ffmpeg
 
 # Stage 4: Runtime - Minimal image with only runtime dependencies
 FROM python:3.13-slim AS runner
@@ -144,7 +141,22 @@ ENV PYTHONUNBUFFERED=1
 # Copy application code (chown at copy-time avoids a duplicate /app layer
 # from a later `RUN chown -R`, which would double the on-disk size of /app).
 COPY --chown=dograh:dograh ./api ./api
-COPY --chown=dograh:dograh ./scripts/start_services_docker.sh ./scripts/start_services_docker.sh
+
+# Entrypoint scripts.
+#   start_services_docker.sh — single-container (docker-compose) entrypoint
+#     that runs every service in one process tree.
+#   run_*.sh                 — per-service entrypoints used by the Helm chart,
+#     which runs each workload (web, arq-worker, ari-manager,
+#     campaign-orchestrator, migrate) as its own pod. Keep this list in sync
+#     with the command:[] entries in deploy/helm/dograh/templates/*.yaml.
+COPY --chown=dograh:dograh \
+     ./scripts/start_services_docker.sh \
+     ./scripts/run_migrate.sh \
+     ./scripts/run_web.sh \
+     ./scripts/run_arq_worker.sh \
+     ./scripts/run_ari_manager.sh \
+     ./scripts/run_campaign_orchestrator.sh \
+     ./scripts/
 
 # ts_validator Node deps (built in ts-deps stage with full node:22-slim image).
 # The validator runs as a short-lived subprocess from api/mcp_server/ts_bridge.py.

api/alembic/versions/00b0201ad918_backfill_org_model_configuration_v2.py

@@ -0,0 +1,215 @@
+"""backfill org model configuration v2 from legacy user rows
+
+Revision ID: 00b0201ad918
+Revises: c52dc3cccfb0
+Create Date: 2026-07-09 12:00:00.000000
+
+"""
+
+import json
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "00b0201ad918"
+down_revision: Union[str, None] = "c52dc3cccfb0"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+# Organizations created before MODEL_CONFIGURATION_V2 only have per-user legacy
+# MODEL_CONFIGURATION rows, and the application no longer falls back to those.
+# This migration converts one member's legacy row per organization into an
+# org-level v2 row. The conversion below is a frozen copy of
+# convert_legacy_ai_model_configuration_to_v2 (api/services/configuration/
+# ai_model_configuration.py at this revision) operating on raw JSON, so the
+# migration never imports application code. API keys are deliberately carried
+# over without validation: a dead key fails at call time exactly as it did
+# before this migration.
+
+_SPEED_MIN = 0.5
+_SPEED_MAX = 2.0
+_DEFAULT_VOICE = "default"
+_DEFAULT_LANGUAGE = "multi"
+
+# One candidate row per (organization, member with a legacy config row).
+# Members are ordered the way the hosted migration script picked its
+# representative user: users whose selected organization is this org first,
+# then earliest created. The first member whose row converts wins.
+_CANDIDATE_ROWS_SQL = """
+SELECT ou.organization_id,
+       uc.configuration,
+       uc.last_validated_at
+FROM organization_users ou
+JOIN users u ON u.id = ou.user_id
+JOIN user_configurations uc
+  ON uc.user_id = u.id AND uc.key = 'MODEL_CONFIGURATION'
+WHERE NOT EXISTS (
+        SELECT 1
+        FROM organization_configurations oc
+        WHERE oc.organization_id = ou.organization_id
+          AND oc.key = 'MODEL_CONFIGURATION_V2'
+      )
+ORDER BY ou.organization_id,
+         (u.selected_organization_id IS NOT DISTINCT FROM ou.organization_id) DESC,
+         u.created_at ASC NULLS LAST,
+         u.id ASC
+"""
+
+_INSERT_SQL = """
+INSERT INTO organization_configurations
+    (organization_id, key, value, created_at, updated_at, last_validated_at)
+VALUES
+    (:organization_id, 'MODEL_CONFIGURATION_V2', CAST(:value AS json),
+     now(), now(), :last_validated_at)
+ON CONFLICT ON CONSTRAINT _organization_key_uc DO NOTHING
+"""
+
+
+def _section(configuration: dict, name: str) -> dict | None:
+    value = configuration.get(name)
+    return value if isinstance(value, dict) else None
+
+
+def _single_api_key(service: dict) -> str | None:
+    key = service.get("api_key")
+    if isinstance(key, str) and key:
+        return key
+    if isinstance(key, list) and len(key) == 1 and isinstance(key[0], str) and key[0]:
+        return key[0]
+    return None
+
+
+def _first_dograh_api_key(configuration: dict) -> str | None:
+    for name in ("llm", "tts", "stt", "embeddings", "realtime"):
+        service = _section(configuration, name)
+        if service is None or service.get("provider") != "dograh":
+            continue
+        key = _single_api_key(service)
+        if key:
+            return key
+    return None
+
+
+def _has_dograh_provider(*services: dict | None) -> bool:
+    return any(
+        service is not None and service.get("provider") == "dograh"
+        for service in services
+    )
+
+
+def _sanitized_speed(tts: dict | None) -> float:
+    speed = (tts or {}).get("speed", 1.0)
+    try:
+        speed = float(speed)
+    except (TypeError, ValueError):
+        return 1.0
+    if not _SPEED_MIN <= speed <= _SPEED_MAX:
+        return 1.0
+    return speed
+
+
+def convert_legacy_configuration_to_v2(configuration: dict) -> dict | None:
+    """Convert a legacy MODEL_CONFIGURATION JSON value to a v2 value.
+
+    Returns None when the legacy row is too incomplete to have produced a
+    working pipeline (such rows failed at pipeline startup before v2 too).
+    """
+    llm = _section(configuration, "llm")
+    tts = _section(configuration, "tts")
+    stt = _section(configuration, "stt")
+    embeddings = _section(configuration, "embeddings")
+    realtime = _section(configuration, "realtime")
+
+    dograh_key = _first_dograh_api_key(configuration)
+    if dograh_key:
+        return {
+            "version": 2,
+            "mode": "dograh",
+            "dograh": {
+                "api_key": dograh_key,
+                "voice": (tts or {}).get("voice") or _DEFAULT_VOICE,
+                "speed": _sanitized_speed(tts),
+                "language": (stt or {}).get("language") or _DEFAULT_LANGUAGE,
+            },
+        }
+
+    if configuration.get("is_realtime"):
+        # BYOK schemas reject dograh providers; a dograh provider without a
+        # single resolvable key cannot be represented in v2.
+        if realtime is None or llm is None or _has_dograh_provider(llm, embeddings):
+            return None
+        section: dict = {"realtime": realtime, "llm": llm}
+        if embeddings is not None:
+            section["embeddings"] = embeddings
+        return {
+            "version": 2,
+            "mode": "byok",
+            "byok": {"mode": "realtime", "realtime": section},
+        }
+
+    if llm is None or tts is None or stt is None:
+        return None
+    if _has_dograh_provider(llm, tts, stt, embeddings):
+        return None
+    section = {"llm": llm, "tts": tts, "stt": stt}
+    if embeddings is not None:
+        section["embeddings"] = embeddings
+    return {
+        "version": 2,
+        "mode": "byok",
+        "byok": {"mode": "pipeline", "pipeline": section},
+    }
+
+
+def upgrade() -> None:
+    connection = op.get_bind()
+    rows = connection.execute(sa.text(_CANDIDATE_ROWS_SQL)).mappings().all()
+
+    backfilled: set[int] = set()
+    seen: set[int] = set()
+    for row in rows:
+        organization_id = row["organization_id"]
+        seen.add(organization_id)
+        if organization_id in backfilled:
+            continue
+
+        configuration = row["configuration"]
+        if isinstance(configuration, str):
+            try:
+                configuration = json.loads(configuration)
+            except ValueError:
+                continue
+        if not isinstance(configuration, dict):
+            continue
+
+        try:
+            v2_value = convert_legacy_configuration_to_v2(configuration)
+        except Exception:
+            v2_value = None
+        if v2_value is None:
+            continue
+
+        connection.execute(
+            sa.text(_INSERT_SQL),
+            {
+                "organization_id": organization_id,
+                "value": json.dumps(v2_value),
+                "last_validated_at": row["last_validated_at"],
+            },
+        )
+        backfilled.add(organization_id)
+
+    skipped = sorted(seen - backfilled)
+    print(
+        f"Backfilled MODEL_CONFIGURATION_V2 for {len(backfilled)} organization(s); "
+        f"skipped {len(skipped)} with no convertible legacy configuration"
+        + (f": {skipped}" if skipped else "")
+    )
+
+
+def downgrade() -> None:
+    # Backfilled rows are indistinguishable from rows written by the
+    # application; leaving them in place is safe for older code.
+    pass

api/alembic/versions/b7e3c9a1d2f4_add_webhook_deliveries.py

@@ -0,0 +1,119 @@
+"""add webhook_deliveries
+
+Durable, retrying outbound webhook delivery so a transient network error can't
+permanently drop a workflow's final webhook.
+
+Revision ID: b7e3c9a1d2f4
+Revises: 91cc6ba3e1c7
+Create Date: 2026-06-28 19:40:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "b7e3c9a1d2f4"
+down_revision: Union[str, None] = "91cc6ba3e1c7"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "webhook_deliveries",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("delivery_uuid", sa.String(length=36), nullable=False),
+        sa.Column("workflow_run_id", sa.Integer(), nullable=False),
+        sa.Column("organization_id", sa.Integer(), nullable=False),
+        sa.Column("webhook_name", sa.String(), nullable=True),
+        sa.Column("endpoint_url", sa.String(), nullable=False),
+        sa.Column(
+            "http_method",
+            sa.String(),
+            nullable=False,
+        ),
+        sa.Column(
+            "payload",
+            sa.JSON(),
+            nullable=False,
+        ),
+        sa.Column("custom_headers", sa.JSON(), nullable=True),
+        sa.Column("credential_uuid", sa.String(length=36), nullable=True),
+        sa.Column("webhook_node_id", sa.String(), nullable=False),
+        sa.Column(
+            "status",
+            sa.Enum(
+                "pending",
+                "succeeded",
+                "dead_letter",
+                name="webhook_delivery_status",
+            ),
+            server_default=sa.text("'pending'"),
+            nullable=False,
+        ),
+        sa.Column(
+            "attempt_count",
+            sa.Integer(),
+            server_default=sa.text("0"),
+            nullable=False,
+        ),
+        sa.Column(
+            "max_attempts",
+            sa.Integer(),
+            server_default=sa.text("5"),
+            nullable=False,
+        ),
+        sa.Column("scheduled_for", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_status_code", sa.Integer(), nullable=True),
+        sa.Column("last_error", sa.Text(), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["workflow_run_id"], ["workflow_runs.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["organization_id"], ["organizations.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "workflow_run_id",
+            "webhook_node_id",
+            name="uq_webhook_deliveries_run_node",
+        ),
+    )
+    op.create_index(
+        "ix_webhook_deliveries_delivery_uuid",
+        "webhook_deliveries",
+        ["delivery_uuid"],
+        unique=True,
+    )
+    op.create_index(
+        "idx_webhook_deliveries_run",
+        "webhook_deliveries",
+        ["workflow_run_id"],
+        unique=False,
+    )
+    # Partial index for the sweeper's hot path: due pending deliveries.
+    op.create_index(
+        "idx_webhook_deliveries_pending_scheduled",
+        "webhook_deliveries",
+        ["scheduled_for"],
+        unique=False,
+        postgresql_where=sa.text("status = 'pending'"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "idx_webhook_deliveries_pending_scheduled",
+        table_name="webhook_deliveries",
+    )
+    op.drop_index("idx_webhook_deliveries_run", table_name="webhook_deliveries")
+    op.drop_index(
+        "ix_webhook_deliveries_delivery_uuid", table_name="webhook_deliveries"
+    )
+    op.drop_table("webhook_deliveries")
+    op.execute("DROP TYPE IF EXISTS webhook_delivery_status")

api/alembic/versions/c52dc3cccfb0_add_last_validated_at_on_org_config.py

@@ -0,0 +1,29 @@
+"""add last_validated_at on org config
+
+Revision ID: c52dc3cccfb0
+Revises: b7e3c9a1d2f4
+Create Date: 2026-07-09 19:18:29.550267
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "c52dc3cccfb0"
+down_revision: Union[str, None] = "b7e3c9a1d2f4"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "organization_configurations",
+        sa.Column("last_validated_at", sa.DateTime(timezone=True), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("organization_configurations", "last_validated_at")

api/constants.py

@@ -19,7 +19,23 @@ LANGFUSE_PUBLIC_KEY = os.getenv("LANGFUSE_PUBLIC_KEY")
 LANGFUSE_SECRET_KEY = os.getenv("LANGFUSE_SECRET_KEY")
 
 # URLs for deployment
-BACKEND_API_ENDPOINT = os.getenv("BACKEND_API_ENDPOINT", "http://localhost:8000")
+#
+# PUBLIC_BASE_URL is the single canonical origin a deployment is reached at
+# (scheme + host, e.g. https://203-0-113-10.sslip.io). For a standard single-host
+# install it is the only endpoint value an operator sets — the per-subsystem URLs
+# below derive from it (and from PUBLIC_HOST for the TURN/ICE host). Each derived
+# var can still be set explicitly to override it for a split deployment.
+PUBLIC_BASE_URL = os.getenv("PUBLIC_BASE_URL") or None
+PUBLIC_HOST = os.getenv("PUBLIC_HOST") or None
+
+# Public URL the backend builds webhook/callback/embed links from. Derives from
+# PUBLIC_BASE_URL (public IP / domain), falling back to localhost for local dev.
+# When this is a non-public address (localhost or a private/reserved IP) the host
+# isn't reachable from the internet, so get_backend_endpoints() resolves a running
+# Cloudflare tunnel's URL at runtime instead (see api/utils/common.py).
+BACKEND_API_ENDPOINT = (
+    os.getenv("BACKEND_API_ENDPOINT") or PUBLIC_BASE_URL or "http://localhost:8000"
+)
 UI_APP_URL = os.getenv("UI_APP_URL", "http://localhost:3010")
 
 DATABASE_URL = os.environ["DATABASE_URL"]
@@ -30,6 +46,7 @@ CORS_ALLOWED_ORIGINS = [
     o.strip() for o in os.getenv("CORS_ALLOWED_ORIGINS", "").split(",") if o.strip()
 ]
 AUTH_PROVIDER = os.getenv("AUTH_PROVIDER", "local")
+ENABLE_SIGNUP = os.getenv("ENABLE_SIGNUP", "true").lower() == "true"
 # Stack Auth public client config. These are safe to expose to the browser (the
 # publishable client key is public by design, and the project id is non-sensitive),
 # and are served to the UI at runtime via /api/v1/health so the frontend no longer
@@ -38,13 +55,19 @@ STACK_AUTH_PROJECT_ID = os.getenv("STACK_AUTH_PROJECT_ID")
 STACK_PUBLISHABLE_CLIENT_KEY = os.getenv("STACK_PUBLISHABLE_CLIENT_KEY")
 DOGRAH_MPS_SECRET_KEY = os.getenv("DOGRAH_MPS_SECRET_KEY", None)
 MPS_API_URL = os.getenv("MPS_API_URL", "https://services.dograh.com")
+DOGRAH_DEVOPS_SECRET = os.getenv("DOGRAH_DEVOPS_SECRET") or None
 
 # Storage Configuration
 ENABLE_AWS_S3 = os.getenv("ENABLE_AWS_S3", "false").lower() == "true"
 
 # MinIO Configuration
 MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT", "localhost:9000")
-MINIO_PUBLIC_ENDPOINT = os.getenv("MINIO_PUBLIC_ENDPOINT")
+# Full URL (scheme + host) browsers use to reach object storage. Derives from
+# PUBLIC_BASE_URL (remote nginx proxies /voice-audio/ to MinIO); set explicitly
+# only to point object storage at a separate origin.
+MINIO_PUBLIC_ENDPOINT = (
+    os.getenv("MINIO_PUBLIC_ENDPOINT") or PUBLIC_BASE_URL or "http://localhost:9000"
+)
 MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY", "minioadmin")
 MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY", "minioadmin")
 MINIO_BUCKET = os.getenv("MINIO_BUCKET", "voice-audio")
@@ -53,6 +76,17 @@ MINIO_SECURE = os.getenv("MINIO_SECURE", "false").lower() == "true"
 # AWS S3 Configuration
 S3_BUCKET = os.environ.get("S3_BUCKET")
 S3_REGION = os.environ.get("S3_REGION", "us-east-1")
+# Optional overrides for S3-compatible backends (e.g. MinIO, rustfs, Ceph).
+# S3_ENDPOINT_URL: full URL of a custom S3 endpoint (e.g. "https://s3.example.com").
+#   Leave unset to use AWS's default endpoint resolution.
+# S3_SIGNATURE_VERSION: botocore signature version used to sign requests and
+#   presigned URLs. Defaults to None (botocore's default, currently SigV2 for
+#   presigned URLs). Set to "s3v4" for S3-compatible servers that require SigV4.
+# S3_ADDRESSING_STYLE: "auto" (default), "path", or "virtual". Many S3-compatible
+#   servers and TLS setups require "path".
+S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL")
+S3_SIGNATURE_VERSION = os.environ.get("S3_SIGNATURE_VERSION")
+S3_ADDRESSING_STYLE = os.environ.get("S3_ADDRESSING_STYLE")
 
 # Sentry configuration
 SENTRY_DSN = os.getenv("SENTRY_DSN")
@@ -73,7 +107,7 @@ LOG_LEVEL = os.getenv("LOG_LEVEL", "DEBUG").upper()
 LOG_ROTATION_SIZE = os.getenv("LOG_ROTATION_SIZE", "100 MB")
 LOG_RETENTION = os.getenv("LOG_RETENTION", "7 days")
 LOG_COMPRESSION = os.getenv("LOG_COMPRESSION", "gz")
-ENABLE_TELEMETRY = os.getenv("ENABLE_TELEMETRY", "false").lower() == "true"
+ENABLE_TELEMETRY = os.getenv("ENABLE_TELEMETRY", "true").lower() == "true"
 
 
 def _get_version() -> str:
@@ -117,7 +151,11 @@ COUNTRY_CODES = {
     "IE": "353",  # Ireland
 }
 
-DEFAULT_ORG_CONCURRENCY_LIMIT = os.getenv("DEFAULT_ORG_CONCURRENCY_LIMIT", 2)
+# Floor at 1 so a misconfigured env var (0 or negative) can't silently block
+# every call in the deployment.
+DEFAULT_ORG_CONCURRENCY_LIMIT = max(
+    1, int(os.getenv("DEFAULT_ORG_CONCURRENCY_LIMIT", "10"))
+)
 DEFAULT_CAMPAIGN_RETRY_CONFIG = {
     "enabled": True,
     "max_retries": 1,
@@ -128,6 +166,18 @@ DEFAULT_CAMPAIGN_RETRY_CONFIG = {
 }
 
 
+# Outbound webhook delivery: bounded retry with exponential backoff.
+# Delivery is persisted (see WebhookDeliveryModel) and retried by an ARQ task so a
+# transient network error can't permanently drop a final webhook. After
+# ``max_attempts`` transient failures the delivery is parked as ``dead_letter``.
+DEFAULT_WEBHOOK_DELIVERY_CONFIG = {
+    "max_attempts": int(os.getenv("WEBHOOK_DELIVERY_MAX_ATTEMPTS", 5)),
+    "base_delay_seconds": int(os.getenv("WEBHOOK_DELIVERY_BASE_DELAY_SECONDS", 30)),
+    "max_delay_seconds": int(os.getenv("WEBHOOK_DELIVERY_MAX_DELAY_SECONDS", 600)),
+    "timeout_seconds": int(os.getenv("WEBHOOK_DELIVERY_TIMEOUT_SECONDS", 30)),
+}
+
+
 # Circuit breaker defaults for campaign call failure detection
 DEFAULT_CIRCUIT_BREAKER_CONFIG = {
     "enabled": True,
@@ -138,7 +188,9 @@ DEFAULT_CIRCUIT_BREAKER_CONFIG = {
 
 
 TURN_SECRET = os.getenv("TURN_SECRET")
-TURN_HOST = os.getenv("TURN_HOST", "localhost")
+# Host browsers dial for TURN/ICE. Derives from PUBLIC_HOST; set explicitly only
+# when the TURN server runs on a separate host from the app.
+TURN_HOST = os.getenv("TURN_HOST") or PUBLIC_HOST or "localhost"
 TURN_PORT = int(os.getenv("TURN_PORT", "3478"))
 TURN_TLS_PORT = int(os.getenv("TURN_TLS_PORT", "5349"))
 TURN_CREDENTIAL_TTL = int(os.getenv("TURN_CREDENTIAL_TTL", "86400"))

api/db/db_client.py

@@ -14,6 +14,7 @@ from api.db.telephony_phone_number_client import TelephonyPhoneNumberClient
 from api.db.tool_client import ToolClient
 from api.db.user_client import UserClient
 from api.db.webhook_credential_client import WebhookCredentialClient
+from api.db.webhook_delivery_client import WebhookDeliveryClient
 from api.db.workflow_client import WorkflowClient
 from api.db.workflow_recording_client import WorkflowRecordingClient
 from api.db.workflow_run_client import WorkflowRunClient
@@ -37,6 +38,7 @@ class DBClient(
     EmbedTokenClient,
     AgentTriggerClient,
     WebhookCredentialClient,
+    WebhookDeliveryClient,
     ToolClient,
     KnowledgeBaseClient,
     WorkflowRecordingClient,
@@ -62,6 +64,7 @@ class DBClient(
     - EmbedTokenClient: handles embed token and session operations
     - AgentTriggerClient: handles agent trigger operations for API-based call triggering
     - WebhookCredentialClient: handles webhook credential operations
+    - WebhookDeliveryClient: handles durable outbound webhook delivery records
     - ToolClient: handles tool operations for reusable HTTP API tools
     - KnowledgeBaseClient: handles knowledge base document and vector search operations
     - FolderClient: handles folder operations for grouping workflows (agents)

api/db/knowledge_base_client.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import List, Optional
 
 from loguru import logger
-from sqlalchemy import select
+from sqlalchemy import delete, select
 from sqlalchemy.orm import selectinload
 
 from api.db.base_client import BaseDBClient
@@ -300,6 +300,31 @@ class KnowledgeBaseClient(BaseDBClient):
             logger.info(f"Created {len(chunks)} chunks")
             return chunks
 
+    async def replace_chunks_for_document(
+        self,
+        document_id: int,
+        organization_id: int,
+        chunks: List[KnowledgeBaseChunkModel],
+    ) -> List[KnowledgeBaseChunkModel]:
+        """Replace all chunks for a document with a new precomputed batch."""
+        async with self.async_session() as session:
+            await session.execute(
+                delete(KnowledgeBaseChunkModel).where(
+                    KnowledgeBaseChunkModel.document_id == document_id,
+                    KnowledgeBaseChunkModel.organization_id == organization_id,
+                )
+            )
+            session.add_all(chunks)
+            await session.commit()
+
+            for chunk in chunks:
+                await session.refresh(chunk)
+
+            logger.info(
+                f"Replaced chunks for document {document_id}: {len(chunks)} chunks"
+            )
+            return chunks
+
     async def get_chunks_for_document(
         self,
         document_id: int,

api/db/models.py

@@ -205,11 +205,8 @@ class OrganizationConfigurationModel(Base):
     key = Column(String, nullable=False)
     value = Column(JSON, nullable=False, default=dict)
     created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
-    updated_at = Column(
-        DateTime(timezone=True),
-        default=lambda: datetime.now(UTC),
-        onupdate=lambda: datetime.now(UTC),
-    )
+    updated_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
+    last_validated_at = Column(DateTime(timezone=True), nullable=True)
 
     # Relationships
     organization = relationship("OrganizationModel", back_populates="configurations")
@@ -1022,6 +1019,103 @@ class ExternalCredentialModel(Base):
     )
 
 
+class WebhookDeliveryModel(Base):
+    """Durable record of an outbound webhook delivery attempt.
+
+    Final webhooks (e.g. a workflow's "Final Webhook" node) must not be lost to a
+    single transient network error. Instead of firing the HTTP request inline and
+    forgetting it, we persist one row per webhook node per workflow run and let an
+    ARQ task drive delivery with bounded, backed-off retries. The row is the source
+    of truth: it survives worker restarts and a periodic sweeper re-enqueues any
+    ``pending`` delivery whose ``scheduled_for`` is overdue. After ``max_attempts``
+    transient failures (or on a permanent 4xx) the row is parked as ``dead_letter``
+    for inspection rather than retried forever.
+
+    Mirrors the campaign retry pattern (``QueuedRunModel``): persisted state,
+    ``scheduled_for`` gating, a hard attempt ceiling, and a terminal failure state.
+    """
+
+    __tablename__ = "webhook_deliveries"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    # Stable idempotency key sent to the receiver so it can dedupe retries.
+    delivery_uuid = Column(
+        String(36),
+        unique=True,
+        nullable=False,
+        index=True,
+        default=lambda: str(uuid.uuid4()),
+    )
+
+    workflow_run_id = Column(
+        Integer,
+        ForeignKey("workflow_runs.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    organization_id = Column(
+        Integer, ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False
+    )
+
+    # Frozen request definition. The payload is rendered once at enqueue time so
+    # retries are deterministic. Secrets are NOT stored here: the auth header is
+    # re-resolved from ``credential_uuid`` at send time (honours rotation/revocation).
+    webhook_name = Column(String, nullable=True)
+    endpoint_url = Column(String, nullable=False)
+    http_method = Column(String, nullable=False, default="POST")
+    payload = Column(JSON, nullable=False, default=dict)
+    custom_headers = Column(JSON, nullable=True)
+    credential_uuid = Column(String(36), nullable=True)
+
+    # Workflow node that produced this delivery. Combined with workflow_run_id it
+    # is the per-run/per-node idempotency key, so a retried run_integrations does
+    # not create (and send) a duplicate delivery for the same node. Non-nullable:
+    # a NULL would be distinct under the unique constraint and defeat the dedupe.
+    webhook_node_id = Column(String, nullable=False)
+
+    status = Column(
+        Enum(
+            "pending",
+            "succeeded",
+            "dead_letter",
+            name="webhook_delivery_status",
+        ),
+        nullable=False,
+        default="pending",
+        server_default="pending",
+    )
+    attempt_count = Column(Integer, nullable=False, default=0, server_default=text("0"))
+    max_attempts = Column(Integer, nullable=False, default=5, server_default=text("5"))
+    # When the next attempt becomes due. NULL once terminal (succeeded/dead_letter).
+    scheduled_for = Column(DateTime(timezone=True), nullable=True)
+
+    last_status_code = Column(Integer, nullable=True)
+    last_error = Column(Text, nullable=True)
+
+    created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
+    updated_at = Column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(UTC),
+        onupdate=lambda: datetime.now(UTC),
+    )
+
+    __table_args__ = (
+        # Sweeper lookup: due pending deliveries.
+        Index(
+            "idx_webhook_deliveries_pending_scheduled",
+            "scheduled_for",
+            postgresql_where=text("status = 'pending'"),
+        ),
+        Index("idx_webhook_deliveries_run", "workflow_run_id"),
+        # Per-run/per-node idempotency: one delivery per webhook node per run.
+        UniqueConstraint(
+            "workflow_run_id",
+            "webhook_node_id",
+            name="uq_webhook_deliveries_run_node",
+        ),
+    )
+
+
 class ToolModel(Base):
     """Model for storing reusable tools that can be invoked during workflows.
 

api/db/organization_client.py

@@ -1,6 +1,7 @@
 from datetime import datetime, timezone
 from typing import Optional
 
+from sqlalchemy import exists
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.future import select
 
@@ -8,6 +9,7 @@ from api.db.base_client import BaseDBClient
 from api.db.models import (
     APIKeyModel,
     OrganizationModel,
+    UserModel,
     organization_users_association,
 )
 from api.utils.api_key import generate_api_key
@@ -24,6 +26,22 @@ class OrganizationClient(BaseDBClient):
             )
             return result.scalars().first()
 
+    async def get_organization_users(self, organization_id: int) -> list[UserModel]:
+        """Get all users linked to an organization (many-to-many)."""
+        async with self.async_session() as session:
+            result = await session.execute(
+                select(UserModel)
+                .join(
+                    organization_users_association,
+                    organization_users_association.c.user_id == UserModel.id,
+                )
+                .where(
+                    organization_users_association.c.organization_id == organization_id
+                )
+                .order_by(UserModel.id)
+            )
+            return list(result.scalars().all())
+
     async def get_or_create_organization_by_provider_id(
         self, org_provider_id: str, user_id: int
     ) -> tuple[OrganizationModel, bool]:
@@ -91,6 +109,24 @@ class OrganizationClient(BaseDBClient):
                 return organization, was_created
             return organization, False
 
+    async def is_user_member_of_organization(
+        self, user_id: int, organization_id: int
+    ) -> bool:
+        """Return True if the user belongs to the given organization."""
+        async with self.async_session() as session:
+            result = await session.execute(
+                select(
+                    exists().where(
+                        (organization_users_association.c.user_id == user_id)
+                        & (
+                            organization_users_association.c.organization_id
+                            == organization_id
+                        )
+                    )
+                )
+            )
+            return bool(result.scalar())
+
     async def add_user_to_organization(
         self, user_id: int, organization_id: int
     ) -> None:

api/db/organization_configuration_client.py

@@ -1,3 +1,4 @@
+from datetime import UTC, datetime
 from typing import Any, Dict, List, Optional
 
 from sqlalchemy.future import select
@@ -33,10 +34,15 @@ class OrganizationConfigurationClient(BaseDBClient):
             return result.scalars().all()
 
     async def upsert_configuration(
-        self, organization_id: int, key: str, value: Any
+        self,
+        organization_id: int,
+        key: str,
+        value: Any,
+        last_validated_at: datetime | None = None,
     ) -> OrganizationConfigurationModel:
         """Create or update a configuration for an organization."""
         async with self.async_session() as session:
+            now = datetime.now(UTC)
             # First try to get existing configuration
             result = await session.execute(
                 select(OrganizationConfigurationModel).where(
@@ -49,12 +55,16 @@ class OrganizationConfigurationClient(BaseDBClient):
             if config:
                 # Update existing configuration
                 config.value = value
+                config.updated_at = now
+                config.last_validated_at = last_validated_at
             else:
                 # Create new configuration
                 config = OrganizationConfigurationModel(
                     organization_id=organization_id,
                     key=key,
                     value=value,
+                    updated_at=now,
+                    last_validated_at=last_validated_at,
                 )
                 session.add(config)
 
@@ -66,6 +76,30 @@ class OrganizationConfigurationClient(BaseDBClient):
             await session.refresh(config)
             return config
 
+    async def mark_configuration_validated(
+        self, organization_id: int, key: str
+    ) -> Optional[OrganizationConfigurationModel]:
+        """Update the validation timestamp for an existing organization configuration."""
+        async with self.async_session() as session:
+            result = await session.execute(
+                select(OrganizationConfigurationModel).where(
+                    OrganizationConfigurationModel.organization_id == organization_id,
+                    OrganizationConfigurationModel.key == key,
+                )
+            )
+            config = result.scalars().first()
+            if not config:
+                return None
+
+            config.last_validated_at = datetime.now(UTC)
+            try:
+                await session.commit()
+            except Exception as e:
+                await session.rollback()
+                raise e
+            await session.refresh(config)
+            return config
+
     async def delete_configuration(self, organization_id: int, key: str) -> bool:
         """Delete a configuration for an organization."""
         async with self.async_session() as session:

api/db/organization_usage_client.py

@@ -13,13 +13,10 @@ from api.db.models import (
     OrganizationConfigurationModel,
     OrganizationModel,
     OrganizationUsageCycleModel,
-    UserConfigurationModel,
-    UserModel,
     WorkflowModel,
     WorkflowRunModel,
 )
-from api.enums import OrganizationConfigurationKey, UserConfigurationKey
-from api.schemas.ai_model_configuration import EffectiveAIModelConfiguration
+from api.enums import OrganizationConfigurationKey
 from api.utils.recording_artifacts import get_recording_storage_key
 
 
@@ -139,9 +136,8 @@ class OrganizationUsageClient(BaseDBClient):
             query = (
                 select(WorkflowRunModel)
                 .join(WorkflowModel, WorkflowRunModel.workflow_id == WorkflowModel.id)
-                .join(UserModel, WorkflowModel.user_id == UserModel.id)
                 .where(
-                    UserModel.selected_organization_id == organization_id,
+                    WorkflowModel.organization_id == organization_id,
                     WorkflowRunModel.usage_info.isnot(None),
                 )
                 .order_by(WorkflowRunModel.created_at.desc())
@@ -277,9 +273,8 @@ class OrganizationUsageClient(BaseDBClient):
                     WorkflowRunModel.public_access_token,
                 )
                 .join(WorkflowModel, WorkflowRunModel.workflow_id == WorkflowModel.id)
-                .join(UserModel, WorkflowModel.user_id == UserModel.id)
                 .where(
-                    UserModel.selected_organization_id == organization_id,
+                    WorkflowModel.organization_id == organization_id,
                     WorkflowRunModel.usage_info.isnot(None),
                 )
                 .order_by(WorkflowRunModel.created_at.desc())
@@ -316,7 +311,6 @@ class OrganizationUsageClient(BaseDBClient):
         start_date: datetime,
         end_date: datetime,
         price_per_second_usd: float,
-        user_id: Optional[int] = None,
     ) -> dict:
         """Get daily usage breakdown for an organization with pricing."""
 
@@ -344,22 +338,6 @@ class OrganizationUsageClient(BaseDBClient):
             if pref_obj and pref_obj.value:
                 user_timezone = pref_obj.value.get("timezone") or user_timezone
 
-            if user_id:
-                config_result = await session.execute(
-                    select(UserConfigurationModel).where(
-                        UserConfigurationModel.user_id == user_id,
-                        UserConfigurationModel.key
-                        == UserConfigurationKey.MODEL_CONFIGURATION.value,
-                    )
-                )
-                config_obj = config_result.scalar_one_or_none()
-                if config_obj and config_obj.configuration:
-                    effective_config = EffectiveAIModelConfiguration.model_validate(
-                        config_obj.configuration
-                    )
-                    if effective_config.timezone and user_timezone == "UTC":
-                        user_timezone = effective_config.timezone
-
             # Validate timezone string
             try:
                 # Test if timezone is valid
@@ -382,9 +360,8 @@ class OrganizationUsageClient(BaseDBClient):
                     func.count(WorkflowRunModel.id).label("call_count"),
                 )
                 .join(WorkflowModel, WorkflowModel.id == WorkflowRunModel.workflow_id)
-                .join(UserModel, UserModel.id == WorkflowModel.user_id)
                 .where(
-                    UserModel.selected_organization_id == organization_id,
+                    WorkflowModel.organization_id == organization_id,
                     WorkflowRunModel.created_at >= start_date,
                     WorkflowRunModel.created_at <= end_date,
                     WorkflowRunModel.is_completed == True,

api/db/telephony_configuration_client.py

@@ -103,6 +103,30 @@ class TelephonyConfigurationClient(BaseDBClient):
             )
             return int(result.scalar() or 0)
 
+    async def count_vonage_configs_missing_signature_secret(
+        self, organization_id: int
+    ) -> int:
+        """Count Vonage configs in this org with no signature_secret."""
+        async with self.async_session() as session:
+            result = await session.execute(
+                select(func.count(TelephonyConfigurationModel.id)).where(
+                    TelephonyConfigurationModel.organization_id == organization_id,
+                    TelephonyConfigurationModel.provider == "vonage",
+                    (
+                        TelephonyConfigurationModel.credentials.op("->>")(
+                            "signature_secret"
+                        ).is_(None)
+                    )
+                    | (
+                        TelephonyConfigurationModel.credentials.op("->>")(
+                            "signature_secret"
+                        )
+                        == ""
+                    ),
+                )
+            )
+            return int(result.scalar() or 0)
+
     async def list_all_telephony_configurations_by_provider(
         self, provider: str
     ) -> List[TelephonyConfigurationModel]:

api/db/webhook_delivery_client.py

@@ -0,0 +1,244 @@
+"""Database client for durable outbound webhook deliveries.
+
+Persists one row per webhook node per workflow run and exposes the state
+transitions the delivery task and sweeper need: create (pending), succeed,
+schedule the next retry, and park as dead-letter. Mirrors the campaign retry
+pattern -- the row is the source of truth, ``scheduled_for`` gates due work.
+"""
+
+from datetime import UTC, datetime, timedelta
+from typing import List, Optional, Tuple
+
+from loguru import logger
+from sqlalchemy import or_, select, update
+from sqlalchemy.exc import IntegrityError
+
+from api.db.base_client import BaseDBClient
+from api.db.models import WebhookDeliveryModel, WorkflowModel, WorkflowRunModel
+
+
+class WebhookDeliveryClient(BaseDBClient):
+    """Client for managing persisted webhook delivery records."""
+
+    async def create_webhook_delivery(
+        self,
+        workflow_run_id: int,
+        organization_id: int,
+        endpoint_url: str,
+        payload: dict,
+        max_attempts: int,
+        http_method: str = "POST",
+        webhook_name: Optional[str] = None,
+        custom_headers: Optional[list] = None,
+        credential_uuid: Optional[str] = None,
+        webhook_node_id: Optional[str] = None,
+        scheduled_for: Optional[datetime] = None,
+    ) -> Tuple[WebhookDeliveryModel, bool]:
+        """Get-or-create the ``pending`` delivery for this run + webhook node.
+
+        Idempotent on ``(workflow_run_id, webhook_node_id)``: a retried
+        ``run_integrations`` returns the existing row instead of creating (and
+        sending) a duplicate. Returns ``(delivery, created)`` so the caller only
+        enqueues a send for a freshly-created row.
+        """
+        async with self.async_session() as session:
+            run_scope_result = await session.execute(
+                select(WorkflowRunModel.id, WorkflowModel.organization_id)
+                .join(WorkflowModel, WorkflowRunModel.workflow_id == WorkflowModel.id)
+                .where(WorkflowRunModel.id == workflow_run_id)
+            )
+            run_scope = run_scope_result.one_or_none()
+            if run_scope is None:
+                raise ValueError(f"Workflow run {workflow_run_id} not found")
+
+            _, run_organization_id = run_scope
+            if run_organization_id is None:
+                raise ValueError(
+                    f"Workflow run {workflow_run_id} is not associated with an organization"
+                )
+            if run_organization_id != organization_id:
+                raise ValueError(
+                    f"Workflow run {workflow_run_id} belongs to organization "
+                    f"{run_organization_id}, not {organization_id}"
+                )
+
+            delivery = WebhookDeliveryModel(
+                workflow_run_id=workflow_run_id,
+                organization_id=organization_id,
+                webhook_name=webhook_name,
+                webhook_node_id=webhook_node_id,
+                endpoint_url=endpoint_url,
+                http_method=http_method,
+                payload=payload,
+                custom_headers=custom_headers,
+                credential_uuid=credential_uuid,
+                max_attempts=max_attempts,
+                status="pending",
+                attempt_count=0,
+                scheduled_for=scheduled_for or datetime.now(UTC),
+            )
+            session.add(delivery)
+            try:
+                await session.commit()
+            except IntegrityError:
+                await session.rollback()
+                existing = await session.execute(
+                    select(WebhookDeliveryModel).where(
+                        WebhookDeliveryModel.workflow_run_id == workflow_run_id,
+                        WebhookDeliveryModel.webhook_node_id == webhook_node_id,
+                    )
+                )
+                row = existing.scalar_one_or_none()
+                if row is not None:
+                    return row, False
+                # The violation was not the run+node uniqueness -- re-raise.
+                raise
+            await session.refresh(delivery)
+            return delivery, True
+
+    async def get_webhook_delivery(
+        self, delivery_id: int
+    ) -> Optional[WebhookDeliveryModel]:
+        async with self.async_session() as session:
+            result = await session.execute(
+                select(WebhookDeliveryModel).where(
+                    WebhookDeliveryModel.id == delivery_id
+                )
+            )
+            return result.scalar_one_or_none()
+
+    async def claim_webhook_delivery(
+        self, delivery_id: int, lease_seconds: int
+    ) -> Optional[WebhookDeliveryModel]:
+        """Atomically claim a pending, due delivery for one worker to process.
+
+        A conditional UPDATE pushes ``scheduled_for`` out by a short lease. Only
+        one concurrent worker can win -- the others re-evaluate the WHERE after
+        the first commits, see the future ``scheduled_for``, match nothing, and
+        get ``None``. This prevents the non-atomic ``status == 'pending'`` read
+        from letting two workers double-send the same delivery. If the winning
+        worker crashes mid-send, the lease expires and the sweeper re-enqueues it.
+
+        Returns the claimed row, or ``None`` if it was not claimable (already
+        claimed, not pending, or not yet due).
+        """
+        now = datetime.now(UTC)
+        lease_until = now + timedelta(seconds=lease_seconds)
+        async with self.async_session() as session:
+            result = await session.execute(
+                update(WebhookDeliveryModel)
+                .where(
+                    WebhookDeliveryModel.id == delivery_id,
+                    WebhookDeliveryModel.status == "pending",
+                    or_(
+                        WebhookDeliveryModel.scheduled_for.is_(None),
+                        WebhookDeliveryModel.scheduled_for <= now,
+                    ),
+                )
+                .values(scheduled_for=lease_until, updated_at=now)
+            )
+            await session.commit()
+            if result.rowcount == 0:
+                return None
+            fetched = await session.execute(
+                select(WebhookDeliveryModel).where(
+                    WebhookDeliveryModel.id == delivery_id
+                )
+            )
+            return fetched.scalar_one_or_none()
+
+    async def mark_webhook_delivery_succeeded(
+        self, delivery_id: int, attempt_count: int, status_code: Optional[int]
+    ) -> None:
+        async with self.async_session() as session:
+            await session.execute(
+                update(WebhookDeliveryModel)
+                .where(WebhookDeliveryModel.id == delivery_id)
+                .values(
+                    status="succeeded",
+                    attempt_count=attempt_count,
+                    last_status_code=status_code,
+                    last_error=None,
+                    scheduled_for=None,
+                    updated_at=datetime.now(UTC),
+                )
+            )
+            await session.commit()
+
+    async def schedule_webhook_delivery_retry(
+        self,
+        delivery_id: int,
+        attempt_count: int,
+        scheduled_for: datetime,
+        last_error: str,
+        last_status_code: Optional[int],
+    ) -> None:
+        """Record a transient failure and set when the next attempt is due."""
+        async with self.async_session() as session:
+            await session.execute(
+                update(WebhookDeliveryModel)
+                .where(WebhookDeliveryModel.id == delivery_id)
+                .values(
+                    status="pending",
+                    attempt_count=attempt_count,
+                    scheduled_for=scheduled_for,
+                    last_error=last_error[:2000] if last_error else last_error,
+                    last_status_code=last_status_code,
+                    updated_at=datetime.now(UTC),
+                )
+            )
+            await session.commit()
+
+    async def mark_webhook_delivery_dead_letter(
+        self,
+        delivery_id: int,
+        attempt_count: int,
+        last_error: str,
+        last_status_code: Optional[int],
+    ) -> None:
+        """Terminal failure: parked for inspection, never retried again."""
+        async with self.async_session() as session:
+            await session.execute(
+                update(WebhookDeliveryModel)
+                .where(WebhookDeliveryModel.id == delivery_id)
+                .values(
+                    status="dead_letter",
+                    attempt_count=attempt_count,
+                    last_error=last_error[:2000] if last_error else last_error,
+                    last_status_code=last_status_code,
+                    scheduled_for=None,
+                    updated_at=datetime.now(UTC),
+                )
+            )
+            await session.commit()
+            logger.warning(
+                f"Webhook delivery {delivery_id} dead-lettered after "
+                f"{attempt_count} attempts: {last_error}"
+            )
+
+    async def get_due_webhook_deliveries(
+        self, now: Optional[datetime] = None, limit: int = 100, after_id: int = 0
+    ) -> List[WebhookDeliveryModel]:
+        """One page of pending deliveries whose next attempt is due.
+
+        Used by the periodic sweeper to re-enqueue deliveries whose ARQ job was
+        lost (worker restart, Redis flush). The delivery task is idempotent, so a
+        spurious re-enqueue is harmless. Ordered by ``id`` and gated on
+        ``after_id`` for keyset pagination -- re-enqueuing does not change a row's
+        due state, so the sweeper pages by id to drain the whole backlog instead
+        of re-reading the same first page forever.
+        """
+        cutoff = now or datetime.now(UTC)
+        async with self.async_session() as session:
+            result = await session.execute(
+                select(WebhookDeliveryModel)
+                .where(
+                    WebhookDeliveryModel.status == "pending",
+                    WebhookDeliveryModel.scheduled_for.isnot(None),
+                    WebhookDeliveryModel.scheduled_for <= cutoff,
+                    WebhookDeliveryModel.id > after_id,
+                )
+                .order_by(WebhookDeliveryModel.id)
+                .limit(limit)
+            )
+            return list(result.scalars().all())

api/db/workflow_client.py

@@ -606,11 +606,9 @@ class WorkflowClient(BaseDBClient):
         """Get workflows by IDs for a specific organization"""
         async with self.async_session() as session:
             result = await session.execute(
-                select(WorkflowModel)
-                .join(WorkflowModel.user)
-                .where(
+                select(WorkflowModel).where(
                     WorkflowModel.id.in_(workflow_ids),
-                    WorkflowModel.user.has(selected_organization_id=organization_id),
+                    WorkflowModel.organization_id == organization_id,
                 )
             )
             return result.scalars().all()

api/db/workflow_run_client.py

@@ -40,14 +40,14 @@ class WorkflowRunClient(BaseDBClient):
             workflow_query = (
                 select(WorkflowModel)
                 .options(joinedload(WorkflowModel.user))
-                .where(
-                    WorkflowModel.id == workflow_id, WorkflowModel.user_id == user_id
-                )
+                .where(WorkflowModel.id == workflow_id)
             )
             if organization_id is not None:
                 workflow_query = workflow_query.where(
                     WorkflowModel.organization_id == organization_id
                 )
+            elif user_id is not None:
+                workflow_query = workflow_query.where(WorkflowModel.user_id == user_id)
 
             workflow = await session.execute(workflow_query)
             workflow = workflow.scalars().first()
@@ -93,12 +93,17 @@ class WorkflowRunClient(BaseDBClient):
                 else workflow.template_context_variables
             )
 
+            merged_initial_context = {
+                **(default_context or {}),
+                **(initial_context or {}),
+            }
+
             new_run = WorkflowRunModel(
                 name=name,
                 workflow=workflow,
                 mode=mode,
                 definition_id=target_def.id if target_def else None,
-                initial_context=initial_context or default_context,
+                initial_context=merged_initial_context,
                 gathered_context=gathered_context or {},
                 logs=logs or {},
                 campaign_id=campaign_id,
@@ -433,10 +438,10 @@ class WorkflowRunClient(BaseDBClient):
             if not workflow_run:
                 return None, None
 
-            if not workflow_run.workflow or not workflow_run.workflow.user:
+            if not workflow_run.workflow:
                 return workflow_run, None
 
-            organization_id = workflow_run.workflow.user.selected_organization_id
+            organization_id = workflow_run.workflow.organization_id
             return workflow_run, organization_id
 
     async def ensure_public_access_token(self, workflow_run_id: int) -> Optional[str]:

api/enums.py

@@ -17,6 +17,32 @@ class CallType(Enum):
     OUTBOUND = "outbound"
 
 
+class TelephonyCallStatus(str, Enum):
+    INITIATED = "initiated"
+    RINGING = "ringing"
+    IN_PROGRESS = "in-progress"
+    ANSWERED = "answered"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    BUSY = "busy"
+    NO_ANSWER = "no-answer"
+    CANCELED = "canceled"
+    ERROR = "error"
+
+    @classmethod
+    def from_raw(cls, value: object) -> "TelephonyCallStatus | None":
+        if isinstance(value, cls):
+            return value
+
+        if value in (None, ""):
+            return None
+
+        try:
+            return cls(str(value).lower())
+        except ValueError:
+            return None
+
+
 class WorkflowRunMode(Enum):
     ARI = "ari"
     PLIVO = "plivo"
@@ -77,8 +103,6 @@ class WorkflowRunStatus(Enum):
 
 
 class OrganizationConfigurationKey(Enum):
-    DISPOSITION_CODE_MAPPING = "DISPOSITION_CODE_MAPPING"
-    DISPOSITION_MESSAGE_TEMPLATE = "DISPOSITION_MESSAGE_TEMPLATE"
     CONCURRENT_CALL_LIMIT = "CONCURRENT_CALL_LIMIT"
     TELEPHONY_CONFIGURATION = (
         "TELEPHONY_CONFIGURATION"  # Stores all providers + active one
@@ -176,3 +200,5 @@ class PostHogEvent(str, Enum):
     SIGNED_IN = "signed_in"
     ORGANIZATION_CREATED = "organization_created"
     ORGANIZATION_USER_ASSOCIATED = "organization_user_associated"
+    # usage_* events track orgs hitting capacity/limit boundaries
+    USAGE_CONCURRENT_CALL_LIMIT_REACHED = "usage_concurrent_call_limit_reached"

api/errors/telephony_errors.py

@@ -14,6 +14,7 @@ class TelephonyError(Enum):
     ACCOUNT_VALIDATION_FAILED = "ACCOUNT_VALIDATION_FAILED"
     PHONE_NUMBER_NOT_CONFIGURED = "PHONE_NUMBER_NOT_CONFIGURED"
     SIGNATURE_VALIDATION_FAILED = "SIGNATURE_VALIDATION_FAILED"
+    CONCURRENT_CALL_LIMIT = "CONCURRENT_CALL_LIMIT"
     QUOTA_EXCEEDED = "QUOTA_EXCEEDED"
     GENERAL_AUTH_FAILED = "GENERAL_AUTH_FAILED"
     VALID = "VALID"
@@ -26,6 +27,7 @@ TELEPHONY_ERROR_MESSAGES = {
     TelephonyError.ACCOUNT_VALIDATION_FAILED: "Authentication error: Account credentials do not match. Please verify your account SID configuration in the dashboard matches your telephony provider settings.",
     TelephonyError.PHONE_NUMBER_NOT_CONFIGURED: "Phone number not configured: This number is not set up for inbound calls in your account. Please add this number to your telephony configuration.",
     TelephonyError.SIGNATURE_VALIDATION_FAILED: "Security error: Webhook signature validation failed. Please verify your auth token configuration and ensure requests are coming from your telephony provider.",
+    TelephonyError.CONCURRENT_CALL_LIMIT: "Service temporarily unavailable: Your account has reached its concurrent call limit. Please try again later.",
     TelephonyError.QUOTA_EXCEEDED: "Service temporarily unavailable: Your account has exceeded usage limits. Please contact your administrator or upgrade your plan to continue receiving calls.",
     TelephonyError.GENERAL_AUTH_FAILED: "Authentication failed: Please check your webhook URL configuration and ensure your telephony provider settings match your dashboard configuration.",
 }

api/mcp_server/instructions.py

@@ -20,26 +20,26 @@ mistake the system has seen at least once.
 """
 
 DOGRAH_MCP_INSTRUCTIONS = """\
-You build and edit Dograh voice-AI workflows by emitting TypeScript that uses the `@dograh/sdk` package. Workflows are stored as JSON; this server projects them to TypeScript for editing and parses them back on save.
+You build and edit Dograh voice-AI workflows **interactively** with the user using TypeScript that uses the `@dograh/sdk` package. Workflows are stored as JSON; this server projects them to TypeScript for editing and parses them back on save.
 
-## Stages
+## Planning and workflow creation
 
-Every authoring session runs through three stages. Inject the right guidance at each by calling `get_voice_prompting_guide` before you write or revise prompts. Do not skip plan when creating; do not skip review when editing prompt-bearing fields.
+Every authoring session runs through three stages. Inject the right guidance at each by calling `get_voice_prompting_guide` before you write or revise prompts. You must go through plan phase to gather context from the builder before attempting to build the agent.
 
-1. **Plan** — call `get_voice_prompting_guide` with `stage="plan"` first. Decide persona, ordered node list, edges, exit conditions, and tools/credentials needed. Enumerate available `list_node_types`, `list_tools`, `list_credentials`, `list_documents`, `list_recordings` as needed. Present a structured plan to the user and wait for confirmation before writing any code.
+1. **Plan** — call `get_voice_prompting_guide` with `stage="plan"` first. Ask the relevant contextual questions to the user and reinforce with guidelines from the prompting guide. Decide persona, ordered node list, edges, exit conditions, and tools/credentials needed. Present a structured plan to the user and wait for confirmation before writing any code.
 
-2. **Create** — call `get_voice_prompting_guide` with `stage="create"` and (when applicable) `node_type=<type>` before writing each node type's prompts. Drill into specific topics via `get_voice_prompting_guide` with `topic=<id>` only when complexity warrants it. Then emit TypeScript and call `create_workflow` (new) or `save_workflow` (edit).
+2. **Create** — **after** you have an approved plan from the user, get into this stage. call `get_voice_prompting_guide` with `stage="create"` and (when applicable) `node_type=<type>` before writing each node type's prompts. For a `globalNode`, you must then call `get_voice_prompting_guide` with `topic="common_guidelines"` and put that content in the global node as close to verbatim as possible, changing only details the user has updated. Drill into other topics via `get_voice_prompting_guide` with `topic=<id>` only when complexity warrants it. Then emit TypeScript and call `create_workflow` (new) or `save_workflow` (edit). Enumerate available `list_node_types`, `list_tools`, `list_credentials`, `list_documents`, `list_recordings` as needed.
 
-3. **Review** — after a successful save, read any `tips[]` returned and surface them to the user with proposed fixes. Call `get_voice_prompting_guide` with `stage="review"` to enumerate review-time concerns (instruction collision, missing handoff cues, success-criteria gaps).
+3. **Review** — **after** a successful save, call `get_voice_prompting_guide` with `stage="review"` to enumerate review-time concerns (instruction collision, missing handoff cues, success-criteria gaps).
 
-The guide tool is the authoritative source for prompt-authoring craft (turn-taking, persona, readback, disfluencies). Product-mechanics questions (how a node type works at runtime, what `template_variables` resolve to) belong in `search_docs` / `read_doc` instead — don't conflate the two.
+The guide tool is the authoritative source for prompt-authoring craft (global guidelines, turn-taking, tool calls, success criteria, guardrails). Product-mechanics questions (how a node type works at runtime, what `template_variables` resolve to) belong in `search_docs` / `read_doc` instead — don't conflate the two.
 
-## Call order
+## Helpers after planning stage to create workflow
 
-### Creating a reusable tool
+### Creating a reusable tool and using it in workflow
 1. If authentication is needed, call `list_credentials` and use an existing `credential_uuid`; the user creates credential secrets in the UI.
 2. Build a typed tool definition and call `create_tool`. The request schema is authoritative for allowed tool categories and config fields.
-3. Use the returned `tool_uuid` in workflow node `tool_uuids`, then call `create_workflow` or `save_workflow`.
+3. Use the returned `tool_uuid` in workflow node `tool_uuids`, then call `create_workflow` for a new workflow or `save_workflow` when editing an existing workflow.
 
 ### Reading documentation
 1. `search_docs` — use first for keyword or acronym lookup when the user is asking how Dograh works or how to configure something.
@@ -50,7 +50,7 @@ The guide tool is the authoritative source for prompt-authoring craft (turn-taki
 1. `list_workflows` — locate the target workflow.
 2. `get_workflow_code` — fetch the current source for that workflow.
 3. (optional) `list_node_types` / `get_node_type` — consult before adding or editing a node type whose fields aren't already visible in the current code.
-4. (optional) `get_voice_prompting_guide` with `stage="create"` and `node_type=<type>` — call before revising any node's prompt field.
+4. (optional) `get_voice_prompting_guide` with `stage="create"` and `node_type=<type>` — call before revising any node's prompt field. When revising a `globalNode`, also call `get_voice_prompting_guide` with `topic="common_guidelines"` and preserve that content's structure and wording unless the user supplied a targeted change.
 5. Mutate the code in place. Preserve existing nodes, edges, and variable names unless the task requires removing or renaming them.
 6. `save_workflow` — persist as a new draft. The published version is untouched.
 
@@ -58,7 +58,7 @@ The guide tool is the authoritative source for prompt-authoring craft (turn-taki
 1. Run the plan stage (see above) before any code.
 2. Create a simple 1-node workflow with only `startCall` if the user just wants a starter. The user can iteratively add complexity by editing it.
 3. `list_node_types` / `get_node_type` — consult to learn the fields available on the node types you intend to use.
-4. `get_voice_prompting_guide` with `stage="create"` and `node_type=<type>` — call before writing each node's prompt.
+4. `get_voice_prompting_guide` with `stage="create"` and `node_type=<type>` — call before writing each node's prompt. For a `globalNode`, also call `get_voice_prompting_guide` with `topic="common_guidelines"` and place that content in the global node nearly verbatim, adapting only user-provided details such as language, persona, company, transfer target, or qualification scope.
 5. Author SDK TypeScript from scratch. The `new Workflow({ name: "..." })` call is required — `name` becomes the workflow's display name.
 6. `create_workflow` — persists a new workflow as version 1 (published). Returns the new `workflow_id`. For subsequent edits use `save_workflow` (which writes a draft).
 

api/mcp_server/tools/voice_prompting_guide.py

@@ -36,8 +36,9 @@ async def get_voice_prompting_guide(
     """Fetch staged voice-prompting guidance for authoring Dograh workflows.
 
     Call this BEFORE composing or revising any prompt field on a node. The
-    guide is the authoritative source for prompt-authoring craft (turn-taking,
-    persona, readback rules, disfluencies); product-mechanics questions
+    guide is the authoritative source for prompt-authoring craft (global
+    guidelines, turn-taking, tool calls, success criteria, guardrails);
+    product-mechanics questions
     (how a node type works at runtime) belong in `search_docs` / `read_doc`.
 
     Args:
@@ -60,7 +61,9 @@ async def get_voice_prompting_guide(
 
     Briefings are designed to be cheap — read the lens, decide what to
     drill into, then ask for full content for the 1–3 topics that matter
-    for the prompt you're about to write. Do not pull every topic.
+    for the prompt you're about to write. Always drill into
+    topic="common_guidelines" before writing or revising a globalNode so the
+    template content is actually read. Do not pull every topic.
     """
     await authenticate_mcp_request()
 

api/pyproject.toml

@@ -1,5 +1,5 @@
 [project]
 name = "dograh-api"
-version = "1.37.0"
+version = "1.42.0"
 description = "Backend API for Dograh voice AI platform"
 requires-python = ">=3.13,<3.14"

api/requirements.txt

@@ -18,5 +18,5 @@ bcrypt==5.0.0
 email-validator==2.3.0
 posthog==7.19.1
 fastmcp==3.2.4
-tuner-pipecat-sdk==0.2.0
+tuner-pipecat-sdk==0.2.4
 PyNaCl==1.6.2

api/routes/agent_stream.py

@@ -1,19 +1,15 @@
 """Agent-stream WebSocket endpoint.
 
-A single ``/agent-stream/{workflow_uuid}`` socket where a caller can drive
-an agent run by passing everything inline in the query string — including
-provider credentials. The standard ``/telephony/ws/...`` path requires a
-``TelephonyConfigurationModel`` row stored in the org; this one does not.
+A single ``/agent-stream/{provider_name}/{workflow_uuid}`` socket where a
+caller can drive an agent run. The provider is part of the URL path;
+provider-specific call metadata is read from that provider's stream protocol.
 
 Auth: the workflow UUID itself acts as the identifier — no API key.
-Routing: when ``?provider=<registered>`` matches a telephony provider, we
-dispatch to that provider's ``handle_external_websocket``. The raw-audio
-branch (no provider) is reserved for a future protocol decision and
-currently rejects with 1011.
+Routing: when ``/{provider_name}`` matches a telephony provider, we
+dispatch to that provider's ``handle_external_websocket``.
 """
 
 import uuid
-from typing import Optional
 
 from fastapi import APIRouter, WebSocket
 from loguru import logger
@@ -22,38 +18,30 @@ from starlette.websockets import WebSocketDisconnect
 
 from api.db import db_client
 from api.enums import CallType, WorkflowRunState
+from api.services.call_concurrency import (
+    CallConcurrencyLimitError,
+    call_concurrency,
+)
 from api.services.quota_service import authorize_workflow_run_start
 from api.services.telephony import registry as telephony_registry
 
 router = APIRouter(prefix="/agent-stream")
 
 
-@router.websocket("/{workflow_uuid}")
+@router.websocket("/{provider_name}/{workflow_uuid}")
 async def agent_stream_websocket(
     websocket: WebSocket,
+    provider_name: str,
     workflow_uuid: str,
 ):
     """Generic agent-stream WebSocket.
 
-    Query params:
-        provider: registered telephony provider name (e.g. ``cloudonix``)
-        from / to / callId: call metadata persisted on the workflow run
-        ...: provider-specific credentials/identifiers (e.g. ``session``,
-             ``AccountSid``, ``CallSid`` for cloudonix)
-
-    Without ``provider`` the raw-audio branch is currently not implemented.
+    ``provider_name`` is the registered telephony provider name
+    (e.g. ``cloudonix``).
     """
     await websocket.accept()
     params = dict(websocket.query_params)
-    provider_name: Optional[str] = params.get("provider")
-
-    if not provider_name:
-        logger.warning(
-            f"agent-stream raw audio branch not yet supported "
-            f"(workflow_uuid={workflow_uuid})"
-        )
-        await websocket.close(code=1011, reason="Raw audio stream not yet implemented")
-        return
+    params.pop("provider", None)
 
     spec = telephony_registry.get_optional(provider_name)
     if spec is None:
@@ -67,36 +55,44 @@ async def agent_stream_websocket(
         await websocket.close(code=1008, reason="Workflow not found")
         return
 
+    try:
+        concurrency_slot = await call_concurrency.acquire_org_slot(
+            workflow.organization_id,
+            source=f"agent_stream:{provider_name}",
+            timeout=0,
+        )
+    except CallConcurrencyLimitError:
+        await websocket.close(code=1008, reason="Concurrent call limit reached")
+        return
+
     numeric_suffix = int(str(uuid.uuid4()).replace("-", "")[:8], 16) % 100000000
     workflow_run_name = f"WR-AGS-{numeric_suffix:08d}"
-    call_id = params.get("callId") or params.get("CallSid")
     initial_context = {
         **(workflow.template_context_variables or {}),
         "provider": provider_name,
-        "caller_number": params.get("from"),
-        "called_number": params.get("to"),
         "direction": "inbound",
     }
-    workflow_run = await db_client.create_workflow_run(
-        workflow_run_name,
-        workflow.id,
-        provider_name,
-        user_id=workflow.user_id,
-        call_type=CallType.INBOUND,
-        initial_context=initial_context,
-        gathered_context={"call_id": call_id} if call_id else {},
-        logs={
-            "inbound_webhook": {
-                "domain": params.get("Domain"),
-            },
-        },
-    )
+    try:
+        workflow_run = await db_client.create_workflow_run(
+            workflow_run_name,
+            workflow.id,
+            provider_name,
+            user_id=workflow.user_id,
+            call_type=CallType.INBOUND,
+            initial_context=initial_context,
+            organization_id=workflow.organization_id,
+        )
+        await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run.id)
+    except Exception:
+        await call_concurrency.release_slot(concurrency_slot)
+        raise
 
     set_current_run_id(workflow_run.id)
     set_current_org_id(workflow.organization_id)
 
     quota_result = await authorize_workflow_run_start(
         workflow_id=workflow.id,
+        organization_id=workflow.organization_id,
         workflow_run_id=workflow_run.id,
     )
     if not quota_result.has_quota:
@@ -104,36 +100,39 @@ async def agent_stream_websocket(
             f"agent-stream quota exceeded for user {workflow.user_id}: "
             f"{quota_result.error_message}"
         )
+        await call_concurrency.release_workflow_run_slot(workflow_run.id)
         await websocket.close(
             code=1008, reason=quota_result.error_message or "Quota exceeded"
         )
         return
 
-    await db_client.update_workflow_run(
-        run_id=workflow_run.id, state=WorkflowRunState.RUNNING.value
-    )
-
-    provider_instance = spec.provider_cls({})
     try:
-        await provider_instance.handle_external_websocket(
-            websocket,
-            organization_id=workflow.organization_id,
-            workflow_id=workflow.id,
-            user_id=workflow.user_id,
-            workflow_run_id=workflow_run.id,
-            params=params,
+        await db_client.update_workflow_run(
+            run_id=workflow_run.id, state=WorkflowRunState.RUNNING.value
         )
-    except NotImplementedError as e:
-        logger.warning(f"agent-stream provider {provider_name} not supported: {e}")
+
+        provider_instance = spec.provider_cls({})
         try:
-            await websocket.close(code=1011, reason=str(e))
-        except RuntimeError:
-            pass
-    except WebSocketDisconnect as e:
-        logger.info(f"agent-stream disconnected: code={e.code} reason={e.reason}")
-    except Exception as e:
-        logger.error(f"agent-stream error for run {workflow_run.id}: {e}")
-        try:
-            await websocket.close(1011, "Internal server error")
-        except RuntimeError:
-            pass
+            await provider_instance.handle_external_websocket(
+                websocket,
+                organization_id=workflow.organization_id,
+                workflow_id=workflow.id,
+                workflow_run_id=workflow_run.id,
+                params=params,
+            )
+        except NotImplementedError as e:
+            logger.warning(f"agent-stream provider {provider_name} not supported: {e}")
+            try:
+                await websocket.close(code=1011, reason=str(e))
+            except RuntimeError:
+                pass
+        except WebSocketDisconnect as e:
+            logger.info(f"agent-stream disconnected: code={e.code} reason={e.reason}")
+        except Exception as e:
+            logger.error(f"agent-stream error for run {workflow_run.id}: {e}")
+            try:
+                await websocket.close(1011, "Internal server error")
+            except RuntimeError:
+                pass
+    finally:
+        await call_concurrency.unregister_active_call(workflow_run.id)

api/routes/auth.py

@@ -1,11 +1,16 @@
 from fastapi import APIRouter, Depends, HTTPException
 from loguru import logger
 
+from api.constants import ENABLE_SIGNUP
 from api.db import db_client
 from api.db.models import UserModel
 from api.enums import OrganizationConfigurationKey, PostHogEvent
 from api.schemas.auth import AuthResponse, LoginRequest, SignupRequest, UserResponse
-from api.services.auth.depends import create_user_configuration_with_mps_key, get_user
+from api.services.auth.depends import (
+    create_user_configuration_with_mps_key,
+    get_user,
+    require_local_auth,
+)
 from api.services.configuration.ai_model_configuration import (
     convert_legacy_ai_model_configuration_to_v2,
 )
@@ -18,8 +23,15 @@ router = APIRouter(
 )
 
 
-@router.post("/signup", response_model=AuthResponse)
+@router.post(
+    "/signup",
+    response_model=AuthResponse,
+    dependencies=[Depends(require_local_auth)],
+)
 async def signup(request: SignupRequest):
+    if not ENABLE_SIGNUP:
+        raise HTTPException(status_code=403, detail="Signup is disabled")
+
     # Check if email is already taken
     existing_user = await db_client.get_user_by_email(request.email)
     if existing_user:
@@ -85,7 +97,11 @@ async def signup(request: SignupRequest):
     )
 
 
-@router.post("/login", response_model=AuthResponse)
+@router.post(
+    "/login",
+    response_model=AuthResponse,
+    dependencies=[Depends(require_local_auth)],
+)
 async def login(request: LoginRequest):
     # Look up user by email
     user = await db_client.get_user_by_email(request.email)

api/routes/campaign.py

@@ -351,9 +351,12 @@ async def create_campaign(
 ) -> CampaignResponse:
     """Create a new campaign"""
     # Verify workflow exists and belongs to organization
-    workflow_name = await db_client.get_workflow_name(request.workflow_id, user.id)
-    if not workflow_name:
+    workflow = await db_client.get_workflow(
+        request.workflow_id, organization_id=user.selected_organization_id
+    )
+    if not workflow:
         raise HTTPException(status_code=404, detail="Workflow not found")
+    workflow_name = workflow.name
 
     # Validate source data (phone_number column and format)
     sync_service = get_sync_service(request.source_type)
@@ -364,9 +367,6 @@ async def create_campaign(
         raise HTTPException(status_code=400, detail=validation_result.error.message)
 
     # Validate template variables against source data columns
-    workflow = await db_client.get_workflow(
-        request.workflow_id, organization_id=user.selected_organization_id
-    )
     if workflow:
         from api.services.workflow.dto import ReactFlowDTO
         from api.services.workflow.workflow_graph import WorkflowGraph
@@ -512,7 +512,9 @@ async def get_campaign(
     if not campaign:
         raise HTTPException(status_code=404, detail="Campaign not found")
 
-    workflow_name = await db_client.get_workflow_name(campaign.workflow_id, user.id)
+    workflow_name = await db_client.get_workflow_name(
+        campaign.workflow_id, organization_id=user.selected_organization_id
+    )
 
     executed, total = await _get_campaign_stats(campaign.id)
     cfg_name = await _get_telephony_configuration_name(
@@ -552,6 +554,7 @@ async def start_campaign(
     # model_overrides so we evaluate the keys this campaign will use).
     quota_result = await authorize_workflow_run_start(
         workflow_id=campaign.workflow_id,
+        organization_id=user.selected_organization_id,
         actor_user=user,
     )
     if not quota_result.has_quota:
@@ -565,7 +568,9 @@ async def start_campaign(
 
     # Get updated campaign
     campaign = await db_client.get_campaign(campaign_id, user.selected_organization_id)
-    workflow_name = await db_client.get_workflow_name(campaign.workflow_id, user.id)
+    workflow_name = await db_client.get_workflow_name(
+        campaign.workflow_id, organization_id=user.selected_organization_id
+    )
 
     executed, total = await _get_campaign_stats(campaign.id)
     cfg_name = await _get_telephony_configuration_name(
@@ -599,7 +604,9 @@ async def pause_campaign(
 
     # Get updated campaign
     campaign = await db_client.get_campaign(campaign_id, user.selected_organization_id)
-    workflow_name = await db_client.get_workflow_name(campaign.workflow_id, user.id)
+    workflow_name = await db_client.get_workflow_name(
+        campaign.workflow_id, organization_id=user.selected_organization_id
+    )
 
     executed, total = await _get_campaign_stats(campaign.id)
     cfg_name = await _get_telephony_configuration_name(
@@ -669,7 +676,9 @@ async def update_campaign(
 
     # Re-fetch to return updated data
     campaign = await db_client.get_campaign(campaign_id, user.selected_organization_id)
-    workflow_name = await db_client.get_workflow_name(campaign.workflow_id, user.id)
+    workflow_name = await db_client.get_workflow_name(
+        campaign.workflow_id, organization_id=user.selected_organization_id
+    )
 
     executed, total = await _get_campaign_stats(campaign.id)
     cfg_name = await _get_telephony_configuration_name(
@@ -838,7 +847,9 @@ async def redial_campaign(
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
-    workflow_name = await db_client.get_workflow_name(child.workflow_id, user.id)
+    workflow_name = await db_client.get_workflow_name(
+        child.workflow_id, organization_id=user.selected_organization_id
+    )
     executed, total = await _get_campaign_stats(child.id)
     cfg_name = await _get_telephony_configuration_name(
         child.telephony_configuration_id, user.selected_organization_id
@@ -877,6 +888,7 @@ async def resume_campaign(
     # model_overrides so we evaluate the keys this campaign will use).
     quota_result = await authorize_workflow_run_start(
         workflow_id=campaign.workflow_id,
+        organization_id=user.selected_organization_id,
         actor_user=user,
     )
     if not quota_result.has_quota:
@@ -890,7 +902,9 @@ async def resume_campaign(
 
     # Get updated campaign
     campaign = await db_client.get_campaign(campaign_id, user.selected_organization_id)
-    workflow_name = await db_client.get_workflow_name(campaign.workflow_id, user.id)
+    workflow_name = await db_client.get_workflow_name(
+        campaign.workflow_id, organization_id=user.selected_organization_id
+    )
 
     executed, total = await _get_campaign_stats(campaign.id)
     cfg_name = await _get_telephony_configuration_name(

api/routes/knowledge_base.py

@@ -373,15 +373,10 @@ async def search_chunks(
             apply_managed_embeddings_base_url,
             get_resolved_ai_model_configuration,
         )
-        from api.services.configuration.registry import ServiceProviders
-        from api.services.gen_ai import (
-            AzureOpenAIEmbeddingService,
-            OpenAIEmbeddingService,
-        )
+        from api.services.gen_ai import build_embedding_service
 
-        # Try to get user's embeddings configuration
+        # Try to get the organization's embeddings configuration
         resolved_config = await get_resolved_ai_model_configuration(
-            user_id=user.id,
             organization_id=user.selected_organization_id,
         )
         effective_config = resolved_config.effective
@@ -405,22 +400,18 @@ async def search_chunks(
                 effective_config.embeddings, "api_version", None
             )
 
-        # Initialize embedding service based on provider
-        if embeddings_provider == ServiceProviders.AZURE.value and embeddings_endpoint:
-            embedding_service = AzureOpenAIEmbeddingService(
-                db_client=db_client,
-                api_key=embeddings_api_key,
-                endpoint=embeddings_endpoint,
-                model_id=embeddings_model or "text-embedding-3-small",
-                api_version=embeddings_api_version or "2024-02-15-preview",
-            )
-        else:
-            embedding_service = OpenAIEmbeddingService(
-                db_client=db_client,
-                api_key=embeddings_api_key,
-                model_id=embeddings_model or "text-embedding-3-small",
-                base_url=embeddings_base_url,
-            )
+        # Manual search runs outside any workflow run, so resolve the MPS
+        # correlation id here.
+        embedding_service = await build_embedding_service(
+            db_client=db_client,
+            provider=embeddings_provider,
+            api_key=embeddings_api_key,
+            model=embeddings_model,
+            base_url=embeddings_base_url,
+            endpoint=embeddings_endpoint,
+            api_version=embeddings_api_version,
+            resolve_correlation=True,
+        )
 
         # Perform search
         results = await embedding_service.search_similar_chunks(

api/routes/main.py

@@ -1,4 +1,7 @@
-from fastapi import APIRouter
+import secrets
+from typing import Annotated
+
+from fastapi import APIRouter, Header, HTTPException, status
 from loguru import logger
 from pydantic import BaseModel
 
@@ -68,10 +71,15 @@ class HealthResponse(BaseModel):
     status: str
     version: str
     backend_api_endpoint: str
+    # Public URL the deployment is reachable at when it sits behind a Cloudflare
+    # tunnel (the host has no public IP). null for a directly-reachable deployment.
+    # The UI shows this so operators know the URL telephony providers should call.
+    tunnel_url: str | None = None
     deployment_mode: str
     auth_provider: str
     turn_enabled: bool
     force_turn_relay: bool
+    signup_enabled: bool
     # Public Stack Auth client config — only populated when auth_provider == "stack".
     # The UI reads these at runtime to initialize Stack, so they no longer need to
     # be baked into the browser bundle at build time. Both are public values.
@@ -84,27 +92,90 @@ async def health() -> HealthResponse:
     from api.constants import (
         APP_VERSION,
         AUTH_PROVIDER,
+        BACKEND_API_ENDPOINT,
         DEPLOYMENT_MODE,
+        ENABLE_SIGNUP,
         FORCE_TURN_RELAY,
         STACK_AUTH_PROJECT_ID,
         STACK_PUBLISHABLE_CLIENT_KEY,
         TURN_SECRET,
     )
-    from api.utils.common import get_backend_endpoints
+    from api.utils.common import get_backend_endpoints, is_local_or_private_url
 
     logger.debug("Health endpoint called")
     backend_endpoint, _ = await get_backend_endpoints()
+    # tunnel_url is set only when a Cloudflare tunnel was actually resolved: the
+    # configured address isn't publicly reachable, but get_backend_endpoints found
+    # a public tunnel URL for it. This is the URL the UI shows for inbound webhooks.
+    # It stays null for a directly-reachable (public IP / domain) deployment, where
+    # backend_api_endpoint itself is the public URL.
+    tunnel_url = (
+        backend_endpoint
+        if is_local_or_private_url(BACKEND_API_ENDPOINT)
+        and not is_local_or_private_url(backend_endpoint)
+        else None
+    )
     is_stack = AUTH_PROVIDER == "stack"
     return HealthResponse(
         status="ok",
         version=APP_VERSION,
-        backend_api_endpoint=backend_endpoint,
+        backend_api_endpoint=BACKEND_API_ENDPOINT,
+        tunnel_url=tunnel_url,
         deployment_mode=DEPLOYMENT_MODE,
         auth_provider=AUTH_PROVIDER,
         turn_enabled=bool(TURN_SECRET),
         force_turn_relay=FORCE_TURN_RELAY,
+        signup_enabled=ENABLE_SIGNUP,
         stack_project_id=STACK_AUTH_PROJECT_ID if is_stack else None,
         stack_publishable_client_key=(
             STACK_PUBLISHABLE_CLIENT_KEY if is_stack else None
         ),
     )
+
+
+class ActiveCallsResponse(BaseModel):
+    active_calls: int
+
+
+DOGRAH_DEVOPS_SECRET_HEADER = "X-Dograh-Devops-Secret"
+
+
+def _verify_devops_secret(
+    configured_secret: str | None,
+    provided_secret: str | None,
+) -> None:
+    if not configured_secret:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="Devops secret is not configured",
+        )
+    if not provided_secret or not secrets.compare_digest(
+        provided_secret,
+        configured_secret,
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Forbidden",
+        )
+
+
+@router.get("/health/active-calls", response_model=ActiveCallsResponse)
+async def active_calls(
+    x_dograh_devops_secret: Annotated[
+        str | None,
+        Header(alias=DOGRAH_DEVOPS_SECRET_HEADER),
+    ] = None,
+) -> ActiveCallsResponse:
+    """In-flight call count for THIS worker — the drain signal for deploys.
+
+    A deploy orchestrator polls this per worker and waits for zero before
+    sending SIGTERM, because uvicorn force-closes live call WebSockets (close
+    code 1012) on SIGTERM and would cut calls mid-conversation otherwise. The
+    count is per-process: one uvicorn per VM port (scripts/rolling_update.sh)
+    or per Kubernetes pod (preStop hook). See api/services/pipecat/active_calls.py.
+    """
+    from api.constants import DOGRAH_DEVOPS_SECRET
+    from api.services.pipecat.active_calls import active_call_count
+
+    _verify_devops_secret(DOGRAH_DEVOPS_SECRET, x_dograh_devops_secret)
+    return ActiveCallsResponse(active_calls=active_call_count())

api/routes/organization.py

@@ -42,7 +42,6 @@ from api.schemas.telephony_phone_number import (
     ProviderSyncStatus,
 )
 from api.services.auth.depends import (
-    _sync_posthog_organization_mps_billing_v2_status,
     get_user,
     get_user_with_selected_organization,
 )
@@ -61,6 +60,7 @@ from api.services.configuration.check_validity import UserConfigurationValidator
 from api.services.configuration.defaults import DEFAULT_SERVICE_PROVIDERS
 from api.services.configuration.masking import is_mask_of, mask_key, mask_user_config
 from api.services.configuration.registry import (
+    DOGRAH_MULTILINGUAL_AUTODETECT_LANGUAGES,
     DOGRAH_STT_LANGUAGES,
     REGISTRY,
     DograhTTSService,
@@ -68,6 +68,7 @@ from api.services.configuration.registry import (
     ServiceType,
 )
 from api.services.mps_billing import ensure_hosted_mps_billing_account_v2
+from api.services.mps_service_key_client import mps_service_key_client
 from api.services.organization_context import (
     OrganizationContextResponse,
     get_organization_context,
@@ -144,6 +145,23 @@ class TelephonyConfigWarningsResponse(BaseModel):
     """
 
     telnyx_missing_webhook_public_key_count: int
+    vonage_missing_signature_secret_count: int
+
+
+class ModelConfigurationMetricPrice(BaseModel):
+    metric_code: str
+    display_name: str
+    unit: str
+    price_per_minute: float
+    currency: str
+    rounding_policy: str
+
+
+class ModelConfigurationPricingResponse(BaseModel):
+    """MPS-owned effective prices relevant to model configuration choices."""
+
+    platform_usage: ModelConfigurationMetricPrice | None = None
+    dograh_model: ModelConfigurationMetricPrice | None = None
 
 
 @router.get("/context", response_model=OrganizationContextResponse)
@@ -199,8 +217,7 @@ async def get_telephony_providers_metadata(user: UserModel = Depends(get_user)):
 async def get_telephony_config_warnings(user: UserModel = Depends(get_user)):
     """Return aggregated warning counts for the current org's telephony configs.
 
-    Today this surfaces only Telnyx configs missing ``webhook_public_key``;
-    additional warning types should be added as new fields on the response.
+    Surfaces provider configs missing webhook-verification credentials.
     """
     if not user.selected_organization_id:
         raise HTTPException(status_code=400, detail="No organization selected")
@@ -208,8 +225,12 @@ async def get_telephony_config_warnings(user: UserModel = Depends(get_user)):
     telnyx_missing = await db_client.count_telnyx_configs_missing_webhook_public_key(
         user.selected_organization_id
     )
+    vonage_missing = await db_client.count_vonage_configs_missing_signature_secret(
+        user.selected_organization_id
+    )
     return TelephonyConfigWarningsResponse(
         telnyx_missing_webhook_public_key_count=telnyx_missing,
+        vonage_missing_signature_secret_count=vonage_missing,
     )
 
 
@@ -239,7 +260,6 @@ async def _model_configuration_v2_response(
     configuration: OrganizationAIModelConfigurationV2 | None = None,
 ) -> OrganizationAIModelConfigurationResponse:
     resolved = await get_resolved_ai_model_configuration(
-        user_id=user.id,
         organization_id=user.selected_organization_id,
     )
     raw_configuration = (
@@ -274,6 +294,7 @@ async def get_model_configuration_v2_defaults(
                 "step": DOGRAH_SPEED_STEP,
             },
             "languages": DOGRAH_STT_LANGUAGES,
+            "multilingual_languages": DOGRAH_MULTILINGUAL_AUTODETECT_LANGUAGES,
             "defaults": {
                 "voice": DOGRAH_DEFAULT_VOICE,
                 "speed": 1.0,
@@ -308,6 +329,34 @@ async def get_model_configuration_v2(
     return await _model_configuration_v2_response(user=user)
 
 
+@router.get(
+    "/model-configurations/v2/pricing",
+    response_model=ModelConfigurationPricingResponse,
+)
+async def get_model_configuration_pricing(
+    user: UserModel = Depends(get_user_with_selected_organization),
+) -> ModelConfigurationPricingResponse:
+    """Return the hosted organization prices shown in Model Configurations."""
+    if DEPLOYMENT_MODE == "oss":
+        return ModelConfigurationPricingResponse()
+
+    try:
+        pricing = await mps_service_key_client.get_billing_pricing(
+            user.selected_organization_id,
+        )
+        return ModelConfigurationPricingResponse.model_validate(pricing)
+    except Exception as exc:
+        logger.error(
+            "Failed to get MPS model-configuration pricing for organization {}: {}",
+            user.selected_organization_id,
+            exc,
+        )
+        raise HTTPException(
+            status_code=502,
+            detail="Failed to retrieve model configuration pricing",
+        ) from exc
+
+
 @router.put(
     "/model-configurations/v2",
     response_model=OrganizationAIModelConfigurationResponse,
@@ -385,22 +434,21 @@ async def migrate_model_configuration_v2(
     except ValueError as exc:
         raise HTTPException(status_code=422, detail=exc.args[0])
 
-    billing_account_status = None
     if DEPLOYMENT_MODE != "oss":
         try:
-            billing_account_status = await ensure_hosted_mps_billing_account_v2(
+            await ensure_hosted_mps_billing_account_v2(
                 organization_id,
                 created_by=str(user.provider_id),
             )
         except Exception as exc:
             logger.error(
-                "Failed to initialize MPS billing v2 account for organization {}: {}",
+                "Failed to initialize MPS billing account for organization {}: {}",
                 organization_id,
                 exc,
             )
             raise HTTPException(
                 status_code=502,
-                detail="Failed to initialize MPS billing v2 account",
+                detail="Failed to initialize MPS billing account",
             )
 
     await upsert_organization_ai_model_configuration_v2(
@@ -411,14 +459,6 @@ async def migrate_model_configuration_v2(
         organization_id=organization_id,
         fallback_user_config=legacy,
     )
-    if DEPLOYMENT_MODE != "oss":
-        _sync_posthog_organization_mps_billing_v2_status(
-            organization_id,
-            uses_mps_billing_v2=bool(
-                billing_account_status
-                and billing_account_status.get("billing_mode") == "v2"
-            ),
-        )
     return await _model_configuration_v2_response(
         user=user,
         configuration=configuration,

api/routes/organization_usage.py

@@ -1,6 +1,6 @@
 import json
 from datetime import datetime, timedelta
-from typing import Any, Dict, List, Literal, Optional
+from typing import Any, Dict, List, Optional
 
 from fastapi import APIRouter, Depends, HTTPException, Query
 from fastapi.responses import StreamingResponse
@@ -29,12 +29,6 @@ class CurrentUsageResponse(BaseModel):
     price_per_second_usd: Optional[float] = None
 
 
-class MPSCreditsResponse(BaseModel):
-    total_credits_used: float
-    remaining_credits: float
-    total_quota: float
-
-
 class MPSCreditPurchaseUrlResponse(BaseModel):
     checkout_url: str
 
@@ -69,7 +63,6 @@ class MPSCreditLedgerEntryResponse(BaseModel):
 
 
 class MPSBillingCreditsResponse(BaseModel):
-    billing_version: Literal["legacy", "v2"]
     total_credits_used: float = 0.0
     remaining_credits: float = 0.0
     total_quota: float = 0.0
@@ -162,69 +155,13 @@ async def get_current_period_usage(user: UserModel = Depends(get_user)):
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@router.get("/usage/mps-credits", response_model=MPSCreditsResponse)
-async def get_mps_credits(user: UserModel = Depends(get_user)):
-    """Get aggregated usage and quota from MPS.
-
-    OSS users: queries by provider_id (created_by).
-    Hosted users: queries by organization_id.
-    """
-    try:
-        if DEPLOYMENT_MODE == "oss":
-            usage = await mps_service_key_client.get_usage_by_created_by(
-                str(user.provider_id)
-            )
-        else:
-            if not user.selected_organization_id:
-                raise HTTPException(status_code=400, detail="No organization selected")
-            usage = await mps_service_key_client.get_usage_by_organization(
-                user.selected_organization_id
-            )
-
-        total_used = usage.get("total_credits_used", 0.0)
-        total_remaining = usage.get("remaining_credits", 0.0)
-
-        return MPSCreditsResponse(
-            total_credits_used=total_used,
-            remaining_credits=total_remaining,
-            total_quota=total_used + total_remaining,
-        )
-    except HTTPException:
-        raise
-    except Exception as e:
-        logger.error(f"Failed to fetch MPS credits: {e}")
-        raise HTTPException(status_code=500, detail=str(e))
-
-
-async def _get_mps_billing_account_status(
-    user: UserModel, organization_id: int
-) -> Optional[dict]:
-    return await mps_service_key_client.get_billing_account_status(
-        organization_id=organization_id,
-        created_by=str(user.provider_id),
-    )
-
-
-def _is_mps_billing_v2(account: Optional[dict]) -> bool:
-    return bool(account and account.get("billing_mode") == "v2")
-
-
-async def _legacy_mps_credits_response(user: UserModel) -> MPSBillingCreditsResponse:
-    if DEPLOYMENT_MODE == "oss":
-        usage = await mps_service_key_client.get_usage_by_created_by(
-            str(user.provider_id)
-        )
-    else:
-        if not user.selected_organization_id:
-            raise HTTPException(status_code=400, detail="No organization selected")
-        usage = await mps_service_key_client.get_usage_by_organization(
-            user.selected_organization_id
-        )
+async def _oss_mps_credits_response(user: UserModel) -> MPSBillingCreditsResponse:
+    """Aggregate per-key MPS credits for OSS deployments (no billing account)."""
+    usage = await mps_service_key_client.get_usage_by_created_by(str(user.provider_id))
 
     total_used = float(usage.get("total_credits_used", 0.0))
     total_remaining = float(usage.get("remaining_credits", 0.0))
     return MPSBillingCreditsResponse(
-        billing_version="legacy",
         total_credits_used=total_used,
         remaining_credits=total_remaining,
         total_quota=total_used + total_remaining,
@@ -237,16 +174,15 @@ async def get_billing_credits(
     limit: int = Query(50, ge=1, le=100),
     user: UserModel = Depends(get_user),
 ):
-    """Return legacy MPS credits or paginated v2 billing ledger details for the org."""
+    """Return per-key MPS credits (OSS) or the org's paginated billing ledger."""
     try:
-        if DEPLOYMENT_MODE == "oss" or not user.selected_organization_id:
-            return await _legacy_mps_credits_response(user)
+        if DEPLOYMENT_MODE == "oss":
+            return await _oss_mps_credits_response(user)
+
+        if not user.selected_organization_id:
+            raise HTTPException(status_code=400, detail="No organization selected")
 
         organization_id = user.selected_organization_id
-        account_status = await _get_mps_billing_account_status(user, organization_id)
-        if not _is_mps_billing_v2(account_status):
-            return await _legacy_mps_credits_response(user)
-
         ledger = await mps_service_key_client.get_credit_ledger(
             organization_id=organization_id,
             page=page,
@@ -287,7 +223,6 @@ async def get_billing_credits(
             total_debits = float(ledger["total_debits_credits"])
 
         return MPSBillingCreditsResponse(
-            billing_version="v2",
             total_credits_used=total_debits,
             remaining_credits=balance,
             total_quota=balance + total_debits,
@@ -350,7 +285,7 @@ async def get_billing_credits(
 async def create_mps_credit_purchase_url(
     user: UserModel = Depends(get_user_with_selected_organization),
 ):
-    """Create a checkout URL for organizations using Dograh-managed MPS v2."""
+    """Create a checkout URL for purchasing organization credits."""
     if DEPLOYMENT_MODE == "oss":
         raise HTTPException(
             status_code=404,
@@ -359,14 +294,6 @@ async def create_mps_credit_purchase_url(
 
     organization_id = user.selected_organization_id
     assert organization_id is not None
-    account_status = await _get_mps_billing_account_status(user, organization_id)
-    if not _is_mps_billing_v2(account_status):
-        raise HTTPException(
-            status_code=403,
-            detail=(
-                "Credit purchases are available only for organizations using billing v2"
-            ),
-        )
 
     try:
         session = await mps_service_key_client.create_credit_purchase_url(
@@ -585,7 +512,6 @@ async def get_daily_usage_breakdown(
             start_date,
             end_date,
             org.price_per_second_usd,
-            user_id=user.id,
         )
 
         return breakdown

api/routes/public_agent.py

@@ -14,6 +14,10 @@ from pydantic import BaseModel
 
 from api.db import db_client
 from api.enums import TriggerState, WorkflowStatus
+from api.services.call_concurrency import (
+    CallConcurrencyLimitError,
+    call_concurrency,
+)
 from api.services.quota_service import authorize_workflow_run_start
 from api.services.telephony.factory import (
     get_default_telephony_provider,
@@ -243,15 +247,32 @@ async def _execute_resolved_target(
         initial_context["api_key_created_by"] = api_key_created_by
     initial_context.update(request.initial_context or {})
 
-    workflow_run = await db_client.create_workflow_run(
-        name=workflow_run_name,
-        workflow_id=target.workflow.id,
-        mode=workflow_run_mode,
-        initial_context=initial_context,
-        user_id=execution_user_id,
-        use_draft=use_draft,
-        organization_id=target.organization_id,
-    )
+    try:
+        concurrency_slot = await call_concurrency.acquire_org_slot(
+            target.organization_id,
+            source="public_agent",
+            timeout=0,
+        )
+    except CallConcurrencyLimitError:
+        raise HTTPException(
+            status_code=429,
+            detail="Concurrent call limit reached",
+        )
+
+    try:
+        workflow_run = await db_client.create_workflow_run(
+            name=workflow_run_name,
+            workflow_id=target.workflow.id,
+            mode=workflow_run_mode,
+            initial_context=initial_context,
+            user_id=execution_user_id,
+            use_draft=use_draft,
+            organization_id=target.organization_id,
+        )
+        await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run.id)
+    except Exception:
+        await call_concurrency.release_slot(concurrency_slot)
+        raise
 
     logger.info(
         f"Created workflow run {workflow_run.id} for public agent "
@@ -264,25 +285,30 @@ async def _execute_resolved_target(
     # the MPS correlation id before the provider starts the call.
     quota_result = await authorize_workflow_run_start(
         workflow_id=target.workflow.id,
+        organization_id=target.organization_id,
         workflow_run_id=workflow_run.id,
     )
     if not quota_result.has_quota:
+        await call_concurrency.release_workflow_run_slot(workflow_run.id)
         raise HTTPException(status_code=402, detail=quota_result.error_message)
 
     # 9. Construct webhook URL for telephony provider callback
-    backend_endpoint, _ = await get_backend_endpoints()
+    try:
+        backend_endpoint, _ = await get_backend_endpoints()
+    except Exception:
+        await call_concurrency.release_workflow_run_slot(workflow_run.id)
+        raise
     webhook_endpoint = provider.WEBHOOK_ENDPOINT
 
     webhook_url = (
         f"{backend_endpoint}/api/v1/telephony/{webhook_endpoint}"
         f"?workflow_id={target.workflow.id}"
-        f"&user_id={execution_user_id}"
         f"&workflow_run_id={workflow_run.id}"
         f"&organization_id={target.organization_id}"
     )
 
-    # 10. Initiate call via telephony provider. workflow_id and user_id are
-    # required by providers that build the media WebSocket URL at dial time
+    # 10. Initiate call via telephony provider. workflow_id and organization_id
+    # are required by providers that build the media WebSocket URL at dial time
     # (e.g. Telnyx, Cloudonix); without them the URL contains "None/None" and
     # the stream connection fails.
     try:
@@ -291,12 +317,13 @@ async def _execute_resolved_target(
             webhook_url=webhook_url,
             workflow_run_id=workflow_run.id,
             workflow_id=target.workflow.id,
-            user_id=execution_user_id,
+            organization_id=target.organization_id,
         )
     except Exception as e:
         logger.warning(
             f"Failed to initiate call for workflow run {workflow_run.id}: {e}"
         )
+        await call_concurrency.release_workflow_run_slot(workflow_run.id)
         raise HTTPException(
             status_code=400,
             detail=f"Failed to initiate call: {e}",

api/routes/public_embed.py

@@ -309,7 +309,11 @@ async def initialize_embed_session(
             workflow_id=embed_token.workflow_id,
             mode=WorkflowRunMode.SMALLWEBRTC.value,
             user_id=embed_token.created_by,  # Use token creator as run owner
-            initial_context=init_request.context_variables,
+            organization_id=embed_token.organization_id,
+            initial_context={
+                **(init_request.context_variables or {}),
+                "provider": WorkflowRunMode.SMALLWEBRTC.value,
+            },
         )
     except Exception as e:
         logger.error(f"Failed to create workflow run: {e}")

api/routes/superuser.py

@@ -8,7 +8,11 @@ from pydantic import BaseModel
 from api.db import db_client
 from api.db.models import UserModel
 from api.services.auth.depends import get_superuser
-from api.services.auth.stack_auth import stackauth
+from api.services.auth.stack_auth import (
+    StackAuthSessionError,
+    StackAuthUserSearchError,
+    stackauth,
+)
 
 router = APIRouter(prefix="/superuser", tags=["superuser"])
 
@@ -16,12 +20,14 @@ router = APIRouter(prefix="/superuser", tags=["superuser"])
 class ImpersonateRequest(BaseModel):
     """Request payload for superadmin impersonation.
 
-    Either ``provider_user_id`` **or** ``user_id`` must be supplied. If both are
-    provided, ``provider_user_id`` takes precedence.
+    ``provider_user_id``, ``user_id``, or ``email`` may be supplied. If more
+    than one is provided, ``provider_user_id`` takes precedence, followed by
+    ``user_id`` and then ``email``.
     """
 
     provider_user_id: str | None = None
     user_id: int | None = None
+    email: str | None = None
 
 
 class ImpersonateResponse(BaseModel):
@@ -65,32 +71,79 @@ async def impersonate(
     to create an impersonation session.
     """
 
-    provider_user_id: str | None = request.provider_user_id
+    provider_user_id = (
+        request.provider_user_id.strip() if request.provider_user_id else None
+    ) or None
+    email = request.email.strip().lower() if request.email else None
 
     # ------------------------------------------------------------------
-    # Fallback: resolve provider_user_id from internal ``user_id``
+    # Fallback: resolve provider_user_id from internal ``user_id`` or email.
     # ------------------------------------------------------------------
     if provider_user_id is None:
-        if request.user_id is None:
+        if request.user_id is not None:
+            db_user = await db_client.get_user_by_id(request.user_id)
+
+            if db_user is None:
+                raise HTTPException(
+                    status_code=status.HTTP_404_NOT_FOUND,
+                    detail=f"User with ID {request.user_id} not found.",
+                )
+
+            provider_user_id = db_user.provider_id
+        elif email:
+            db_user = await db_client.get_user_by_email(email)
+
+            if db_user is not None:
+                provider_user_id = db_user.provider_id
+            else:
+                try:
+                    stack_users = await stackauth.find_users_by_email(email)
+                except StackAuthUserSearchError as exc:
+                    raise HTTPException(
+                        status_code=status.HTTP_502_BAD_GATEWAY,
+                        detail="Failed to search Stack Auth users.",
+                    ) from exc
+
+                if len(stack_users) == 1 and isinstance(stack_users[0].get("id"), str):
+                    provider_user_id = stack_users[0]["id"]
+                elif len(stack_users) > 1:
+                    raise HTTPException(
+                        status_code=status.HTTP_409_CONFLICT,
+                        detail="Multiple Stack Auth users matched that email.",
+                    )
+                else:
+                    raise HTTPException(
+                        status_code=status.HTTP_404_NOT_FOUND,
+                        detail=f"User with email {email} not found.",
+                    )
+        else:
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,
-                detail="Either 'provider_user_id' or 'user_id' must be provided.",
+                detail=(
+                    "One of 'provider_user_id', 'user_id', or 'email' must be provided."
+                ),
             )
 
-        db_user = await db_client.get_user_by_id(request.user_id)
-
-        if db_user is None:
-            raise HTTPException(
-                status_code=status.HTTP_404_NOT_FOUND,
-                detail=f"User with ID {request.user_id} not found.",
-            )
-
-        provider_user_id = db_user.provider_id
-
     # ------------------------------------------------------------------
     # Call Stack Auth to create the impersonation session
     # ------------------------------------------------------------------
-    session = await stackauth.impersonate(provider_user_id)
+    try:
+        session = await stackauth.impersonate(provider_user_id)
+    except StackAuthSessionError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail="Failed to create Stack Auth impersonation session.",
+        ) from exc
+
+    if (
+        not isinstance(session, dict)
+        or "refresh_token" not in session
+        or "access_token" not in session
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail="Failed to create Stack Auth impersonation session.",
+        )
 
     return ImpersonateResponse(
         refresh_token=session["refresh_token"],

api/routes/telephony.py

@@ -21,10 +21,15 @@ from starlette.websockets import WebSocketDisconnect
 
 from api.db import db_client
 from api.db.models import UserModel
-from api.enums import CallType, WorkflowRunState
+from api.enums import CallType, WorkflowRunMode, WorkflowRunState
 from api.errors.telephony_errors import TelephonyError
 from api.sdk_expose import sdk_expose
 from api.services.auth.depends import get_user
+from api.services.call_concurrency import (
+    CallConcurrencyLimitError,
+    WorkflowRunSlotAlreadyBoundError,
+    call_concurrency,
+)
 from api.services.quota_service import authorize_workflow_run_start
 from api.services.telephony.call_transfer_manager import get_call_transfer_manager
 from api.services.telephony.factory import (
@@ -139,70 +144,6 @@ async def initiate_call(
     # Determine the workflow run mode based on provider type
     workflow_run_mode = provider.PROVIDER_NAME
 
-    workflow_run_id = request.workflow_run_id
-
-    if not workflow_run_id:
-        # Merge template context variables (e.g. caller_number, called_number
-        # set in workflow settings for testing pre-call data fetch).
-        template_vars = workflow.template_context_variables or {}
-
-        numeric_suffix = int(str(uuid.uuid4()).replace("-", "")[:8], 16) % 100000000
-        workflow_run_name = f"WR-TEL-OUT-{numeric_suffix:08d}"
-        workflow_run = await db_client.create_workflow_run(
-            workflow_run_name,
-            workflow.id,
-            workflow_run_mode,
-            user_id=execution_user_id,
-            call_type=CallType.OUTBOUND,
-            initial_context={
-                **template_vars,
-                "phone_number": phone_number,
-                "called_number": phone_number,
-                "provider": provider.PROVIDER_NAME,
-                "telephony_configuration_id": telephony_configuration_id,
-            },
-            use_draft=True,
-            organization_id=user.selected_organization_id,
-        )
-        workflow_run_id = workflow_run.id
-    else:
-        workflow_run = await db_client.get_workflow_run(
-            workflow_run_id, organization_id=user.selected_organization_id
-        )
-        if not workflow_run:
-            raise HTTPException(status_code=400, detail="Workflow run not found")
-        if workflow_run.workflow_id != workflow.id:
-            raise HTTPException(
-                status_code=400,
-                detail="workflow_run_workflow_mismatch",
-            )
-        workflow_run_name = workflow_run.name
-
-    # Check Dograh quota after the run exists so hosted v2 can mint and store
-    # the MPS correlation id before initiating the call.
-    quota_result = await authorize_workflow_run_start(
-        workflow_id=workflow.id,
-        workflow_run_id=workflow_run_id,
-        actor_user=user,
-    )
-    if not quota_result.has_quota:
-        raise HTTPException(status_code=402, detail=quota_result.error_message)
-
-    # Construct webhook URL based on provider type
-    backend_endpoint, _ = await get_backend_endpoints()
-
-    webhook_endpoint = provider.WEBHOOK_ENDPOINT
-
-    webhook_url = (
-        f"{backend_endpoint}/api/v1/telephony/{webhook_endpoint}"
-        f"?workflow_id={workflow.id}"
-        f"&user_id={execution_user_id}"
-        f"&workflow_run_id={workflow_run_id}"
-        f"&organization_id={user.selected_organization_id}"
-    )
-
-    keywords = {"workflow_id": workflow.id, "user_id": execution_user_id}
-
     # Resolve optional caller-ID. The config has already been validated against
     # the user's organization, so filtering by config_id is sufficient for
     # tenant isolation.
@@ -220,14 +161,105 @@ async def initiate_call(
             raise HTTPException(status_code=400, detail="from_phone_number_not_found")
         from_number = phone_row.address_normalized
 
-    # Initiate call via provider
-    result = await provider.initiate_call(
-        to_number=phone_number,
-        webhook_url=webhook_url,
+    workflow_run_id = request.workflow_run_id
+    try:
+        concurrency_slot = await call_concurrency.acquire_org_slot(
+            user.selected_organization_id,
+            source="telephony_outbound",
+            timeout=0,
+        )
+    except CallConcurrencyLimitError:
+        raise HTTPException(status_code=429, detail="Concurrent call limit reached")
+
+    try:
+        if not workflow_run_id:
+            # Merge template context variables (e.g. caller_number, called_number
+            # set in workflow settings for testing pre-call data fetch).
+            template_vars = workflow.template_context_variables or {}
+
+            numeric_suffix = int(str(uuid.uuid4()).replace("-", "")[:8], 16) % 100000000
+            workflow_run_name = f"WR-TEL-OUT-{numeric_suffix:08d}"
+            workflow_run = await db_client.create_workflow_run(
+                workflow_run_name,
+                workflow.id,
+                workflow_run_mode,
+                user_id=execution_user_id,
+                call_type=CallType.OUTBOUND,
+                initial_context={
+                    **template_vars,
+                    "phone_number": phone_number,
+                    "called_number": phone_number,
+                    "provider": provider.PROVIDER_NAME,
+                    "telephony_configuration_id": telephony_configuration_id,
+                },
+                use_draft=True,
+                organization_id=user.selected_organization_id,
+            )
+            workflow_run_id = workflow_run.id
+        else:
+            workflow_run = await db_client.get_workflow_run(
+                workflow_run_id, organization_id=user.selected_organization_id
+            )
+            if not workflow_run:
+                raise HTTPException(status_code=400, detail="Workflow run not found")
+            if workflow_run.workflow_id != workflow.id:
+                raise HTTPException(
+                    status_code=400,
+                    detail="workflow_run_workflow_mismatch",
+                )
+            workflow_run_name = workflow_run.name
+
+        await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run_id)
+    except WorkflowRunSlotAlreadyBoundError:
+        raise HTTPException(
+            status_code=409,
+            detail="Workflow run already has an active call",
+        )
+    except Exception:
+        await call_concurrency.release_slot(concurrency_slot)
+        raise
+
+    # Check Dograh quota after the run exists so hosted v2 can mint and store
+    # the MPS correlation id before initiating the call.
+    quota_result = await authorize_workflow_run_start(
+        workflow_id=workflow.id,
+        organization_id=user.selected_organization_id,
         workflow_run_id=workflow_run_id,
-        from_number=from_number,
-        **keywords,
+        actor_user=user,
     )
+    if not quota_result.has_quota:
+        await call_concurrency.release_workflow_run_slot(workflow_run_id)
+        raise HTTPException(status_code=402, detail=quota_result.error_message)
+
+    try:
+        # Construct webhook URL based on provider type
+        backend_endpoint, _ = await get_backend_endpoints()
+
+        webhook_endpoint = provider.WEBHOOK_ENDPOINT
+
+        webhook_url = (
+            f"{backend_endpoint}/api/v1/telephony/{webhook_endpoint}"
+            f"?workflow_id={workflow.id}"
+            f"&workflow_run_id={workflow_run_id}"
+            f"&organization_id={user.selected_organization_id}"
+        )
+
+        keywords = {
+            "workflow_id": workflow.id,
+            "organization_id": user.selected_organization_id,
+        }
+
+        # Initiate call via provider
+        result = await provider.initiate_call(
+            to_number=phone_number,
+            webhook_url=webhook_url,
+            workflow_run_id=workflow_run_id,
+            from_number=from_number,
+            **keywords,
+        )
+    except Exception:
+        await call_concurrency.release_workflow_run_slot(workflow_run_id)
+        raise
 
     # Store provider metadata and caller_number in workflow run context
     gathered_context = {
@@ -421,6 +453,7 @@ async def _validate_inbound_request(
 async def _create_inbound_workflow_run(
     workflow_id: int,
     user_id: int,
+    organization_id: int,
     provider: str,
     normalized_data,
     telephony_configuration_id: int,
@@ -456,6 +489,7 @@ async def _create_inbound_workflow_run(
                 "raw_webhook_data": normalized_data.raw_data,
             },
         },
+        organization_id=organization_id,
     )
 
     logger.info(
@@ -511,13 +545,13 @@ async def websocket_ari_endpoint(websocket: WebSocket):
     query params (appended by the v() dial string option in externalMedia).
     """
     workflow_id = websocket.query_params.get("workflow_id")
-    user_id = websocket.query_params.get("user_id")
+    organization_id = websocket.query_params.get("organization_id")
     workflow_run_id = websocket.query_params.get("workflow_run_id")
 
-    if not workflow_id or not user_id or not workflow_run_id:
+    if not workflow_id or not organization_id or not workflow_run_id:
         logger.error(
-            f"ARI WebSocket missing query params: "
-            f"workflow_id={workflow_id}, user_id={user_id}, workflow_run_id={workflow_run_id}"
+            f"ARI WebSocket missing query params: workflow_id={workflow_id}, "
+            f"organization_id={organization_id}, workflow_run_id={workflow_run_id}"
         )
         await websocket.close(code=4400, reason="Missing required query params")
         return
@@ -527,41 +561,64 @@ async def websocket_ari_endpoint(websocket: WebSocket):
     await websocket.accept(subprotocol="media")
 
     await _handle_telephony_websocket(
-        websocket, int(workflow_id), int(user_id), int(workflow_run_id)
+        websocket, int(workflow_id), int(organization_id), int(workflow_run_id)
     )
 
 
-@router.websocket("/ws/{workflow_id}/{user_id}/{workflow_run_id}")
+@router.websocket("/ws/{workflow_id}/{organization_id}/{workflow_run_id}")
 async def websocket_endpoint(
-    websocket: WebSocket, workflow_id: int, user_id: int, workflow_run_id: int
+    websocket: WebSocket, workflow_id: int, organization_id: int, workflow_run_id: int
 ):
     """WebSocket endpoint for real-time call handling - routes to provider-specific handlers."""
     await websocket.accept()
-    await _handle_telephony_websocket(websocket, workflow_id, user_id, workflow_run_id)
+    await _handle_telephony_websocket(
+        websocket, workflow_id, organization_id, workflow_run_id
+    )
 
 
 async def _handle_telephony_websocket(
-    websocket: WebSocket, workflow_id: int, user_id: int, workflow_run_id: int
+    websocket: WebSocket, workflow_id: int, organization_id: int, workflow_run_id: int
 ):
-    """Shared WebSocket handler logic (connection already accepted)."""
+    """Shared WebSocket handler logic (connection already accepted).
+
+    TODO(security): ``organization_id`` arrives in the URL the provider dials
+    back, so it is caller-supplied and unauthenticated — this socket has no
+    signature check, and the id triple is a guessable bearer capability.
+    Scoping the lookups below by it prevents an accidental cross-org mismatch,
+    not a deliberate one. The real fix is a one-shot capability token minted at
+    run creation and redeemed here through an atomic
+    ``initialized -> running`` compare-and-swap, which would also close the
+    read-then-write race on the state check further down.
+    """
     try:
         # Set the run context
         set_current_run_id(workflow_run_id)
 
         # Get workflow run to determine provider type
-        workflow_run = await db_client.get_workflow_run(workflow_run_id)
+        workflow_run = await db_client.get_workflow_run(
+            workflow_run_id, organization_id=organization_id
+        )
         if not workflow_run:
-            logger.error(f"Workflow run {workflow_run_id} not found")
+            logger.error(
+                f"Workflow run {workflow_run_id} not found for org {organization_id}"
+            )
             await websocket.close(code=4404, reason="Workflow run not found")
             return
 
-        # Get workflow for organization info. System lookup keyed only on the
-        # workflow_id (org is derived below) — use the explicit unscoped variant.
-        workflow = await db_client.get_workflow_by_id(workflow_id)
+        workflow = await db_client.get_workflow(
+            workflow_id, organization_id=organization_id
+        )
         if not workflow:
-            logger.error(f"Workflow {workflow_id} not found")
+            logger.error(f"Workflow {workflow_id} not found for org {organization_id}")
             await websocket.close(code=4404, reason="Workflow not found")
             return
+        if workflow_run.workflow_id != workflow.id:
+            logger.error(
+                f"Workflow run {workflow_run_id} belongs to workflow "
+                f"{workflow_run.workflow_id}, not {workflow.id}"
+            )
+            await websocket.close(code=4400, reason="workflow_run_workflow_mismatch")
+            return
 
         # Check workflow run state - only allow 'initialized' state
         if workflow_run.state != WorkflowRunState.INITIALIZED.value:
@@ -584,12 +641,36 @@ async def _handle_telephony_websocket(
             provider_type = workflow_run.initial_context.get("provider")
             logger.info(f"Extracted provider_type: {provider_type}")
 
+        if (
+            workflow_run.mode == WorkflowRunMode.SMALLWEBRTC.value
+            or provider_type == WorkflowRunMode.SMALLWEBRTC.value
+        ):
+            logger.warning(
+                f"SmallWebRTC workflow run {workflow_run_id} reached telephony "
+                f"websocket; mode={workflow_run.mode}, provider={provider_type}"
+            )
+            await websocket.close(
+                code=4400,
+                reason=(
+                    "smallwebrtc runs connect through the WebRTC signaling endpoint, "
+                    "not the telephony websocket"
+                ),
+            )
+            return
+
         if not provider_type:
             logger.error(
                 f"No provider type found in workflow run {workflow_run_id}. "
                 f"gathered_context: {workflow_run.gathered_context}, mode: {workflow_run.mode}"
             )
-            await websocket.close(code=4400, reason="Provider type not found")
+            await websocket.close(
+                code=4400,
+                reason=(
+                    f"No provider type found for workflow run {workflow_run_id} "
+                    f"(mode: {workflow_run.mode}); telephony websocket requires "
+                    "a telephony provider"
+                ),
+            )
             return
 
         logger.info(
@@ -619,7 +700,7 @@ async def _handle_telephony_websocket(
 
         # Delegate to provider-specific handler
         await provider.handle_websocket(
-            websocket, workflow_id, user_id, workflow_run_id
+            websocket, workflow_id, organization_id, workflow_run_id
         )
 
     except WebSocketDisconnect as e:
@@ -660,7 +741,7 @@ async def handle_inbound_run(request: Request):
             logger.error("Unable to detect provider for /inbound/run webhook")
             return generic_hangup_response()
 
-        normalized_data = normalize_webhook_data(provider_class, webhook_data)
+        normalized_data = normalize_webhook_data(provider_class, webhook_data, headers)
         logger.info(
             f"/inbound/run normalized data — provider={normalized_data.provider} "
             f"to={normalized_data.to_number} from={normalized_data.from_number}"
@@ -741,40 +822,68 @@ async def handle_inbound_run(request: Request):
                 TelephonyError.SIGNATURE_VALIDATION_FAILED
             )
 
-        # 5. Create workflow run + authorize quota before returning provider
-        # stream instructions.
-        workflow_run_id = await _create_inbound_workflow_run(
-            workflow_id,
-            user_id,
-            provider_class.PROVIDER_NAME,
-            normalized_data,
-            telephony_configuration_id=telephony_configuration_id,
-            from_phone_number_id=phone_row.id,
-        )
-        quota_result = await authorize_workflow_run_start(
-            workflow_id=workflow_id,
-            workflow_run_id=workflow_run_id,
-        )
-        if not quota_result.has_quota:
-            logger.warning(
-                f"User {user_id} has exceeded quota: {quota_result.error_message}"
+        try:
+            concurrency_slot = await call_concurrency.acquire_org_slot(
+                config.organization_id,
+                source=f"inbound:{provider_class.PROVIDER_NAME}",
+                timeout=0,
             )
+        except CallConcurrencyLimitError:
             return provider_class.generate_validation_error_response(
-                TelephonyError.QUOTA_EXCEEDED
+                TelephonyError.CONCURRENT_CALL_LIMIT
             )
 
-        backend_endpoint, wss_backend_endpoint = await get_backend_endpoints()
-        websocket_url = (
-            f"{wss_backend_endpoint}/api/v1/telephony/ws/"
-            f"{workflow_id}/{user_id}/{workflow_run_id}"
-        )
+        workflow_run_id = None
+        try:
+            # 5. Create workflow run + authorize quota before returning provider
+            # stream instructions.
+            workflow_run_id = await _create_inbound_workflow_run(
+                workflow_id,
+                user_id,
+                config.organization_id,
+                provider_class.PROVIDER_NAME,
+                normalized_data,
+                telephony_configuration_id=telephony_configuration_id,
+                from_phone_number_id=phone_row.id,
+            )
+            await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run_id)
 
-        return await provider_instance.start_inbound_stream(
-            websocket_url=websocket_url,
-            workflow_run_id=workflow_run_id,
-            normalized_data=normalized_data,
-            backend_endpoint=backend_endpoint,
-        )
+            quota_result = await authorize_workflow_run_start(
+                workflow_id=workflow_id,
+                organization_id=config.organization_id,
+                workflow_run_id=workflow_run_id,
+            )
+            if not quota_result.has_quota:
+                logger.warning(
+                    f"User {user_id} has exceeded quota: {quota_result.error_message}"
+                )
+                await call_concurrency.release_workflow_run_slot(workflow_run_id)
+                return provider_class.generate_validation_error_response(
+                    TelephonyError.QUOTA_EXCEEDED
+                )
+
+            backend_endpoint, wss_backend_endpoint = await get_backend_endpoints()
+            websocket_url = (
+                f"{wss_backend_endpoint}/api/v1/telephony/ws/"
+                f"{workflow_id}/{config.organization_id}/{workflow_run_id}"
+            )
+
+            return await provider_instance.start_inbound_stream(
+                websocket_url=websocket_url,
+                workflow_run_id=workflow_run_id,
+                normalized_data=normalized_data,
+                backend_endpoint=backend_endpoint,
+            )
+        except WorkflowRunSlotAlreadyBoundError:
+            return provider_class.generate_validation_error_response(
+                TelephonyError.CONCURRENT_CALL_LIMIT
+            )
+        except Exception:
+            if workflow_run_id:
+                await call_concurrency.release_workflow_run_slot(workflow_run_id)
+            else:
+                await call_concurrency.release_slot(concurrency_slot)
+            raise
 
     except ValueError as e:
         logger.error(f"/inbound/run request parsing error: {e}")
@@ -847,7 +956,7 @@ async def handle_inbound_telephony(
             logger.error("Unable to detect provider for webhook")
             return generic_hangup_response()
 
-        normalized_data = normalize_webhook_data(provider_class, webhook_data)
+        normalized_data = normalize_webhook_data(provider_class, webhook_data, headers)
 
         logger.info(f"Inbound call - Provider: {normalized_data.provider}")
         logger.info(f"Normalized data: {normalized_data}")
@@ -876,38 +985,73 @@ async def handle_inbound_telephony(
             logger.error(f"Request validation failed: {error_type}")
             return provider_class.generate_validation_error_response(error_type)
 
-        # Create workflow run.
         user_id = workflow_context["user_id"]
-        workflow_run_id = await _create_inbound_workflow_run(
-            workflow_id,
-            workflow_context["user_id"],
-            workflow_context["provider"],
-            normalized_data,
-            telephony_configuration_id=workflow_context["telephony_configuration_id"],
-            from_phone_number_id=workflow_context.get("from_phone_number_id"),
-        )
-        quota_result = await authorize_workflow_run_start(
-            workflow_id=workflow_id,
-            workflow_run_id=workflow_run_id,
-        )
-        if not quota_result.has_quota:
-            logger.warning(
-                f"User {user_id} has exceeded quota for inbound calls: {quota_result.error_message}"
+        organization_id = workflow_context["organization_id"]
+        try:
+            concurrency_slot = await call_concurrency.acquire_org_slot(
+                organization_id,
+                source=f"inbound_legacy:{workflow_context['provider']}",
+                timeout=0,
             )
+        except CallConcurrencyLimitError:
             return provider_class.generate_validation_error_response(
-                TelephonyError.QUOTA_EXCEEDED
+                TelephonyError.CONCURRENT_CALL_LIMIT
             )
 
-        # Generate response URLs
-        backend_endpoint, wss_backend_endpoint = await get_backend_endpoints()
-        websocket_url = f"{wss_backend_endpoint}/api/v1/telephony/ws/{workflow_id}/{workflow_context['user_id']}/{workflow_run_id}"
+        workflow_run_id = None
+        try:
+            # Create workflow run.
+            workflow_run_id = await _create_inbound_workflow_run(
+                workflow_id,
+                workflow_context["user_id"],
+                organization_id,
+                workflow_context["provider"],
+                normalized_data,
+                telephony_configuration_id=workflow_context[
+                    "telephony_configuration_id"
+                ],
+                from_phone_number_id=workflow_context.get("from_phone_number_id"),
+            )
+            await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run_id)
 
-        response = await provider_instance.start_inbound_stream(
-            websocket_url=websocket_url,
-            workflow_run_id=workflow_run_id,
-            normalized_data=normalized_data,
-            backend_endpoint=backend_endpoint,
-        )
+            quota_result = await authorize_workflow_run_start(
+                workflow_id=workflow_id,
+                organization_id=organization_id,
+                workflow_run_id=workflow_run_id,
+            )
+            if not quota_result.has_quota:
+                logger.warning(
+                    f"User {user_id} has exceeded quota for inbound calls: "
+                    f"{quota_result.error_message}"
+                )
+                await call_concurrency.release_workflow_run_slot(workflow_run_id)
+                return provider_class.generate_validation_error_response(
+                    TelephonyError.QUOTA_EXCEEDED
+                )
+
+            # Generate response URLs
+            backend_endpoint, wss_backend_endpoint = await get_backend_endpoints()
+            websocket_url = (
+                f"{wss_backend_endpoint}/api/v1/telephony/ws/"
+                f"{workflow_id}/{organization_id}/{workflow_run_id}"
+            )
+
+            response = await provider_instance.start_inbound_stream(
+                websocket_url=websocket_url,
+                workflow_run_id=workflow_run_id,
+                normalized_data=normalized_data,
+                backend_endpoint=backend_endpoint,
+            )
+        except WorkflowRunSlotAlreadyBoundError:
+            return provider_class.generate_validation_error_response(
+                TelephonyError.CONCURRENT_CALL_LIMIT
+            )
+        except Exception:
+            if workflow_run_id:
+                await call_concurrency.release_workflow_run_slot(workflow_run_id)
+            else:
+                await call_concurrency.release_slot(concurrency_slot)
+            raise
 
         logger.info(
             f"Generated {normalized_data.provider} response for call {normalized_data.call_id}"

api/routes/user.py

@@ -10,9 +10,16 @@ from api.db.models import (
     UserModel,
 )
 from api.schemas.onboarding_state import OnboardingState, OnboardingStateUpdate
+from api.schemas.workflow_configurations import (
+    WorkflowConfigurationDefaults,
+    get_default_workflow_configurations,
+)
 from api.services.auth.depends import get_user
 from api.services.configuration.ai_model_configuration import (
+    convert_legacy_ai_model_configuration_to_v2,
     get_resolved_ai_model_configuration,
+    update_organization_ai_model_configuration_last_validated_at,
+    upsert_organization_ai_model_configuration_v2,
 )
 from api.services.configuration.check_validity import (
     APIKeyStatusResponse,
@@ -40,13 +47,14 @@ class AuthUserResponse(TypedDict):
     is_superuser: bool
 
 
-class DefaultConfigurationsResponse(TypedDict):
+class DefaultConfigurationsResponse(BaseModel):
     llm: dict[str, dict]
     tts: dict[str, dict]
     stt: dict[str, dict]
     embeddings: dict[str, dict]
     realtime: dict[str, dict]
     default_providers: dict[str, str]
+    workflow_configurations: WorkflowConfigurationDefaults
 
 
 @router.get("/configurations/defaults")
@@ -73,8 +81,9 @@ async def get_default_configurations() -> DefaultConfigurationsResponse:
             for provider, model_cls in REGISTRY[ServiceType.REALTIME].items()
         },
         "default_providers": DEFAULT_SERVICE_PROVIDERS,
+        "workflow_configurations": get_default_workflow_configurations(),
     }
-    return configurations
+    return DefaultConfigurationsResponse(**configurations)
 
 
 @router.get("/auth/user")
@@ -99,12 +108,29 @@ class UserConfigurationRequestResponseSchema(BaseModel):
     organization_pricing: dict[str, Union[float, str, bool]] | None = None
 
 
+def _is_validation_cache_stale(
+    last_validated_at: datetime | None,
+    validity_ttl_seconds: int,
+) -> bool:
+    if last_validated_at is None:
+        return True
+
+    has_timezone = (
+        last_validated_at.tzinfo is not None
+        and last_validated_at.utcoffset() is not None
+    )
+    if has_timezone:
+        now = datetime.now(last_validated_at.tzinfo)
+    else:
+        now = datetime.now()
+    return last_validated_at < now - timedelta(seconds=validity_ttl_seconds)
+
+
 @router.get("/configurations/user")
 async def get_user_configurations(
     user: UserModel = Depends(get_user),
 ) -> UserConfigurationRequestResponseSchema:
     resolved_config = await get_resolved_ai_model_configuration(
-        user_id=user.id,
         organization_id=user.selected_organization_id,
     )
     masked_config = mask_user_config(resolved_config.effective)
@@ -133,7 +159,11 @@ async def update_user_configurations(
     request: UserConfigurationRequestResponseSchema,
     user: UserModel = Depends(get_user),
 ) -> UserConfigurationRequestResponseSchema:
-    existing_config = await db_client.get_user_configurations(user.id)
+    existing_config = (
+        await get_resolved_ai_model_configuration(
+            organization_id=user.selected_organization_id,
+        )
+    ).effective
 
     incoming_dict = request.model_dump(exclude_none=True)
 
@@ -146,6 +176,9 @@ async def update_user_configurations(
     }
 
     if incoming_dict:
+        if not user.selected_organization_id:
+            raise HTTPException(status_code=400, detail="No organization selected")
+
         # Merge via helper
         try:
             user_configurations = merge_user_configurations(
@@ -169,8 +202,16 @@ async def update_user_configurations(
         except ValueError as e:
             raise HTTPException(status_code=422, detail=e.args[0])
 
-        user_configurations = await db_client.update_user_configuration(
-            user.id, user_configurations
+        try:
+            organization_configuration = convert_legacy_ai_model_configuration_to_v2(
+                user_configurations
+            )
+        except ValueError as e:
+            raise HTTPException(status_code=422, detail=str(e))
+
+        await upsert_organization_ai_model_configuration_v2(
+            user.selected_organization_id,
+            organization_configuration,
         )
     else:
         user_configurations = existing_config
@@ -229,15 +270,13 @@ async def validate_user_configurations(
     user: UserModel = Depends(get_user),
 ) -> APIKeyStatusResponse:
     resolved_config = await get_resolved_ai_model_configuration(
-        user_id=user.id,
         organization_id=user.selected_organization_id,
     )
     configurations = resolved_config.effective
 
-    if (
-        configurations.last_validated_at
-        and configurations.last_validated_at
-        < datetime.now() - timedelta(seconds=validity_ttl_seconds)
+    if _is_validation_cache_stale(
+        configurations.last_validated_at,
+        validity_ttl_seconds,
     ):
         validator = UserConfigurationValidator()
         try:
@@ -246,7 +285,13 @@ async def validate_user_configurations(
                 organization_id=user.selected_organization_id,
                 created_by=user.provider_id,
             )
-            await db_client.update_user_configuration_last_validated_at(user.id)
+            if (
+                resolved_config.source == "organization_v2"
+                and user.selected_organization_id is not None
+            ):
+                await update_organization_ai_model_configuration_last_validated_at(
+                    user.selected_organization_id
+                )
             return status
         except ValueError as e:
             raise HTTPException(status_code=422, detail=e.args[0])

api/routes/webrtc_signaling.py

@@ -40,6 +40,11 @@ from api.routes.turn_credentials import (
     generate_turn_credentials,
 )
 from api.services.auth.depends import get_user_ws
+from api.services.call_concurrency import (
+    CallConcurrencyLimitError,
+    WorkflowRunSlotAlreadyBoundError,
+    call_concurrency,
+)
 from api.services.pipecat.run_pipeline import run_pipeline_smallwebrtc
 from api.services.pipecat.ws_sender_registry import (
     register_ws_sender,
@@ -321,6 +326,9 @@ class SignalingManager:
         workflow_id: int,
         workflow_run_id: int,
         user: UserModel,
+        organization_id: int,
+        enforce_call_concurrency: bool = False,
+        call_concurrency_source: str = "webrtc",
     ):
         """Handle WebSocket connection for signaling."""
         await websocket.accept()
@@ -337,7 +345,10 @@ class SignalingManager:
                     workflow_id,
                     workflow_run_id,
                     user,
+                    organization_id,
                     connection_key,
+                    enforce_call_concurrency,
+                    call_concurrency_source,
                 )
         except WebSocketDisconnect:
             logger.info(f"WebSocket disconnected for {connection_id}")
@@ -378,7 +389,10 @@ class SignalingManager:
         workflow_id: int,
         workflow_run_id: int,
         user: UserModel,
+        organization_id: int,
         connection_key: str,
+        enforce_call_concurrency: bool,
+        call_concurrency_source: str = "webrtc",
     ):
         """Handle incoming WebSocket messages."""
         msg_type = message.get("type")
@@ -386,7 +400,15 @@ class SignalingManager:
 
         if msg_type == "offer":
             await self._handle_offer(
-                ws, payload, workflow_id, workflow_run_id, user, connection_key
+                ws,
+                payload,
+                workflow_id,
+                workflow_run_id,
+                user,
+                organization_id,
+                connection_key,
+                enforce_call_concurrency,
+                call_concurrency_source,
             )
         elif msg_type == "ice-candidate":
             await self._handle_ice_candidate(payload, connection_key)
@@ -400,7 +422,10 @@ class SignalingManager:
         workflow_id: int,
         workflow_run_id: int,
         user: UserModel,
+        organization_id: int,
         connection_key: str,
+        enforce_call_concurrency: bool,
+        call_concurrency_source: str = "webrtc",
     ):
         """Handle offer message and create answer with ICE trickling."""
         pc_id = payload.get("pc_id")
@@ -420,14 +445,13 @@ class SignalingManager:
         # Set run context for logging and tracing. org_id must be set before
         # pc.initialize() so that aiortc's internal tasks inherit it.
         set_current_run_id(workflow_run_id)
-        org_id = await db_client.get_workflow_organization_id(workflow_id)
-        if org_id:
-            set_current_org_id(org_id)
+        set_current_org_id(organization_id)
 
         # Check Dograh quota before initiating the call (apply per-workflow
         # model_overrides so we evaluate the keys this workflow will use).
         quota_result = await authorize_workflow_run_start(
             workflow_id=workflow_id,
+            organization_id=organization_id,
             workflow_run_id=workflow_run_id,
             actor_user=user,
         )
@@ -472,69 +496,124 @@ class SignalingManager:
                 }
             )
         else:
+            concurrency_slot = None
+            concurrency_bound = False
+            pipeline_started = False
+            pc = None
+            if enforce_call_concurrency:
+                try:
+                    concurrency_slot = await call_concurrency.acquire_org_slot(
+                        organization_id,
+                        source=call_concurrency_source,
+                        timeout=0,
+                    )
+                    await call_concurrency.bind_workflow_run(
+                        concurrency_slot,
+                        workflow_run_id,
+                    )
+                    concurrency_bound = True
+                except CallConcurrencyLimitError:
+                    await ws.send_json(
+                        {
+                            "type": "error",
+                            "payload": {
+                                "error_type": "concurrency_limit_exceeded",
+                                "message": "Concurrent call limit reached",
+                            },
+                        }
+                    )
+                    return
+                except WorkflowRunSlotAlreadyBoundError:
+                    await ws.send_json(
+                        {
+                            "type": "error",
+                            "payload": {
+                                "error_type": "workflow_run_already_active",
+                                "message": "Workflow run already has an active call",
+                            },
+                        }
+                    )
+                    return
+
             # Create new connection using correct SmallWebRTC API
             # Generate ICE servers with time-limited TURN credentials for this user
-            user_ice_servers = get_ice_servers(user_id=str(user.id))
-            pc = SmallWebRTCConnection(
-                ice_servers=user_ice_servers, connection_timeout_secs=60
-            )
-            # Set the pc_id before initialization so it's available in get_answer()
-            pc._pc_id = pc_id
-
-            # Initialize connection with offer
-            await pc.initialize(sdp=sdp, type=type_)
-
-            # Store peer connection using client's pc_id
-            self._track_peer_connection(connection_key, pc_id, pc)
-
-            # Register WebSocket sender for real-time feedback
-            async def ws_sender(message: dict):
-                if ws.application_state == WebSocketState.CONNECTED:
-                    await ws.send_json(message)
-
-            register_ws_sender(workflow_run_id, ws_sender)
-
-            # Setup closed handler
-            @pc.event_handler("closed")
-            async def handle_disconnected(webrtc_connection: SmallWebRTCConnection):
-                logger.info(f"PeerConnection closed: {webrtc_connection.pc_id}")
-                owner_connection_id = self._forget_peer_connection(
-                    webrtc_connection.pc_id
+            try:
+                user_ice_servers = get_ice_servers(user_id=str(user.id))
+                pc = SmallWebRTCConnection(
+                    ice_servers=user_ice_servers, connection_timeout_secs=60
                 )
-                if owner_connection_id == connection_key:
-                    await self._notify_call_ended_and_close_websocket(
-                        ws,
-                        workflow_run_id,
-                        webrtc_connection.pc_id,
-                        reason="peer_connection_closed",
+                # Set the pc_id before initialization so it's available in get_answer()
+                pc._pc_id = pc_id
+
+                # Initialize connection with offer
+                await pc.initialize(sdp=sdp, type=type_)
+
+                # Store peer connection using client's pc_id
+                self._track_peer_connection(connection_key, pc_id, pc)
+
+                # Register WebSocket sender for real-time feedback
+                async def ws_sender(message: dict):
+                    if ws.application_state == WebSocketState.CONNECTED:
+                        await ws.send_json(message)
+
+                register_ws_sender(workflow_run_id, ws_sender)
+
+                # Setup closed handler
+                @pc.event_handler("closed")
+                async def handle_disconnected(
+                    webrtc_connection: SmallWebRTCConnection,
+                ):
+                    logger.info(f"PeerConnection closed: {webrtc_connection.pc_id}")
+                    owner_connection_id = self._forget_peer_connection(
+                        webrtc_connection.pc_id
                     )
+                    if owner_connection_id == connection_key:
+                        await self._notify_call_ended_and_close_websocket(
+                            ws,
+                            workflow_run_id,
+                            webrtc_connection.pc_id,
+                            reason="peer_connection_closed",
+                        )
 
-            # Start pipeline in background
-            asyncio.create_task(
-                run_pipeline_smallwebrtc(
-                    pc,
-                    workflow_id,
-                    workflow_run_id,
-                    user.id,
-                    call_context_vars,
-                    user_provider_id=str(user.provider_id),
+                # Start pipeline in background
+                asyncio.create_task(
+                    run_pipeline_smallwebrtc(
+                        pc,
+                        workflow_id,
+                        workflow_run_id,
+                        user.id,
+                        call_context_vars,
+                        user_provider_id=str(user.provider_id),
+                        organization_id=organization_id,
+                    )
                 )
-            )
+                pipeline_started = True
 
-            # Get answer after initialization
-            answer = pc.get_answer()
+                # Get answer after initialization
+                answer = pc.get_answer()
 
-            # Send answer immediately (ICE candidates will be sent separately via trickling)
-            await ws.send_json(
-                {
-                    "type": "answer",
-                    "payload": {
-                        "sdp": filter_outbound_sdp(answer["sdp"]),
-                        "type": answer["type"],
-                        "pc_id": answer["pc_id"],
-                    },
-                }
-            )
+                # Send answer immediately (ICE candidates will be sent separately via trickling)
+                await ws.send_json(
+                    {
+                        "type": "answer",
+                        "payload": {
+                            "sdp": filter_outbound_sdp(answer["sdp"]),
+                            "type": answer["type"],
+                            "pc_id": answer["pc_id"],
+                        },
+                    }
+                )
+            except Exception:
+                if pipeline_started and pc is not None:
+                    try:
+                        await pc.disconnect()
+                    except Exception as e:
+                        logger.debug(f"Failed to disconnect failed offer pc: {e}")
+                elif concurrency_bound:
+                    await call_concurrency.release_workflow_run_slot(workflow_run_id)
+                elif concurrency_slot is not None:
+                    await call_concurrency.release_slot(concurrency_slot)
+                raise
 
     async def _handle_ice_candidate(self, payload: dict, connection_key: str):
         """Handle incoming ICE candidate from client.
@@ -630,13 +709,33 @@ async def signaling_websocket(
     user: UserModel = Depends(get_user_ws),
 ):
     """WebSocket endpoint for WebRTC signaling with ICE trickling."""
-    workflow_run = await db_client.get_workflow_run(workflow_run_id, user.id)
+    if not user.selected_organization_id:
+        raise HTTPException(status_code=400, detail="No organization selected")
+
+    workflow_run = await db_client.get_workflow_run(
+        workflow_run_id, organization_id=user.selected_organization_id
+    )
     if not workflow_run:
-        logger.warning(f"workflow run {workflow_run_id} not found for user {user.id}")
+        logger.warning(
+            f"workflow run {workflow_run_id} not found for org "
+            f"{user.selected_organization_id}"
+        )
         raise HTTPException(status_code=400, detail="Bad workflow_run_id")
+    if workflow_run.workflow_id != workflow_id:
+        logger.warning(
+            f"workflow run {workflow_run_id} belongs to workflow "
+            f"{workflow_run.workflow_id}, not {workflow_id}"
+        )
+        raise HTTPException(status_code=400, detail="workflow_run_workflow_mismatch")
 
     await signaling_manager.handle_websocket(
-        websocket, workflow_id, workflow_run_id, user
+        websocket,
+        workflow_id,
+        workflow_run_id,
+        user,
+        user.selected_organization_id,
+        enforce_call_concurrency=True,
+        call_concurrency_source="webrtc",
     )
 
 
@@ -670,6 +769,17 @@ async def public_signaling_websocket(
         await websocket.close(code=1008, reason="Invalid embed token")
         return
 
+    workflow_run = await db_client.get_workflow_run(
+        embed_session.workflow_run_id,
+        organization_id=embed_token.organization_id,
+    )
+    if not workflow_run:
+        await websocket.close(code=1008, reason="Invalid workflow run")
+        return
+    if workflow_run.workflow_id != embed_token.workflow_id:
+        await websocket.close(code=1008, reason="workflow_run_workflow_mismatch")
+        return
+
     # Enforce the embed token's allowed-domain policy on the public signaling
     # path, mirroring the HTTP embed endpoints (issue #330). Without this a
     # leaked or replayed session token could attach from an arbitrary origin.
@@ -693,5 +803,11 @@ async def public_signaling_websocket(
 
     # Handle the WebSocket connection using the existing signaling manager
     await signaling_manager.handle_websocket(
-        websocket, embed_token.workflow_id, embed_session.workflow_run_id, user
+        websocket,
+        embed_token.workflow_id,
+        embed_session.workflow_run_id,
+        user,
+        embed_token.organization_id,
+        enforce_call_concurrency=True,
+        call_concurrency_source="public_embed",
     )

api/routes/workflow.py

@@ -18,6 +18,7 @@ from api.db.workflow_template_client import WorkflowTemplateClient
 from api.enums import CallType, PostHogEvent, StorageBackend, WorkflowStatus
 from api.schemas.ai_model_configuration import OrganizationAIModelConfigurationV2
 from api.schemas.workflow import WorkflowRunResponseSchema
+from api.schemas.workflow_configurations import WorkflowConfigurationDefaults
 from api.sdk_expose import sdk_expose
 from api.services.auth.depends import get_user
 from api.services.configuration.ai_model_configuration import (
@@ -284,7 +285,10 @@ class UpdateWorkflowRequest(BaseModel):
     name: str | None = None
     workflow_definition: dict | None = None
     template_context_variables: dict | None = None
-    workflow_configurations: dict | None = None
+    # Typed so field constraints (e.g. the max_call_duration cap) are
+    # enforced by FastAPI; extra="allow" keeps passthrough keys like
+    # model_configuration_v2_override intact.
+    workflow_configurations: WorkflowConfigurationDefaults | None = None
 
 
 class WorkflowVersionResponse(BaseModel):
@@ -1039,7 +1043,13 @@ async def update_workflow(
 
         # Validate model overrides. v2 uses a complete workflow-level model
         # configuration; legacy v1 uses partial service overlays.
-        workflow_configurations = request.workflow_configurations
+        # exclude_unset keeps stored configs sparse: keys the request didn't
+        # send stay absent so runtime defaults keep applying to them.
+        workflow_configurations = (
+            request.workflow_configurations.model_dump(exclude_unset=True)
+            if request.workflow_configurations is not None
+            else None
+        )
         if workflow_configurations and workflow_configurations.get(
             WORKFLOW_MODEL_CONFIGURATION_V2_OVERRIDE_KEY
         ):
@@ -1080,7 +1090,6 @@ async def update_workflow(
                 )
                 if existing_v2_override_config is None:
                     resolved_config = await get_resolved_ai_model_configuration(
-                        user_id=user.id,
                         organization_id=user.selected_organization_id,
                     )
                     v2_override = merge_ai_model_configuration_v2_secrets(
@@ -1123,7 +1132,6 @@ async def update_workflow(
                 existing_configs,
             )
             resolved_config = await get_resolved_ai_model_configuration(
-                user_id=user.id,
                 organization_id=user.selected_organization_id,
             )
             effective_config = resolved_config.effective

api/routes/workflow_text_chat.py

@@ -103,6 +103,7 @@ async def _ensure_text_chat_quota(
 ) -> None:
     quota_result = await authorize_workflow_run_start(
         workflow_id=workflow_id,
+        organization_id=user.selected_organization_id,
         workflow_run_id=workflow_run_id,
         actor_user=user,
     )

api/schemas/ai_model_configuration.py

@@ -176,7 +176,7 @@ def _compile_dograh_configuration(
         embeddings=DograhEmbeddingsConfiguration(
             provider=ServiceProviders.DOGRAH,
             api_key=configuration.api_key,
-            model="default",
+            model="dograh_embedding_v1",
         ),
         is_realtime=False,
         managed_service_version=2,

api/schemas/tool.py

@@ -8,7 +8,6 @@ when the same schema is surfaced through MCP or SDK authoring flows.
 
 from __future__ import annotations
 
-import re
 from datetime import datetime
 from typing import Annotated, Any, Dict, List, Literal, Optional, Union
 
@@ -181,14 +180,62 @@ class EndCallConfig(BaseModel):
     )
 
 
+class HttpTransferResolverConfig(BaseModel):
+    """HTTP endpoint used to resolve transfer destination at call time."""
+
+    type: Literal["http"] = Field(default="http", description="Resolver type.")
+    url: str = Field(description="HTTP or HTTPS endpoint for transfer resolution.")
+    headers: Optional[Dict[str, str]] = Field(
+        default=None,
+        description="Static headers to include with every resolver request.",
+    )
+    credential_uuid: Optional[str] = Field(
+        default=None,
+        description="Reference to an external credential for resolver authentication.",
+    )
+    timeout_ms: int = Field(
+        default=3000,
+        ge=500,
+        le=5000,
+        description="Resolver request timeout in milliseconds.",
+    )
+    wait_message: Optional[str] = Field(
+        default=None,
+        description="Optional short message played while Dograh resolves routing.",
+    )
+    parameters: Optional[List[ToolParameter]] = Field(
+        default=None,
+        description="Parameters the model may provide when calling this transfer tool.",
+    )
+    preset_parameters: Optional[List[PresetToolParameter]] = Field(
+        default=None,
+        description=(
+            "Parameters injected by Dograh from fixed values or workflow context "
+            "templates."
+        ),
+    )
+
+    @field_validator("url")
+    @classmethod
+    def validate_url(cls, v: str) -> str:
+        if not isinstance(v, str) or not v.startswith(("http://", "https://")):
+            raise ValueError("config.resolver.url must be an http(s) URL")
+        return v
+
+
 class TransferCallConfig(BaseModel):
     """Configuration for Transfer Call tools."""
 
+    destination_source: Literal["static", "dynamic"] = Field(
+        default="static",
+        description="Whether transfer destination is static/template or resolved by HTTP.",
+    )
     destination: str = Field(
+        default="",
         description=(
-            "Phone number or SIP endpoint to transfer the call to, e.g. "
-            "+1234567890 or PJSIP/1234."
-        )
+            "Phone number, SIP endpoint, or template to transfer the call to, e.g. "
+            "+1234567890, PJSIP/1234, or {{initial_context.transfer_destination}}."
+        ),
     )
     messageType: Literal["none", "custom", "audio"] = Field(
         default="none", description="Type of message to play before transfer."
@@ -205,26 +252,25 @@ class TransferCallConfig(BaseModel):
         le=120,
         description="Maximum seconds to wait for the destination to answer.",
     )
+    parameters: Optional[List[ToolParameter]] = Field(
+        default=None,
+        description=(
+            "Parameters the model may provide when calling this transfer tool, "
+            "for example state, department, or transfer reason."
+        ),
+    )
+    resolver: Optional[HttpTransferResolverConfig] = Field(
+        default=None,
+        description="Optional resolver that determines transfer routing at call time.",
+    )
 
-    @field_validator("destination")
-    @classmethod
-    def validate_destination(cls, v: str) -> str:
-        """Validate that destination is a valid E.164 phone number or SIP endpoint."""
-        if not v.strip():
-            return v
-
-        e164_pattern = r"^\+[1-9]\d{1,14}$"
-        sip_pattern = r"^(PJSIP|SIP)/[\w\-\.@]+$"
-
-        is_valid_e164 = re.match(e164_pattern, v)
-        is_valid_sip = re.match(sip_pattern, v, re.IGNORECASE)
-
-        if not (is_valid_e164 or is_valid_sip):
+    @model_validator(mode="after")
+    def validate_destination_source_config(self):
+        if self.destination_source == "dynamic" and self.resolver is None:
             raise ValueError(
-                "Destination must be a valid E.164 phone number "
-                "(e.g., +1234567890) or SIP endpoint (e.g., PJSIP/1234)"
+                "config.resolver is required when destination_source is dynamic"
             )
-        return v
+        return self
 
 
 class McpToolConfig(BaseModel):

api/schemas/workflow_configurations.py

@@ -0,0 +1,62 @@
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+DEFAULT_MAX_CALL_DURATION_SECONDS = 300
+# Hard ceiling on configurable call duration. Must stay <= the concurrency
+# rate limiter's stale_call_timeout (20 min): a call running past that has
+# its slot purged as stale and the org concurrency limit under-counts.
+MAX_CALL_DURATION_SECONDS = 1200
+DEFAULT_MAX_USER_IDLE_TIMEOUT_SECONDS = 10.0
+DEFAULT_SMART_TURN_STOP_SECS = 2.0
+DEFAULT_TURN_START_STRATEGY = "default"
+DEFAULT_TURN_START_MIN_WORDS = 3
+DEFAULT_PROVISIONAL_VAD_PAUSE_SECS = 1.5
+DEFAULT_TURN_STOP_STRATEGY = "transcription"
+DEFAULT_CONTEXT_COMPACTION_ENABLED = False
+
+
+class AmbientNoiseConfigurationDefaults(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    enabled: bool = False
+    volume: float = 0.3
+
+
+class WorkflowConfigurationDefaults(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    @model_validator(mode="before")
+    @classmethod
+    def _treat_null_as_unset(cls, data):
+        # Stored configs (and older clients) carry explicit JSON nulls for
+        # keys the user never configured; dropping them lets the field
+        # defaults apply instead of failing validation.
+        if isinstance(data, dict):
+            return {k: v for k, v in data.items() if v is not None}
+        return data
+
+    ambient_noise_configuration: AmbientNoiseConfigurationDefaults = Field(
+        default_factory=AmbientNoiseConfigurationDefaults
+    )
+    max_call_duration: int = Field(
+        default=DEFAULT_MAX_CALL_DURATION_SECONDS,
+        gt=0,
+        le=MAX_CALL_DURATION_SECONDS,
+    )
+    max_user_idle_timeout: float = DEFAULT_MAX_USER_IDLE_TIMEOUT_SECONDS
+    smart_turn_stop_secs: float = DEFAULT_SMART_TURN_STOP_SECS
+    turn_start_strategy: Literal["default", "min_words", "provisional_vad"] = (
+        DEFAULT_TURN_START_STRATEGY
+    )
+    turn_start_min_words: int = DEFAULT_TURN_START_MIN_WORDS
+    provisional_vad_pause_secs: float = DEFAULT_PROVISIONAL_VAD_PAUSE_SECS
+    turn_stop_strategy: Literal["transcription", "turn_analyzer"] = (
+        DEFAULT_TURN_STOP_STRATEGY
+    )
+    dictionary: str = ""
+    context_compaction_enabled: bool = DEFAULT_CONTEXT_COMPACTION_ENABLED
+
+
+def get_default_workflow_configurations() -> WorkflowConfigurationDefaults:
+    return WorkflowConfigurationDefaults()

api/services/auth/depends.py

@@ -14,14 +14,24 @@ from api.services.auth.stack_auth import stackauth
 from api.services.configuration.registry import ServiceProviders
 from api.services.mps_billing import ensure_hosted_mps_billing_account_v2
 from api.services.posthog_client import (
+    POSTHOG_ORGANIZATION_GROUP_TYPE,
     capture_event,
     group_identify,
     set_person_properties,
 )
 from api.utils.auth import decode_jwt_token
 
-POSTHOG_ORGANIZATION_GROUP_TYPE = "organization"
-POSTHOG_ORGANIZATION_USES_MPS_BILLING_V2_PROPERTY = "uses_mps_billing_v2"
+
+async def require_local_auth() -> None:
+    """Reject email/password auth requests outside OSS (local) deployments.
+
+    The auth router stays mounted in every mode so the OpenAPI spec — and the
+    clients generated from it — don't vary with AUTH_PROVIDER; the gate has to
+    happen at request time. Without it, the SaaS deployment accepts
+    unauthenticated signups that mint oss_* users bypassing Stack Auth.
+    """
+    if AUTH_PROVIDER != "local":
+        raise HTTPException(status_code=404, detail="Not found")
 
 
 async def get_user(
@@ -180,7 +190,6 @@ def _sync_created_organization_to_posthog(
     organization,
     stack_user: dict | None = None,
     created_by_provider_id: str | None = None,
-    uses_mps_billing_v2: bool | None = None,
 ) -> None:
     """Create/update the PostHog organization group for a newly-created org."""
     try:
@@ -196,10 +205,6 @@ def _sync_created_organization_to_posthog(
         }
         if created_by:
             properties["created_by_provider_id"] = created_by
-        if uses_mps_billing_v2 is not None:
-            properties[POSTHOG_ORGANIZATION_USES_MPS_BILLING_V2_PROPERTY] = (
-                uses_mps_billing_v2
-            )
 
         group_identify(
             POSTHOG_ORGANIZATION_GROUP_TYPE,
@@ -218,50 +223,6 @@ def _sync_created_organization_to_posthog(
         logger.exception("Failed to sync created organization to PostHog")
 
 
-def _sync_posthog_organization_group_properties(
-    *,
-    organization,
-    uses_mps_billing_v2: bool | None = None,
-) -> None:
-    """Update PostHog organization group properties without creating a person."""
-    try:
-        organization_id = int(organization.id)
-        properties = {
-            "organization_id": organization_id,
-            "organization_provider_id": getattr(organization, "provider_id", None),
-            "auth_provider": "stack",
-        }
-        if uses_mps_billing_v2 is not None:
-            properties[POSTHOG_ORGANIZATION_USES_MPS_BILLING_V2_PROPERTY] = (
-                uses_mps_billing_v2
-            )
-
-        group_identify(
-            POSTHOG_ORGANIZATION_GROUP_TYPE,
-            str(organization_id),
-            properties,
-        )
-    except Exception:
-        logger.exception("Failed to sync organization group properties to PostHog")
-
-
-def _sync_posthog_organization_mps_billing_v2_status(
-    organization_id: int,
-    *,
-    uses_mps_billing_v2: bool,
-) -> None:
-    """Update the PostHog organization group with current MPS billing status."""
-    try:
-        organization_id = int(organization_id)
-        group_identify(
-            POSTHOG_ORGANIZATION_GROUP_TYPE,
-            str(organization_id),
-            {POSTHOG_ORGANIZATION_USES_MPS_BILLING_V2_PROPERTY: uses_mps_billing_v2},
-        )
-    except Exception:
-        logger.exception("Failed to sync organization billing status to PostHog")
-
-
 def _associate_user_with_posthog_organization(
     *,
     user: UserModel,
@@ -457,6 +418,11 @@ async def create_user_configuration_with_mps_key(
                         "api_key": [service_key],
                         "model": "default",
                     },
+                    "embeddings": {
+                        "provider": ServiceProviders.DOGRAH.value,
+                        "api_key": [service_key],
+                        "model": "dograh_embedding_v1",
+                    },
                 }
                 effective_config = EffectiveAIModelConfiguration(**configuration)
                 return effective_config

api/services/auth/stack_auth.py

@@ -1,8 +1,17 @@
 import os
+from typing import Any
 
 import aiohttp
 
 
+class StackAuthUserSearchError(Exception):
+    """Raised when Stack Auth user search fails unexpectedly."""
+
+
+class StackAuthSessionError(Exception):
+    """Raised when Stack Auth cannot create an impersonation session."""
+
+
 class StackAuth:
     def __init__(self):
         self.project_id = os.environ.get("STACK_AUTH_PROJECT_ID")
@@ -56,10 +65,56 @@ class StackAuth:
             "is_impersonation": True,
         }
 
-        async with aiohttp.ClientSession() as session:
-            async with session.post(url, headers=headers, json=data) as response:
-                response = await response.json()
-                return response
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(url, headers=headers, json=data) as response:
+                    if response.status >= 400:
+                        raise StackAuthSessionError(
+                            "Stack Auth session creation failed"
+                        )
+
+                    return await response.json()
+        except (aiohttp.ClientError, ValueError) as exc:
+            raise StackAuthSessionError("Stack Auth session creation failed") from exc
+
+    async def find_users_by_email(self, email: str) -> list[dict[str, Any]]:
+        """Return Stack Auth users whose primary email exactly matches."""
+        normalized_email = email.strip().lower()
+        url = os.environ.get("STACK_AUTH_API_URL") + "/api/v1/users"
+        headers = {
+            "x-stack-access-type": "server",
+            "x-stack-project-id": self.project_id,
+            "x-stack-secret-server-key": self.secret_server_key,
+        }
+        params = {
+            "query": normalized_email,
+            "limit": "10",
+        }
+
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, headers=headers, params=params) as response:
+                    if response.status >= 400:
+                        raise StackAuthUserSearchError("Stack Auth user search failed")
+
+                    payload = await response.json()
+        except (aiohttp.ClientError, ValueError) as exc:
+            raise StackAuthUserSearchError("Stack Auth user search failed") from exc
+
+        users = payload.get("items", []) if isinstance(payload, dict) else []
+        if not isinstance(users, list):
+            return []
+
+        return [
+            user
+            for user in users
+            if isinstance(user, dict)
+            and self._stack_user_has_email(user, normalized_email)
+        ]
+
+    def _stack_user_has_email(self, user: dict[str, Any], email: str) -> bool:
+        primary_email = user.get("primary_email")
+        return isinstance(primary_email, str) and primary_email.lower() == email
 
     # ------------------------------------------------------------------
     # Team & user management helpers

api/services/call_concurrency.py

@@ -0,0 +1,282 @@
+import asyncio
+import time
+from dataclasses import dataclass
+
+from loguru import logger
+
+from api.constants import DEFAULT_ORG_CONCURRENCY_LIMIT
+from api.db import db_client
+from api.enums import OrganizationConfigurationKey, PostHogEvent
+from api.services.campaign.rate_limiter import rate_limiter
+from api.services.posthog_client import capture_event
+
+
+@dataclass(frozen=True)
+class CallConcurrencySlot:
+    organization_id: int
+    slot_id: str
+    max_concurrent: int
+    source: str
+    scope_key: str | None = None
+
+
+class CallConcurrencyLimitError(Exception):
+    """Raised when an org has no available concurrent call slots."""
+
+    def __init__(
+        self,
+        *,
+        organization_id: int,
+        source: str,
+        wait_time: float,
+        max_concurrent: int,
+    ):
+        self.organization_id = organization_id
+        self.source = source
+        self.wait_time = wait_time
+        self.max_concurrent = max_concurrent
+        super().__init__(
+            f"Concurrent call limit reached for org {organization_id} "
+            f"(source={source}, limit={max_concurrent}, waited={wait_time:.1f}s)"
+        )
+
+
+class WorkflowRunSlotAlreadyBoundError(Exception):
+    """Raised when a workflow run already owns a concurrent call slot."""
+
+    def __init__(self, workflow_run_id: int):
+        self.workflow_run_id = workflow_run_id
+        super().__init__(
+            f"Workflow run {workflow_run_id} already has an active call slot"
+        )
+
+
+class CallConcurrencyService:
+    def __init__(self):
+        self.default_concurrent_limit = int(DEFAULT_ORG_CONCURRENCY_LIMIT)
+
+    async def get_org_concurrent_limit(self, organization_id: int) -> int:
+        """Get the concurrent call limit for an organization."""
+        try:
+            config = await db_client.get_configuration(
+                organization_id,
+                OrganizationConfigurationKey.CONCURRENT_CALL_LIMIT.value,
+            )
+            if config and config.value:
+                value = config.value.get("value")
+                if value is not None:
+                    return int(value)
+        except Exception as e:
+            logger.warning(
+                f"Error getting concurrent limit for org {organization_id}: {e}"
+            )
+        return self.default_concurrent_limit
+
+    async def acquire_org_slot(
+        self,
+        organization_id: int,
+        *,
+        source: str,
+        timeout: float = 0,
+        scope_key: str | None = None,
+        scope_max_concurrent: int | None = None,
+        retry_interval: float = 1,
+    ) -> CallConcurrencySlot:
+        """Acquire a slot in the org-wide concurrency counter.
+
+        ``scope_key``/``scope_max_concurrent`` additionally bound a secondary
+        counter (e.g. ``campaign:<id>``) so a source can cap its own
+        concurrency without measuring — or being starved by — unrelated calls
+        in the same org.
+        """
+        max_concurrent = await self.get_org_concurrent_limit(organization_id)
+        if scope_max_concurrent is not None:
+            scope_max_concurrent = int(scope_max_concurrent)
+
+        wait_start = time.time()
+        while True:
+            acquisition = await rate_limiter.try_acquire_concurrent_slot_details(
+                organization_id,
+                max_concurrent,
+                scope_key=scope_key,
+                scope_max_concurrent=scope_max_concurrent,
+            )
+            if acquisition:
+                logger.info(
+                    f"Acquired concurrent call slot for org {organization_id}: "
+                    f"source={source}, active_calls="
+                    f"{acquisition.active_count}/{max_concurrent}, "
+                    f"slot_id={acquisition.slot_id}"
+                )
+                return CallConcurrencySlot(
+                    organization_id=organization_id,
+                    slot_id=acquisition.slot_id,
+                    max_concurrent=max_concurrent,
+                    source=source,
+                    scope_key=scope_key,
+                )
+
+            wait_time = time.time() - wait_start
+            if wait_time >= timeout:
+                current_count = await rate_limiter.get_concurrent_count(organization_id)
+                scope_note = (
+                    f", scope={scope_key} (limit={scope_max_concurrent})"
+                    if scope_key
+                    else ""
+                )
+                logger.warning(
+                    f"Concurrent call limit reached for org {organization_id}: "
+                    f"source={source}, active_calls={current_count}/{max_concurrent}"
+                    f"{scope_note}, waited={wait_time:.1f}s"
+                )
+                properties = {
+                    "event_source": "dograh",
+                    "organization_id": organization_id,
+                    "source": source,
+                    "max_concurrent": max_concurrent,
+                    "active_calls": current_count,
+                    "waited_seconds": round(wait_time, 1),
+                }
+                if scope_key:
+                    properties["scope_key"] = scope_key
+                    properties["scope_max_concurrent"] = scope_max_concurrent
+                await self._notify_limit_reached(organization_id, properties)
+                raise CallConcurrencyLimitError(
+                    organization_id=organization_id,
+                    source=source,
+                    wait_time=wait_time,
+                    max_concurrent=max_concurrent,
+                )
+
+            logger.debug(
+                f"Waiting for concurrent call slot for org {organization_id}, "
+                f"source={source}, waited {wait_time:.1f}s"
+            )
+            await asyncio.sleep(min(retry_interval, max(0, timeout - wait_time)))
+
+    async def _notify_limit_reached(
+        self, organization_id: int, properties: dict
+    ) -> None:
+        """Fan the usage event out to every org member's provider_id, matching
+        how MPS emits org-scoped billing events (billing_posthog_service.py)
+        into the shared PostHog project. Never raises.
+
+        NOTE: intentionally NOT attaching ``$groups`` (organization) to this
+        event. PostHog evaluates a $groups event at both person and group
+        scope, which double-triggers person-scoped workflows enrolled on the
+        event. The org is still available as the ``organization_id`` property.
+        """
+        try:
+            members = await db_client.get_organization_users(organization_id)
+            if not members:
+                logger.debug(
+                    f"No users found for org {organization_id}; skipping "
+                    "concurrent-call-limit PostHog event"
+                )
+                return
+            for member in members:
+                capture_event(
+                    distinct_id=str(member.provider_id),
+                    event=PostHogEvent.USAGE_CONCURRENT_CALL_LIMIT_REACHED,
+                    properties=properties,
+                )
+        except Exception:
+            logger.exception(
+                "Failed to send concurrent-call-limit PostHog event for org "
+                f"{organization_id}"
+            )
+
+    async def bind_workflow_run(
+        self, slot: CallConcurrencySlot, workflow_run_id: int
+    ) -> None:
+        stored = await rate_limiter.store_workflow_slot_mapping_if_absent(
+            workflow_run_id,
+            slot.organization_id,
+            slot.slot_id,
+            scope_key=slot.scope_key,
+        )
+        if stored:
+            return
+
+        await self.release_slot(slot)
+        raise WorkflowRunSlotAlreadyBoundError(workflow_run_id)
+
+    async def register_active_call(
+        self,
+        organization_id: int,
+        workflow_run_id: int,
+        *,
+        source: str,
+        timeout: float = 0,
+        scope_key: str | None = None,
+        scope_max_concurrent: int | None = None,
+        retry_interval: float = 1,
+    ) -> CallConcurrencySlot:
+        slot = await self.acquire_org_slot(
+            organization_id,
+            source=source,
+            timeout=timeout,
+            scope_key=scope_key,
+            scope_max_concurrent=scope_max_concurrent,
+            retry_interval=retry_interval,
+        )
+        await self.bind_workflow_run(slot, workflow_run_id)
+        return slot
+
+    async def unregister_active_call(self, workflow_run_id: int) -> bool:
+        """Release the run's slot without ever raising.
+
+        Callers invoke this from ``finally`` blocks during pipeline/socket
+        teardown; a cleanup failure must not mask the original exception.
+        The slot mapping survives a failed release, so a later cleanup path
+        (status callback, StasisEnd) or the Redis stale timeout recovers it.
+        """
+        try:
+            return await self.release_workflow_run_slot(workflow_run_id)
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            logger.warning(
+                f"Failed to release concurrent call slot for workflow run "
+                f"{workflow_run_id}: {e}"
+            )
+            return False
+
+    async def release_slot(self, slot: CallConcurrencySlot | None) -> bool:
+        if slot is None:
+            return False
+        released = await rate_limiter.release_concurrent_slot(
+            slot.organization_id, slot.slot_id, scope_key=slot.scope_key
+        )
+        return bool(released)
+
+    async def release_workflow_run_slot(self, workflow_run_id: int) -> bool:
+        mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run_id)
+        if not mapping:
+            return False
+
+        org_id, slot_id, scope_key = mapping
+        released = await rate_limiter.release_concurrent_slot(
+            org_id, slot_id, scope_key=scope_key
+        )
+        if released is None:
+            # Redis error while releasing — keep the mapping so a later
+            # cleanup path can retry instead of orphaning a live slot until
+            # the stale timeout.
+            logger.warning(
+                f"Failed to release concurrent slot for workflow run "
+                f"{workflow_run_id}; keeping mapping for retry"
+            )
+            return False
+        await rate_limiter.delete_workflow_slot_mapping(workflow_run_id)
+        if released:
+            logger.info(f"Released concurrent slot for workflow run {workflow_run_id}")
+        else:
+            logger.debug(
+                f"Concurrent slot mapping for workflow run {workflow_run_id} "
+                "had no live slot; deleted stale mapping"
+            )
+        return released
+
+
+call_concurrency = CallConcurrencyService()

api/services/campaign/campaign_call_dispatcher.py

@@ -5,10 +5,14 @@ from typing import TYPE_CHECKING, Optional
 
 from loguru import logger
 
-from api.constants import DEFAULT_ORG_CONCURRENCY_LIMIT
 from api.db import db_client
 from api.db.models import QueuedRunModel, WorkflowRunModel
-from api.enums import OrganizationConfigurationKey, WorkflowRunState
+from api.enums import WorkflowRunState
+from api.services.call_concurrency import (
+    CallConcurrencyLimitError,
+    CallConcurrencySlot,
+    call_concurrency,
+)
 from api.services.campaign.circuit_breaker import circuit_breaker
 from api.services.campaign.errors import (
     ConcurrentSlotAcquisitionError,
@@ -29,9 +33,6 @@ if TYPE_CHECKING:
 class CampaignCallDispatcher:
     """Manages rate-limited and concurrent-limited call dispatching"""
 
-    def __init__(self):
-        self.default_concurrent_limit = int(DEFAULT_ORG_CONCURRENCY_LIMIT)
-
     async def get_provider_for_campaign(self, campaign) -> "TelephonyProvider":
         """Get the telephony provider pinned to this campaign's config. Falls back
         to the org's default config for legacy campaigns whose
@@ -53,18 +54,7 @@ class CampaignCallDispatcher:
 
     async def get_org_concurrent_limit(self, organization_id: int) -> int:
         """Get the concurrent call limit for an organization."""
-        try:
-            config = await db_client.get_configuration(
-                organization_id,
-                OrganizationConfigurationKey.CONCURRENT_CALL_LIMIT.value,
-            )
-            if config and config.value:
-                return int(config.value["value"])
-        except Exception as e:
-            logger.warning(
-                f"Error getting concurrent limit for org {organization_id}: {e}"
-            )
-        return self.default_concurrent_limit
+        return await call_concurrency.get_org_concurrent_limit(organization_id)
 
     async def process_batch(self, campaign_id: int, batch_size: int = 10) -> int:
         """
@@ -119,12 +109,14 @@ class CampaignCallDispatcher:
                 )
 
                 # Acquire concurrent slot - waits until a slot is available
-                slot_id = await self.acquire_concurrent_slot(
+                concurrency_slot = await self.acquire_concurrent_slot(
                     campaign.organization_id, campaign
                 )
 
                 # Dispatch the call
-                workflow_run = await self.dispatch_call(queued_run, campaign, slot_id)
+                workflow_run = await self.dispatch_call(
+                    queued_run, campaign, concurrency_slot
+                )
 
                 # Update queued run as processed
                 await db_client.update_queued_run(
@@ -233,68 +225,61 @@ class CampaignCallDispatcher:
             )
 
     async def dispatch_call(
-        self, queued_run: QueuedRunModel, campaign: any, slot_id: str
+        self,
+        queued_run: QueuedRunModel,
+        campaign: any,
+        concurrency_slot: CallConcurrencySlot,
     ) -> Optional[WorkflowRunModel]:
-        """Creates workflow run and initiates call. Requires a pre-acquired slot_id."""
+        """Creates workflow run and initiates call. Requires a pre-acquired slot."""
         from_number = None
+        workflow_run = None
+        slot_bound = False
 
-        # Get workflow details
-        workflow = await db_client.get_workflow_by_id(campaign.workflow_id)
-        if not workflow:
-            # Release slot before raising
-            await rate_limiter.release_concurrent_slot(
-                campaign.organization_id, slot_id
-            )
-            raise ValueError(f"Workflow {campaign.workflow_id} not found")
-
-        # Extract phone number
-        phone_number = queued_run.context_variables.get("phone_number")
-        if not phone_number:
-            # Release slot before raising
-            await rate_limiter.release_concurrent_slot(
-                campaign.organization_id, slot_id
-            )
-            raise ValueError(f"No phone number in queued run {queued_run.id}")
-
-        # Get provider for this campaign's pinned telephony config.
-        provider = await self.get_provider_for_campaign(campaign)
-        workflow_run_mode = provider.PROVIDER_NAME
-
-        # Acquire a unique from_number from the pool scoped to this campaign's
-        # telephony configuration so orgs with multiple configs don't leak
-        # caller IDs across configs.
-        from_number = await self.acquire_from_number(
-            campaign.organization_id,
-            telephony_configuration_id=campaign.telephony_configuration_id,
-        )
-        if from_number is None:
-            # Release concurrent slot before raising
-            await rate_limiter.release_concurrent_slot(
-                campaign.organization_id, slot_id
-            )
-            raise PhoneNumberPoolExhaustedError(
-                organization_id=campaign.organization_id
-            )
-
-        logger.info(f"Provider name: {provider.PROVIDER_NAME}")
-        logger.info(f"Queued run context: {queued_run.context_variables}")
-
-        # Merge context variables (queued_run context already includes retry info if applicable)
-        initial_context = {
-            **queued_run.context_variables,
-            "campaign_id": campaign.id,
-            "provider": provider.PROVIDER_NAME,
-            "source_uuid": queued_run.source_uuid,
-            "caller_number": from_number,
-            "called_number": phone_number,
-            "telephony_configuration_id": campaign.telephony_configuration_id,
-        }
-
-        logger.info(f"Final initial_context: {initial_context}")
-
-        # Create workflow run with queued_run_id tracking
-        workflow_run_name = f"WR-CAMPAIGN-{campaign.id}-{queued_run.id}"
         try:
+            # Get workflow details
+            workflow = await db_client.get_workflow_by_id(campaign.workflow_id)
+            if not workflow:
+                raise ValueError(f"Workflow {campaign.workflow_id} not found")
+
+            # Extract phone number
+            phone_number = queued_run.context_variables.get("phone_number")
+            if not phone_number:
+                raise ValueError(f"No phone number in queued run {queued_run.id}")
+
+            # Get provider for this campaign's pinned telephony config.
+            provider = await self.get_provider_for_campaign(campaign)
+            workflow_run_mode = provider.PROVIDER_NAME
+
+            # Acquire a unique from_number from the pool scoped to this campaign's
+            # telephony configuration so orgs with multiple configs don't leak
+            # caller IDs across configs.
+            from_number = await self.acquire_from_number(
+                campaign.organization_id,
+                telephony_configuration_id=campaign.telephony_configuration_id,
+            )
+            if from_number is None:
+                raise PhoneNumberPoolExhaustedError(
+                    organization_id=campaign.organization_id
+                )
+
+            logger.info(f"Provider name: {provider.PROVIDER_NAME}")
+            logger.info(f"Queued run context: {queued_run.context_variables}")
+
+            # Merge context variables (queued_run context already includes retry info if applicable)
+            initial_context = {
+                **queued_run.context_variables,
+                "campaign_id": campaign.id,
+                "provider": provider.PROVIDER_NAME,
+                "source_uuid": queued_run.source_uuid,
+                "caller_number": from_number,
+                "called_number": phone_number,
+                "telephony_configuration_id": campaign.telephony_configuration_id,
+            }
+
+            logger.info(f"Final initial_context: {initial_context}")
+
+            # Create workflow run with queued_run_id tracking
+            workflow_run_name = f"WR-CAMPAIGN-{campaign.id}-{queued_run.id}"
             workflow_run = await db_client.create_workflow_run(
                 name=workflow_run_name,
                 workflow_id=campaign.workflow_id,
@@ -303,12 +288,10 @@ class CampaignCallDispatcher:
                 initial_context=initial_context,
                 campaign_id=campaign.id,
                 queued_run_id=queued_run.id,  # Link to queued run for retry tracking
+                organization_id=campaign.organization_id,
             )
-
-            # Store slot_id mapping in Redis for cleanup later
-            await rate_limiter.store_workflow_slot_mapping(
-                workflow_run.id, campaign.organization_id, slot_id
-            )
+            await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run.id)
+            slot_bound = True
 
             # Store from_number mapping for cleanup on call completion
             await rate_limiter.store_workflow_from_number_mapping(
@@ -319,9 +302,10 @@ class CampaignCallDispatcher:
             )
         except Exception as e:
             # Release slot and from_number on error
-            await rate_limiter.release_concurrent_slot(
-                campaign.organization_id, slot_id
-            )
+            if slot_bound and workflow_run:
+                await call_concurrency.release_workflow_run_slot(workflow_run.id)
+            else:
+                await call_concurrency.release_slot(concurrency_slot)
             if from_number:
                 await rate_limiter.release_from_number(
                     campaign.organization_id,
@@ -342,6 +326,7 @@ class CampaignCallDispatcher:
 
         quota_result = await authorize_workflow_run_start(
             workflow_id=campaign.workflow_id,
+            organization_id=campaign.organization_id,
             workflow_run_id=workflow_run.id,
         )
         if not quota_result.has_quota:
@@ -357,21 +342,7 @@ class CampaignCallDispatcher:
                 gathered_context={"error": error_message},
             )
 
-            mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run.id)
-            if mapping:
-                org_id, mapped_slot_id = mapping
-                await rate_limiter.release_concurrent_slot(org_id, mapped_slot_id)
-                await rate_limiter.delete_workflow_slot_mapping(workflow_run.id)
-
-            from_number_mapping = await rate_limiter.get_workflow_from_number_mapping(
-                workflow_run.id
-            )
-            if from_number_mapping:
-                fn_org_id, fn_number, fn_tcid = from_number_mapping
-                await rate_limiter.release_from_number(
-                    fn_org_id, fn_number, telephony_configuration_id=fn_tcid
-                )
-                await rate_limiter.delete_workflow_from_number_mapping(workflow_run.id)
+            await self.release_call_slot(workflow_run.id)
 
             raise ValueError(error_message)
 
@@ -383,7 +354,6 @@ class CampaignCallDispatcher:
             webhook_url = (
                 f"{backend_endpoint}/api/v1/telephony/{webhook_endpoint}"
                 f"?workflow_id={campaign.workflow_id}"
-                f"&user_id={campaign.created_by}"
                 f"&workflow_run_id={workflow_run.id}"
                 f"&organization_id={campaign.organization_id}"
             )
@@ -394,7 +364,7 @@ class CampaignCallDispatcher:
                 workflow_run_id=workflow_run.id,
                 from_number=from_number,
                 workflow_id=campaign.workflow_id,
-                user_id=campaign.created_by,
+                organization_id=campaign.organization_id,
             )
 
             # Store provider type and metadata in gathered_context
@@ -446,23 +416,7 @@ class CampaignCallDispatcher:
                 reason="call_initiation_failed",
             )
 
-            # Release concurrent slot on failure
-            mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run.id)
-            if mapping:
-                org_id, slot_id = mapping
-                await rate_limiter.release_concurrent_slot(org_id, slot_id)
-                await rate_limiter.delete_workflow_slot_mapping(workflow_run.id)
-
-            # Release from_number on failure
-            from_number_mapping = await rate_limiter.get_workflow_from_number_mapping(
-                workflow_run.id
-            )
-            if from_number_mapping:
-                fn_org_id, fn_number, fn_tcid = from_number_mapping
-                await rate_limiter.release_from_number(
-                    fn_org_id, fn_number, telephony_configuration_id=fn_tcid
-                )
-                await rate_limiter.delete_workflow_from_number_mapping(workflow_run.id)
+            await self.release_call_slot(workflow_run.id)
 
             raise
 
@@ -501,7 +455,7 @@ class CampaignCallDispatcher:
 
     async def acquire_concurrent_slot(
         self, organization_id: int, campaign: any, timeout: float = 600
-    ) -> str:
+    ) -> CallConcurrencySlot:
         """
         Acquires a concurrent call slot - waits if necessary until a slot is available.
 
@@ -510,54 +464,41 @@ class CampaignCallDispatcher:
             campaign: The campaign object
             timeout: Maximum time to wait for a slot (default 10 minutes)
 
-        Returns the slot_id which must be released when the call completes.
+        Returns the slot which must be released when the call completes.
 
         Raises:
             ConcurrentSlotAcquisitionError: If slot cannot be acquired within timeout
         """
-        # Get concurrent limit for organization
-        org_concurrent_limit = await self.get_org_concurrent_limit(organization_id)
-
-        # Check for campaign-level max_concurrency in orchestrator_metadata
+        # Check for campaign-level max_concurrency in orchestrator_metadata.
+        # It caps this campaign's own concurrent calls via a campaign-scoped
+        # counter — the org-wide limit still applies on top, but calls from
+        # other sources (WebRTC, inbound, other campaigns) don't count
+        # against the campaign's cap.
         campaign_max_concurrency = None
         if campaign.orchestrator_metadata:
             campaign_max_concurrency = campaign.orchestrator_metadata.get(
                 "max_concurrency"
             )
 
-        # Use the lower of campaign limit and org limit
-        if campaign_max_concurrency is not None:
-            max_concurrent = min(campaign_max_concurrency, org_concurrent_limit)
-        else:
-            max_concurrent = org_concurrent_limit
-
-        # Track wait time for alerting
-        wait_start = time.time()
-
-        # Wait until we can acquire a concurrent slot
-        while True:
-            slot_id = await rate_limiter.try_acquire_concurrent_slot(
-                organization_id, max_concurrent
+        try:
+            return await call_concurrency.acquire_org_slot(
+                organization_id,
+                source=f"campaign:{campaign.id}",
+                timeout=timeout,
+                scope_key=(
+                    f"campaign:{campaign.id}"
+                    if campaign_max_concurrency is not None
+                    else None
+                ),
+                scope_max_concurrent=campaign_max_concurrency,
+                retry_interval=1,
             )
-            if slot_id:
-                return slot_id
-
-            # Check if we've been waiting too long
-            wait_time = time.time() - wait_start
-            if wait_time > timeout:
-                raise ConcurrentSlotAcquisitionError(
-                    organization_id=organization_id,
-                    campaign_id=campaign.id,
-                    wait_time=wait_time,
-                )
-
-            logger.debug(
-                f"Attempting to get a slot for {organization_id} {campaign.id}, "
-                f"waited {wait_time:.1f}s"
-            )
-
-            # Wait before retrying
-            await asyncio.sleep(1)
+        except CallConcurrencyLimitError as e:
+            raise ConcurrentSlotAcquisitionError(
+                organization_id=organization_id,
+                campaign_id=campaign.id,
+                wait_time=e.wait_time,
+            ) from e
 
     async def acquire_from_number(
         self,
@@ -602,17 +543,9 @@ class CampaignCallDispatcher:
         Release concurrent slot and from_number when a call completes.
         Called by Twilio webhooks or workflow completion handlers.
         """
-        slot_released = False
-        mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run_id)
-        if mapping:
-            org_id, slot_id = mapping
-            success = await rate_limiter.release_concurrent_slot(org_id, slot_id)
-            if success:
-                await rate_limiter.delete_workflow_slot_mapping(workflow_run_id)
-                logger.info(
-                    f"Released concurrent slot for workflow run {workflow_run_id}"
-                )
-                slot_released = True
+        slot_released = await call_concurrency.release_workflow_run_slot(
+            workflow_run_id
+        )
 
         # Release from_number back to its (org, telephony config) pool
         from_number_mapping = await rate_limiter.get_workflow_from_number_mapping(

api/services/campaign/rate_limiter.py

@@ -1,5 +1,6 @@
 import time
 import uuid
+from dataclasses import dataclass
 from typing import Optional
 
 import redis.asyncio as aioredis
@@ -8,6 +9,12 @@ from loguru import logger
 from api.constants import REDIS_URL
 
 
+@dataclass(frozen=True)
+class ConcurrentSlotAcquisition:
+    slot_id: str
+    active_count: int
+
+
 class RateLimiter:
     """Sliding window rate limiter to enforce strict per-second limits and concurrent call limits"""
 
@@ -100,34 +107,71 @@ class RateLimiter:
         Try to acquire a concurrent call slot.
         Returns a unique slot_id if successful, None if limit reached.
         """
+        acquisition = await self.try_acquire_concurrent_slot_details(
+            organization_id, max_concurrent
+        )
+        return acquisition.slot_id if acquisition else None
+
+    async def try_acquire_concurrent_slot_details(
+        self,
+        organization_id: int,
+        max_concurrent: int = 20,
+        *,
+        scope_key: str | None = None,
+        scope_max_concurrent: int | None = None,
+    ) -> Optional[ConcurrentSlotAcquisition]:
+        """
+        Try to acquire a concurrent call slot.
+        Returns the slot_id and post-acquire active count if successful,
+        or None if the limit is reached.
+
+        When ``scope_key``/``scope_max_concurrent`` are provided, the slot is
+        also registered in a secondary counter (``concurrent_calls:<scope_key>``,
+        e.g. ``campaign:<id>``) and acquisition additionally requires that
+        counter to be below ``scope_max_concurrent``. Both counters are
+        updated atomically. The scope-scoped slot must be released with the
+        same ``scope_key``.
+        """
         redis_client = await self._get_redis()
 
         concurrent_key = f"concurrent_calls:{organization_id}"
+        scope_concurrent_key = f"concurrent_calls:{scope_key}" if scope_key else ""
         now = time.time()
         stale_cutoff = now - self.stale_call_timeout
 
-        # Lua script for atomic operation
+        # Lua script for atomic operation across the org counter and the
+        # optional scope counter (empty scope key = org-only acquisition).
         lua_script = """
         local key = KEYS[1]
+        local scope_key = KEYS[2]
         local now = tonumber(ARGV[1])
         local max_concurrent = tonumber(ARGV[2])
         local stale_cutoff = tonumber(ARGV[3])
         local slot_id = ARGV[4]
-        
-        -- Remove stale entries (older than 30 minutes)
+        local scope_max_concurrent = tonumber(ARGV[5])
+
+        -- Remove stale entries (older than the stale-call timeout)
         redis.call('ZREMRANGEBYSCORE', key, 0, stale_cutoff)
-        
+
         -- Get current count
         local current_count = redis.call('ZCARD', key)
-        
-        if current_count < max_concurrent then
-            -- Add new slot
-            redis.call('ZADD', key, now, slot_id)
-            redis.call('EXPIRE', key, 3600)  -- Expire after 1 hour
-            return slot_id
-        else
+
+        if current_count >= max_concurrent then
             return nil
         end
+
+        if scope_key ~= '' then
+            redis.call('ZREMRANGEBYSCORE', scope_key, 0, stale_cutoff)
+            if redis.call('ZCARD', scope_key) >= scope_max_concurrent then
+                return nil
+            end
+            redis.call('ZADD', scope_key, now, slot_id)
+            redis.call('EXPIRE', scope_key, 3600)
+        end
+
+        redis.call('ZADD', key, now, slot_id)
+        redis.call('EXPIRE', key, 3600)  -- Expire after 1 hour
+        return {slot_id, current_count + 1}
         """
 
         # Generate unique slot ID (timestamp + random component)
@@ -136,22 +180,38 @@ class RateLimiter:
         try:
             result = await redis_client.eval(
                 lua_script,
-                1,
+                2,
                 concurrent_key,
+                scope_concurrent_key,
                 now,
                 max_concurrent,
                 stale_cutoff,
                 slot_id,
+                scope_max_concurrent if scope_max_concurrent is not None else 0,
+            )
+            if not result:
+                return None
+
+            acquired_slot_id, active_count = result
+            return ConcurrentSlotAcquisition(
+                slot_id=str(acquired_slot_id),
+                active_count=int(active_count),
             )
-            return result
         except Exception as e:
             logger.error(f"Concurrent limiter error: {e}")
             return None
 
-    async def release_concurrent_slot(self, organization_id: int, slot_id: str) -> bool:
+    async def release_concurrent_slot(
+        self,
+        organization_id: int,
+        slot_id: str,
+        scope_key: str | None = None,
+    ) -> bool | None:
         """
-        Release a concurrent call slot.
-        Returns True if slot was released, False otherwise.
+        Release a concurrent call slot (and its scope counter entry, if any).
+        Returns True if the slot was released, False if it was already gone
+        (released/stale-expired), or None on a Redis error — callers that
+        track cleanup state should keep it around for retry when None.
         """
         if not slot_id:
             return False
@@ -161,6 +221,8 @@ class RateLimiter:
 
         try:
             removed = await redis_client.zrem(concurrent_key, slot_id)
+            if scope_key:
+                await redis_client.zrem(f"concurrent_calls:{scope_key}", slot_id)
             if removed:
                 logger.debug(
                     f"Released concurrent slot {slot_id} for org {organization_id}"
@@ -168,7 +230,7 @@ class RateLimiter:
             return bool(removed)
         except Exception as e:
             logger.error(f"Error releasing concurrent slot: {e}")
-            return False
+            return None
 
     async def get_concurrent_count(self, organization_id: int) -> int:
         """
@@ -212,12 +274,62 @@ class RateLimiter:
             logger.error(f"Error storing workflow slot mapping: {e}")
             return False
 
+    async def store_workflow_slot_mapping_if_absent(
+        self,
+        workflow_run_id: int,
+        organization_id: int,
+        slot_id: str,
+        scope_key: str | None = None,
+    ) -> bool:
+        """
+        Store the workflow_run_id -> concurrent slot mapping only if no mapping
+        already exists. This prevents duplicate public/WebRTC starts for the
+        same workflow run from overwriting the cleanup pointer.
+        """
+        redis_client = await self._get_redis()
+        mapping_key = f"workflow_slot_mapping:{workflow_run_id}"
+
+        lua_script = """
+        local key = KEYS[1]
+        local org_id = ARGV[1]
+        local slot_id = ARGV[2]
+        local ttl = tonumber(ARGV[3])
+        local scope_key = ARGV[4]
+
+        if redis.call('EXISTS', key) == 1 then
+            return 0
+        end
+
+        redis.call('HSET', key, 'org_id', org_id, 'slot_id', slot_id)
+        if scope_key ~= '' then
+            redis.call('HSET', key, 'scope_key', scope_key)
+        end
+        redis.call('EXPIRE', key, ttl)
+        return 1
+        """
+
+        try:
+            stored = await redis_client.eval(
+                lua_script,
+                1,
+                mapping_key,
+                organization_id,
+                slot_id,
+                self.stale_call_timeout,
+                scope_key or "",
+            )
+            return bool(stored)
+        except Exception as e:
+            logger.error(f"Error storing workflow slot mapping if absent: {e}")
+            return False
+
     async def get_workflow_slot_mapping(
         self, workflow_run_id: int
-    ) -> Optional[tuple[int, str]]:
+    ) -> Optional[tuple[int, str, str | None]]:
         """
         Get the concurrent slot mapping for a workflow run.
-        Returns (organization_id, slot_id) tuple or None if not found.
+        Returns (organization_id, slot_id, scope_key) or None if not found;
+        scope_key is None for slots acquired without a scope counter.
         """
         redis_client = await self._get_redis()
         mapping_key = f"workflow_slot_mapping:{workflow_run_id}"
@@ -225,7 +337,11 @@ class RateLimiter:
         try:
             mapping = await redis_client.hgetall(mapping_key)
             if mapping and "org_id" in mapping and "slot_id" in mapping:
-                return (int(mapping["org_id"]), mapping["slot_id"])
+                return (
+                    int(mapping["org_id"]),
+                    mapping["slot_id"],
+                    mapping.get("scope_key") or None,
+                )
             return None
         except Exception as e:
             logger.error(f"Error getting workflow slot mapping: {e}")

api/services/configuration/ai_model_configuration.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import copy
 from dataclasses import dataclass
+from datetime import UTC, datetime
 from typing import Literal
 
 from loguru import logger
@@ -11,7 +12,11 @@ from sqlalchemy.orm import selectinload
 
 from api.constants import MPS_API_URL
 from api.db import db_client
-from api.db.models import WorkflowDefinitionModel, WorkflowModel
+from api.db.models import (
+    OrganizationConfigurationModel,
+    WorkflowDefinitionModel,
+    WorkflowModel,
+)
 from api.enums import OrganizationConfigurationKey
 from api.schemas.ai_model_configuration import (
     DOGRAH_DEFAULT_LANGUAGE,
@@ -55,35 +60,36 @@ class WorkflowAIModelConfigurationMigrationResult:
 
 async def get_resolved_ai_model_configuration(
     *,
-    user_id: int | None,
     organization_id: int | None,
 ) -> ResolvedAIModelConfiguration:
-    organization_configuration = await get_organization_ai_model_configuration_v2(
-        organization_id
+    """Resolve the effective model configuration for an organization."""
+    organization_configuration_row = (
+        await _get_organization_ai_model_configuration_v2_row(organization_id)
+    )
+    organization_configuration = _parse_organization_ai_model_configuration_v2(
+        organization_configuration_row,
+        organization_id,
     )
     if organization_configuration is not None:
+        effective = compile_ai_model_configuration_v2(organization_configuration)
+        if organization_configuration_row is not None:
+            effective.last_validated_at = (
+                organization_configuration_row.last_validated_at
+            )
         return ResolvedAIModelConfiguration(
-            effective=compile_ai_model_configuration_v2(organization_configuration),
+            effective=effective,
             source="organization_v2",
             organization_configuration=organization_configuration,
         )
 
-    if user_id is None:
-        return ResolvedAIModelConfiguration(
-            effective=EffectiveAIModelConfiguration(),
-            source="empty",
-        )
-
-    legacy = await db_client.get_user_configurations(user_id)
     return ResolvedAIModelConfiguration(
-        effective=legacy,
-        source="legacy_user_v1" if _has_model_services(legacy) else "empty",
+        effective=EffectiveAIModelConfiguration(),
+        source="empty",
     )
 
 
 async def get_effective_ai_model_configuration_for_workflow(
     *,
-    user_id: int | None,
     organization_id: int | None,
     workflow_configurations: dict | None,
 ) -> EffectiveAIModelConfiguration:
@@ -97,7 +103,6 @@ async def get_effective_ai_model_configuration_for_workflow(
         )
 
     resolved_config = await get_resolved_ai_model_configuration(
-        user_id=user_id,
         organization_id=organization_id,
     )
     return resolve_effective_config(
@@ -109,12 +114,34 @@ async def get_effective_ai_model_configuration_for_workflow(
 async def get_organization_ai_model_configuration_v2(
     organization_id: int | None,
 ) -> OrganizationAIModelConfigurationV2 | None:
-    if organization_id is None:
-        return None
-    row = await db_client.get_configuration(
+    row = await _get_organization_ai_model_configuration_v2_row(organization_id)
+    return _parse_organization_ai_model_configuration_v2(row, organization_id)
+
+
+async def update_organization_ai_model_configuration_last_validated_at(
+    organization_id: int,
+) -> None:
+    await db_client.mark_configuration_validated(
         organization_id,
         OrganizationConfigurationKey.MODEL_CONFIGURATION_V2.value,
     )
+
+
+async def _get_organization_ai_model_configuration_v2_row(
+    organization_id: int | None,
+) -> OrganizationConfigurationModel | None:
+    if organization_id is None:
+        return None
+    return await db_client.get_configuration(
+        organization_id,
+        OrganizationConfigurationKey.MODEL_CONFIGURATION_V2.value,
+    )
+
+
+def _parse_organization_ai_model_configuration_v2(
+    row: OrganizationConfigurationModel | None,
+    organization_id: int | None,
+) -> OrganizationAIModelConfigurationV2 | None:
     if row is None or not row.value:
         return None
     try:
@@ -135,6 +162,7 @@ async def upsert_organization_ai_model_configuration_v2(
         organization_id,
         OrganizationConfigurationKey.MODEL_CONFIGURATION_V2.value,
         configuration.model_dump(mode="json", exclude_none=True),
+        last_validated_at=datetime.now(UTC),
     )
     return configuration
 
@@ -145,19 +173,12 @@ async def migrate_workflow_model_configurations_to_v2(
     fallback_user_config: EffectiveAIModelConfiguration,
 ) -> WorkflowAIModelConfigurationMigrationResult:
     workflows = await _list_workflows_for_model_configuration_migration(organization_id)
-    owner_configs: dict[int, EffectiveAIModelConfiguration] = {}
     workflow_updates: list[tuple[int, dict]] = []
     definition_updates: list[tuple[int, dict]] = []
     migrated_workflow_ids: set[int] = set()
 
     for workflow in workflows:
         base_config = fallback_user_config
-        if workflow.user_id is not None:
-            if workflow.user_id not in owner_configs:
-                owner_configs[
-                    workflow.user_id
-                ] = await db_client.get_user_configurations(workflow.user_id)
-            base_config = owner_configs[workflow.user_id]
 
         workflow_configs, workflow_changed = (
             migrate_workflow_configuration_model_override_to_v2(
@@ -316,6 +337,7 @@ def convert_legacy_ai_model_configuration_to_v2(
 
 
 def dograh_embeddings_base_url() -> str:
+    # AsyncOpenAI appends "/embeddings"; MPS exposes that under /api/v1/llm.
     return f"{MPS_API_URL}/api/v1/llm"
 
 
@@ -419,19 +441,6 @@ def _mask_secret_value(value):
     return mask_key(value)
 
 
-def _has_model_services(configuration: EffectiveAIModelConfiguration) -> bool:
-    return any(
-        service is not None
-        for service in (
-            configuration.llm,
-            configuration.tts,
-            configuration.stt,
-            configuration.embeddings,
-            configuration.realtime,
-        )
-    )
-
-
 def _convert_any_dograh_legacy_configuration(
     configuration: EffectiveAIModelConfiguration,
     dograh_key: str,

api/services/configuration/check_validity.py

@@ -64,6 +64,7 @@ class UserConfigurationValidator:
             ServiceProviders.RIME.value: self._check_rime_api_key,
             ServiceProviders.MINIMAX.value: self._check_minimax_api_key,
             ServiceProviders.SMALLEST.value: self._check_smallest_api_key,
+            ServiceProviders.XAI.value: self._check_xai_api_key,
         }
 
     async def validate(
@@ -376,6 +377,32 @@ class UserConfigurationValidator:
     def _check_grok_realtime_api_key(self, model: str, api_key: str) -> bool:
         return True
 
+    def _check_xai_api_key(self, model: str, api_key: str) -> bool:
+        # Use the TTS voices endpoint as a best-effort smoke test. Some xAI keys
+        # can be scoped in ways that block listing voices even though the key is
+        # still intended for TTS usage, so only a clear auth failure rejects save.
+        try:
+            response = httpx.get(
+                "https://api.x.ai/v1/tts/voices",
+                headers={"Authorization": f"Bearer {api_key}"},
+                timeout=10.0,
+            )
+        except httpx.RequestError:
+            raise ValueError(
+                "Could not connect to the xAI API. Please check your network "
+                "connection and try again."
+            )
+        if response.status_code == 200:
+            return True
+        if response.status_code == 401:
+            raise ValueError(
+                "Invalid xAI API key. The key was rejected by the xAI API. "
+                "Please check that your API key is correct and active. "
+                "You can verify your keys at "
+                "https://console.x.ai."
+            )
+        return True
+
     def _check_ultravox_realtime_api_key(self, model: str, api_key: str) -> bool:
         return True
 

api/services/configuration/options/__init__.py

@@ -9,6 +9,12 @@ from .azure import (
     AZURE_SPEECH_TTS_LANGUAGES,
     AZURE_SPEECH_TTS_VOICES,
 )
+from .cartesia import (
+    CARTESIA_INK_2_STT_LANGUAGES,
+    CARTESIA_INK_WHISPER_STT_LANGUAGES,
+    CARTESIA_STT_LANGUAGES,
+    CARTESIA_STT_MODELS,
+)
 from .deepgram import (
     DEEPGRAM_FLUX_MODELS,
     DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGE_OPTIONS,
@@ -16,6 +22,7 @@ from .deepgram import (
     DEEPGRAM_LANGUAGES,
     DEEPGRAM_STT_MODELS,
 )
+from .elevenlabs import ELEVENLABS_STT_LANGUAGES, ELEVENLABS_STT_MODELS
 from .gladia import GLADIA_STT_LANGUAGES, GLADIA_STT_MODELS
 from .google import (
     GOOGLE_MODELS,
@@ -59,6 +66,12 @@ __all__ = [
     "AZURE_SPEECH_STT_LANGUAGES",
     "AZURE_SPEECH_TTS_LANGUAGES",
     "AZURE_SPEECH_TTS_VOICES",
+    "CARTESIA_INK_2_STT_LANGUAGES",
+    "CARTESIA_INK_WHISPER_STT_LANGUAGES",
+    "CARTESIA_STT_LANGUAGES",
+    "CARTESIA_STT_MODELS",
+    "ELEVENLABS_STT_LANGUAGES",
+    "ELEVENLABS_STT_MODELS",
     "DEEPGRAM_FLUX_MODELS",
     "DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGES",
     "DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGE_OPTIONS",

api/services/configuration/options/azure.py

@@ -1,6 +1,10 @@
 AZURE_MODELS = ["gpt-4.1-mini"]
 
-AZURE_REALTIME_MODELS = ["gpt-4o-realtime-preview"]
+AZURE_REALTIME_MODELS = [
+    "gpt-realtime",
+    "gpt-realtime-1.5",
+    "gpt-realtime-mini",
+]
 AZURE_REALTIME_VOICES = [
     "alloy",
     "ash",
@@ -12,6 +16,7 @@ AZURE_REALTIME_VOICES = [
     "verse",
 ]
 AZURE_REALTIME_API_VERSIONS = [
+    "v1",
     "2025-04-01-preview",
     "2024-10-01-preview",
     "2024-12-17",

api/services/configuration/options/cartesia.py

@@ -0,0 +1,105 @@
+CARTESIA_STT_MODELS = ["ink-2", "ink-whisper"]
+CARTESIA_INK_2_STT_LANGUAGES = ("en",)
+CARTESIA_INK_WHISPER_STT_LANGUAGES = (
+    "en",
+    "zh",
+    "de",
+    "es",
+    "ru",
+    "ko",
+    "fr",
+    "ja",
+    "pt",
+    "tr",
+    "pl",
+    "ca",
+    "nl",
+    "ar",
+    "sv",
+    "it",
+    "id",
+    "hi",
+    "fi",
+    "vi",
+    "he",
+    "uk",
+    "el",
+    "ms",
+    "cs",
+    "ro",
+    "da",
+    "hu",
+    "ta",
+    "no",
+    "th",
+    "ur",
+    "hr",
+    "bg",
+    "lt",
+    "la",
+    "mi",
+    "ml",
+    "cy",
+    "sk",
+    "te",
+    "fa",
+    "lv",
+    "bn",
+    "sr",
+    "az",
+    "sl",
+    "kn",
+    "et",
+    "mk",
+    "br",
+    "eu",
+    "is",
+    "hy",
+    "ne",
+    "mn",
+    "bs",
+    "kk",
+    "sq",
+    "sw",
+    "gl",
+    "mr",
+    "pa",
+    "si",
+    "km",
+    "sn",
+    "yo",
+    "so",
+    "af",
+    "oc",
+    "ka",
+    "be",
+    "tg",
+    "sd",
+    "gu",
+    "am",
+    "yi",
+    "lo",
+    "uz",
+    "fo",
+    "ht",
+    "ps",
+    "tk",
+    "nn",
+    "mt",
+    "sa",
+    "lb",
+    "my",
+    "bo",
+    "tl",
+    "mg",
+    "as",
+    "tt",
+    "haw",
+    "ln",
+    "ha",
+    "ba",
+    "jw",
+    "su",
+    "yue",
+)
+CARTESIA_STT_LANGUAGES = CARTESIA_INK_WHISPER_STT_LANGUAGES

api/services/configuration/options/elevenlabs.py

@@ -0,0 +1,95 @@
+ELEVENLABS_STT_MODELS = ("scribe_v2_realtime",)
+
+ELEVENLABS_STT_LANGUAGES = (
+    "auto",
+    "af",
+    "am",
+    "ar",
+    "as",
+    "az",
+    "be",
+    "bg",
+    "bn",
+    "bs",
+    "ca",
+    "ceb",
+    "cs",
+    "cy",
+    "da",
+    "de",
+    "el",
+    "en",
+    "es",
+    "et",
+    "fa",
+    "fi",
+    "fil",
+    "fr",
+    "ga",
+    "gl",
+    "gu",
+    "ha",
+    "he",
+    "hi",
+    "hr",
+    "hu",
+    "hy",
+    "id",
+    "ig",
+    "is",
+    "it",
+    "ja",
+    "jv",
+    "ka",
+    "kk",
+    "km",
+    "kn",
+    "ko",
+    "ku",
+    "ky",
+    "lo",
+    "lt",
+    "lv",
+    "mi",
+    "mk",
+    "ml",
+    "mn",
+    "mr",
+    "ms",
+    "mt",
+    "my",
+    "ne",
+    "nl",
+    "no",
+    "ny",
+    "oc",
+    "or",
+    "pa",
+    "pl",
+    "ps",
+    "pt",
+    "ro",
+    "ru",
+    "sd",
+    "sk",
+    "sl",
+    "sn",
+    "so",
+    "sr",
+    "sv",
+    "sw",
+    "ta",
+    "te",
+    "tg",
+    "th",
+    "tr",
+    "uk",
+    "ur",
+    "uz",
+    "vi",
+    "wo",
+    "xh",
+    "yue",
+    "zh",
+    "zu",
+)

api/services/configuration/options/google.py

@@ -1,6 +1,4 @@
 GOOGLE_MODELS = (
-    "gemini-2.0-flash",
-    "gemini-2.0-flash-lite",
     "gemini-2.5-flash",
     "gemini-2.5-flash-lite",
     "gemini-3.5-flash",

api/services/configuration/registry.py

@@ -14,9 +14,16 @@ from api.services.configuration.options import (
     AZURE_SPEECH_STT_LANGUAGES,
     AZURE_SPEECH_TTS_LANGUAGES,
     AZURE_SPEECH_TTS_VOICES,
+    CARTESIA_INK_2_STT_LANGUAGES,
+    CARTESIA_INK_WHISPER_STT_LANGUAGES,
+    CARTESIA_STT_LANGUAGES,
+    CARTESIA_STT_MODELS,
     DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGE_OPTIONS,
+    DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGES,
     DEEPGRAM_LANGUAGES,
     DEEPGRAM_STT_MODELS,
+    ELEVENLABS_STT_LANGUAGES,
+    ELEVENLABS_STT_MODELS,
     GLADIA_STT_LANGUAGES,
     GLADIA_STT_MODELS,
     GOOGLE_MODELS,
@@ -87,6 +94,7 @@ class ServiceProviders(str, Enum):
     GOOGLE_VERTEX_REALTIME = "google_vertex_realtime"
     AZURE_REALTIME = "azure_realtime"
     SMALLEST = "smallest"
+    XAI = "xai"
 
 
 class BaseServiceConfiguration(BaseModel):
@@ -117,6 +125,7 @@ class BaseServiceConfiguration(BaseModel):
         ServiceProviders.AZURE_REALTIME,
         ServiceProviders.SARVAM,
         ServiceProviders.SMALLEST,
+        ServiceProviders.XAI,
     ]
     api_key: str | list[str]
 
@@ -251,6 +260,7 @@ GOOGLE_VERTEX_REALTIME_PROVIDER_MODEL_CONFIG = provider_model_config(
 DEEPGRAM_PROVIDER_MODEL_CONFIG = provider_model_config("Deepgram")
 ELEVENLABS_PROVIDER_MODEL_CONFIG = provider_model_config("ElevenLabs")
 CARTESIA_PROVIDER_MODEL_CONFIG = provider_model_config("Cartesia")
+XAI_PROVIDER_MODEL_CONFIG = provider_model_config("xAI")
 INWORLD_PROVIDER_MODEL_CONFIG = provider_model_config(
     "Inworld",
     description=(
@@ -315,7 +325,6 @@ OPENROUTER_MODELS = [
     "openai/gpt-4.1-mini",
     "anthropic/claude-sonnet-4",
     "google/gemini-2.5-flash",
-    "google/gemini-2.0-flash",
     "meta-llama/llama-3.3-70b-instruct",
     "deepseek/deepseek-chat-v3-0324",
 ]
@@ -350,7 +359,7 @@ class GoogleLLMService(BaseLLMConfiguration):
     model_config = GOOGLE_PROVIDER_MODEL_CONFIG
     provider: Literal[ServiceProviders.GOOGLE] = ServiceProviders.GOOGLE
     model: str = Field(
-        default="gemini-2.0-flash",
+        default="gemini-2.5-flash",
         description="Gemini model on Google AI Studio (not Vertex).",
         json_schema_extra={"examples": GOOGLE_MODELS, "allow_custom_input": True},
     )
@@ -527,6 +536,7 @@ class HuggingFaceLLMConfiguration(BaseLLMConfiguration):
 MINIMAX_MODELS = [
     "MiniMax-M2.7",
     "MiniMax-M2.7-highspeed",
+    "MiniMax-M3",
 ]
 
 
@@ -611,7 +621,7 @@ class OpenAIRealtimeLLMConfiguration(BaseLLMConfiguration):
 
 
 GROK_REALTIME_MODELS = ["grok-voice-think-fast-1.0"]
-GROK_REALTIME_VOICES = ["Ara", "Rex", "Sal", "Eve", "Leo"]
+GROK_REALTIME_VOICES = ["ara", "rex", "sal", "eve", "leo"]
 ULTRAVOX_REALTIME_MODELS = ["ultravox-v0.7", "fixie-ai/ultravox"]
 
 
@@ -628,7 +638,7 @@ class GrokRealtimeLLMConfiguration(BaseLLMConfiguration):
         },
     )
     voice: str = Field(
-        default="Ara",
+        default="ara",
         description="Voice the model speaks in.",
         json_schema_extra={
             "examples": GROK_REALTIME_VOICES,
@@ -746,7 +756,7 @@ class AzureRealtimeLLMConfiguration(BaseLLMConfiguration):
     model_config = AZURE_REALTIME_PROVIDER_MODEL_CONFIG
     provider: Literal[ServiceProviders.AZURE_REALTIME] = ServiceProviders.AZURE_REALTIME
     model: str = Field(
-        default="gpt-4o-realtime-preview",
+        default="gpt-realtime",
         description="Azure OpenAI realtime deployment name.",
         json_schema_extra={
             "examples": AZURE_REALTIME_MODELS,
@@ -765,8 +775,11 @@ class AzureRealtimeLLMConfiguration(BaseLLMConfiguration):
         },
     )
     api_version: str = Field(
-        default="2025-04-01-preview",
-        description="Azure OpenAI API version.",
+        default="v1",
+        description=(
+            "Azure OpenAI Realtime protocol version. Use 'v1' for the GA API; "
+            "date-based versions select the deprecated preview endpoint."
+        ),
         json_schema_extra={
             "examples": AZURE_REALTIME_API_VERSIONS,
         },
@@ -854,7 +867,10 @@ class ElevenlabsTTSConfiguration(BaseServiceConfiguration):
     model: str = Field(
         default="eleven_flash_v2_5",
         description="ElevenLabs TTS model.",
-        json_schema_extra={"examples": ELEVENLABS_TTS_MODELS},
+        json_schema_extra={
+            "examples": ELEVENLABS_TTS_MODELS,
+            "allow_custom_input": True,
+        },
     )
     base_url: str = Field(
         default="https://api.elevenlabs.io",
@@ -1274,6 +1290,32 @@ class SmallestAITTSConfiguration(BaseTTSConfiguration):
     )
 
 
+XAI_TTS_VOICES = ["eve", "ara", "leo", "rex", "sal"]
+
+
+@register_tts
+class XAITTSConfiguration(BaseServiceConfiguration):
+    model_config = XAI_PROVIDER_MODEL_CONFIG
+    provider: Literal[ServiceProviders.XAI] = ServiceProviders.XAI
+    voice: str = Field(
+        default="eve",
+        description="xAI voice persona.",
+        json_schema_extra={"examples": XAI_TTS_VOICES, "allow_custom_input": True},
+    )
+    language: str = Field(
+        default="en",
+        description="BCP-47 language code for synthesis (e.g. 'en', 'fr', 'de'), or 'auto' for automatic language detection.",
+        json_schema_extra={"allow_custom_input": True},
+    )
+
+    @computed_field
+    @property
+    def model(self) -> str:
+        # xAI TTS has no separate model selector; the voice fully specifies the
+        # output. A constant keeps the shared `.model` contract satisfied.
+        return "xai-tts"
+
+
 TTSConfig = Annotated[
     Union[
         DeepgramTTSConfiguration,
@@ -1290,6 +1332,7 @@ TTSConfig = Annotated[
         MiniMaxTTSConfiguration,
         AzureSpeechTTSConfiguration,
         SmallestAITTSConfiguration,
+        XAITTSConfiguration,
     ],
     Field(discriminator="provider"),
 ]
@@ -1323,9 +1366,6 @@ class DeepgramSTTConfiguration(BaseSTTConfiguration):
     )
 
 
-CARTESIA_STT_MODELS = ["ink-whisper"]
-
-
 @register_stt
 class CartesiaSTTConfiguration(BaseSTTConfiguration):
     model_config = CARTESIA_PROVIDER_MODEL_CONFIG
@@ -1335,6 +1375,17 @@ class CartesiaSTTConfiguration(BaseSTTConfiguration):
         description="Cartesia STT model.",
         json_schema_extra={"examples": CARTESIA_STT_MODELS},
     )
+    language: str = Field(
+        default="en",
+        description="ISO 639-1 language code. ink-2 currently supports English only.",
+        json_schema_extra={
+            "examples": CARTESIA_STT_LANGUAGES,
+            "model_options": {
+                "ink-2": CARTESIA_INK_2_STT_LANGUAGES,
+                "ink-whisper": CARTESIA_INK_WHISPER_STT_LANGUAGES,
+            },
+        },
+    )
 
 
 OPENAI_STT_MODELS = ["gpt-4o-transcribe"]
@@ -1397,6 +1448,10 @@ class GoogleSTTConfiguration(BaseSTTConfiguration):
 # Dograh STT Service
 DOGRAH_STT_MODELS = ["default"]
 DOGRAH_STT_LANGUAGES = DEEPGRAM_LANGUAGES
+# Languages auto-detected when the Dograh STT language is "multi". Dograh STT runs
+# Deepgram Flux multilingual under the hood, which only auto-detects this subset —
+# not the full DOGRAH_STT_LANGUAGES list offered for explicit single-language selection.
+DOGRAH_MULTILINGUAL_AUTODETECT_LANGUAGES = DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGES
 
 
 @register_stt
@@ -1625,6 +1680,39 @@ SMALLEST_STT_LANGUAGES = [
 ]
 
 
+@register_stt
+class ElevenlabsSTTConfiguration(BaseSTTConfiguration):
+    model_config = ELEVENLABS_PROVIDER_MODEL_CONFIG
+    provider: Literal[ServiceProviders.ELEVENLABS] = ServiceProviders.ELEVENLABS
+    model: str = Field(
+        default="scribe_v2_realtime",
+        description="ElevenLabs realtime STT model.",
+        json_schema_extra={
+            "examples": ELEVENLABS_STT_MODELS,
+            "allow_custom_input": True,
+        },
+    )
+    language: str = Field(
+        default="en",
+        description=(
+            "ISO 639-1 language code for transcription. "
+            "Use 'auto' to let ElevenLabs detect the language."
+        ),
+        json_schema_extra={
+            "examples": ELEVENLABS_STT_LANGUAGES,
+            "allow_custom_input": True,
+        },
+    )
+    base_url: str = Field(
+        default="https://api.elevenlabs.io",
+        description=(
+            "ElevenLabs API base URL. Override to use a Data Residency endpoint "
+            "(e.g. https://api.eu.residency.elevenlabs.io) for GDPR / HIPAA / "
+            "regional compliance."
+        ),
+    )
+
+
 @register_stt
 class SmallestAISTTConfiguration(BaseSTTConfiguration):
     model_config = SMALLEST_PROVIDER_MODEL_CONFIG
@@ -1659,6 +1747,7 @@ STTConfig = Annotated[
         GladiaSTTConfiguration,
         AzureSpeechSTTConfiguration,
         SmallestAISTTConfiguration,
+        ElevenlabsSTTConfiguration,
     ],
     Field(discriminator="provider"),
 ]
@@ -1722,7 +1811,7 @@ class AzureOpenAIEmbeddingsConfiguration(BaseEmbeddingsConfiguration):
     )
 
 
-DOGRAH_EMBEDDING_MODELS = ["default"]
+DOGRAH_EMBEDDING_MODELS = ["dograh_embedding_v1"]
 
 
 @register_embeddings
@@ -1730,7 +1819,7 @@ class DograhEmbeddingsConfiguration(BaseEmbeddingsConfiguration):
     model_config = DOGRAH_PROVIDER_MODEL_CONFIG
     provider: Literal[ServiceProviders.DOGRAH] = ServiceProviders.DOGRAH
     model: str = Field(
-        default="default",
+        default="dograh_embedding_v1",
         description="Dograh-managed embedding model.",
         json_schema_extra={"examples": DOGRAH_EMBEDDING_MODELS},
     )

api/services/filesystem/base.py

@@ -1,23 +1,51 @@
 from abc import ABC, abstractmethod
-from typing import Any, BinaryIO, Dict, Optional
+from typing import Any, Dict, Optional, Protocol
+
+
+class AsyncReadable(Protocol):
+    """Anything exposing ``await .read() -> bytes`` (aiofiles handles, in-memory wrappers)."""
+
+    async def read(self) -> bytes: ...
+
+
+class _AsyncBytesReader:
+    """Async file-like wrapper over in-memory bytes for acreate_file()."""
+
+    def __init__(self, data: bytes):
+        self._data = data
+
+    async def read(self) -> bytes:
+        return self._data
 
 
 class BaseFileSystem(ABC):
     """Abstract base class for filesystem operations."""
 
     @abstractmethod
-    async def acreate_file(self, file_path: str, content: BinaryIO) -> bool:
+    async def acreate_file(self, file_path: str, content: AsyncReadable) -> bool:
         """Create a new file with the given content.
 
         Args:
             file_path: Path where the file should be created
-            content: File content as a binary stream
+            content: File content readable via ``await content.read()``
 
         Returns:
             bool: True if file was created successfully, False otherwise
         """
         pass
 
+    async def acreate_file_from_bytes(self, file_path: str, data: bytes) -> bool:
+        """Create a file directly from in-memory bytes (no local file needed).
+
+        Args:
+            file_path: Path where the file should be created
+            data: File content as bytes
+
+        Returns:
+            bool: True if file was created successfully, False otherwise
+        """
+        return await self.acreate_file(file_path, _AsyncBytesReader(data))
+
     @abstractmethod
     async def aupload_file(self, local_path: str, destination_path: str) -> bool:
         """Upload a file from local path to destination.

api/services/filesystem/local.py

@@ -1,11 +1,11 @@
 import asyncio
 import os
 from datetime import datetime
-from typing import BinaryIO, Optional
+from typing import Optional
 
 import aiofiles
 
-from .base import BaseFileSystem
+from .base import AsyncReadable, BaseFileSystem
 
 
 class LocalFileSystem(BaseFileSystem):
@@ -24,7 +24,7 @@ class LocalFileSystem(BaseFileSystem):
         """Get the full path by joining with base path."""
         return os.path.join(self.base_path, file_path)
 
-    async def acreate_file(self, file_path: str, content: BinaryIO) -> bool:
+    async def acreate_file(self, file_path: str, content: AsyncReadable) -> bool:
         try:
             full_path = self._get_full_path(file_path)
             os.makedirs(os.path.dirname(full_path), exist_ok=True)

api/services/filesystem/minio.py

@@ -1,12 +1,13 @@
 import asyncio
+import io
 import json
-from typing import Any, BinaryIO, Dict, Optional
+from typing import Any, Dict, Optional
 
 from loguru import logger
 from minio import Minio
 from minio.error import S3Error
 
-from .base import BaseFileSystem
+from .base import AsyncReadable, BaseFileSystem
 
 
 class MinioFileSystem(BaseFileSystem):
@@ -89,15 +90,16 @@ class MinioFileSystem(BaseFileSystem):
             logger.debug(f"Bucket setup note: {e}")
             pass
 
-    async def acreate_file(self, file_path: str, content: BinaryIO) -> bool:
+    async def acreate_file(self, file_path: str, content: AsyncReadable) -> bool:
         try:
             data = await content.read()
 
             def _put():
+                # The MinIO SDK requires a stream with .read(), not raw bytes.
                 self.client.put_object(
                     self.bucket_name,
                     file_path,
-                    data=bytes(data),
+                    data=io.BytesIO(data),
                     length=len(data),
                 )
 

api/services/filesystem/null.py

@@ -1,6 +1,6 @@
-from typing import Any, BinaryIO, Dict, NoReturn, Optional
+from typing import Any, Dict, NoReturn, Optional
 
-from .base import BaseFileSystem
+from .base import AsyncReadable, BaseFileSystem
 
 
 class NullFileSystem(BaseFileSystem):
@@ -16,7 +16,7 @@ class NullFileSystem(BaseFileSystem):
             "Set ENVIRONMENT to a non-test value or inject a real filesystem fixture."
         )
 
-    async def acreate_file(self, file_path: str, content: BinaryIO) -> bool:
+    async def acreate_file(self, file_path: str, content: AsyncReadable) -> bool:
         self._fail("acreate_file")
 
     async def aupload_file(self, local_path: str, destination_path: str) -> bool:

api/services/filesystem/s3.py

@@ -1,30 +1,65 @@
-from typing import Any, BinaryIO, Dict, Optional
+from typing import Any, Dict, Optional
 
 import aioboto3
+from botocore.config import Config
 from botocore.exceptions import ClientError
 
-from .base import BaseFileSystem
+from .base import AsyncReadable, BaseFileSystem
 
 
 class S3FileSystem(BaseFileSystem):
     """S3 implementation of the filesystem interface."""
 
-    def __init__(self, bucket_name: str, region_name: str = "us-east-1"):
+    def __init__(
+        self,
+        bucket_name: str,
+        region_name: str = "us-east-1",
+        endpoint_url: Optional[str] = None,
+        signature_version: Optional[str] = None,
+        addressing_style: Optional[str] = None,
+    ):
         """Initialize S3 filesystem.
 
         Args:
             bucket_name: Name of the S3 bucket
             region_name: AWS region name
+            endpoint_url: Optional custom S3 endpoint (e.g. for MinIO/rustfs).
+                ``None`` uses AWS's default endpoint resolution.
+            signature_version: Optional botocore signature version (e.g.
+                ``"s3v4"``). ``None`` keeps botocore's default signing behavior.
+            addressing_style: Optional S3 addressing style (``"path"`` /
+                ``"virtual"`` / ``"auto"``). ``None`` keeps botocore's default.
         """
         self.bucket_name = bucket_name
         self.region_name = region_name
+        self.endpoint_url = endpoint_url
         self.session = aioboto3.Session()
 
-    async def acreate_file(self, file_path: str, content: BinaryIO) -> bool:
+        # Build a botocore Config only when an override is requested so that the
+        # default behavior is byte-for-byte unchanged when no env vars are set.
+        config_kwargs: Dict[str, Any] = {}
+        if signature_version:
+            config_kwargs["signature_version"] = signature_version
+        if addressing_style:
+            config_kwargs["s3"] = {"addressing_style": addressing_style}
+        self._config = Config(**config_kwargs) if config_kwargs else None
+
+    def _client_kwargs(self) -> Dict[str, Any]:
+        """Common kwargs for every ``session.client("s3", ...)`` call.
+
+        Only includes ``endpoint_url`` / ``config`` when configured, so default
+        deployments behave exactly as before.
+        """
+        kwargs: Dict[str, Any] = {"region_name": self.region_name}
+        if self.endpoint_url:
+            kwargs["endpoint_url"] = self.endpoint_url
+        if self._config is not None:
+            kwargs["config"] = self._config
+        return kwargs
+
+    async def acreate_file(self, file_path: str, content: AsyncReadable) -> bool:
         try:
-            async with self.session.client(
-                "s3", region_name=self.region_name
-            ) as s3_client:
+            async with self.session.client("s3", **self._client_kwargs()) as s3_client:
                 await s3_client.put_object(
                     Bucket=self.bucket_name, Key=file_path, Body=await content.read()
                 )
@@ -34,9 +69,7 @@ class S3FileSystem(BaseFileSystem):
 
     async def aupload_file(self, local_path: str, destination_path: str) -> bool:
         try:
-            async with self.session.client(
-                "s3", region_name=self.region_name
-            ) as s3_client:
+            async with self.session.client("s3", **self._client_kwargs()) as s3_client:
                 await s3_client.upload_file(
                     local_path, self.bucket_name, destination_path
                 )
@@ -59,9 +92,7 @@ class S3FileSystem(BaseFileSystem):
         disposition on the response.
         """
         try:
-            async with self.session.client(
-                "s3", region_name=self.region_name
-            ) as s3_client:
+            async with self.session.client("s3", **self._client_kwargs()) as s3_client:
                 params = {"Bucket": self.bucket_name, "Key": file_path}
 
                 # Make artifacts viewable inline in the browser when requested
@@ -100,9 +131,7 @@ class S3FileSystem(BaseFileSystem):
     async def aget_file_metadata(self, file_path: str) -> Optional[Dict[str, Any]]:
         """Get S3 object metadata."""
         try:
-            async with self.session.client(
-                "s3", region_name=self.region_name
-            ) as s3_client:
+            async with self.session.client("s3", **self._client_kwargs()) as s3_client:
                 response = await s3_client.head_object(
                     Bucket=self.bucket_name, Key=file_path
                 )
@@ -126,9 +155,7 @@ class S3FileSystem(BaseFileSystem):
     ) -> Optional[str]:
         """Generate a presigned PUT URL for direct file upload."""
         try:
-            async with self.session.client(
-                "s3", region_name=self.region_name
-            ) as s3_client:
+            async with self.session.client("s3", **self._client_kwargs()) as s3_client:
                 url = await s3_client.generate_presigned_url(
                     "put_object",
                     Params={
@@ -145,9 +172,7 @@ class S3FileSystem(BaseFileSystem):
     async def adownload_file(self, source_path: str, local_path: str) -> bool:
         """Download a file from S3 to local path."""
         try:
-            async with self.session.client(
-                "s3", region_name=self.region_name
-            ) as s3_client:
+            async with self.session.client("s3", **self._client_kwargs()) as s3_client:
                 await s3_client.download_file(self.bucket_name, source_path, local_path)
             return True
         except ClientError:
@@ -156,9 +181,7 @@ class S3FileSystem(BaseFileSystem):
     async def acopy_file(self, source_path: str, destination_path: str) -> bool:
         """Copy a file within S3 (server-side copy)."""
         try:
-            async with self.session.client(
-                "s3", region_name=self.region_name
-            ) as s3_client:
+            async with self.session.client("s3", **self._client_kwargs()) as s3_client:
                 await s3_client.copy_object(
                     Bucket=self.bucket_name,
                     Key=destination_path,

api/services/gen_ai/__init__.py

@@ -4,8 +4,11 @@ from .embedding import (
     AzureEmbeddingAPIKeyNotConfiguredError,
     AzureOpenAIEmbeddingService,
     BaseEmbeddingService,
+    DograhEmbeddingService,
     EmbeddingAPIKeyNotConfiguredError,
     OpenAIEmbeddingService,
+    build_embedding_service,
+    resolve_embedding_correlation_id,
 )
 from .json_parser import parse_llm_json
 
@@ -13,7 +16,10 @@ __all__ = [
     "AzureEmbeddingAPIKeyNotConfiguredError",
     "AzureOpenAIEmbeddingService",
     "BaseEmbeddingService",
+    "DograhEmbeddingService",
     "EmbeddingAPIKeyNotConfiguredError",
     "OpenAIEmbeddingService",
+    "build_embedding_service",
+    "resolve_embedding_correlation_id",
     "parse_llm_json",
 ]

api/services/gen_ai/embedding/__init__.py

@@ -5,12 +5,17 @@ from .azure_openai_service import (
     AzureOpenAIEmbeddingService,
 )
 from .base import BaseEmbeddingService
+from .dograh_service import DograhEmbeddingService
+from .factory import build_embedding_service, resolve_embedding_correlation_id
 from .openai_service import EmbeddingAPIKeyNotConfiguredError, OpenAIEmbeddingService
 
 __all__ = [
     "AzureEmbeddingAPIKeyNotConfiguredError",
     "AzureOpenAIEmbeddingService",
     "BaseEmbeddingService",
+    "DograhEmbeddingService",
     "EmbeddingAPIKeyNotConfiguredError",
     "OpenAIEmbeddingService",
+    "build_embedding_service",
+    "resolve_embedding_correlation_id",
 ]

api/services/gen_ai/embedding/dograh_service.py

@@ -0,0 +1,69 @@
+"""Dograh-managed embedding service.
+
+Routes embeddings through Dograh's managed proxy (MPS). This mirrors the managed
+voice services (``DograhLLMService`` / ``DograhTTSService``): when a server-minted
+MPS correlation id is present, it forwards the MPS billing v2 protocol
+(``correlation_id`` + ``mps_billing_version``) in the request body so MPS can
+authorize and attribute the call. With no correlation id (e.g. a v1 org) it
+behaves like a plain OpenAI-compatible call, which MPS accepts.
+
+Keeping this in a subclass keeps ``OpenAIEmbeddingService`` a generic
+OpenAI-compatible client; only the managed path carries MPS-specific metadata,
+so BYOK OpenAI/Azure requests never ship MPS fields to the real provider.
+"""
+
+from typing import Any, Dict, Optional
+
+from api.db.db_client import DBClient
+
+from .openai_service import DEFAULT_MODEL_ID, OpenAIEmbeddingService
+
+# Protocol contract with MPS (see model_services
+# api/services/model_service_correlations.py). Kept local to avoid coupling the
+# app layer to the pipecat package, which defines its own copy for voice.
+MPS_BILLING_VERSION_KEY = "mps_billing_version"
+MPS_BILLING_VERSION_V2 = "2"
+
+
+class DograhEmbeddingService(OpenAIEmbeddingService):
+    """OpenAI-compatible embedding client pointed at Dograh's managed proxy."""
+
+    def __init__(
+        self,
+        db_client: DBClient,
+        api_key: Optional[str] = None,
+        model_id: str = DEFAULT_MODEL_ID,
+        base_url: Optional[str] = None,
+        correlation_id: Optional[str] = None,
+    ):
+        """Initialize the managed embedding service.
+
+        Args:
+            db_client: Database client for vector similarity search.
+            api_key: Dograh-managed MPS service key.
+            model_id: Embedding model/tier id (default: text-embedding-3-small).
+            base_url: MPS embeddings base URL.
+            correlation_id: Server-minted MPS correlation id. When set, the MPS
+                billing v2 protocol is forwarded with each request. When None,
+                requests are sent without the protocol (valid for v1 orgs).
+        """
+        super().__init__(
+            db_client=db_client,
+            api_key=api_key,
+            model_id=model_id,
+            base_url=base_url,
+        )
+        self._correlation_id = correlation_id
+
+    def _request_kwargs(self) -> Dict[str, Any]:
+        """Forward the MPS billing v2 protocol when a correlation id is present."""
+        if not self._correlation_id:
+            return {}
+        return {
+            "extra_body": {
+                "metadata": {
+                    "correlation_id": self._correlation_id,
+                    MPS_BILLING_VERSION_KEY: MPS_BILLING_VERSION_V2,
+                }
+            }
+        }

api/services/gen_ai/embedding/factory.py

@@ -0,0 +1,107 @@
+"""Factory for embedding services, including the Dograh-managed (MPS) path.
+
+Centralizes the provider branching (Azure BYOK / Dograh-managed / OpenAI-compatible
+BYOK) that was previously duplicated across document ingestion, the search route,
+and the RAG tool, and resolves the MPS correlation id the same way the voice
+path does.
+"""
+
+from typing import Optional
+
+from loguru import logger
+
+from api.db.db_client import DBClient
+
+from .azure_openai_service import AzureOpenAIEmbeddingService
+from .base import BaseEmbeddingService
+from .dograh_service import DograhEmbeddingService
+from .openai_service import OpenAIEmbeddingService
+
+DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small"
+DEFAULT_AZURE_API_VERSION = "2024-02-15-preview"
+
+
+async def resolve_embedding_correlation_id(
+    *,
+    service_key: Optional[str],
+) -> Optional[str]:
+    """Mint an MPS correlation id for a managed embedding call made outside a run.
+
+    Matches the voice path's ``_authorize_oss_managed_v2_correlation``: the
+    correlation is minted via the bearer service-key endpoint, so it works for
+    hosted orgs and OSS keys alike. Returns ``None`` when minting fails; MPS
+    accepts un-correlated embedding calls.
+    """
+    if not service_key:
+        return None
+
+    # Imported lazily to avoid import-time cycles between the gen_ai and service
+    # layers (matches the inline-import convention used elsewhere in the app).
+    from api.services.mps_service_key_client import mps_service_key_client
+
+    try:
+        minted = await mps_service_key_client.create_correlation_id(
+            service_key=service_key
+        )
+        return minted.get("correlation_id")
+    except Exception as e:
+        logger.warning(
+            "Could not resolve MPS correlation id for managed embeddings; "
+            "sending without it: {}",
+            e,
+        )
+        return None
+
+
+async def build_embedding_service(
+    *,
+    db_client: DBClient,
+    provider: Optional[str],
+    api_key: Optional[str],
+    model: Optional[str],
+    base_url: Optional[str] = None,
+    endpoint: Optional[str] = None,
+    api_version: Optional[str] = None,
+    correlation_id: Optional[str] = None,
+    resolve_correlation: bool = False,
+) -> BaseEmbeddingService:
+    """Construct the right embedding service for a provider/config.
+
+    Args:
+        correlation_id: A correlation id already available in context (e.g. the
+            running workflow's MPS correlation id). Used for the Dograh provider.
+        resolve_correlation: When True and no ``correlation_id`` is supplied, resolve
+            one for the Dograh provider via ``resolve_embedding_correlation_id``
+            (for calls made outside a workflow run: ingestion, manual search).
+    """
+    from api.services.configuration.registry import ServiceProviders
+
+    model_id = model or DEFAULT_EMBEDDING_MODEL
+
+    if provider == ServiceProviders.AZURE.value and endpoint:
+        return AzureOpenAIEmbeddingService(
+            db_client=db_client,
+            api_key=api_key,
+            endpoint=endpoint,
+            model_id=model_id,
+            api_version=api_version or DEFAULT_AZURE_API_VERSION,
+        )
+
+    if provider == ServiceProviders.DOGRAH.value:
+        cid = correlation_id
+        if cid is None and resolve_correlation:
+            cid = await resolve_embedding_correlation_id(service_key=api_key)
+        return DograhEmbeddingService(
+            db_client=db_client,
+            api_key=api_key,
+            model_id=model_id,
+            base_url=base_url,
+            correlation_id=cid,
+        )
+
+    return OpenAIEmbeddingService(
+        db_client=db_client,
+        api_key=api_key,
+        model_id=model_id,
+        base_url=base_url,
+    )

api/services/gen_ai/embedding/openai_service.py

@@ -85,6 +85,14 @@ class OpenAIEmbeddingService(BaseEmbeddingService):
         if not self._api_key_configured or self.client is None:
             raise EmbeddingAPIKeyNotConfiguredError()
 
+    def _request_kwargs(self) -> Dict[str, Any]:
+        """Extra kwargs merged into every embeddings.create() call.
+
+        Override hook for subclasses (e.g. DograhEmbeddingService injects the MPS
+        billing protocol here). The base service adds nothing.
+        """
+        return {}
+
     async def embed_texts(self, texts: List[str]) -> List[List[float]]:
         """Embed a batch of texts using OpenAI API.
 
@@ -97,6 +105,7 @@ class OpenAIEmbeddingService(BaseEmbeddingService):
             response = await self.client.embeddings.create(
                 input=texts,
                 model=self.model_id,
+                **self._request_kwargs(),
             )
             return [item.embedding for item in response.data]
         except Exception as e:

api/services/integrations/paygent/__init__.py

@@ -0,0 +1,32 @@
+"""Paygent integration package.
+
+Self-registers on import via ``register_package``.  Auto-discovered by
+``api/services/integrations/loader.py`` (scans all submodules of
+``api.services.integrations`` except ``base``, ``loader``, and ``registry``).
+
+Provides:
+- ``PaygentNodeData`` – Pydantic config node shown in the Dograh UI under
+  INTEGRATIONS → "Paygent"
+- ``create_runtime_sessions`` – live-call observer that accumulates usage data
+- ``run_completion`` – post-call REST delivery to the Paygent API
+"""
+
+from __future__ import annotations
+
+from api.services.integrations.base import IntegrationPackageSpec
+from api.services.integrations.registry import register_package
+
+from .completion import run_completion
+from .node import NODE
+from .runtime import create_runtime_sessions
+
+PACKAGE = register_package(
+    IntegrationPackageSpec(
+        name="paygent",
+        nodes=(NODE,),
+        create_runtime_sessions=create_runtime_sessions,
+        run_completion=run_completion,
+    )
+)
+
+__all__ = ["PACKAGE"]

api/services/integrations/paygent/client.py

@@ -0,0 +1,315 @@
+"""Paygent REST API client (pure httpx, no SDK).
+
+All network I/O goes through ``post_paygent`` which is the single delivery
+coroutine used by the completion handler.  The individual tracker functions
+(session, STT, TTS, LLM, STS, indicator) mirror the exact shape of the
+Paygent REST API documented in ``paygent_sdk/voice_client.py``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+from pydantic import BaseModel, field_validator
+
+_DEFAULT_BASE_URL = "https://cp-api.withpaygent.com"
+_REQUEST_TIMEOUT = 15  # seconds – generous for post-call delivery
+
+
+# ---------------------------------------------------------------------------
+# Config model
+# ---------------------------------------------------------------------------
+
+
+class PaygentDeliveryConfig(BaseModel):
+    """Validated delivery configuration, filled from the node data."""
+
+    base_url: str = _DEFAULT_BASE_URL
+    api_key: str
+    agent_id: str
+    customer_id: str
+
+    @field_validator("api_key", "agent_id", "customer_id")
+    @classmethod
+    def _must_not_be_empty(cls, value: str) -> str:
+        if not value or not value.strip():
+            raise ValueError("must not be empty")
+        return value.strip()
+
+    @field_validator("base_url")
+    @classmethod
+    def _normalise_base_url(cls, value: str) -> str:
+        return (value or _DEFAULT_BASE_URL).rstrip("/")
+
+
+# ---------------------------------------------------------------------------
+# Live-call snapshot (collected during the call, delivered after)
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class PaygentCallSnapshot:
+    """Immutable snapshot produced at call-finish; passed to ``deliver``."""
+
+    session_id: str
+    agent_id: str
+    customer_id: str
+    is_realtime: bool
+
+    # Usage buckets filled from PipelineMetricsAggregator + user_config
+    stt_provider: str = ""
+    stt_model: str = ""
+    stt_audio_seconds: float = 0.0
+
+    llm_provider: str = ""
+    llm_model: str = ""
+    llm_prompt_tokens: int = 0
+    llm_completion_tokens: int = 0
+    llm_cached_tokens: int = 0
+
+    tts_provider: str = ""
+    tts_model: str = ""
+    tts_characters: int = 0
+
+    sts_provider: str = ""
+    sts_model: str = ""
+    sts_usage_metadata: dict[str, Any] | None = None
+
+    # Final call status / total duration seconds
+    call_disposition: str = "completed"
+    total_duration_seconds: int = 0
+    indicator: str = "per-minute-call"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "session_id": self.session_id,
+            "agent_id": self.agent_id,
+            "customer_id": self.customer_id,
+            "is_realtime": self.is_realtime,
+            "stt": {
+                "provider": self.stt_provider,
+                "model": self.stt_model,
+                "audio_seconds": self.stt_audio_seconds,
+            },
+            "llm": {
+                "provider": self.llm_provider,
+                "model": self.llm_model,
+                "prompt_tokens": self.llm_prompt_tokens,
+                "completion_tokens": self.llm_completion_tokens,
+                "cached_tokens": self.llm_cached_tokens,
+            },
+            "tts": {
+                "provider": self.tts_provider,
+                "model": self.tts_model,
+                "characters": self.tts_characters,
+            },
+            "sts": {
+                "provider": self.sts_provider,
+                "model": self.sts_model,
+                "usage_metadata": self.sts_usage_metadata,
+            },
+            "call_disposition": self.call_disposition,
+            "total_duration_seconds": self.total_duration_seconds,
+            "indicator": self.indicator,
+        }
+
+
+# ---------------------------------------------------------------------------
+# REST delivery helpers
+# ---------------------------------------------------------------------------
+
+
+def _headers(api_key: str) -> dict[str, str]:
+    return {
+        "Content-Type": "application/json",
+        "paygent-api-key": api_key,
+    }
+
+
+async def _post(
+    client: httpx.AsyncClient,
+    url: str,
+    api_key: str,
+    payload: dict[str, Any],
+    *,
+    label: str,
+) -> None:
+    """POST ``payload`` to ``url``; raises on 4xx/5xx or network failure.
+
+    Intentionally non-swallowing: callers in ``deliver()`` each wrap this in
+    their own try/except to build the ``errors`` list and the ``status`` field.
+    """
+    resp = await client.post(url, json=payload, headers=_headers(api_key))
+    resp.raise_for_status()
+
+
+async def deliver(
+    config: PaygentDeliveryConfig,
+    snapshot: PaygentCallSnapshot,
+) -> dict[str, Any]:
+    """
+    Execute the full Paygent REST call sequence for one completed call:
+
+    1. initialize_voice_session
+    2. track_stt            (if STT is used, i.e. not realtime-only)
+    3. track_llm
+    4. track_tts            (if TTS is used, i.e. not realtime-only)
+    5. track_sts            (if realtime / STS model used)
+    6. set_indicator        (always; marks end of session)
+
+    Returns a result dict merged into ``workflow_run.annotations``.
+    """
+    base = config.base_url
+    api_key = config.api_key
+    session_id = snapshot.session_id
+
+    delivered_steps: list[str] = []
+    errors: list[str] = []
+
+    async with httpx.AsyncClient(timeout=_REQUEST_TIMEOUT) as client:
+        # 1. Initialize voice session ----------------------------------------
+        try:
+            await _post(
+                client,
+                f"{base}/api/v1/voice/session",
+                api_key,
+                {
+                    "sessionId": session_id,
+                    "agentId": snapshot.agent_id,
+                    "customerId": snapshot.customer_id,
+                },
+                label="initialize_voice_session",
+            )
+            delivered_steps.append("session_init")
+        except Exception as exc:
+            errors.append(f"session_init: {exc}")
+
+        # 2. Track STT (only for non-realtime pipelines) ---------------------
+        if not snapshot.is_realtime and snapshot.stt_audio_seconds > 0:
+            try:
+                await _post(
+                    client,
+                    f"{base}/api/v1/voice/stt",
+                    api_key,
+                    {
+                        "sessionId": session_id,
+                        "audioMinutes": snapshot.stt_audio_seconds / 60.0,
+                        "provider": snapshot.stt_provider,
+                        "model": snapshot.stt_model,
+                        "plan": "",
+                    },
+                    label="track_stt",
+                )
+                delivered_steps.append("track_stt")
+            except Exception as exc:
+                errors.append(f"track_stt: {exc}")
+
+        # 3. Track LLM -------------------------------------------------------
+        if snapshot.llm_prompt_tokens > 0 or snapshot.llm_completion_tokens > 0:
+            llm_payload: dict[str, Any] = {
+                "sessionId": session_id,
+                "provider": snapshot.llm_provider,
+                "model": snapshot.llm_model,
+                "plan": "",
+                "promptTokens": snapshot.llm_prompt_tokens,
+                "completionTokens": snapshot.llm_completion_tokens,
+            }
+            if snapshot.llm_cached_tokens > 0:
+                llm_payload["cachedTokens"] = snapshot.llm_cached_tokens
+            try:
+                await _post(
+                    client,
+                    f"{base}/api/v1/voice/llm",
+                    api_key,
+                    llm_payload,
+                    label="track_llm",
+                )
+                delivered_steps.append("track_llm")
+            except Exception as exc:
+                errors.append(f"track_llm: {exc}")
+
+        # 4. Track TTS (only for non-realtime pipelines) ---------------------
+        if not snapshot.is_realtime and snapshot.tts_characters > 0:
+            try:
+                await _post(
+                    client,
+                    f"{base}/api/v1/voice/tts",
+                    api_key,
+                    {
+                        "sessionId": session_id,
+                        "provider": snapshot.tts_provider,
+                        "model": snapshot.tts_model,
+                        "plan": "",
+                        "characters": snapshot.tts_characters,
+                    },
+                    label="track_tts",
+                )
+                delivered_steps.append("track_tts")
+            except Exception as exc:
+                errors.append(f"track_tts: {exc}")
+
+        # 5. Track STS (Speech-to-Speech) for Realtime Models ----------------
+        if snapshot.is_realtime:
+            metadata = snapshot.sts_usage_metadata or {}
+            # Only append connection minutes if we don't already have a rich token payload
+            # (e.g. from OpenAI Realtime or Gemini Live)
+            if (
+                "connection" not in metadata
+                and "prompt_tokens" not in metadata
+                and "input" not in metadata
+            ):
+                metadata["connection"] = {
+                    "minutes": snapshot.total_duration_seconds / 60.0
+                }
+
+            try:
+                await _post(
+                    client,
+                    f"{base}/api/v1/voice/speech-to-speech",
+                    api_key,
+                    {
+                        "sessionId": session_id,
+                        "provider": snapshot.sts_provider,
+                        "model": snapshot.sts_model,
+                        "plan": "",
+                        "usageMetadata": metadata,
+                    },
+                    label="track_sts",
+                )
+                delivered_steps.append("track_sts")
+            except Exception as exc:
+                errors.append(f"track_sts: {exc}")
+
+        # 6. Set indicator (end-of-session marker) ---------------------------
+        try:
+            await _post(
+                client,
+                f"{base}/api/v1/voice/indicator",
+                api_key,
+                {
+                    "sessionId": session_id,
+                    "indicator": snapshot.indicator,
+                    "totalDuration": snapshot.total_duration_seconds / 60.0,
+                },
+                label="set_indicator",
+            )
+            delivered_steps.append("set_indicator")
+        except Exception as exc:
+            errors.append(f"set_indicator: {exc}")
+
+    return _result(session_id, delivered_steps, errors)
+
+
+def _result(
+    session_id: str,
+    delivered_steps: list[str],
+    errors: list[str],
+) -> dict[str, Any]:
+    return {
+        "session_id": session_id,
+        "delivered_steps": delivered_steps,
+        "errors": errors,
+        "status": "ok" if not errors else ("partial" if delivered_steps else "failed"),
+    }

api/services/integrations/paygent/collector.py

@@ -0,0 +1,704 @@
+"""Paygent live-call collector.
+
+Attaches to the pipecat pipeline as a ``BaseObserver`` to accumulate per-call
+usage metrics (STT audio seconds, LLM tokens, TTS characters, STS metadata)
+in memory during the call.  No network I/O happens here; all delivery is
+deferred to the post-call completion handler.
+
+Design mirrors ``api/services/integrations/tuner/collector.py`` exactly:
+- Attach to the task in ``PaygentRuntimeSession.attach``
+- Build a serialisable snapshot in ``build_snapshot``
+- Return it from ``on_call_finished`` so it lands in ``workflow_run.logs``
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict
+
+from loguru import logger
+from pipecat.frames.frames import (
+    CancelFrame,
+    EndFrame,
+    MetricsFrame,
+    StartFrame,
+    TTSTextFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
+)
+from pipecat.metrics.metrics import (
+    LLMTokenUsage,
+    LLMUsageMetricsData,
+    TTSUsageMetricsData,
+)
+from pipecat.observers.base_observer import BaseObserver, FramePushed
+from pipecat.processors.frame_processor import FrameDirection
+
+
+def _detect_provider(name: str, fallback: str = "unknown") -> str:
+    """Map a processor/model name to a canonical Paygent provider slug dynamically."""
+    if not name:
+        return fallback
+    clean_name = name.lower().strip()
+    if "gemini" in clean_name:
+        return "google"
+    suffixes = [
+        "service",
+        "multimodallive",
+        "realtime",
+        "vertex",
+        "llm",
+        "tts",
+        "stt",
+        "helper",
+        "transport",
+    ]
+    changed = True
+    while changed:
+        changed = False
+        for suffix in suffixes:
+            if clean_name.endswith(suffix):
+                clean_name = clean_name[: -len(suffix)].rstrip("_").rstrip("-")
+                changed = True
+                break
+    return clean_name or fallback
+
+
+@dataclass
+class _UsageAccumulator:
+    """In-memory accumulator for per-call usage data."""
+
+    # STT
+    stt_audio_seconds: float = 0.0
+
+    # LLM (aggregated across all turns)
+    llm_prompt_tokens: int = 0
+    llm_completion_tokens: int = 0
+    llm_cached_tokens: int = 0
+
+    # TTS
+    tts_characters: int = 0
+    _has_tts_metrics: bool = False
+
+    # STS / realtime (last seen usage_metadata dict; callers merge these)
+    sts_usage_metadata: dict[str, Any] | None = None
+
+    # Call timing
+    call_start_abs_ns: int = field(default_factory=time.time_ns)
+    call_end_abs_ns: int | None = None
+    # STT: timestamp of when user started speaking; None when not speaking
+    _user_started_speaking_ns: int | None = field(default=None, repr=False)
+
+    @property
+    def total_duration_seconds(self) -> int:
+        if self.call_end_abs_ns is None:
+            return int((time.time_ns() - self.call_start_abs_ns) / 1_000_000_000)
+        return int((self.call_end_abs_ns - self.call_start_abs_ns) / 1_000_000_000)
+
+    def get_stt_audio_seconds(self) -> float:
+        """Return measured STT audio seconds accumulated from the pipeline.
+
+        NOTE: This is the real measured STT audio duration collected from the
+        pipeline's STT metrics frames, NOT the total call wall-clock duration.
+        The call wall-clock duration is available separately via
+        ``total_duration_seconds``.
+        """
+        return self.stt_audio_seconds
+
+    def add_llm(self, usage: LLMTokenUsage) -> None:
+        self.llm_prompt_tokens += usage.prompt_tokens or 0
+        self.llm_completion_tokens += usage.completion_tokens or 0
+        self.llm_cached_tokens += (usage.cache_read_input_tokens or 0) + (
+            usage.cache_creation_input_tokens or 0
+        )
+
+    def add_tts_metrics(self, data: Any) -> None:
+        if not self._has_tts_metrics:
+            self._has_tts_metrics = True
+            self.tts_characters = 0  # Ignore manual count if metrics emit natively
+
+        # Extremely robust extraction
+        val = 0
+        if isinstance(data, (int, float)):
+            val = data
+        elif hasattr(data, "value"):
+            val = getattr(data, "value", 0) or 0
+        elif hasattr(data, "characters"):
+            val = getattr(data, "characters", 0) or 0
+        elif isinstance(data, dict):
+            val = data.get("value") or data.get("characters") or 0
+
+        try:
+            self.tts_characters += int(val or 0)
+        except Exception as exc:
+            logger.warning(
+                "[paygent] Failed to accumulate TTS characters (val={!r}): {}", val, exc
+            )
+
+    def add_tts_manual(self, text: str) -> None:
+        if not self._has_tts_metrics:
+            self.tts_characters += len(text)
+
+    def on_user_started_speaking(self) -> None:
+        """Mark the start of a user utterance for STT audio metering."""
+        if self._user_started_speaking_ns is None:
+            self._user_started_speaking_ns = time.time_ns()
+
+    def on_user_stopped_speaking(self) -> None:
+        """Accumulate the completed utterance duration into stt_audio_seconds."""
+        if self._user_started_speaking_ns is not None:
+            elapsed_s = (
+                time.time_ns() - self._user_started_speaking_ns
+            ) / 1_000_000_000
+            self.stt_audio_seconds += elapsed_s
+            self._user_started_speaking_ns = None
+
+    def finalize(self) -> None:
+        if self.call_end_abs_ns is None:
+            self.call_end_abs_ns = time.time_ns()
+        # If user was mid-utterance when the call ended, close the interval.
+        self.on_user_stopped_speaking()
+
+
+def _google_live_usage_to_sts_metadata(usage: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Pure Python translation of Google GenAI Live usage_metadata to
+    Paygent's canonical speech-to-speech /api/v1/voice/speech-to-speech API schema.
+    """
+    if not usage:
+        return {"schemaVersion": 1}
+
+    def _get_val(obj, *keys):
+        if not obj:
+            return None
+        for k in keys:
+            if isinstance(obj, dict):
+                if k in obj:
+                    return obj[k]
+            else:
+                if hasattr(obj, k):
+                    return getattr(obj, k)
+        return None
+
+    def _get_list(obj, *keys):
+        val = _get_val(obj, *keys)
+        if val is None:
+            return None
+        return list(val) if not isinstance(val, list) else val
+
+    def _optional_int(obj, *keys):
+        val = _get_val(obj, *keys)
+        if val is not None:
+            try:
+                return int(val)
+            except (TypeError, ValueError):
+                return None
+        return None
+
+    def _modality_token_count(details, modality_name):
+        if not details:
+            return 0
+        want = modality_name.upper()
+        total = 0
+        for d in details:
+            try:
+                mod = _get_val(d, "modality")
+                if mod is None:
+                    continue
+                label = _get_val(mod, "name") or _get_val(mod, "value") or mod
+                if str(label).upper() != want:
+                    continue
+                tc = _get_val(d, "token_count", "tokenCount")
+                total += int(tc or 0)
+            except Exception:
+                continue
+        return total
+
+    prompt_details = _get_list(usage, "prompt_tokens_details", "promptTokensDetails")
+    response_details = _get_list(
+        usage, "response_tokens_details", "responseTokensDetails"
+    )
+    tool_details = _get_list(
+        usage, "tool_use_prompt_tokens_details", "toolUsePromptTokensDetails"
+    )
+    cache_details = _get_list(usage, "cache_tokens_details", "cacheTokensDetails")
+
+    # input side: TEXT + DOCUMENT + AUDIO + IMAGE + VIDEO
+    text_in = _modality_token_count(prompt_details, "TEXT") + _modality_token_count(
+        tool_details, "TEXT"
+    )
+    audio_in = _modality_token_count(prompt_details, "AUDIO") + _modality_token_count(
+        tool_details, "AUDIO"
+    )
+    image_in = _modality_token_count(prompt_details, "IMAGE") + _modality_token_count(
+        tool_details, "IMAGE"
+    )
+    video_in = _modality_token_count(prompt_details, "VIDEO") + _modality_token_count(
+        tool_details, "VIDEO"
+    )
+    doc_as_text = _modality_token_count(
+        prompt_details, "DOCUMENT"
+    ) + _modality_token_count(tool_details, "DOCUMENT")
+    text_in += doc_as_text
+
+    # fallback aggregate mapping
+    tutc = _optional_int(
+        usage, "tool_use_prompt_token_count", "toolUsePromptTokenCount"
+    )
+    if tutc is not None and not tool_details:
+        text_in += int(tutc)
+
+    ptc = _optional_int(usage, "prompt_token_count", "promptTokenCount")
+    if ptc is not None and not prompt_details and not tool_details:
+        text_in += int(ptc)
+
+    # output side: TEXT + DOCUMENT + AUDIO + VIDEO + THINKING
+    text_out = _modality_token_count(response_details, "TEXT") + _modality_token_count(
+        response_details, "DOCUMENT"
+    )
+    audio_out = _modality_token_count(
+        response_details, "AUDIO"
+    ) + _modality_token_count(response_details, "VIDEO")
+
+    rtc = _optional_int(usage, "response_token_count", "responseTokenCount")
+    if text_out == 0 and audio_out == 0 and rtc is not None:
+        # Default fallback to audio output for STS audio connection
+        audio_out = int(rtc)
+
+    # Thinking / reasoning tokens (Gemini 2.5+ thinking models).
+    # Emitted as a separate output modality so Paygent has full billing visibility.
+    thinking_tokens = (
+        _optional_int(
+            usage,
+            "thoughts_token_count",
+            "thoughtsTokenCount",
+            "thinking_token_count",
+            "thinkingTokenCount",
+        )
+        or 0
+    )
+
+    # Cache breakdowns
+    cached_text = _modality_token_count(cache_details, "TEXT") + _modality_token_count(
+        cache_details, "DOCUMENT"
+    )
+    cached_audio = _modality_token_count(
+        cache_details, "AUDIO"
+    ) + _modality_token_count(cache_details, "VIDEO")
+    cached_image = _modality_token_count(cache_details, "IMAGE")
+    cached_legacy = _optional_int(
+        usage, "cached_content_token_count", "cachedContentTokenCount"
+    )
+
+    # Build response payload
+    out = {"schemaVersion": 1}
+
+    # Input Side
+    inp = {}
+    if text_in > 0:
+        inp["text"] = {"tokens": text_in}
+    if audio_in > 0:
+        inp["audio"] = {"tokens": audio_in}
+    if image_in > 0:
+        inp["image"] = {"tokens": image_in}
+    if video_in > 0:
+        inp["video"] = {"tokens": video_in}
+    if inp:
+        out["input"] = inp
+
+    # Output Side
+    o = {}
+    if text_out > 0:
+        o["text"] = {"tokens": text_out}
+    if audio_out > 0:
+        o["audio"] = {"tokens": audio_out}
+    if thinking_tokens > 0:
+        o["thinking"] = {"tokens": thinking_tokens}
+    if o:
+        out["output"] = o
+
+    # Cached breakdown
+    has_split = bool(cached_text or cached_audio or cached_image)
+    if cached_legacy is not None and cached_legacy > 0 and not has_split:
+        out["cached"] = {"tokens": int(cached_legacy)}
+    elif has_split:
+        cd = {}
+        if cached_text > 0:
+            cd["text"] = {"tokens": cached_text}
+        if cached_audio > 0:
+            cd["audio"] = {"tokens": cached_audio}
+        if cached_image > 0:
+            cd["image"] = {"tokens": cached_image}
+        if cd:
+            out["cached"] = cd
+
+    return out
+
+
+def _openai_realtime_usage_to_sts_metadata(usage: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Pure Python translation of OpenAI Realtime usage_metadata to
+    Paygent's canonical speech-to-speech /api/v1/voice/speech-to-speech API schema.
+    """
+    if not usage:
+        return {"schemaVersion": 1}
+
+    def _get_val(obj, *keys):
+        if not obj:
+            return None
+        for k in keys:
+            if isinstance(obj, dict):
+                if k in obj:
+                    return obj[k]
+            else:
+                if hasattr(obj, k):
+                    return getattr(obj, k)
+        return None
+
+    total_in = int(_get_val(usage, "input_tokens", "inputTokens") or 0)
+    total_out = int(_get_val(usage, "output_tokens", "outputTokens") or 0)
+
+    in_details = _get_val(usage, "input_token_details", "inputTokenDetails") or {}
+    out_details = _get_val(usage, "output_token_details", "outputTokenDetails") or {}
+
+    audio_in = int(_get_val(in_details, "audio_tokens", "audioTokens") or 0)
+    text_in = int(_get_val(in_details, "text_tokens", "textTokens") or 0)
+    image_in = int(_get_val(in_details, "image_tokens", "imageTokens") or 0)
+
+    cached_total = int(
+        _get_val(usage, "cached_tokens", "cachedTokens")
+        or _get_val(in_details, "cached_tokens", "cachedTokens")
+        or 0
+    )
+
+    cached_details = (
+        _get_val(in_details, "cached_tokens_details", "cachedTokensDetails") or {}
+    )
+    cached_audio = int(_get_val(cached_details, "audio_tokens", "audioTokens") or 0)
+    cached_text = int(_get_val(cached_details, "text_tokens", "textTokens") or 0)
+    cached_image = int(_get_val(cached_details, "image_tokens", "imageTokens") or 0)
+
+    if not (cached_audio or cached_text or cached_image):
+        cached_audio = int(
+            _get_val(in_details, "cached_audio_tokens", "cachedAudioTokens") or 0
+        )
+        cached_text = int(
+            _get_val(in_details, "cached_text_tokens", "cachedTextTokens") or 0
+        )
+        cached_image = int(
+            _get_val(in_details, "cached_image_tokens", "cachedImageTokens") or 0
+        )
+
+    audio_out = int(_get_val(out_details, "audio_tokens", "audioTokens") or 0)
+    text_out = int(_get_val(out_details, "text_tokens", "textTokens") or 0)
+
+    if not (text_in or audio_in or image_in) and total_in > 0:
+        text_in = total_in - cached_total
+
+    out = {"schemaVersion": 1}
+    inp = {}
+    if text_in > 0:
+        inp["text"] = {"tokens": text_in}
+    if audio_in > 0:
+        inp["audio"] = {"tokens": audio_in}
+    if image_in > 0:
+        inp["image"] = {"tokens": image_in}
+    if inp:
+        out["input"] = inp
+
+    o = {}
+    if text_out > 0:
+        o["text"] = {"tokens": text_out}
+    if audio_out > 0:
+        o["audio"] = {"tokens": audio_out}
+    if o:
+        out["output"] = o
+
+    has_split = bool(cached_text or cached_audio or cached_image)
+    if cached_total > 0 and not has_split:
+        out["cached"] = {"tokens": int(cached_total)}
+    elif has_split:
+        cd = {}
+        if cached_text > 0:
+            cd["text"] = {"tokens": cached_text}
+        if cached_audio > 0:
+            cd["audio"] = {"tokens": cached_audio}
+        if cached_image > 0:
+            cd["image"] = {"tokens": cached_image}
+        if cd:
+            out["cached"] = cd
+
+    return out
+
+
+def _merge_sts_metadata(existing: dict, new: dict) -> dict:
+    if not existing:
+        return new
+    out = {"schemaVersion": 1}
+    for key in ("input", "output", "cached"):
+        e_val = existing.get(key, {})
+        n_val = new.get(key, {})
+        if not e_val and not n_val:
+            continue
+
+        merged_cat: dict = {}
+
+        # Prefer per-modality merge when either side has per-modality detail.
+        # Only use the flat aggregate{"tokens": N} form when neither side has
+        # any per-modality breakdown at all (e.g. legacy schema).
+        e_has_modalities = any(
+            m in e_val for m in ("text", "audio", "image", "video", "thinking")
+        )
+        n_has_modalities = any(
+            m in n_val for m in ("text", "audio", "image", "video", "thinking")
+        )
+
+        if e_has_modalities or n_has_modalities:
+            for modality in ("text", "audio", "image", "video", "thinking"):
+                e_mod = e_val.get(modality, {}).get("tokens", 0)
+                n_mod = n_val.get(modality, {}).get("tokens", 0)
+                total = e_mod + n_mod
+                if total > 0:
+                    merged_cat[modality] = {"tokens": total}
+            # Also sum any lingering aggregate total so no tokens are lost
+            e_agg = e_val.get("tokens", 0) if not e_has_modalities else 0
+            n_agg = n_val.get("tokens", 0) if not n_has_modalities else 0
+            if e_agg or n_agg:
+                # Incorporate the unbroken-down side into the "text" bucket as
+                # a best-effort attribution rather than silently dropping it.
+                existing_text = merged_cat.get("text", {}).get("tokens", 0)
+                merged_cat["text"] = {"tokens": existing_text + e_agg + n_agg}
+        elif "tokens" in e_val or "tokens" in n_val:
+            merged_cat["tokens"] = e_val.get("tokens", 0) + n_val.get("tokens", 0)
+
+        if merged_cat:
+            out[key] = merged_cat
+
+    # retain any other keys, summing up numeric ones to keep metadata consistent
+    for k, v in existing.items():
+        if k not in ("schemaVersion", "input", "output", "cached"):
+            out[k] = v
+    for k, v in new.items():
+        if k not in ("schemaVersion", "input", "output", "cached"):
+            if (
+                k in out
+                and isinstance(out[k], (int, float))
+                and isinstance(v, (int, float))
+            ):
+                out[k] = out[k] + v
+            else:
+                out[k] = v
+
+    return out
+
+
+class PaygentCollector(BaseObserver):
+    """Pipecat observer that accumulates usage data for a single call.
+
+    Accumulates:
+    - LLM token usage from ``MetricsFrame / LLMUsageMetricsData``
+    - TTS character usage from ``MetricsFrame / TTSUsageMetricsData``
+    - STT audio seconds from ``MetricsFrame`` (when exposed by the pipeline)
+    - Call start / end timestamps for ``total_duration_seconds``
+
+    Does **not** do any network I/O.
+    """
+
+    def __init__(
+        self,
+        *,
+        workflow_run_id: int,
+        is_realtime: bool,
+        stt_provider: str = "",
+        stt_model: str = "",
+        llm_provider: str = "",
+        llm_model: str = "",
+        tts_provider: str = "",
+        tts_model: str = "",
+        sts_provider: str = "",
+        sts_model: str = "",
+    ) -> None:
+        super().__init__()
+        self._workflow_run_id = workflow_run_id
+        self._is_realtime = is_realtime
+        self._stt_provider = stt_provider
+        self._stt_model = stt_model
+        self._llm_provider = llm_provider
+        self._llm_model = llm_model
+        self._tts_provider = tts_provider
+        self._tts_model = tts_model
+        self._sts_provider = sts_provider
+        self._sts_model = sts_model
+        self._acc = _UsageAccumulator()
+        self._call_disposition: str = "completed"
+        # Dedup guard: pipecat can re-deliver frames. This collector is created
+        # fresh per call (see create_runtime_sessions) so the set size is bounded
+        # by call duration. We intentionally do NOT trim the set: trimming would
+        # evict old IDs and allow re-delivered frames to be processed a second time.
+        self._seen_frame_ids: set[int] = set()
+
+    # ------------------------------------------------------------------
+    # Public hooks
+    # ------------------------------------------------------------------
+
+    def set_call_disposition(self, disposition: str | None) -> None:
+        if disposition:
+            self._call_disposition = disposition
+
+    def build_snapshot(self) -> dict[str, Any]:
+        """Return a JSON-serialisable dict stored in ``workflow_run.logs``."""
+        self._acc.finalize()
+        stt_audio_sec = self._acc.get_stt_audio_seconds()
+
+        return {
+            "workflow_run_id": self._workflow_run_id,
+            "is_realtime": self._is_realtime,
+            "stt_provider": self._stt_provider,
+            "stt_model": self._stt_model,
+            "stt_audio_seconds": stt_audio_sec,
+            "llm_provider": self._llm_provider,
+            "llm_model": self._llm_model,
+            "llm_prompt_tokens": self._acc.llm_prompt_tokens,
+            "llm_completion_tokens": self._acc.llm_completion_tokens,
+            "llm_cached_tokens": self._acc.llm_cached_tokens,
+            "tts_provider": self._tts_provider,
+            "tts_model": self._tts_model,
+            "tts_characters": self._acc.tts_characters,
+            "sts_provider": self._sts_provider,
+            "sts_model": self._sts_model,
+            "sts_usage_metadata": self._acc.sts_usage_metadata,
+            "call_disposition": self._call_disposition,
+            "total_duration_seconds": self._acc.total_duration_seconds,
+        }
+
+    # ------------------------------------------------------------------
+    # BaseObserver implementation
+    # ------------------------------------------------------------------
+
+    async def on_push_frame(self, data: FramePushed) -> None:  # type: ignore[override]
+        try:
+            # Only process downstream frames; ignore upstream (mic → STT direction)
+            if data.direction != FrameDirection.DOWNSTREAM:
+                return
+
+            frame = data.frame
+
+            # Dedup: per-call set; grows with the call but is GC’d when the
+            # call ends. Never trim — trimming would reopen a re-delivery window.
+            if frame.id in self._seen_frame_ids:
+                return
+            self._seen_frame_ids.add(frame.id)
+
+            if isinstance(frame, StartFrame):
+                self._acc.call_start_abs_ns = time.time_ns()
+
+            elif isinstance(frame, MetricsFrame):
+                for item in frame.data:
+                    if isinstance(item, LLMUsageMetricsData):
+                        is_sts_frame = False
+                        proc_lower = (item.processor or "").lower()
+                        if getattr(self, "_is_realtime", False):
+                            if "realtime" in proc_lower or "live" in proc_lower:
+                                is_sts_frame = True
+
+                        if is_sts_frame:
+                            # Normalise the raw provider slug so that variants like
+                            # "openai_realtime", "azure_realtime", etc. route correctly.
+                            raw_provider = getattr(
+                                self, "_sts_provider", ""
+                            ) or getattr(self, "_llm_provider", "")
+                            provider = (
+                                _detect_provider(raw_provider)
+                                if raw_provider
+                                else "unknown"
+                            )
+                            if provider not in ("grok", "ultravox"):
+                                usage = item.value
+                                raw_metadata = getattr(
+                                    usage, "raw_usage_metadata", None
+                                )
+                                if raw_metadata:
+                                    # OpenAI Realtime and Azure Realtime (azure→openai via _detect_provider)
+                                    # share the same wire format.
+                                    if provider in ("openai", "azure"):
+                                        new_meta = (
+                                            _openai_realtime_usage_to_sts_metadata(
+                                                raw_metadata
+                                            )
+                                        )
+                                    else:
+                                        new_meta = _google_live_usage_to_sts_metadata(
+                                            raw_metadata
+                                        )
+                                else:
+                                    prompt_tokens = (
+                                        getattr(usage, "prompt_tokens", 0) or 0
+                                    )
+                                    completion_tokens = (
+                                        getattr(usage, "completion_tokens", 0) or 0
+                                    )
+                                    cached_tokens = (
+                                        getattr(usage, "cache_read_input_tokens", 0)
+                                        or getattr(usage, "cached_tokens", 0)
+                                        or 0
+                                    )
+                                    new_meta = {"schemaVersion": 1}
+                                    if prompt_tokens > 0:
+                                        new_meta.setdefault("input", {})["text"] = {
+                                            "tokens": prompt_tokens
+                                        }
+                                    if completion_tokens > 0:
+                                        new_meta.setdefault("output", {})["text"] = {
+                                            "tokens": completion_tokens
+                                        }
+                                    if cached_tokens > 0:
+                                        new_meta["cached"] = {"tokens": cached_tokens}
+
+                                    if hasattr(usage, "__dict__"):
+                                        for k, v in vars(usage).items():
+                                            if (
+                                                not k.startswith("_")
+                                                and v is not None
+                                                and k not in new_meta
+                                            ):
+                                                new_meta[k] = v
+
+                                self._acc.sts_usage_metadata = _merge_sts_metadata(
+                                    self._acc.sts_usage_metadata or {}, new_meta
+                                )
+                        else:
+                            self._acc.add_llm(item.value)
+                    elif isinstance(item, TTSUsageMetricsData):
+                        chars_val = getattr(item, "value", 0) or 0
+                        self._acc.add_tts_metrics(chars_val)
+                    # STT usage is exposed as a float in TTSUsageMetricsData-like
+                    # structure by some providers; we also pull from the aggregator
+                    # snapshot at call-finish (see runtime.py) for robustness.
+
+            elif isinstance(frame, TTSTextFrame):
+                # Fallback character counting for providers that don't emit native TTS metrics.
+                # TTSTextFrame carries only the text actually sent to the TTS engine;
+                # using base TextFrame would incorrectly include user transcriptions.
+                self._acc.add_tts_manual(frame.text)
+
+            elif isinstance(frame, UserStartedSpeakingFrame):
+                # Measure real STT audio seconds from VAD events rather than
+                # relying on wall-clock time. Skipped for realtime pipelines
+                # which have no separate STT stage.
+                if not self._is_realtime:
+                    self._acc.on_user_started_speaking()
+
+            elif isinstance(frame, UserStoppedSpeakingFrame):
+                if not self._is_realtime:
+                    self._acc.on_user_stopped_speaking()
+
+            elif isinstance(frame, (EndFrame, CancelFrame)):
+                self._acc.finalize()
+        except Exception as exc:
+            logger.warning(
+                "[paygent] Unexpected error processing frame {!r} in collector: {}",
+                type(data.frame).__name__,
+                exc,
+                exc_info=True,
+            )

api/services/integrations/paygent/completion.py

@@ -0,0 +1,193 @@
+"""Paygent post-call completion handler.
+
+Reads the ``paygent_snapshot`` that the runtime session stored in
+``workflow_run.logs``, reconstructs the full ``PaygentCallSnapshot``, and
+drives the ordered REST delivery sequence via ``client.deliver()``.
+
+Mirrors ``tuner/completion.py`` exactly:
+- validate each node with Pydantic
+- skip disabled nodes
+- read runtime snapshot from ``context.workflow_run.logs``
+- build a ``PaygentDeliveryConfig`` per node
+- call ``deliver(config, snapshot)``
+- collect results keyed by ``paygent_{node_id}``
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+from loguru import logger
+
+from api.services.integrations.base import IntegrationCompletionContext
+
+from .client import PaygentCallSnapshot, PaygentDeliveryConfig, deliver
+from .node import PaygentNodeData
+
+_DEFAULT_BASE_URL = "https://cp-api.withpaygent.com"
+
+
+def _build_snapshot(
+    raw: dict[str, Any],
+    *,
+    workflow_run_id: int,
+) -> PaygentCallSnapshot:
+    """Reconstruct a ``PaygentCallSnapshot`` from the persisted log dict."""
+    return PaygentCallSnapshot(
+        # session_id is always the authoritative workflow_run_id; the persisted
+        # snapshot value is never used to override it, preventing billing drift
+        # if the log is stale or corrupted.
+        session_id=str(workflow_run_id),
+        agent_id=raw.get("agent_id", ""),  # filled from node config below
+        customer_id=raw.get("customer_id", ""),  # filled from node config below
+        is_realtime=raw.get("is_realtime", False),
+        stt_provider=raw.get("stt_provider", ""),
+        stt_model=raw.get("stt_model", ""),
+        stt_audio_seconds=float(raw.get("stt_audio_seconds", 0.0)),
+        llm_provider=raw.get("llm_provider", ""),
+        llm_model=raw.get("llm_model", ""),
+        llm_prompt_tokens=int(raw.get("llm_prompt_tokens", 0)),
+        llm_completion_tokens=int(raw.get("llm_completion_tokens", 0)),
+        llm_cached_tokens=int(raw.get("llm_cached_tokens", 0)),
+        tts_provider=raw.get("tts_provider", ""),
+        tts_model=raw.get("tts_model", ""),
+        tts_characters=int(raw.get("tts_characters", 0)),
+        sts_provider=raw.get("sts_provider", ""),
+        sts_model=raw.get("sts_model", ""),
+        sts_usage_metadata=raw.get("sts_usage_metadata"),
+        call_disposition=raw.get("call_disposition", "completed"),
+        total_duration_seconds=int(raw.get("total_duration_seconds", 0)),
+    )
+
+
+async def run_completion(
+    nodes: list[dict[str, Any]],
+    context: IntegrationCompletionContext,
+) -> dict[str, Any]:
+    """Post-call completion handler: deliver usage data to Paygent REST API."""
+    results: dict[str, Any] = {}
+
+    raw_snapshot: dict[str, Any] | None = (context.workflow_run.logs or {}).get(
+        "paygent_snapshot"
+    )
+
+    for node in nodes:
+        node_id = node.get("id", "unknown")
+
+        # ---- Validate the node config via Pydantic -------------------------
+        try:
+            node_data = PaygentNodeData.model_validate(node.get("data", {}))
+        except Exception:
+            results[f"paygent_{node_id}"] = {"error": "validation_failed"}
+            continue
+
+        if not node_data.paygent_enabled:
+            continue
+
+        # ---- Guard: runtime snapshot must exist ----------------------------
+        if not raw_snapshot:
+            results[f"paygent_{node_id}"] = {"error": "missing_runtime_snapshot"}
+            continue
+
+        # ---- Build typed objects -------------------------------------------
+        snapshot = _build_snapshot(
+            raw_snapshot, workflow_run_id=context.workflow_run_id
+        )
+        # Inject node-level credentials into the snapshot
+        snapshot.agent_id = (node_data.paygent_agent_id or "").strip()
+        snapshot.customer_id = (node_data.paygent_customer_id or "").strip()
+        snapshot.indicator = (node_data.paygent_indicator or "per-minute-call").strip()
+
+        # Fallback to usage_info if snapshot has 0s (Pipecat metrics might be missing)
+        usage_info = context.workflow_run.usage_info or {}
+        try:
+            # Only fallback to pipeline-level llm usage if this is NOT a realtime pipeline.
+            # In realtime pipelines, the collector properly segregates STS and LLM tokens;
+            # falling back here would duplicate the STS tokens into the LLM bucket.
+            if (
+                snapshot.llm_prompt_tokens == 0
+                and snapshot.llm_completion_tokens == 0
+                and not snapshot.is_realtime
+            ):
+                llm_providers: list[str] = []
+                llm_models: list[str] = []
+                for key, val in usage_info.get("llm", {}).items():
+                    # Skip post-call QA analysis entries — they must not be billed
+                    # as in-conversation LLM usage.
+                    if key.startswith("QAAnalysis|||"):
+                        continue
+                    snapshot.llm_prompt_tokens += val.get("prompt_tokens", 0)
+                    snapshot.llm_completion_tokens += val.get("completion_tokens", 0)
+                    snapshot.llm_cached_tokens += val.get(
+                        "cache_read_input_tokens", 0
+                    ) + val.get("cache_creation_input_tokens", 0)
+                    parts = key.split("|||")
+                    if len(parts) == 2:
+                        llm_providers.append(parts[0])
+                        llm_models.append(parts[1])
+                if not snapshot.llm_provider and llm_providers:
+                    snapshot.llm_provider = ",".join(dict.fromkeys(llm_providers))
+                if not snapshot.llm_model and llm_models:
+                    snapshot.llm_model = ",".join(dict.fromkeys(llm_models))
+
+            if snapshot.tts_characters == 0:
+                tts_providers: list[str] = []
+                tts_models: list[str] = []
+                for key, val in usage_info.get("tts", {}).items():
+                    snapshot.tts_characters += val
+                    parts = key.split("|||")
+                    if len(parts) == 2:
+                        tts_providers.append(parts[0])
+                        tts_models.append(parts[1])
+                if not snapshot.tts_provider and tts_providers:
+                    snapshot.tts_provider = ",".join(dict.fromkeys(tts_providers))
+                if not snapshot.tts_model and tts_models:
+                    snapshot.tts_model = ",".join(dict.fromkeys(tts_models))
+
+            if snapshot.stt_audio_seconds == 0:
+                stt_providers: list[str] = []
+                stt_models: list[str] = []
+                for key, val in usage_info.get("stt", {}).items():
+                    snapshot.stt_audio_seconds += val
+                    parts = key.split("|||")
+                    if len(parts) == 2:
+                        stt_providers.append(parts[0])
+                        stt_models.append(parts[1])
+                if not snapshot.stt_provider and stt_providers:
+                    snapshot.stt_provider = ",".join(dict.fromkeys(stt_providers))
+                if not snapshot.stt_model and stt_models:
+                    snapshot.stt_model = ",".join(dict.fromkeys(stt_models))
+                # Note: if STT audio seconds remain 0 after all fallbacks, we do NOT
+                # substitute total_duration_seconds — that would overbill wall-clock time
+                # (silence, hold, agent speech) as STT input.
+        except Exception as exc:
+            logger.warning(
+                "[paygent] Failed to apply usage_info fallback for run {}: {}",
+                context.workflow_run_id,
+                exc,
+            )
+
+        try:
+            config = PaygentDeliveryConfig(
+                api_key=(node_data.paygent_api_key or "").strip(),
+                agent_id=snapshot.agent_id,
+                customer_id=snapshot.customer_id,
+            )
+        except Exception as exc:
+            results[f"paygent_{node_id}"] = {"error": f"invalid_config: {exc}"}
+            continue
+
+        # ---- REST delivery -------------------------------------------------
+        try:
+            delivery_result = await deliver(config, snapshot)
+            results[f"paygent_{node_id}"] = {
+                **delivery_result,
+                "agent_id": snapshot.agent_id,
+                "customer_id": snapshot.customer_id,
+                "exported_at": datetime.now(UTC).isoformat(),
+            }
+        except Exception as exc:
+            results[f"paygent_{node_id}"] = {"error": str(exc)}
+
+    return results

api/services/integrations/paygent/node.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+from pydantic import model_validator
+
+from api.services.integrations.base import IntegrationNodeRegistration
+from api.services.workflow.node_data import BaseNodeData
+from api.services.workflow.node_specs._base import (
+    GraphConstraints,
+    NodeCategory,
+    NodeExample,
+    PropertyType,
+)
+from api.services.workflow.node_specs.model_spec import (
+    build_spec,
+    node_spec,
+    spec_field,
+)
+
+
+@node_spec(
+    name="paygent",
+    display_name="Paygent",
+    description="Cost Tracking and Billing",
+    llm_hint=(
+        "Paygent is a post-call usage-tracking and billing integration. "
+        "It does not participate in the conversation graph and should not be connected to other nodes."
+    ),
+    docs_url="https://docs.dograh.com/integrations/paygent",
+    category=NodeCategory.integration,
+    icon="CreditCard",
+    examples=[
+        NodeExample(
+            name="paygent_tracking",
+            data={
+                "name": "Paygent Tracking",
+                "paygent_enabled": True,
+                "paygent_api_key": "pg_live_xxxxxxxxxxxxxxxx",
+                "paygent_agent_id": "my-voice-agent-prod",
+                "paygent_customer_id": "org-123",
+                "paygent_indicator": "per-minute-call",
+            },
+        )
+    ],
+    graph_constraints=GraphConstraints(
+        min_incoming=0, max_incoming=0, min_outgoing=0, max_outgoing=0, max_instances=1
+    ),
+    property_order=(
+        "name",
+        "paygent_enabled",
+        "paygent_api_key",
+        "paygent_agent_id",
+        "paygent_customer_id",
+        "paygent_indicator",
+    ),
+    field_overrides={
+        "name": {
+            "spec_default": "Paygent",
+            "description": "Short identifier for this Paygent configuration.",
+        },
+        "paygent_enabled": {
+            "display_name": "Enabled",
+            "description": "When false, Dograh skips all Paygent tracking for this call.",
+        },
+        "paygent_api_key": {
+            "display_name": "Paygent API Key",
+            "description": "API key used to authenticate requests to the Paygent REST API.",
+            "required": True,
+        },
+        "paygent_agent_id": {
+            "display_name": "Agent ID",
+            "description": "The agent identifier registered in your Paygent account.",
+            "required": True,
+        },
+        "paygent_customer_id": {
+            "display_name": "Customer ID",
+            "description": "Your Paygent customer / organisation ID.",
+            "required": True,
+        },
+        "paygent_indicator": {
+            "display_name": "Indicator",
+            "description": "The indicator event name sent at the end of the call (e.g. per-minute-call).",
+            "required": True,
+            "spec_default": "per-minute-call",
+        },
+    },
+)
+class PaygentNodeData(BaseNodeData):
+    paygent_enabled: bool = spec_field(
+        default=True,
+        ui_type=PropertyType.boolean,
+        display_name="Enabled",
+        description="When false, Dograh skips all Paygent tracking for this call.",
+    )
+    paygent_api_key: str | None = spec_field(
+        default=None,
+        ui_type=PropertyType.string,
+        display_name="Paygent API Key",
+        description="API key used to authenticate requests to the Paygent REST API.",
+    )
+    paygent_agent_id: str | None = spec_field(
+        default=None,
+        ui_type=PropertyType.string,
+        display_name="Agent ID",
+        description="The agent identifier registered in your Paygent account.",
+    )
+    paygent_customer_id: str | None = spec_field(
+        default=None,
+        ui_type=PropertyType.string,
+        display_name="Customer ID",
+        description="Your Paygent customer / organisation ID.",
+    )
+    paygent_indicator: str = spec_field(
+        default="per-minute-call",
+        ui_type=PropertyType.string,
+        display_name="Indicator",
+        description="The indicator event name sent at the end of the call (e.g. per-minute-call).",
+    )
+
+    @model_validator(mode="after")
+    def _validate_enabled_config(self) -> "PaygentNodeData":
+        if not self.paygent_enabled:
+            return self
+
+        missing: list[str] = []
+        if not self.paygent_api_key or not self.paygent_api_key.strip():
+            missing.append("paygent_api_key")
+        if not self.paygent_agent_id or not self.paygent_agent_id.strip():
+            missing.append("paygent_agent_id")
+        if not self.paygent_customer_id or not self.paygent_customer_id.strip():
+            missing.append("paygent_customer_id")
+        if not self.paygent_indicator or not self.paygent_indicator.strip():
+            missing.append("paygent_indicator")
+
+        if missing:
+            fields = ", ".join(missing)
+            raise ValueError(
+                f"Paygent node is enabled but missing required fields: {fields}"
+            )
+
+        return self
+
+
+SPEC = build_spec(PaygentNodeData)
+
+NODE = IntegrationNodeRegistration(
+    type_name="paygent",
+    data_model=PaygentNodeData,
+    node_spec=SPEC,
+    sensitive_fields=("paygent_api_key",),
+)

api/services/integrations/paygent/runtime.py

@@ -0,0 +1,139 @@
+"""Paygent runtime session.
+
+Wires the ``PaygentCollector`` into the live pipecat pipeline exactly the way
+``TunerRuntimeSession`` wires ``TunerCollector``.
+
+Lifecycle:
+  1. ``create_runtime_sessions`` scans the workflow graph for an enabled
+     ``paygent`` node and, if found, builds a collector from context metadata.
+  2. ``attach`` hooks the collector into the task as a pipeline observer so it
+     receives all ``MetricsFrame`` events during the call.
+  3. ``on_call_finished`` seals the snapshot and returns it to the generic
+     integration framework, which persists it in ``workflow_run.logs`` under
+     the key ``"paygent_snapshot"``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from api.services.integrations.base import (
+    IntegrationRuntimeContext,
+    IntegrationRuntimeSession,
+)
+
+from .collector import PaygentCollector
+
+
+def _label(provider: str | None, model: str | None) -> str:
+    """Compose a human-readable ``provider/model`` label."""
+    if provider and model:
+        return f"{provider}/{model}"
+    return model or provider or ""
+
+
+def _resolve_model_labels(
+    context: IntegrationRuntimeContext,
+) -> tuple[str, str, str, str, str, str, str, str]:
+    """Return (stt_provider, stt_model, llm_provider, llm_model,
+               tts_provider, tts_model, sts_provider, sts_model).
+
+    Mirrors the logic in ``tuner/runtime.py:_resolve_model_labels``.
+    """
+    user_config = context.user_config
+
+    if context.is_realtime and user_config.realtime:
+        realtime_provider = getattr(user_config.realtime, "provider", "") or ""
+        realtime_model = getattr(user_config.realtime, "model", "") or ""
+        llm_provider = getattr(user_config.llm, "provider", "") or ""
+        llm_model = getattr(user_config.llm, "model", "") or ""
+        return (
+            "",  # stt_provider  (no separate STT in realtime)
+            "",  # stt_model
+            llm_provider,
+            llm_model,
+            "",  # tts_provider  (no separate TTS in realtime)
+            "",  # tts_model
+            realtime_provider,
+            realtime_model,
+        )
+
+    return (
+        getattr(user_config.stt, "provider", "") or "",
+        getattr(user_config.stt, "model", "") or "",
+        getattr(user_config.llm, "provider", "") or "",
+        getattr(user_config.llm, "model", "") or "",
+        getattr(user_config.tts, "provider", "") or "",
+        getattr(user_config.tts, "model", "") or "",
+        "",  # sts_provider
+        "",  # sts_model
+    )
+
+
+class PaygentRuntimeSession(IntegrationRuntimeSession):
+    """Thin wrapper that connects the collector to the pipeline task."""
+
+    name = "paygent"
+
+    def __init__(self, collector: PaygentCollector) -> None:
+        self._collector = collector
+
+    # --- IntegrationRuntimeSession protocol --------------------------------
+
+    def attach(self, task: Any) -> None:
+        """Register the collector as a pipeline observer."""
+        task.add_observer(self._collector)
+
+    async def on_call_finished(
+        self,
+        *,
+        gathered_context: dict[str, Any],
+    ) -> dict[str, Any] | None:
+        """Seal the snapshot and hand it to the framework for persistence."""
+        self._collector.set_call_disposition(gathered_context.get("call_disposition"))
+        snapshot = self._collector.build_snapshot()
+        return {"paygent_snapshot": snapshot}
+
+
+# ---------------------------------------------------------------------------
+# Runtime session factory (called by the generic integration framework)
+# ---------------------------------------------------------------------------
+
+
+def create_runtime_sessions(
+    context: IntegrationRuntimeContext,
+) -> list[IntegrationRuntimeSession]:
+    """Return a ``PaygentRuntimeSession`` if a live, enabled paygent node exists."""
+    paygent_nodes = [
+        node
+        for node in context.workflow_graph.nodes.values()
+        if node.node_type == "paygent" and getattr(node.data, "paygent_enabled", True)
+    ]
+    if not paygent_nodes:
+        return []
+
+    (
+        stt_provider,
+        stt_model,
+        llm_provider,
+        llm_model,
+        tts_provider,
+        tts_model,
+        sts_provider,
+        sts_model,
+    ) = _resolve_model_labels(context)
+
+    collector = PaygentCollector(
+        workflow_run_id=context.workflow_run_id,
+        is_realtime=context.is_realtime,
+        stt_provider=stt_provider,
+        stt_model=stt_model,
+        llm_provider=llm_provider,
+        llm_model=llm_model,
+        tts_provider=tts_provider,
+        tts_model=tts_model,
+        sts_provider=sts_provider,
+        sts_model=sts_model,
+    )
+
+    return [PaygentRuntimeSession(collector)]

api/services/integrations/registry.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 from typing import Any
 
+from loguru import logger
+
 from api.services.integrations.base import (
     IntegrationCompletionContext,
     IntegrationNodeRegistration,
@@ -122,7 +124,17 @@ async def run_completion_handlers(
     for package, nodes in iter_completion_packages(context.workflow_definition):
         if package.run_completion is None:
             continue
-        package_result = await package.run_completion(nodes, context)
+        try:
+            package_result = await package.run_completion(nodes, context)
+        except Exception as exc:
+            logger.exception(
+                f"Integration completion handler failed for package "
+                f"{package.name!r}: {exc}"
+            )
+            results[f"integration_{package.name}"] = {
+                "error": "completion_handler_failed"
+            }
+            continue
         if package_result:
             results.update(package_result)
     return results

api/services/integrations/tuner/collector.py

@@ -1,48 +1,21 @@
 from __future__ import annotations
 
-import time
-from collections import deque
-from dataclasses import dataclass
-from typing import Any, Callable
+from typing import Any
 
-from loguru import logger
-from pipecat.frames.frames import (
-    BotStartedSpeakingFrame,
-    BotStoppedSpeakingFrame,
-    CancelFrame,
-    EndFrame,
-    FunctionCallInProgressFrame,
-    FunctionCallResultFrame,
-    MetricsFrame,
-    StartFrame,
-    UserStartedSpeakingFrame,
-    UserStoppedSpeakingFrame,
-    VADUserStoppedSpeakingFrame,
-)
-from pipecat.observers.base_observer import BaseObserver, FramePushed
-from pipecat.observers.turn_tracking_observer import TurnTrackingObserver
-from pipecat.observers.user_bot_latency_observer import UserBotLatencyObserver
-from pipecat.processors.frame_processor import FrameDirection
-from pipecat.utils.context.message_sanitization import strip_thought_ids_from_messages
-from tuner_pipecat_sdk.accumulator import CallAccumulator
-from tuner_pipecat_sdk.payload_builder import build_payload
+from tuner_pipecat_sdk import Observer
 
 from api.enums import WorkflowRunMode
 
 TUNER_RECORDING_PLACEHOLDER = "pipecat://no-recording"
 
-
-@dataclass(frozen=True)
-class _PayloadConfig:
-    call_id: str
-    call_type: str
-    recording_url: str
-    asr_model: str
-    llm_model: str
-    tts_model: str
-    sip_call_id: str | None = None
-    sip_headers: dict[str, str] | None = None
-    agent_version: int | None = None
+# Placeholder credentials for the SDK Observer's TunerConfig. Real BYOK credentials
+# (api_key / workspace_id / agent_id) are per tuner node and are applied later during
+# the deferred delivery phase (completion.py), so they are not known here. TunerConfig
+# validators require a non-empty api_key/agent_id and a positive workspace_id, hence
+# these placeholders.
+_DEFERRED_API_KEY = "deferred"
+_DEFERRED_WORKSPACE_ID = 1
+_DEFERRED_AGENT_ID = "deferred"
 
 
 def mode_to_tuner_call_type(mode: str | None) -> str:
@@ -54,8 +27,15 @@ def mode_to_tuner_call_type(mode: str | None) -> str:
     return "phone_call"
 
 
-class TunerCollector(BaseObserver):
-    """Collect runtime call metadata and build a deferred Tuner payload."""
+class DeferredTunerObserver(Observer):
+    """SDK ``Observer`` that builds the Tuner payload from the live frame stream but
+    defers delivery to the completion phase instead of POSTing on call end.
+
+    The SDK ``Observer`` normally fire-and-forgets ``post_call`` when the call ends.
+    Dograh instead snapshots the payload into ``workflow_run.logs`` and delivers it
+    later (``completion.py``) — once per tuner node with that node's BYOK credentials,
+    after injecting the real ``recording_url`` and a locally-computed ``call_cost``.
+    """
 
     def __init__(
         self,
@@ -66,126 +46,33 @@ class TunerCollector(BaseObserver):
         llm_model: str = "",
         tts_model: str = "",
         agent_version: int | None = None,
-        max_frames: int = 500,
     ) -> None:
-        super().__init__()
-        self._call_id = str(workflow_run_id)
-        self._call_type = call_type
-        self._asr_model = asr_model
-        self._llm_model = llm_model
-        self._tts_model = tts_model
-        self._agent_version = agent_version
-        self._acc = CallAccumulator()
-        self._acc.call_start_abs_ns = time.time_ns()
-        self._pipeline_start_rel_ns: int | None = None
-        self._context_provider: Callable[[], list[dict[str, Any]]] | None = None
-        self._processed_frames: set[int] = set()
-        self._frame_history: deque[int] = deque(maxlen=max_frames)
+        super().__init__(
+            api_key=_DEFERRED_API_KEY,
+            workspace_id=_DEFERRED_WORKSPACE_ID,
+            agent_id=_DEFERRED_AGENT_ID,
+            call_id=str(workflow_run_id),
+            call_type=call_type,
+            recording_url=TUNER_RECORDING_PLACEHOLDER,
+            asr_model=asr_model,
+            llm_model=llm_model,
+            tts_model=tts_model,
+            agent_version=agent_version,
+        )
 
-    def attach_context(self, provider: Callable[[], list[dict[str, Any]]]) -> None:
-        self._context_provider = provider
+    async def _flush(self) -> None:
+        # Suppress the SDK's runtime post_call; delivery is deferred (see class docstring).
+        return None
 
     def set_disconnection_reason(self, reason: str | None) -> None:
         if reason:
             self._acc.set_disconnection_reason(reason)
 
-    def attach_turn_tracking_observer(
-        self, turn_tracker: TurnTrackingObserver | None
-    ) -> None:
-        if turn_tracker is None:
-            return
-
-        @turn_tracker.event_handler("on_turn_started")
-        async def _on_turn_started(_tracker: Any, turn_number: int) -> None:
-            self._acc.on_turn_started(turn_number, time.time_ns())
-
-        @turn_tracker.event_handler("on_turn_ended")
-        async def _on_turn_ended(
-            _tracker: Any, turn_number: int, _duration: float, was_interrupted: bool
-        ) -> None:
-            self._acc.on_turn_ended(turn_number, was_interrupted)
-
-    def attach_latency_observer(
-        self, latency_observer: UserBotLatencyObserver | None
-    ) -> None:
-        if latency_observer is None:
-            return
-
-        @latency_observer.event_handler("on_latency_measured")
-        async def _on_latency_measured(_observer: Any, latency: float) -> None:
-            self._acc.on_latency_measured(latency)
-
-        @latency_observer.event_handler("on_latency_breakdown")
-        async def _on_latency_breakdown(_observer: Any, breakdown: Any) -> None:
-            self._acc.on_latency_breakdown(breakdown)
-
-    async def on_push_frame(self, data: FramePushed):
-        if data.direction != FrameDirection.DOWNSTREAM:
-            return
-
-        if data.frame.id in self._processed_frames:
-            return
-
-        self._processed_frames.add(data.frame.id)
-        self._frame_history.append(data.frame.id)
-        if len(self._processed_frames) > len(self._frame_history):
-            self._processed_frames = set(self._frame_history)
-
-        frame = data.frame
-
-        # data.timestamp is a pipeline-relative clock (ns since pipeline start).
-        # Convert to absolute ns so the accumulator's _rel_ms() works correctly.
-        if self._pipeline_start_rel_ns is None:
-            self._pipeline_start_rel_ns = data.timestamp
-        timestamp_ns = self._acc.call_start_abs_ns + (
-            data.timestamp - self._pipeline_start_rel_ns
-        )
-
-        if isinstance(frame, StartFrame):
-            self._acc.on_start(timestamp_ns)
-        elif isinstance(frame, FunctionCallInProgressFrame):
-            self._acc.on_function_call_in_progress(frame, timestamp_ns)
-        elif isinstance(frame, FunctionCallResultFrame):
-            self._acc.on_function_call_result(frame.tool_call_id, timestamp_ns)
-        elif isinstance(frame, MetricsFrame):
-            self._acc.on_metrics_frame(frame)
-        elif isinstance(frame, UserStartedSpeakingFrame):
-            self._acc.on_user_started_speaking(timestamp_ns)
-        elif isinstance(frame, UserStoppedSpeakingFrame):
-            self._acc.on_user_stopped_speaking(timestamp_ns)
-            self._acc.on_user_turn_stopped(timestamp_ns)
-        elif isinstance(frame, BotStartedSpeakingFrame):
-            self._acc.on_bot_started_speaking(timestamp_ns)
-        elif isinstance(frame, BotStoppedSpeakingFrame):
-            self._acc.on_bot_stopped(timestamp_ns)
-        elif isinstance(frame, VADUserStoppedSpeakingFrame):
-            self._acc.on_vad_stopped(timestamp_ns)
-        elif isinstance(frame, (CancelFrame, EndFrame)):
-            self._acc.on_call_end(timestamp_ns)
-
     def build_payload_snapshot(
         self,
         *,
         recording_url: str = TUNER_RECORDING_PLACEHOLDER,
     ) -> dict[str, Any] | None:
-        if self._context_provider is None:
-            logger.warning(
-                "[tuner] no context provider attached; skipping payload snapshot"
-            )
-            return None
-
-        transcript = strip_thought_ids_from_messages(list(self._context_provider()))
-        payload = build_payload(
-            self._acc,
-            _PayloadConfig(
-                call_id=self._call_id,
-                call_type=self._call_type,
-                recording_url=recording_url,
-                asr_model=self._asr_model,
-                llm_model=self._llm_model,
-                tts_model=self._tts_model,
-                agent_version=self._agent_version,
-            ),
-            transcript,
-        )
+        self._config.recording_url = recording_url
+        payload = self._acc.build_payload(self._config, None)
         return payload.to_dict()

api/services/integrations/tuner/completion.py

@@ -11,6 +11,7 @@ from api.services.integrations.base import IntegrationCompletionContext
 
 from .client import TunerDeliveryConfig, post_call
 from .collector import TUNER_RECORDING_PLACEHOLDER
+from .cost import compute_call_cost_cents
 from .node import TunerNodeData
 
 
@@ -55,6 +56,14 @@ async def run_completion(
         payload = copy.deepcopy(payload_snapshot)
         payload["recording_url"] = recording_url
 
+        call_cost = compute_call_cost_cents(
+            tuner_data,
+            context.workflow_run.usage_info,
+            transcript_segments=payload.get("transcript_with_tool_calls"),
+        )
+        if call_cost is not None:
+            payload["call_cost"] = call_cost
+
         try:
             config = TunerDeliveryConfig(
                 base_url=TUNER_BASE_URL,
@@ -67,6 +76,7 @@ async def run_completion(
                 **delivery,
                 "workspace_id": tuner_data.tuner_workspace_id,
                 "agent_id": tuner_data.tuner_agent_id,
+                **({"call_cost": call_cost} if call_cost is not None else {}),
                 "exported_at": datetime.now(UTC).isoformat(),
             }
         except Exception as exc:

api/services/integrations/tuner/cost.py

@@ -0,0 +1,131 @@
+"""Per-call cost computation for the Tuner export.
+
+Dograh no longer rates calls locally, so when a user wants Tuner to show a
+cost they provide their own per-unit prices on the Tuner node (the "bring your
+own keys" model). This module turns those rates plus the call's measured usage
+(`workflow_run.usage_info`) into a single `call_cost` value in cents, which is
+what Tuner's public API stores.
+
+Rates are optional: a blank rate contributes nothing. Usage metrics come from
+the pipeline aggregator and are reliable for LLM tokens and TTS characters.
+STT seconds are not measured, so the STT and telephony rates are applied
+per-minute against the call's wall-clock duration (an approximation).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .node import TunerNodeData
+
+
+def _sum_llm_tokens(usage_info: dict[str, Any]) -> tuple[int, int, int]:
+    """Sum prompt, completion, and cached-input tokens across all llm entries.
+
+    Cached-input tokens (``cache_read_input_tokens``) are reported as a discounted
+    subset of ``prompt_tokens`` (OpenAI convention), not in addition to it.
+    """
+    prompt_tokens = 0
+    completion_tokens = 0
+    cached_tokens = 0
+    for entry in (usage_info.get("llm") or {}).values():
+        if isinstance(entry, dict):
+            prompt_tokens += entry.get("prompt_tokens") or 0
+            completion_tokens += entry.get("completion_tokens") or 0
+            cached_tokens += entry.get("cache_read_input_tokens") or 0
+    return prompt_tokens, completion_tokens, cached_tokens
+
+
+def _sum_tts_characters(usage_info: dict[str, Any]) -> int:
+    """Sum TTS characters across every tts processor/model entry."""
+    total = 0
+    for value in (usage_info.get("tts") or {}).values():
+        if isinstance(value, (int, float)):
+            total += value
+    return int(total)
+
+
+# Transcript roles that represent bot-spoken text sent to TTS. Excludes
+# "user" (STT input) and "agent_function"/"agent_result" (tool calls).
+_SPOKEN_ROLES = {"agent", "assistant", "bot"}
+
+
+def _count_transcript_tts_characters(
+    transcript_segments: list[dict[str, Any]] | None,
+) -> int:
+    """Count characters of bot-spoken transcript turns (TTS proxy).
+
+    Used when the pipeline did not measure TTS characters directly (e.g. the
+    Deepgram websocket TTS service does not emit usage metrics). The spoken
+    transcript text closely matches what was sent to the TTS engine.
+    """
+    if not transcript_segments:
+        return 0
+    total = 0
+    for segment in transcript_segments:
+        if isinstance(segment, dict) and segment.get("role") in _SPOKEN_ROLES:
+            total += len(segment.get("text") or "")
+    return total
+
+
+def compute_call_cost_cents(
+    tuner_data: "TunerNodeData",
+    usage_info: dict[str, Any] | None,
+    transcript_segments: list[dict[str, Any]] | None = None,
+) -> float | None:
+    """Compute the call cost in cents from node rates and measured usage.
+
+    Returns ``None`` when cost calculation is disabled or no rates are
+    configured, so the caller can omit ``call_cost`` from the payload entirely
+    rather than report a misleading zero.
+    """
+    if not tuner_data.cost_calculation_enabled:
+        return None
+
+    raw_rates = (
+        tuner_data.cost_llm_input_rate,
+        tuner_data.cost_llm_cached_input_rate,
+        tuner_data.cost_llm_output_rate,
+        tuner_data.cost_tts_rate,
+        tuner_data.cost_stt_rate,
+        tuner_data.cost_telephony_rate,
+    )
+    if all(rate is None for rate in raw_rates):
+        return None
+
+    usage_info = usage_info or {}
+    prompt_tokens, completion_tokens, cached_tokens = _sum_llm_tokens(usage_info)
+    # Prefer the pipeline-measured TTS characters; fall back to the spoken
+    # transcript when the TTS service did not report usage (e.g. Deepgram websocket).
+    tts_characters = _sum_tts_characters(usage_info)
+    if tts_characters == 0:
+        tts_characters = _count_transcript_tts_characters(transcript_segments)
+    duration_minutes = (usage_info.get("call_duration_seconds") or 0) / 60.0
+
+    llm_input_rate = tuner_data.cost_llm_input_rate or 0.0
+    cached_input_rate = tuner_data.cost_llm_cached_input_rate
+    llm_output_rate = tuner_data.cost_llm_output_rate or 0.0
+    tts_rate = tuner_data.cost_tts_rate or 0.0
+    stt_rate = tuner_data.cost_stt_rate or 0.0
+    telephony_rate = tuner_data.cost_telephony_rate or 0.0
+
+    # Cached tokens are a discounted subset of prompt tokens. Only split them out
+    # when a cached rate is configured; otherwise bill all prompt tokens normally.
+    if cached_input_rate is not None:
+        uncached_prompt_tokens = max(prompt_tokens - cached_tokens, 0)
+        llm_input_usd = (
+            uncached_prompt_tokens * llm_input_rate + cached_tokens * cached_input_rate
+        ) / 1_000_000
+    else:
+        llm_input_usd = prompt_tokens * llm_input_rate / 1_000_000
+
+    cost_usd = (
+        llm_input_usd
+        + completion_tokens * llm_output_rate / 1_000_000
+        + tts_characters * tts_rate / 1_000
+        + duration_minutes * stt_rate
+        + duration_minutes * telephony_rate
+    )
+
+    return round(cost_usd * 100, 4)

api/services/integrations/tuner/node.py

@@ -5,9 +5,13 @@ from pydantic import model_validator
 from api.services.integrations.base import IntegrationNodeRegistration
 from api.services.workflow.node_data import BaseNodeData
 from api.services.workflow.node_specs._base import (
+    DisplayOptions,
     GraphConstraints,
     NodeCategory,
     NodeExample,
+    NumberInputOptions,
+    PropertyLayoutOptions,
+    PropertyRendererOptions,
     PropertyType,
 )
 from api.services.workflow.node_specs.model_spec import (
@@ -16,6 +20,13 @@ from api.services.workflow.node_specs.model_spec import (
     spec_field,
 )
 
+# Cost rate fields are only shown once the user turns on cost calculation.
+_COST_FIELDS_VISIBLE = DisplayOptions(show={"cost_calculation_enabled": [True]})
+_COST_RATE_RENDERER_OPTIONS = PropertyRendererOptions(
+    layout=PropertyLayoutOptions(column_span=6),
+    number_input=NumberInputOptions(fractional=True),
+)
+
 
 @node_spec(
     name="tuner",
@@ -25,6 +36,7 @@ from api.services.workflow.node_specs.model_spec import (
         "Tuner is a post-call observability export. It does not participate in the "
         "conversation graph and should not be connected to other nodes."
     ),
+    docs_url="https://docs.dograh.com/integrations/tuner",
     category=NodeCategory.integration,
     icon="Activity",
     examples=[
@@ -48,6 +60,13 @@ from api.services.workflow.node_specs.model_spec import (
         "tuner_agent_id",
         "tuner_workspace_id",
         "tuner_api_key",
+        "cost_calculation_enabled",
+        "cost_llm_input_rate",
+        "cost_llm_cached_input_rate",
+        "cost_llm_output_rate",
+        "cost_tts_rate",
+        "cost_stt_rate",
+        "cost_telephony_rate",
     ),
     field_overrides={
         "name": {
@@ -103,6 +122,73 @@ class TunerNodeData(BaseNodeData):
         description="Bearer token used when posting completed calls to Tuner.",
     )
 
+    cost_calculation_enabled: bool = spec_field(
+        default=False,
+        ui_type=PropertyType.boolean,
+        display_name="Calculate cost",
+        description="Send a per-call cost to Tuner, computed from your own provider rates (BYOK). All rates below are optional.",
+    )
+    cost_llm_input_rate: float | None = spec_field(
+        default=None,
+        ge=0,
+        le=1000,
+        ui_type=PropertyType.number,
+        display_name="LLM input",
+        description="USD per 1M tokens",
+        display_options=_COST_FIELDS_VISIBLE,
+        renderer_options=_COST_RATE_RENDERER_OPTIONS,
+    )
+    cost_llm_cached_input_rate: float | None = spec_field(
+        default=None,
+        ge=0,
+        le=1000,
+        ui_type=PropertyType.number,
+        display_name="LLM cached input",
+        description="USD per 1M cached tokens",
+        display_options=_COST_FIELDS_VISIBLE,
+        renderer_options=_COST_RATE_RENDERER_OPTIONS,
+    )
+    cost_llm_output_rate: float | None = spec_field(
+        default=None,
+        ge=0,
+        le=1000,
+        ui_type=PropertyType.number,
+        display_name="LLM output",
+        description="USD per 1M tokens",
+        display_options=_COST_FIELDS_VISIBLE,
+        renderer_options=_COST_RATE_RENDERER_OPTIONS,
+    )
+    cost_tts_rate: float | None = spec_field(
+        default=None,
+        ge=0,
+        le=100,
+        ui_type=PropertyType.number,
+        display_name="TTS",
+        description="USD per 1K characters",
+        display_options=_COST_FIELDS_VISIBLE,
+        renderer_options=_COST_RATE_RENDERER_OPTIONS,
+    )
+    cost_stt_rate: float | None = spec_field(
+        default=None,
+        ge=0,
+        le=100,
+        ui_type=PropertyType.number,
+        display_name="STT",
+        description="USD per minute",
+        display_options=_COST_FIELDS_VISIBLE,
+        renderer_options=_COST_RATE_RENDERER_OPTIONS,
+    )
+    cost_telephony_rate: float | None = spec_field(
+        default=None,
+        ge=0,
+        le=100,
+        ui_type=PropertyType.number,
+        display_name="Telephony",
+        description="USD per minute",
+        display_options=_COST_FIELDS_VISIBLE,
+        renderer_options=_COST_RATE_RENDERER_OPTIONS,
+    )
+
     @model_validator(mode="after")
     def _validate_enabled_config(self):
         if not self.tuner_enabled:

api/services/integrations/tuner/runtime.py

@@ -8,7 +8,7 @@ from api.services.integrations.base import (
     IntegrationRuntimeSession,
 )
 
-from .collector import TunerCollector, mode_to_tuner_call_type
+from .collector import DeferredTunerObserver, mode_to_tuner_call_type
 
 
 def _format_model_label(provider: str | None, model: str | None) -> str:
@@ -53,23 +53,25 @@ def _resolve_model_labels(context: IntegrationRuntimeContext) -> tuple[str, str,
 class TunerRuntimeSession(IntegrationRuntimeSession):
     name = "tuner"
 
-    def __init__(self, collector: TunerCollector) -> None:
-        self._collector = collector
+    def __init__(self, observer: DeferredTunerObserver) -> None:
+        self._observer = observer
 
     def attach(self, task: Any) -> None:
-        self._collector.attach_turn_tracking_observer(task.turn_tracking_observer)
-        self._collector.attach_latency_observer(task.user_bot_latency_observer)
-        task.add_observer(self._collector)
+        self._observer.attach_turn_tracking_observer(task.turn_tracking_observer)
+        task.add_observer(self._observer)
+        # The SDK Observer wires latency into the accumulator via its own latency
+        # observer, which must itself be registered to receive frames.
+        task.add_observer(self._observer.latency_observer)
 
     async def on_call_finished(
         self,
         *,
         gathered_context: dict[str, Any],
     ) -> dict[str, Any] | None:
-        self._collector.set_disconnection_reason(
+        self._observer.set_disconnection_reason(
             gathered_context.get("call_disposition")
         )
-        payload = self._collector.build_payload_snapshot()
+        payload = self._observer.build_payload_snapshot()
         if payload is None:
             return None
         return {"tuner_payload": payload}
@@ -88,7 +90,7 @@ def create_runtime_sessions(
 
     asr_model, llm_model, tts_model = _resolve_model_labels(context)
 
-    collector = TunerCollector(
+    observer = DeferredTunerObserver(
         workflow_run_id=context.workflow_run_id,
         call_type=mode_to_tuner_call_type(context.workflow_run.mode),
         asr_model=asr_model,
@@ -96,6 +98,5 @@ def create_runtime_sessions(
         tts_model=tts_model,
         agent_version=getattr(context.run_definition, "version_number", None),
     )
-    collector.attach_context(context.context_messages_provider)
 
-    return [TunerRuntimeSession(collector)]
+    return [TunerRuntimeSession(observer)]

api/services/mps_service_key_client.py

@@ -241,19 +241,12 @@ class MPSServiceKeyClient:
                 )
                 return False
 
-    async def check_service_key_usage(
-        self,
-        service_key: str,
-        organization_id: Optional[int] = None,
-        created_by: Optional[str] = None,
-    ) -> dict:
+    async def check_service_key_usage(self, service_key: str) -> dict:
         """
         Check the usage and quota of a service key.
 
         Args:
             service_key: The service key to check usage for
-            organization_id: Organization ID (for authenticated mode)
-            created_by: User provider ID (for OSS mode)
 
         Returns:
             Dictionary containing:
@@ -321,39 +314,6 @@ class MPSServiceKeyClient:
                     response=response,
                 )
 
-    async def get_usage_by_organization(self, organization_id: int) -> dict:
-        """
-        Get aggregated usage for all service keys belonging to an organization (hosted mode).
-
-        Args:
-            organization_id: The organization's ID
-
-        Returns:
-            Dictionary containing total_credits_used and remaining_credits
-        """
-        async with httpx.AsyncClient(timeout=self.timeout) as client:
-            response = await client.post(
-                f"{self.base_url}/api/v1/service-keys/usage/organization",
-                json={"organization_id": organization_id},
-                headers=self._get_headers(organization_id=organization_id),
-            )
-
-            if response.status_code == 200:
-                data = response.json()
-                return {
-                    "total_credits_used": data.get("total_credits_used", 0.0),
-                    "remaining_credits": data.get("remaining_credits", 0.0),
-                }
-            else:
-                logger.error(
-                    f"Failed to get usage by organization: {response.status_code} - {response.text}"
-                )
-                raise httpx.HTTPStatusError(
-                    f"Failed to get usage by organization: {response.text}",
-                    request=response.request,
-                    response=response,
-                )
-
     async def create_credit_purchase_url(
         self,
         organization_id: int,
@@ -422,30 +382,26 @@ class MPSServiceKeyClient:
                 response=response,
             )
 
-    async def get_billing_account_status(
-        self,
-        organization_id: int,
-        created_by: Optional[str] = None,
-    ) -> Optional[dict]:
-        """Get an existing MPS v2 billing account without creating one."""
+    async def get_billing_pricing(self, organization_id: int) -> dict:
+        """Return MPS-owned effective platform and Dograh model prices for an org."""
+        if DEPLOYMENT_MODE == "oss":
+            raise ValueError("OSS deployments do not fetch hosted billing prices")
+
         async with httpx.AsyncClient(timeout=self.timeout) as client:
             response = await client.get(
-                f"{self.base_url}/api/v1/billing/accounts/{organization_id}/status",
-                headers=self._get_headers(
-                    organization_id=organization_id,
-                    created_by=created_by,
-                ),
+                f"{self.base_url}/api/v1/billing/accounts/{organization_id}/pricing",
+                headers=self._get_headers(organization_id=organization_id),
             )
 
             if response.status_code == 200:
                 return response.json()
 
             logger.error(
-                "Failed to get MPS billing account status: "
+                "Failed to get MPS billing pricing: "
                 f"{response.status_code} - {response.text}"
             )
             raise httpx.HTTPStatusError(
-                f"Failed to get MPS billing account status: {response.text}",
+                f"Failed to get MPS billing pricing: {response.text}",
                 request=response.request,
                 response=response,
             )

api/services/organization_context.py

@@ -31,7 +31,6 @@ async def get_organization_context(user: UserModel) -> OrganizationContextRespon
     )
 
     resolved = await get_resolved_ai_model_configuration(
-        user_id=user.id,
         organization_id=organization_id,
     )
     managed_service_version = resolved.effective.managed_service_version

api/services/pipecat/active_calls.py

@@ -0,0 +1,35 @@
+"""In-process registry of active pipeline runs (live voice calls).
+
+Each uvicorn worker tracks the calls it is currently running so a deploy
+orchestrator can *drain* the worker before stopping it: poll the count, wait for
+zero, then send SIGTERM. Sending SIGTERM while calls are live makes uvicorn
+force-close their WebSockets (close code 1012), which cuts the calls instead of
+letting them finish — so the wait has to happen first.
+
+The registry is deliberately per-process. That is exactly the unit that gets
+drained: one uvicorn process per VM port (see ``scripts/rolling_update.sh``) or
+one uvicorn process per Kubernetes pod (drained via a ``preStop`` hook). The
+count is exposed read-only at ``GET /api/v1/health/active-calls`` and is also a
+natural autoscaling signal (concurrent calls per worker).
+
+Access is single-threaded (asyncio event loop), so no lock is needed. A set of
+run ids — rather than a bare counter — keeps register/unregister idempotent and
+makes the in-flight runs inspectable for debugging.
+"""
+
+_active_run_ids: set[int] = set()
+
+
+def register_active_call(workflow_run_id: int) -> None:
+    """Mark a pipeline run as active in this worker."""
+    _active_run_ids.add(workflow_run_id)
+
+
+def unregister_active_call(workflow_run_id: int) -> None:
+    """Mark a pipeline run as finished in this worker."""
+    _active_run_ids.discard(workflow_run_id)
+
+
+def active_call_count() -> int:
+    """Number of pipeline runs currently active in this worker."""
+    return len(_active_run_ids)

api/services/pipecat/event_handlers.py

@@ -14,8 +14,10 @@ from api.services.pipecat.in_memory_buffers import (
 )
 from api.services.pipecat.pipeline_metrics_aggregator import PipelineMetricsAggregator
 from api.services.pipecat.tracing_config import get_trace_url
+from api.services.pipecat.transcript_log_coordinator import TranscriptLogCoordinator
 from api.services.posthog_client import capture_event
 from api.services.workflow.pipecat_engine import PipecatEngine
+from api.services.workflow_run_artifacts import upload_workflow_run_artifacts
 from api.tasks.arq import enqueue_job
 from api.tasks.function_names import FunctionNames
 from pipecat.frames.frames import (
@@ -64,11 +66,13 @@ def register_event_handlers(
     engine: PipecatEngine,
     audio_buffer: AudioBufferProcessor,
     in_memory_logs_buffer: InMemoryLogsBuffer,
+    transcript_log_coordinator: TranscriptLogCoordinator,
     pipeline_metrics_aggregator: PipelineMetricsAggregator,
     audio_config=AudioConfig,
     pre_call_fetch_task: asyncio.Task | None = None,
     user_provider_id: str | None = None,
     integration_runtime_sessions: list[IntegrationRuntimeSession] | None = None,
+    include_transcript_end_timestamps: bool = False,
 ):
     """Register all event handlers for transport and task events.
 
@@ -221,7 +225,12 @@ def register_event_handlers(
         task: PipelineWorker,
         _frame: Frame,
     ):
-        logger.debug(f"In on_pipeline_finished callback handler")
+        logger.debug("In on_pipeline_finished callback handler")
+
+        # Turn and feedback observers run on independent queues. Drain them
+        # before finalizing immutable transcripts and taking the DB snapshot.
+        await task.wait_for_observers()
+        await transcript_log_coordinator.flush()
 
         workflow_run = await db_client.get_workflow_run_by_id(workflow_run_id)
 
@@ -361,50 +370,51 @@ def register_event_handlers(
             except Exception as e:
                 logger.error(f"Error saving workflow run logs: {e}", exc_info=True)
 
-        # Write buffers to temp files and enqueue combined processing task
-        audio_temp_path = None
-        user_audio_temp_path = None
-        bot_audio_temp_path = None
-        transcript_temp_path = None
-
+        # Upload artifacts straight from the in-memory buffers so nothing has
+        # to cross a process/host boundary via temp files. Must complete
+        # before the completion job is enqueued so QA and webhooks see the
+        # artifacts in storage.
         try:
+            mixed_audio_wav = None
+            user_audio_wav = None
+            bot_audio_wav = None
+
             if not in_memory_audio_buffers.mixed.is_empty:
-                audio_temp_path = (
-                    await in_memory_audio_buffers.mixed.write_to_temp_file()
-                )
+                mixed_audio_wav = await in_memory_audio_buffers.mixed.to_wav_bytes()
             else:
                 logger.debug("Audio buffer is empty, skipping upload")
 
             if not in_memory_audio_buffers.user.is_empty:
-                user_audio_temp_path = (
-                    await in_memory_audio_buffers.user.write_to_temp_file()
-                )
+                user_audio_wav = await in_memory_audio_buffers.user.to_wav_bytes()
             else:
                 logger.debug("User audio buffer is empty, skipping upload")
 
             if not in_memory_audio_buffers.bot.is_empty:
-                bot_audio_temp_path = (
-                    await in_memory_audio_buffers.bot.write_to_temp_file()
-                )
+                bot_audio_wav = await in_memory_audio_buffers.bot.to_wav_bytes()
             else:
                 logger.debug("Bot audio buffer is empty, skipping upload")
 
-            transcript_temp_path = in_memory_logs_buffer.write_transcript_to_temp_file()
-            if not transcript_temp_path:
+            transcript_text = in_memory_logs_buffer.generate_transcript_text(
+                include_end_timestamps=include_transcript_end_timestamps
+            )
+            if not transcript_text:
                 logger.debug("No transcript events in logs buffer, skipping upload")
 
+            await upload_workflow_run_artifacts(
+                workflow_run_id,
+                mixed_audio_wav=mixed_audio_wav,
+                user_audio_wav=user_audio_wav,
+                bot_audio_wav=bot_audio_wav,
+                transcript_text=transcript_text,
+            )
         except Exception as e:
-            logger.error(f"Error preparing buffers for S3 upload: {e}", exc_info=True)
+            logger.error(f"Error uploading call artifacts: {e}", exc_info=True)
 
-        # Combined task: uploads artifacts, runs integrations (including QA),
-        # then calculates cost (so QA token usage is captured in usage_info)
+        # Combined task: runs integrations (including QA), then calculates
+        # cost (so QA token usage is captured in usage_info)
         await enqueue_job(
             FunctionNames.PROCESS_WORKFLOW_COMPLETION,
             workflow_run_id,
-            audio_temp_path,
-            transcript_temp_path,
-            user_audio_temp_path,
-            bot_audio_temp_path,
         )
 
     # Return the buffer so it can be passed to other handlers

api/services/pipecat/gemini_json_schema_adapter.py

@@ -0,0 +1,39 @@
+"""Dograh-specific Gemini adapter customizations."""
+
+from typing import Any
+
+from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
+from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter
+
+
+class DograhGeminiJSONSchemaAdapter(GeminiLLMAdapter):
+    """Use Gemini's full JSON Schema tool parameter field.
+
+    Pipecat's default Gemini adapter maps ``FunctionSchema.parameters`` into
+    ``FunctionDeclaration.parameters``, which is backed by Google GenAI's
+    stricter OpenAPI-style ``Schema`` model. MCP and imported tools may contain
+    valid JSON Schema keywords such as ``const`` and ``not`` that are rejected
+    by that model. ``parameters_json_schema`` is the Google GenAI field intended
+    for full JSON Schema payloads.
+    """
+
+    def to_provider_tools_format(
+        self, tools_schema: ToolsSchema
+    ) -> list[dict[str, Any]]:
+        functions_schema = tools_schema.standard_tools
+        if functions_schema:
+            formatted_functions = []
+            for func in functions_schema:
+                func_dict = func.to_default_dict()
+                parameters = func_dict.pop("parameters")
+                func_dict["parameters_json_schema"] = parameters
+                formatted_functions.append(func_dict)
+            formatted_standard_tools = [{"function_declarations": formatted_functions}]
+        else:
+            formatted_standard_tools = []
+
+        custom_gemini_tools = []
+        if tools_schema.custom_tools:
+            custom_gemini_tools = tools_schema.custom_tools.get(AdapterType.GEMINI, [])
+
+        return formatted_standard_tools + custom_gemini_tools

api/services/pipecat/in_memory_buffers.py

@@ -1,6 +1,7 @@
 import asyncio
-import tempfile
+import io
 import wave
+from copy import deepcopy
 from datetime import UTC, datetime
 from typing import List, Optional
 
@@ -15,7 +16,7 @@ from pipecat.utils.enums import RealtimeFeedbackType
 
 
 class InMemoryAudioBuffer:
-    """Buffer audio data in memory during a call, then write to temp file on disconnect."""
+    """Buffer audio data in memory during a call, then encode to WAV bytes on disconnect."""
 
     def __init__(self, workflow_run_id: int, sample_rate: int, num_channels: int = 1):
         self._workflow_run_id = workflow_run_id
@@ -41,28 +42,30 @@ class InMemoryAudioBuffer:
                 f"Appended {len(pcm_data)} bytes to audio buffer. Total size: {self._total_size}"
             )
 
-    async def write_to_temp_file(self) -> str:
-        """Write audio data to a temporary WAV file and return the path."""
+    async def to_wav_bytes(self) -> bytes:
+        """Encode the buffered PCM data as an in-memory WAV file."""
         async with self._lock:
-            temp_file = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
-            logger.debug(
-                f"Writing audio buffer to temp file {temp_file.name} for workflow {self._workflow_run_id}"
-            )
+            chunks = list(self._chunks)
 
-            # Write WAV header and PCM data
-            with wave.open(temp_file.name, "wb") as wf:
+        def _encode() -> bytes:
+            wav_io = io.BytesIO()
+            with wave.open(wav_io, "wb") as wf:
                 wf.setnchannels(self._num_channels)
                 wf.setsampwidth(2)  # 16-bit audio
                 wf.setframerate(self._sample_rate)
 
                 # Concatenate all chunks
-                for chunk in self._chunks:
+                for chunk in chunks:
                     wf.writeframes(chunk)
+            return wav_io.getvalue()
 
-            logger.info(
-                f"Successfully wrote {self._total_size} bytes of audio to {temp_file.name}"
-            )
-            return temp_file.name
+        # Encoding is mostly memcpy but can touch ~100MB; keep it off the event loop
+        data = await asyncio.to_thread(_encode)
+        logger.info(
+            f"Encoded {self._total_size} bytes of audio to {len(data)} WAV bytes "
+            f"for workflow {self._workflow_run_id}"
+        )
+        return data
 
     @property
     def is_empty(self) -> bool:
@@ -102,7 +105,7 @@ class InMemoryLogsBuffer:
     def __init__(self, workflow_run_id: int):
         self._workflow_run_id = workflow_run_id
         self._events: List[dict] = []
-        self._turn_counter = 0
+        self._current_turn: Optional[int] = None
         self._current_node_id: Optional[str] = None
         self._current_node_name: Optional[str] = None
 
@@ -121,36 +124,44 @@ class InMemoryLogsBuffer:
         """Get the current node name."""
         return self._current_node_name
 
-    async def append(self, event: dict):
-        """Append a feedback event to the buffer with timestamp and current node."""
+    def set_current_turn(self, turn: int) -> None:
+        """Set the fallback turn for non-transcript events."""
+        self._current_turn = turn
+
+    async def append(
+        self,
+        event: dict,
+        *,
+        timestamp: Optional[str] = None,
+        turn: Optional[int] = None,
+        node_id: Optional[str] = None,
+        node_name: Optional[str] = None,
+        use_current_node: bool = True,
+    ):
+        """Append an immutable event with optional correlation metadata."""
+        if use_current_node:
+            node_id = self._current_node_id if node_id is None else node_id
+            node_name = self._current_node_name if node_name is None else node_name
         timestamped_event = stamp_realtime_feedback_event(
-            event,
-            timestamp=datetime.now(UTC).isoformat(),
-            turn=self._turn_counter,
-            node_id=self._current_node_id,
-            node_name=self._current_node_name,
+            deepcopy(event),
+            timestamp=timestamp or datetime.now(UTC).isoformat(timespec="milliseconds"),
+            turn=self._current_turn if turn is None else turn,
+            node_id=node_id,
+            node_name=node_name,
         )
         self._events.append(timestamped_event)
         logger.trace(
             f"Appended event {event.get('type')} to logs buffer for workflow {self._workflow_run_id}"
         )
 
-    def increment_turn(self):
-        """Increment turn counter (called on user transcription completion)."""
-        self._turn_counter += 1
-        logger.trace(
-            f"Incremented turn counter to {self._turn_counter} for workflow {self._workflow_run_id}"
-        )
-
     def _sorted_events(self) -> List[dict]:
-        # Stable sort by the realtime (payload) timestamp when available, falling
-        # back to the buffer-append timestamp. Python's sort is stable, so events
-        # sharing a key retain their original insertion order — this keeps
-        # consecutive bot-text chunks of a single turn contiguous.
+        # Stable sort by the top-level event timestamp used by the persisted
+        # realtime feedback schema. Legacy events without one fall back to their
+        # payload timestamp. Events sharing a key retain insertion order.
         return sorted(self._events, key=realtime_feedback_event_sort_key)
 
     def get_events(self) -> List[dict]:
-        """Get all events for final storage, ordered by realtime timestamp."""
+        """Get all events for final storage, ordered by event timestamp."""
         return self._sorted_events()
 
     def contains_user_speech(self) -> bool:
@@ -164,34 +175,15 @@ class InMemoryLogsBuffer:
                 return True
         return False
 
-    def generate_transcript_text(self) -> str:
+    def generate_transcript_text(self, *, include_end_timestamps: bool = False) -> str:
         """Generate transcript text from logged events.
 
         Filters for rtf-user-transcription (final) and rtf-bot-text events,
         formats them as '[timestamp] user/assistant: text\\n'.
         """
-        return _generate_transcript_text(self._sorted_events())
-
-    def write_transcript_to_temp_file(self) -> Optional[str]:
-        """Write transcript to a temporary text file and return the path.
-
-        Returns None if there are no transcript events.
-        """
-        content = self.generate_transcript_text()
-        if not content:
-            return None
-
-        temp_file = tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False)
-        logger.debug(
-            f"Writing transcript to temp file {temp_file.name} for workflow {self._workflow_run_id}"
+        return _generate_transcript_text(
+            self._sorted_events(), include_end_timestamps=include_end_timestamps
         )
-        temp_file.write(content)
-        temp_file.close()
-
-        logger.info(
-            f"Successfully wrote {len(content)} chars of transcript to {temp_file.name}"
-        )
-        return temp_file.name
 
     @property
     def is_empty(self) -> bool:

api/services/pipecat/pipeline_engine_callbacks_processor.py

@@ -3,6 +3,7 @@ from typing import Awaitable, Callable, Optional
 
 from loguru import logger
 
+from api.schemas.workflow_configurations import DEFAULT_MAX_CALL_DURATION_SECONDS
 from pipecat.frames.frames import (
     Frame,
     HeartbeatFrame,
@@ -23,7 +24,7 @@ class PipelineEngineCallbacksProcessor(FrameProcessor):
 
     def __init__(
         self,
-        max_call_duration_seconds: int = 300,
+        max_call_duration_seconds: int = DEFAULT_MAX_CALL_DURATION_SECONDS,
         max_duration_end_task_callback: Optional[Callable[[], Awaitable[None]]] = None,
         generation_started_callback: Optional[Callable[[], Awaitable[None]]] = None,
         llm_text_frame_callback: Optional[Callable[[str], Awaitable[None]]] = None,

api/services/pipecat/realtime/azure_realtime.py

@@ -1,9 +1,9 @@
 """Dograh subclass of pipecat's Azure OpenAI Realtime LLM service.
 
 Layers Dograh engine integration quirks (mute gating, TTSSpeakFrame greeting
-trigger, LLMMessagesAppendFrame handling, deferred tool calls) onto pipecat's
-AzureRealtimeLLMService, mirroring what DograhOpenAIRealtimeLLMService does
-for the standard OpenAI Realtime endpoint.
+trigger, LLMMessagesAppendFrame handling, workflow-control deferral) onto
+pipecat's AzureRealtimeLLMService, mirroring what
+DograhOpenAIRealtimeLLMService does for the standard OpenAI Realtime endpoint.
 """
 
 import json
@@ -11,6 +11,7 @@ from typing import Any
 
 from loguru import logger
 
+from api.services.pipecat.realtime.static_greeting import format_static_greeting_prompt
 from pipecat.frames.frames import (
     BotStartedSpeakingFrame,
     BotStoppedSpeakingFrame,
@@ -39,7 +40,7 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
     - User-mute audio gating
     - TTSSpeakFrame as initial-response trigger
     - One-off LLMMessagesAppendFrame handling
-    - Deferred tool calls until bot finishes speaking
+    - Workflow-control calls deferred until bot finishes speaking
     - finalized=True on TranscriptionFrame for consistency
     """
 
@@ -48,7 +49,8 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
         self._user_is_muted: bool = False
         self._handled_initial_context: bool = False
         self._bot_is_speaking: bool = False
-        self._deferred_function_calls: list[FunctionCallFromLLM] = []
+        self._deferred_node_transition_function_calls: list[FunctionCallFromLLM] = []
+        self._pending_initial_greeting_text: str | None = None
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         if isinstance(frame, UserMuteStartedFrame):
@@ -61,7 +63,11 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
             return
         if isinstance(frame, TTSSpeakFrame):
             if not self._handled_initial_context:
-                await self._handle_context(self._context)
+                greeting_text = frame.text.strip() if frame.text else ""
+                if greeting_text:
+                    await self._handle_initial_greeting(self._context, greeting_text)
+                else:
+                    await self._handle_context(self._context)
             else:
                 logger.warning(
                     f"{self}: TTSSpeakFrame after initial context already handled — "
@@ -75,7 +81,7 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
             self._bot_is_speaking = True
         elif isinstance(frame, BotStoppedSpeakingFrame):
             self._bot_is_speaking = False
-            await self._run_pending_function_calls()
+            await self._run_pending_node_transition_function_calls()
         await super().process_frame(frame, direction)
 
     async def _handle_messages_append(self, frame: LLMMessagesAppendFrame):
@@ -118,6 +124,57 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
             self._context = context
             await self._process_completed_function_calls(send_new_results=True)
 
+    async def _handle_initial_greeting(self, context: LLMContext, greeting_text: str):
+        if context is None:
+            logger.warning(
+                f"{self}: received initial greeting trigger before context was set"
+            )
+            return
+
+        self._handled_initial_context = True
+        self._context = context
+        await self._create_initial_greeting_response(greeting_text)
+
+    async def _create_initial_greeting_response(self, greeting_text: str):
+        if self._disconnecting:
+            return
+
+        if not self._api_session_ready:
+            self._pending_initial_greeting_text = greeting_text
+            self._run_llm_when_api_session_ready = True
+            return
+
+        self._pending_initial_greeting_text = None
+        await self._ensure_conversation_setup()
+        await self._send_manual_response_create(
+            instructions=format_static_greeting_prompt(greeting_text),
+            tool_choice="none",
+        )
+
+    async def _ensure_conversation_setup(self):
+        if not self._llm_needs_conversation_setup:
+            return
+
+        adapter = self.get_llm_adapter()
+        llm_invocation_params = adapter.get_llm_invocation_params(self._context)
+        for item in llm_invocation_params["messages"]:
+            evt = events.ConversationItemCreateEvent(item=item)
+            self._messages_added_manually[evt.item.id] = True
+            await self.send_client_event(evt)
+
+        await self._send_session_update()
+        self._llm_needs_conversation_setup = False
+
+    async def _handle_evt_session_updated(self, evt):
+        self._api_session_ready = True
+        if self._pending_initial_greeting_text is not None:
+            greeting_text = self._pending_initial_greeting_text
+            self._run_llm_when_api_session_ready = False
+            await self._create_initial_greeting_response(greeting_text)
+        elif self._run_llm_when_api_session_ready:
+            self._run_llm_when_api_session_ready = False
+            await self._create_response()
+
     async def _send_user_audio(self, frame):
         if self._user_is_muted:
             return
@@ -171,30 +228,38 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
             return "\n".join(parts) if parts else None
         return None
 
-    async def _send_manual_response_create(self):
+    async def _send_manual_response_create(
+        self,
+        *,
+        instructions: str | None = None,
+        tool_choice: str | None = None,
+    ):
         await self.push_frame(LLMFullResponseStartFrame())
         await self.start_processing_metrics()
         await self.start_ttfb_metrics()
         await self.send_client_event(
             events.ResponseCreateEvent(
                 response=events.ResponseProperties(
-                    output_modalities=self._get_enabled_modalities()
+                    output_modalities=self._get_enabled_modalities(),
+                    instructions=instructions,
+                    tool_choice=tool_choice,
                 )
             )
         )
 
-    async def _run_pending_function_calls(self):
-        if not self._deferred_function_calls:
+    async def _run_pending_node_transition_function_calls(self):
+        if not self._deferred_node_transition_function_calls:
             return
-        function_calls = self._deferred_function_calls
-        self._deferred_function_calls = []
+        function_calls = self._deferred_node_transition_function_calls
+        self._deferred_node_transition_function_calls = []
         logger.debug(
-            f"{self}: executing {len(function_calls)} deferred function call(s) "
-            "after bot turn ended"
+            f"{self}: executing {len(function_calls)} deferred workflow-control "
+            "call(s) after bot turn ended"
         )
         await self.run_function_calls(function_calls)
 
     async def _handle_evt_function_call_arguments_done(self, evt):
+        """Run ordinary tools immediately and defer workflow-control calls."""
         try:
             args = json.loads(evt.arguments)
 
@@ -211,10 +276,14 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
                     )
                 ]
 
-                if self._bot_is_speaking:
-                    self._deferred_function_calls.extend(function_calls)
+                is_node_transition = self._function_is_node_transition(
+                    function_call_item.name
+                )
+                if self._bot_is_speaking and is_node_transition:
+                    self._deferred_node_transition_function_calls.extend(function_calls)
                     logger.debug(
-                        f"{self}: deferring function call {function_call_item.name} "
+                        f"{self}: deferring workflow-control call "
+                        f"{function_call_item.name} "
                         "until bot stops speaking"
                     )
                 else:

api/services/pipecat/realtime/gemini_live.py

@@ -9,8 +9,8 @@ Layers Dograh engine integration quirks onto upstream-pristine
 - **Reconnect on node transitions.** Gemini Live cannot update
   ``system_instruction`` mid-session, so a setting change triggers a
   reconnect (deferred until the bot turn ends if currently responding).
-- **Function-call deferral.** Tool calls emitted mid-turn are queued and run
-  when the bot stops speaking, to avoid racing the turn's audio.
+- **Node-transition deferral.** Node-transition calls emitted mid-turn are
+  queued and run when the bot stops speaking, to avoid cutting off its audio.
 - **User-mute audio gating.** ``UserMuteStarted/StoppedFrame`` from the
   user aggregator gates whether incoming audio is forwarded to Gemini.
 - **TTSSpeakFrame as greeting trigger.** The engine queues a TTSSpeakFrame
@@ -18,10 +18,16 @@ Layers Dograh engine integration quirks onto upstream-pristine
   it and runs the initial-context path.
 """
 
+import asyncio
 from typing import Any
 
+from google.genai.types import Content, Part
 from loguru import logger
 
+from api.services.pipecat.gemini_json_schema_adapter import (
+    DograhGeminiJSONSchemaAdapter,
+)
+from api.services.pipecat.realtime.static_greeting import format_static_greeting_prompt
 from pipecat.frames.frames import (
     BotStoppedSpeakingFrame,
     Frame,
@@ -39,6 +45,18 @@ from pipecat.utils.tracing.service_decorators import traced_gemini_live
 class DograhGeminiLiveLLMService(GeminiLiveLLMService):
     """Gemini Live with Dograh engine integration quirks. See module docstring."""
 
+    # Gemini input transcription is delivered independently from tool calls.
+    # Give late transcription messages a small window to arrive before running
+    # a node-transition function and tearing down the current Live connection.
+    _NODE_TRANSITION_TRANSCRIPTION_GRACE_SECONDS = 0.5
+
+    # Route tool schemas through Gemini's ``parameters_json_schema`` field so
+    # MCP/imported tools that use JSON Schema keywords (``const``, ``not``,
+    # nested ``anyOf``) rejected by the strict ``Schema`` model are accepted.
+    # Mirrors the non-realtime ``DograhGoogleLLMService`` fix;
+    # ``DograhGeminiLiveVertexLLMService`` inherits this via MRO.
+    adapter_class = DograhGeminiJSONSchemaAdapter
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         # User-mute state, driven by broadcast UserMute{Started,Stopped}Frames.
@@ -47,12 +65,19 @@ class DograhGeminiLiveLLMService(GeminiLiveLLMService):
         # Guards initial-response triggering against double-firing across the
         # initial TTSSpeakFrame and any LLMContextFrame that may arrive.
         self._handled_initial_context: bool = False
-        # When a system_instruction change arrives mid-bot-turn, the reconnect
-        # is queued and drained when the turn ends.
-        self._reconnect_pending: bool = False
-        # Function calls emitted by Gemini mid-bot-turn are deferred here and
-        # invoked when the turn ends, so they don't race the turn's audio.
-        self._pending_function_calls: list[FunctionCallFromLLM] = []
+        # Node-transition calls emitted mid-bot-turn are deferred here so the
+        # transition does not tear down Gemini while it is still producing audio.
+        self._pending_node_transition_function_calls: list[FunctionCallFromLLM] = []
+        # Text greeting captured from the first TTSSpeakFrame while the Gemini
+        # session is still connecting.
+        self._pending_initial_greeting_text: str | None = None
+        self._transition_function_call_task: asyncio.Task | None = None
+        # Intentional node changes use a fresh, context-seeded connection rather
+        # than a potentially stale session-resumption handle. The new connection
+        # remains gated until the function-call result has landed in LLMContext.
+        self._awaiting_node_transition_context: bool = False
+        self._node_transition_context_received: bool = False
+        self._node_transition_context_seed_started: bool = False
 
     # ------------------------------------------------------------------
     # Hooks from upstream GeminiLiveLLMService
@@ -63,33 +88,109 @@ class DograhGeminiLiveLLMService(GeminiLiveLLMService):
         # lets pre-call fetch populate template variables first.
         return bool(self._settings.system_instruction)
 
+    def _requires_node_transition_context_aggregation(self) -> bool:
+        # A node transition replaces the current Gemini Live connection and
+        # seeds the new one from our local LLMContext. Wait for the upstream
+        # user aggregator to commit any final TranscriptionFrame before
+        # set_node() changes the prompt and starts that reconnect.
+        return True
+
+    async def cleanup(self) -> None:
+        """Cancel a delayed transition before tearing down the Live session."""
+        if self._transition_function_call_task:
+            await self.cancel_task(self._transition_function_call_task)
+            self._transition_function_call_task = None
+        await super().cleanup()
+
     async def _handle_changed_settings(self, changed: dict[str, Any]) -> set[str]:
         if "system_instruction" not in changed:
             return set()
+
+        # PipecatEngine updates system_instruction only from set_node(). The
+        # first set_node happens before a Live session exists; every later one
+        # is a node transition whose tool call has already been deferred until
+        # the current bot turn finishes.
         if not self._session:
             # First-time setting after deferred-connect.
             await self._connect()
-        elif self._bot_is_responding:
-            # Bot is mid-turn — drain the reconnect when it ends so we don't
-            # cut the bot off mid-utterance.
-            self._reconnect_pending = True
         else:
-            await self._reconnect()
+            await self._reconnect_for_node_transition()
         return {"system_instruction"}
 
     async def _run_or_defer_function_calls(
         self, function_calls_llm: list[FunctionCallFromLLM]
     ):
+        if not self._contains_node_transition(function_calls_llm):
+            await super()._run_or_defer_function_calls(function_calls_llm)
+            return
+
+        # Keep a provider tool-call batch together. Splitting a mixed batch here
+        # would discard Pipecat's shared function-call group and could trigger an
+        # LLM run before every result from the original batch has arrived.
         if self._bot_is_responding:
             # Latest batch wins; Gemini emits tool calls as one batch per
             # tool_call message, so this overwrite is intentional.
-            self._pending_function_calls = function_calls_llm
+            self._pending_node_transition_function_calls = function_calls_llm
             logger.debug(
-                f"{self}: deferring {len(function_calls_llm)} function call(s) "
+                f"{self}: deferring {len(function_calls_llm)} node-transition "
+                "function call(s) "
                 "until bot turn ends"
             )
             return
-        await super()._run_or_defer_function_calls(function_calls_llm)
+
+        self._schedule_node_transition_function_calls(function_calls_llm)
+
+    def _contains_node_transition(
+        self, function_calls_llm: list[FunctionCallFromLLM]
+    ) -> bool:
+        return any(self._is_node_transition(fc) for fc in function_calls_llm)
+
+    def _is_node_transition(self, function_call: FunctionCallFromLLM) -> bool:
+        return self._function_is_node_transition(function_call.function_name)
+
+    def _schedule_node_transition_function_calls(
+        self, function_calls_llm: list[FunctionCallFromLLM]
+    ) -> None:
+        """Run transition calls after late input transcription has settled."""
+        if (
+            self._transition_function_call_task
+            and not self._transition_function_call_task.done()
+        ):
+            logger.warning(
+                f"{self}: node-transition function call already pending; "
+                "ignoring duplicate batch"
+            )
+            return
+
+        async def _run_after_transcription_grace() -> None:
+            try:
+                await asyncio.sleep(self._NODE_TRANSITION_TRANSCRIPTION_GRACE_SECONDS)
+                await self._flush_pending_user_transcription()
+                await self.run_function_calls(function_calls_llm)
+            finally:
+                self._transition_function_call_task = None
+
+        self._transition_function_call_task = self.create_task(
+            _run_after_transcription_grace(),
+            name=f"{self}::node-transition-function-calls",
+        )
+
+    async def _flush_pending_user_transcription(self) -> None:
+        """Publish any punctuationless user transcript before a node handoff."""
+        if self._transcription_timeout_task:
+            if not self._transcription_timeout_task.done():
+                await self.cancel_task(self._transcription_timeout_task)
+            self._transcription_timeout_task = None
+
+        if not self._user_transcription_buffer:
+            return
+
+        text = self._user_transcription_buffer
+        self._user_transcription_buffer = ""
+        logger.debug(
+            f"{self}: flushing pending user transcription before node transition"
+        )
+        await self._push_user_transcription(text, result=None)
 
     # ------------------------------------------------------------------
     # State-transition side effects
@@ -99,22 +200,35 @@ class DograhGeminiLiveLLMService(GeminiLiveLLMService):
         was_responding = self._bot_is_responding
         await super()._set_bot_is_responding(responding)
         if was_responding and not responding:
-            await self._run_pending_function_calls()
-            if self._reconnect_pending:
-                self._reconnect_pending = False
-                await self._reconnect()
+            await self._run_pending_node_transition_function_calls()
 
-    async def _run_pending_function_calls(self):
-        """Run any function calls deferred during the bot's last turn."""
-        if not self._pending_function_calls:
+    async def _run_pending_node_transition_function_calls(self):
+        """Run any node-transition calls deferred during the bot's last turn."""
+        if not self._pending_node_transition_function_calls:
             return
-        fcs = self._pending_function_calls
-        self._pending_function_calls = []
+        fcs = self._pending_node_transition_function_calls
+        self._pending_node_transition_function_calls = []
         logger.debug(
-            f"{self}: executing {len(fcs)} deferred function call(s) "
+            f"{self}: executing {len(fcs)} deferred node-transition call(s) "
             "after bot turn ended"
         )
-        await self.run_function_calls(fcs)
+        self._schedule_node_transition_function_calls(fcs)
+
+    async def _reconnect_for_node_transition(self) -> None:
+        """Start a fresh connection and wait to seed the completed context.
+
+        Gemini can report ``resumable=False`` while generating or executing a
+        function call. A workflow transition happens at exactly that boundary,
+        so using the last (older) resumption handle can omit the triggering user
+        turn. Use the local LLMContext as the source of truth for this intentional
+        handoff instead.
+        """
+        self._awaiting_node_transition_context = True
+        self._node_transition_context_received = False
+        self._node_transition_context_seed_started = False
+        self._session_resumption_handle = None
+        await self._disconnect()
+        await self._connect(session_resumption_handle=None)
 
     # ------------------------------------------------------------------
     # Frame handling: mute, TTSSpeakFrame, BotStoppedSpeakingFrame flush
@@ -132,10 +246,15 @@ class DograhGeminiLiveLLMService(GeminiLiveLLMService):
         if isinstance(frame, TTSSpeakFrame):
             # Greeting trigger: the engine queues a TTSSpeakFrame to start the
             # bot's first turn after node setup. Gemini Live renders its own
-            # audio, so we don't pass the frame through — we re-enter
-            # _handle_context to kick off the initial response.
+            # audio, so we don't pass the frame through. For configured static
+            # text greetings, ask Gemini to say the exact greeting; otherwise
+            # re-enter _handle_context to kick off the normal initial response.
             if not self._handled_initial_context:
-                await self._handle_context(self._context)
+                greeting_text = frame.text.strip() if frame.text else ""
+                if greeting_text:
+                    await self._handle_initial_greeting(self._context, greeting_text)
+                else:
+                    await self._handle_context(self._context)
             else:
                 logger.warning(
                     f"{self}: TTSSpeakFrame after initial context already "
@@ -145,9 +264,9 @@ class DograhGeminiLiveLLMService(GeminiLiveLLMService):
         if isinstance(frame, BotStoppedSpeakingFrame):
             # Belt-and-suspenders: the main drain happens in
             # _set_bot_is_responding(False), but if Gemini delays turn_complete
-            # past the audible end of the turn, flushing here ensures pending
-            # function calls fire promptly.
-            await self._run_pending_function_calls()
+            # past the audible end of the turn, flushing here ensures a pending
+            # node transition fires promptly.
+            await self._run_pending_node_transition_function_calls()
             # Fall through to super for the actual push.
         await super().process_frame(frame, direction)
 
@@ -165,6 +284,11 @@ class DograhGeminiLiveLLMService(GeminiLiveLLMService):
     # ------------------------------------------------------------------
 
     async def _handle_context(self, context: LLMContext):
+        if self._awaiting_node_transition_context:
+            self._context = context
+            self._node_transition_context_received = True
+            await self._maybe_seed_node_transition_context()
+            return
         if not self._handled_initial_context:
             self._handled_initial_context = True
             self._context = context
@@ -173,6 +297,49 @@ class DograhGeminiLiveLLMService(GeminiLiveLLMService):
             self._context = context
             await self._process_completed_function_calls(send_new_results=True)
 
+    async def _handle_initial_greeting(self, context: LLMContext, greeting_text: str):
+        """Trigger the first Gemini turn with an exact static text greeting."""
+        if context is None:
+            logger.warning(
+                f"{self}: received initial greeting trigger before context was set"
+            )
+            return
+
+        self._handled_initial_context = True
+        self._context = context
+        await self._create_initial_greeting_response(greeting_text)
+
+    async def _create_initial_greeting_response(self, greeting_text: str):
+        """Ask Gemini Live to speak the configured greeting exactly once."""
+        if self._disconnecting:
+            return
+
+        if not self._session:
+            self._pending_initial_greeting_text = greeting_text
+            self._run_llm_when_session_ready = True
+            return
+
+        self._pending_initial_greeting_text = None
+        prompt = format_static_greeting_prompt(greeting_text)
+        turn = Content(role="user", parts=[Part(text=prompt)])
+
+        logger.debug("Creating Gemini Live initial response from static greeting")
+
+        await self.start_ttfb_metrics()
+
+        try:
+            await self._session.send_client_content(
+                turns=[turn],
+                turn_complete=True,
+            )
+            # Gemini 3.x also needs a realtime-input nudge to begin inference.
+            if self._is_gemini_3:
+                await self._session.send_realtime_input(text=" ")
+        except Exception as e:
+            await self._handle_send_error(e)
+
+        self._ready_for_realtime_input = True
+
     # ------------------------------------------------------------------
     # Session lifecycle: drop upstream's automatic reconnect-seed and
     # initial-context-seed paths. The TTSSpeakFrame trigger and the
@@ -186,15 +353,48 @@ class DograhGeminiLiveLLMService(GeminiLiveLLMService):
             f"In _handle_session_ready self._run_llm_when_session_ready: {self._run_llm_when_session_ready}"
         )
         self._session = session
+        if self._awaiting_node_transition_context:
+            # Do not accept realtime input until the function-call result frame
+            # has updated the shared context and that complete history is seeded.
+            self._ready_for_realtime_input = False
+            await self._maybe_seed_node_transition_context()
+            return
         self._ready_for_realtime_input = True
         if self._run_llm_when_session_ready:
             # Context arrived before session was ready — fulfil the queued
             # initial response now.
             self._run_llm_when_session_ready = False
-            await self._create_initial_response()
+            if self._pending_initial_greeting_text is not None:
+                await self._create_initial_greeting_response(
+                    self._pending_initial_greeting_text
+                )
+            else:
+                await self._create_initial_response()
         await self._drain_pending_tool_results()
         # Otherwise: no automatic seed. Reconnect after a session-resumption
         # update relies on the server-side restored state; reconnects without
         # a handle (e.g. node transitions before any handle was issued) are
         # followed by a function-call-result LLMContextFrame which feeds the
         # updated-context branch in _handle_context.
+
+    async def _maybe_seed_node_transition_context(self) -> None:
+        if (
+            not self._awaiting_node_transition_context
+            or not self._node_transition_context_received
+            or not self._session
+            or self._node_transition_context_seed_started
+        ):
+            return
+
+        self._node_transition_context_seed_started = True
+        try:
+            # The complete tool result is already present in the history being
+            # seeded, so mark it delivered locally instead of sending a provider
+            # tool response for a call that the fresh session never issued.
+            await self._process_completed_function_calls(send_new_results=False)
+            await self._create_initial_response()
+            self._awaiting_node_transition_context = False
+            self._node_transition_context_received = False
+            await self._drain_pending_tool_results()
+        finally:
+            self._node_transition_context_seed_started = False

api/services/pipecat/realtime/grok_realtime.py

@@ -11,8 +11,9 @@ Adds:
   flow kicks off the bot's first response.
 - **One-off LLMMessagesAppendFrame handling** for ephemeral realtime prompts
   like user-idle checks, without mutating Dograh's local ``LLMContext``.
-- **Function-call deferral** until the bot finishes speaking, to avoid racing
-  tool execution with the active audio turn.
+- **Workflow-control deferral** so node transitions, call termination, and
+  transfers wait for any current bot audio to finish while ordinary tools run
+  immediately.
 - **finalized=True on TranscriptionFrame** for parity with Dograh's other
   realtime providers.
 """
@@ -22,6 +23,7 @@ from typing import Any
 
 from loguru import logger
 
+from api.services.pipecat.realtime.static_greeting import format_static_greeting_prompt
 from pipecat.frames.frames import (
     BotStartedSpeakingFrame,
     BotStoppedSpeakingFrame,
@@ -49,7 +51,8 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
         self._user_is_muted: bool = False
         self._handled_initial_context: bool = False
         self._bot_is_speaking: bool = False
-        self._deferred_function_calls: list[FunctionCallFromLLM] = []
+        self._deferred_node_transition_function_calls: list[FunctionCallFromLLM] = []
+        self._pending_initial_greeting_text: str | None = None
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         if isinstance(frame, UserMuteStartedFrame):
@@ -62,7 +65,11 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
             return
         if isinstance(frame, TTSSpeakFrame):
             if not self._handled_initial_context:
-                await self._handle_context(self._context)
+                greeting_text = frame.text.strip() if frame.text else ""
+                if greeting_text:
+                    await self._handle_initial_greeting(self._context, greeting_text)
+                else:
+                    await self._handle_context(self._context)
             else:
                 logger.warning(
                     f"{self}: TTSSpeakFrame after initial context already "
@@ -76,7 +83,7 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
             self._bot_is_speaking = True
         elif isinstance(frame, BotStoppedSpeakingFrame):
             self._bot_is_speaking = False
-            await self._run_pending_function_calls()
+            await self._run_pending_node_transition_function_calls()
         await super().process_frame(frame, direction)
 
     async def _handle_messages_append(self, frame: LLMMessagesAppendFrame):
@@ -120,6 +127,67 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
             self._context = context
             await self._process_completed_function_calls(send_new_results=True)
 
+    async def _handle_initial_greeting(self, context: LLMContext, greeting_text: str):
+        if context is None:
+            logger.warning(
+                f"{self}: received initial greeting trigger before context was set"
+            )
+            return
+
+        self._handled_initial_context = True
+        self._context = context
+        await self._create_initial_greeting_response(greeting_text)
+
+    async def _create_initial_greeting_response(self, greeting_text: str):
+        if self._disconnecting:
+            return
+
+        if not self._api_session_ready:
+            self._pending_initial_greeting_text = greeting_text
+            self._run_llm_when_api_session_ready = True
+            return
+
+        self._pending_initial_greeting_text = None
+        await self._ensure_conversation_setup()
+        item = events.ConversationItem(
+            type="message",
+            role="user",
+            content=[
+                events.ItemContent(
+                    type="input_text",
+                    text=format_static_greeting_prompt(greeting_text),
+                )
+            ],
+        )
+        evt = events.ConversationItemCreateEvent(item=item)
+        self._messages_added_manually[evt.item.id] = True
+        await self.send_client_event(evt)
+        await self._send_manual_response_create()
+
+    async def _ensure_conversation_setup(self):
+        if not self._llm_needs_conversation_setup:
+            return
+
+        adapter = self.get_llm_adapter()
+        llm_invocation_params = adapter.get_llm_invocation_params(self._context)
+        for item in llm_invocation_params["messages"]:
+            evt = events.ConversationItemCreateEvent(item=item)
+            self._messages_added_manually[evt.item.id] = True
+            await self.send_client_event(evt)
+
+        await self._send_session_update()
+        self._llm_needs_conversation_setup = False
+
+    async def _handle_evt_session_updated(self, evt):
+        self._api_session_ready = True
+        if self._pending_initial_greeting_text is not None:
+            greeting_text = self._pending_initial_greeting_text
+            self._run_llm_when_api_session_ready = False
+            await self._create_initial_greeting_response(greeting_text)
+        elif self._run_llm_when_api_session_ready:
+            self._run_llm_when_api_session_ready = False
+            await self._create_response()
+
     async def _send_user_audio(self, frame):
         if self._user_is_muted:
             return
@@ -184,19 +252,19 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
             )
         )
 
-    async def _run_pending_function_calls(self):
-        if not self._deferred_function_calls:
+    async def _run_pending_node_transition_function_calls(self):
+        if not self._deferred_node_transition_function_calls:
             return
-        function_calls = self._deferred_function_calls
-        self._deferred_function_calls = []
+        function_calls = self._deferred_node_transition_function_calls
+        self._deferred_node_transition_function_calls = []
         logger.debug(
-            f"{self}: executing {len(function_calls)} deferred function call(s) "
-            "after bot turn ended"
+            f"{self}: executing {len(function_calls)} deferred workflow-control "
+            "call(s) after bot turn ended"
         )
         await self.run_function_calls(function_calls)
 
     async def _handle_evt_function_call_arguments_done(self, evt):
-        """Process or defer tool calls until the bot finishes speaking."""
+        """Run ordinary tools immediately and defer workflow-control calls."""
         try:
             args = json.loads(evt.arguments)
 
@@ -214,10 +282,11 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
                     )
                 ]
 
-                if self._bot_is_speaking:
-                    self._deferred_function_calls.extend(function_calls)
+                is_node_transition = self._function_is_node_transition(function_name)
+                if self._bot_is_speaking and is_node_transition:
+                    self._deferred_node_transition_function_calls.extend(function_calls)
                     logger.debug(
-                        f"{self}: deferring function call {function_name} "
+                        f"{self}: deferring workflow-control call {function_name} "
                         "until bot stops speaking"
                     )
                 else:

api/services/pipecat/realtime/openai_realtime.py

@@ -3,8 +3,8 @@
 Layers Dograh engine integration quirks onto upstream-pristine
 :class:`OpenAIRealtimeLLMService`. Substantially smaller than the Gemini
 subclass because OpenAI Realtime supports runtime ``session.update`` for
-both ``system_instruction`` and tools — no reconnect/defer-tool-call
-machinery needed.
+both ``system_instruction`` and tools, so node changes do not require a
+reconnect.
 
 Adds:
 
@@ -13,6 +13,9 @@ Adds:
   flow kicks off the bot's first response.
 - **One-off LLMMessagesAppendFrame handling** for ephemeral realtime prompts
   like user-idle checks, without mutating Dograh's local ``LLMContext``.
+- **Workflow-control deferral** so node transitions, call termination, and
+  transfers wait for any current bot audio to finish while ordinary tools run
+  immediately.
 - **finalized=True on TranscriptionFrame** because every OpenAI
   transcription via the ``completed`` event is final by construction.
 """
@@ -22,6 +25,7 @@ from typing import Any
 
 from loguru import logger
 
+from api.services.pipecat.realtime.static_greeting import format_static_greeting_prompt
 from pipecat.frames.frames import (
     BotStartedSpeakingFrame,
     BotStoppedSpeakingFrame,
@@ -52,10 +56,11 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
         # LLMContextFrame arrives, so upstream's "first arrival means
         # self._context is None" check no longer works.
         self._handled_initial_context: bool = False
-        # Track bot speech locally so tool calls can be deferred until the bot
-        # has finished speaking, matching Dograh's Gemini Live behavior.
+        # Track bot speech locally so workflow-control calls can wait until the
+        # bot has finished speaking without delaying ordinary tools.
         self._bot_is_speaking: bool = False
-        self._deferred_function_calls: list[FunctionCallFromLLM] = []
+        self._deferred_node_transition_function_calls: list[FunctionCallFromLLM] = []
+        self._pending_initial_greeting_text: str | None = None
 
     # ------------------------------------------------------------------
     # Frame handling: mute, TTSSpeakFrame as greeting trigger
@@ -73,11 +78,16 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
         if isinstance(frame, TTSSpeakFrame):
             # Greeting trigger: the engine queues a TTSSpeakFrame after node
             # setup. OpenAI Realtime renders its own audio, so we don't pass
-            # the frame to TTS. Route through _handle_context so the initial
-            # response and later tool-result turns share the same context
-            # lifecycle even when Dograh has already pre-populated self._context.
+            # the frame to TTS. For configured static text greetings, ask the
+            # model to say the exact greeting; otherwise route through
+            # _handle_context so the initial response and later tool-result
+            # turns share the same context lifecycle.
             if not self._handled_initial_context:
-                await self._handle_context(self._context)
+                greeting_text = frame.text.strip() if frame.text else ""
+                if greeting_text:
+                    await self._handle_initial_greeting(self._context, greeting_text)
+                else:
+                    await self._handle_context(self._context)
             else:
                 logger.warning(
                     f"{self}: TTSSpeakFrame after initial context already "
@@ -93,7 +103,7 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
             self._bot_is_speaking = True
         elif isinstance(frame, BotStoppedSpeakingFrame):
             self._bot_is_speaking = False
-            await self._run_pending_function_calls()
+            await self._run_pending_node_transition_function_calls()
         await super().process_frame(frame, direction)
 
     async def _handle_messages_append(self, frame: LLMMessagesAppendFrame):
@@ -137,6 +147,57 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
             self._context = context
             await self._process_completed_function_calls(send_new_results=True)
 
+    async def _handle_initial_greeting(self, context: LLMContext, greeting_text: str):
+        if context is None:
+            logger.warning(
+                f"{self}: received initial greeting trigger before context was set"
+            )
+            return
+
+        self._handled_initial_context = True
+        self._context = context
+        await self._create_initial_greeting_response(greeting_text)
+
+    async def _create_initial_greeting_response(self, greeting_text: str):
+        if self._disconnecting:
+            return
+
+        if not self._api_session_ready:
+            self._pending_initial_greeting_text = greeting_text
+            self._run_llm_when_api_session_ready = True
+            return
+
+        self._pending_initial_greeting_text = None
+        await self._ensure_conversation_setup()
+        await self._send_manual_response_create(
+            instructions=format_static_greeting_prompt(greeting_text),
+            tool_choice="none",
+        )
+
+    async def _ensure_conversation_setup(self):
+        if not self._llm_needs_conversation_setup:
+            return
+
+        adapter = self.get_llm_adapter()
+        llm_invocation_params = adapter.get_llm_invocation_params(self._context)
+        for item in llm_invocation_params["messages"]:
+            evt = events.ConversationItemCreateEvent(item=item)
+            self._messages_added_manually[evt.item.id] = True
+            await self.send_client_event(evt)
+
+        await self._send_session_update()
+        self._llm_needs_conversation_setup = False
+
+    async def _handle_evt_session_updated(self, evt):
+        self._api_session_ready = True
+        if self._pending_initial_greeting_text is not None:
+            greeting_text = self._pending_initial_greeting_text
+            self._run_llm_when_api_session_ready = False
+            await self._create_initial_greeting_response(greeting_text)
+        elif self._run_llm_when_api_session_ready:
+            self._run_llm_when_api_session_ready = False
+            await self._create_response()
+
     async def _send_user_audio(self, frame):
         if self._user_is_muted:
             return
@@ -190,7 +251,12 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
             return "\n".join(parts) if parts else None
         return None
 
-    async def _send_manual_response_create(self):
+    async def _send_manual_response_create(
+        self,
+        *,
+        instructions: str | None = None,
+        tool_choice: str | None = None,
+    ):
         """Trigger inference after manually appending conversation items."""
         await self.push_frame(LLMFullResponseStartFrame())
         await self.start_processing_metrics()
@@ -198,24 +264,26 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
         await self.send_client_event(
             events.ResponseCreateEvent(
                 response=events.ResponseProperties(
-                    output_modalities=self._get_enabled_modalities()
+                    output_modalities=self._get_enabled_modalities(),
+                    instructions=instructions,
+                    tool_choice=tool_choice,
                 )
             )
         )
 
-    async def _run_pending_function_calls(self):
-        if not self._deferred_function_calls:
+    async def _run_pending_node_transition_function_calls(self):
+        if not self._deferred_node_transition_function_calls:
             return
-        function_calls = self._deferred_function_calls
-        self._deferred_function_calls = []
+        function_calls = self._deferred_node_transition_function_calls
+        self._deferred_node_transition_function_calls = []
         logger.debug(
-            f"{self}: executing {len(function_calls)} deferred function call(s) "
-            "after bot turn ended"
+            f"{self}: executing {len(function_calls)} deferred workflow-control "
+            "call(s) after bot turn ended"
         )
         await self.run_function_calls(function_calls)
 
     async def _handle_evt_function_call_arguments_done(self, evt):
-        """Process or defer tool calls until the bot finishes speaking."""
+        """Run ordinary tools immediately and defer workflow-control calls."""
         try:
             args = json.loads(evt.arguments)
 
@@ -232,10 +300,14 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
                     )
                 ]
 
-                if self._bot_is_speaking:
-                    self._deferred_function_calls.extend(function_calls)
+                is_node_transition = self._function_is_node_transition(
+                    function_call_item.name
+                )
+                if self._bot_is_speaking and is_node_transition:
+                    self._deferred_node_transition_function_calls.extend(function_calls)
                     logger.debug(
-                        f"{self}: deferring function call {function_call_item.name} "
+                        f"{self}: deferring workflow-control call "
+                        f"{function_call_item.name} "
                         "until bot stops speaking"
                     )
                 else:

api/services/pipecat/realtime/static_greeting.py

@@ -0,0 +1,8 @@
+def format_static_greeting_prompt(greeting_text: str) -> str:
+    return (
+        "The phone call has just connected. Greet the caller now: "
+        "say the following opening line out loud, exactly as written, "
+        "in a natural spoken voice, and then stop and wait for the "
+        "caller to respond. Do not add anything before or after it.\n\n"
+        f'"{greeting_text}"'
+    )

api/services/pipecat/realtime/ultravox_realtime.py

@@ -1,19 +1,17 @@
 """Dograh subclass of pipecat's Ultravox realtime LLM service.
 
-Ultravox is audio-native and realtime, but prompt and tool configuration is
-bound to call creation. Dograh therefore cannot lean on in-session updates or
-Gemini-style session resumption handles. This wrapper adapts Ultravox to the
-Dograh engine contract by:
+Ultravox is audio-native and realtime. Its native call stages allow a client
+tool result to atomically change the system prompt and tools while preserving
+the call's server-side conversation history. This wrapper adapts that model to
+the Dograh engine contract by:
 
 - deferring the first call creation until the engine queues the initial node
   opening via ``TTSSpeakFrame`` or ``LLMContextFrame``
-- marking the call for recreation when ``system_instruction`` changes across
-  node transitions, then rebuilding it on the follow-up ``LLMContextFrame``
-  so the transition tool result is present in ``initialMessages``
-- reconstructing Ultravox ``initialMessages`` from Dograh context when the
-  call must be recreated after a node transition
-- appending a transient resumptive user nudge to recreated ``initialMessages``
-  after tool-result transitions, without mutating Dograh's stored context
+- returning node-transition tool results with ``responseType="new-stage"`` so
+  the existing call keeps its complete audio-native history
+- updating the next stage's system prompt and selected tools without a
+  disconnect/reconnect cycle
+- deferring workflow-control tools until any active Ultravox response ends
 - handling Dograh-only frames such as user mute and idle append prompts
 - tagging user transcripts with ``finalized=True`` for downstream parity
 """
@@ -34,12 +32,7 @@ from pipecat.frames.frames import (
     UserMuteStartedFrame,
     UserMuteStoppedFrame,
 )
-from pipecat.processors.aggregators import async_tool_messages
-from pipecat.processors.aggregators.llm_context import (
-    LLMContext,
-    LLMSpecificMessage,
-    is_given,
-)
+from pipecat.processors.aggregators.llm_context import LLMContext, is_given
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.llm_service import LLMService
 from pipecat.services.settings import _NotGiven, assert_given
@@ -58,10 +51,6 @@ class DograhUltravoxOneShotInputParams(OneShotInputParams):
 
 
 _ULTRAVOX_MAX_TOOL_TIMEOUT_SECS = 40.0
-_RESUMPTION_USER_MESSAGE = (
-    "IMPORTANT: We are resuming an existing conversation. You are given previous turns ONLY for your reference. "
-    "Do not use that to frame your response. Follow your ORIGINAL INSTRUCTIONS ONLY."
-)
 
 
 class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
@@ -72,12 +61,19 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
         self._context: LLMContext | None = None
         self._selected_tools = None
         self._user_is_muted: bool = False
-        self._call_system_instruction: str | None = None
-        self._reconnect_required: bool = False
         self._call_started: bool = False
-        self._has_connected_once: bool = False
-        self._pending_reconnect_system_instruction: str | None = None
-        self._pending_initial_messages: list[dict[str, Any]] | None = None
+        self._stage_update_required: bool = False
+        # Ultravox applies a stage update on the matching client tool result,
+        # so retain the provider invocation ID until that result reaches us via
+        # the context aggregator. Unlike Gemini, this ID is part of the wire
+        # protocol needed to update the existing call without reconnecting.
+        self._pending_node_transition_tool_call_ids: set[str] = set()
+        # A stage result can replace the active prompt and tools immediately.
+        # Hold transition invocations separately so ordinary tools can still
+        # run during speech while workflow control waits for response end.
+        self._deferred_node_transition_tool_invocations: list[
+            tuple[str, str, dict[str, Any]]
+        ] = []
         self._pending_user_text_messages: list[str] = []
 
     async def start(self, frame):
@@ -96,9 +92,7 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
         if isinstance(frame, TTSSpeakFrame):
             if not self._socket:
                 await self._connect_call(
-                    system_instruction=self._current_system_instruction(),
                     greeting_text=frame.text,
-                    initial_messages=None,
                     agent_speaks_first=True,
                 )
             else:
@@ -116,18 +110,15 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
         changed = await super(UltravoxRealtimeLLMService, self)._update_settings(delta)
         if "output_medium" in changed:
             await self._update_output_medium(assert_given(self._settings.output_medium))
-        if "system_instruction" in changed and self._has_connected_once:
-            # Mirror Gemini's "settings change means reconnect" intent, but
-            # defer the actual new-call creation until the subsequent
-            # LLMContextFrame arrives with the transition tool result. Ultravox
-            # cannot accept that historical tool result over a formal
-            # post-connect tool-response channel the way Gemini can.
-            self._reconnect_required = True
+        if "system_instruction" in changed and self._socket:
+            # The updated instruction is included in the native new-stage
+            # response when the transition tool result reaches _handle_context.
+            self._stage_update_required = True
         handled = {"output_medium", "system_instruction"}
         self._warn_unhandled_updated_settings(changed.keys() - handled)
         return changed
 
-    async def _disconnect(self, preserve_completed_tool_calls: bool = True):
+    async def _disconnect(self):
         self._disconnecting = True
         await self.stop_all_metrics()
         if self._socket:
@@ -136,10 +127,11 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
         if self._receive_task:
             await self.cancel_task(self._receive_task, timeout=1.0)
             self._receive_task = None
-        if not preserve_completed_tool_calls:
-            self._completed_tool_calls = set()
+        self._completed_tool_calls = set()
         self._call_started = False
         self._started_placeholder_sent = set()
+        self._pending_node_transition_tool_call_ids = set()
+        self._deferred_node_transition_tool_invocations = []
         self._disconnecting = False
 
     async def _send_user_audio(self, frame):
@@ -149,39 +141,20 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
 
     async def _handle_context(self, context: LLMContext):
         self._context = context
-        system_instruction = self._current_system_instruction()
 
-        if self._socket and not self._reconnect_required:
-            await super()._handle_context(context)
+        if not self._socket:
+            await self._connect_call(
+                greeting_text=None,
+                agent_speaks_first=True,
+            )
             return
 
-        initial_messages, history_tool_call_ids = self._build_initial_messages(context)
-        if history_tool_call_ids:
-            self._completed_tool_calls.update(history_tool_call_ids)
-
-        if self._bot_responding:
-            self._pending_reconnect_system_instruction = system_instruction
-            self._pending_initial_messages = initial_messages
-            return
-
-        await self._reconnect_with_context(
-            system_instruction=system_instruction,
-            initial_messages=initial_messages,
-        )
-
-    async def _handle_response_end(self):
-        await super()._handle_response_end()
-        if self._pending_reconnect_system_instruction is None:
-            return
-
-        system_instruction = self._pending_reconnect_system_instruction
-        initial_messages = self._pending_initial_messages
-        self._pending_reconnect_system_instruction = None
-        self._pending_initial_messages = None
-        await self._reconnect_with_context(
-            system_instruction=system_instruction,
-            initial_messages=initial_messages,
-        )
+        current_tools = self._current_tools_schema(context)
+        if self._pending_node_transition_tool_call_ids and self._tools_changed(
+            current_tools
+        ):
+            self._stage_update_required = True
+        await super()._handle_context(context)
 
     async def _handle_messages_append(self, frame: LLMMessagesAppendFrame):
         texts = [
@@ -199,9 +172,7 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
         if not self._socket:
             self._pending_user_text_messages.extend(texts)
             await self._connect_call(
-                system_instruction=self._current_system_instruction(),
                 greeting_text=None,
-                initial_messages=None,
                 agent_speaks_first=False,
             )
             return
@@ -229,17 +200,93 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
             finalized=True,
         )
 
+    def _requires_node_transition_context_aggregation(self) -> bool:
+        """Commit any received final user transcript before changing stages.
+
+        Ultravox preserves its own audio-native history across a stage change,
+        but Dograh's local context still needs the final transcript before the
+        transition handler updates the workflow node.
+        """
+        return True
+
+    async def _handle_tool_invocation(
+        self, tool_name: str, invocation_id: str, parameters: dict[str, Any]
+    ):
+        if self._function_is_node_transition(tool_name):
+            self._pending_node_transition_tool_call_ids.add(invocation_id)
+            if self._bot_responding:
+                self._deferred_node_transition_tool_invocations.append(
+                    (tool_name, invocation_id, parameters)
+                )
+                logger.debug(
+                    f"{self}: deferring workflow-control call {tool_name} "
+                    "until bot turn ends"
+                )
+                return
+        await super()._handle_tool_invocation(tool_name, invocation_id, parameters)
+
+    async def _handle_response_end(self):
+        """Close the current response before applying queued workflow control."""
+        await super()._handle_response_end()
+        await self._run_deferred_node_transition_tool_invocations()
+
+    async def _run_deferred_node_transition_tool_invocations(self):
+        if not self._deferred_node_transition_tool_invocations:
+            return
+
+        invocations = self._deferred_node_transition_tool_invocations
+        self._deferred_node_transition_tool_invocations = []
+        logger.debug(
+            f"{self}: executing {len(invocations)} deferred workflow-control "
+            "call(s) after bot turn ended"
+        )
+        for tool_name, invocation_id, parameters in invocations:
+            await super()._handle_tool_invocation(tool_name, invocation_id, parameters)
+
+    async def _send_tool_result(self, tool_call_id: str, result: str):
+        is_node_transition = tool_call_id in self._pending_node_transition_tool_call_ids
+        try:
+            if is_node_transition and self._stage_update_required:
+                await self._send_node_transition_stage_result(tool_call_id, result)
+            else:
+                await super()._send_tool_result(tool_call_id, result)
+        finally:
+            if is_node_transition:
+                self._pending_node_transition_tool_call_ids.discard(tool_call_id)
+
+    async def _send_node_transition_stage_result(self, tool_call_id: str, result: str):
+        """Apply node settings using Ultravox's native call-stage protocol."""
+        next_tools = self._current_tools_schema(self._context)
+        stage = {
+            "systemPrompt": self._current_system_instruction(),
+            "selectedTools": self._selected_tools_payload(next_tools),
+            # Keep the workflow handler's result as the tool-result message in
+            # the inherited conversation history for the next generation.
+            "toolResultText": result,
+        }
+        logger.debug(
+            f"{self}: updating Ultravox call stage for tool_call_id={tool_call_id} "
+            f"with {len(stage['selectedTools'])} selected tool(s)"
+        )
+        await self._send(
+            {
+                "type": "client_tool_result",
+                "invocationId": tool_call_id,
+                "result": json.dumps(stage, ensure_ascii=True, default=str),
+                "responseType": "new-stage",
+            }
+        )
+        self._selected_tools = next_tools
+        self._stage_update_required = False
+
     async def _connect_call(
         self,
         *,
-        system_instruction: str | None,
         greeting_text: str | None,
-        initial_messages: list[dict[str, Any]] | None,
         agent_speaks_first: bool,
     ):
         params = self._build_one_shot_params(
             greeting_text=greeting_text,
-            initial_messages=initial_messages,
             agent_speaks_first=agent_speaks_first,
         )
         self._params = params
@@ -265,9 +312,7 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
             logger.info(f"Joining Ultravox Realtime call via URL: {join_url}")
             self._socket = await websocket_client.connect(join_url)
             self._receive_task = self.create_task(self._receive_messages())
-            self._call_system_instruction = system_instruction
             self._call_started = False
-            self._has_connected_once = True
         except Exception as e:
             logger.error(
                 f"{self}: Ultravox call creation/join failed "
@@ -365,40 +410,17 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
         for pending_text in pending_texts:
             await self._send_user_text(pending_text)
 
-    async def _reconnect_with_context(
-        self,
-        *,
-        system_instruction: str | None,
-        initial_messages: list[dict[str, Any]] | None,
-    ):
-        call_initial_messages = self._initial_messages_for_call(initial_messages)
-        logger.debug(
-            f"{self}: reconnecting Ultravox call with initialMessages="
-            f"{json.dumps(call_initial_messages, ensure_ascii=True, default=str)}"
-        )
-        if self._socket:
-            await self._disconnect(preserve_completed_tool_calls=True)
-
-        await self._connect_call(
-            system_instruction=system_instruction,
-            greeting_text=None,
-            initial_messages=initial_messages,
-            agent_speaks_first=self._should_agent_speak_first(initial_messages),
-        )
-        self._reconnect_required = False
-
     def _build_one_shot_params(
         self,
         *,
         greeting_text: str | None,
-        initial_messages: list[dict[str, Any]] | None,
         agent_speaks_first: bool,
     ) -> DograhUltravoxOneShotInputParams:
         current_params = self._params
         extra = {
             key: value
             for key, value in current_params.extra.items()
-            if key not in {"firstSpeakerSettings", "initialMessages"}
+            if key != "firstSpeakerSettings"
         }
 
         if greeting_text is not None:
@@ -407,10 +429,6 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
             extra["firstSpeakerSettings"] = {"agent": {}}
         else:
             extra["firstSpeakerSettings"] = {"user": {}}
-        call_initial_messages = self._initial_messages_for_call(initial_messages)
-        if call_initial_messages:
-            extra["initialMessages"] = call_initial_messages
-
         output_medium = self._settings.output_medium
         if isinstance(output_medium, _NotGiven):
             output_medium = current_params.output_medium
@@ -432,6 +450,14 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
             return None
         return context.tools
 
+    def _selected_tools_payload(self, tools: Any) -> list[dict[str, Any]]:
+        return self._to_selected_tools(tools) if tools else []
+
+    def _tools_changed(self, tools: Any) -> bool:
+        return self._selected_tools_payload(tools) != self._selected_tools_payload(
+            self._selected_tools
+        )
+
     def _to_selected_tools(self, tool: Any) -> list[dict[str, Any]]:
         selected_tools = super()._to_selected_tools(tool)
         for selected_tool in selected_tools:
@@ -462,156 +488,6 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
         timeout_secs = min(float(item.timeout_secs), _ULTRAVOX_MAX_TOOL_TIMEOUT_SECS)
         return f"{timeout_secs:g}s"
 
-    def _initial_messages_for_call(
-        self, initial_messages: list[dict[str, Any]] | None
-    ) -> list[dict[str, Any]] | None:
-        if not initial_messages:
-            return None
-        if not self._should_add_resumption_user_message(initial_messages):
-            return initial_messages
-
-        return [
-            *initial_messages,
-            {
-                "role": "MESSAGE_ROLE_USER",
-                "text": _RESUMPTION_USER_MESSAGE,
-            },
-        ]
-
-    def _build_initial_messages(
-        self, context: LLMContext
-    ) -> tuple[list[dict[str, Any]] | None, set[str]]:
-        initial_messages: list[dict[str, Any]] = []
-        tool_call_id_to_name: dict[str, str] = {}
-        completed_tool_call_ids: set[str] = set()
-
-        for message in context.get_messages():
-            if isinstance(message, LLMSpecificMessage):
-                continue
-
-            async_payload = async_tool_messages.parse_message(message)
-            if async_payload is not None:
-                if async_payload.kind == "intermediate":
-                    logger.error(
-                        f"{self}: Ultravox does not support streamed async tool results; "
-                        f"dropping intermediate result from initialMessages for "
-                        f"tool_call_id={async_payload.tool_call_id}."
-                    )
-                    continue
-                if async_payload.kind == "final":
-                    initial_message = self._build_ultravox_message(
-                        role="MESSAGE_ROLE_TOOL_RESULT",
-                        text=async_payload.result or "",
-                        invocation_id=async_payload.tool_call_id,
-                        tool_name=tool_call_id_to_name.get(async_payload.tool_call_id),
-                    )
-                    if initial_message is not None:
-                        initial_messages.append(initial_message)
-                    completed_tool_call_ids.add(async_payload.tool_call_id)
-                continue
-
-            role = message.get("role")
-            if role == "user":
-                initial_message = self._build_ultravox_message(
-                    role="MESSAGE_ROLE_USER",
-                    text=self._extract_text_content(message.get("content")),
-                )
-                if initial_message is not None:
-                    initial_messages.append(initial_message)
-            elif role == "assistant":
-                text = self._extract_text_content(message.get("content"))
-                initial_message = self._build_ultravox_message(
-                    role="MESSAGE_ROLE_AGENT",
-                    text=text,
-                )
-                if initial_message is not None:
-                    initial_messages.append(initial_message)
-
-                tool_calls = message.get("tool_calls")
-                if isinstance(tool_calls, list):
-                    for tool_call in tool_calls:
-                        if not isinstance(tool_call, dict):
-                            continue
-                        tool_id = tool_call.get("id")
-                        function = tool_call.get("function")
-                        tool_name = (
-                            function.get("name") if isinstance(function, dict) else None
-                        )
-                        if isinstance(tool_id, str) and isinstance(tool_name, str):
-                            tool_call_id_to_name[tool_id] = tool_name
-                            initial_message = self._build_ultravox_message(
-                                role="MESSAGE_ROLE_TOOL_CALL",
-                                text="",
-                                invocation_id=tool_id,
-                                tool_name=tool_name,
-                            )
-                            if initial_message is not None:
-                                initial_messages.append(initial_message)
-            elif (
-                role == "tool"
-                and message.get("content") != "IN_PROGRESS"
-                and message.get("content") != "CANCELLED"
-            ):
-                tool_call_id = message.get("tool_call_id")
-                initial_message = self._build_ultravox_message(
-                    role="MESSAGE_ROLE_TOOL_RESULT",
-                    text=self._stringify_tool_result(message.get("content")),
-                    invocation_id=tool_call_id
-                    if isinstance(tool_call_id, str)
-                    else None,
-                    tool_name=(
-                        tool_call_id_to_name.get(tool_call_id)
-                        if isinstance(tool_call_id, str)
-                        else None
-                    ),
-                )
-                if initial_message is not None:
-                    initial_messages.append(initial_message)
-                if isinstance(tool_call_id, str):
-                    completed_tool_call_ids.add(tool_call_id)
-
-        return (initial_messages or None), completed_tool_call_ids
-
-    @staticmethod
-    def _build_ultravox_message(
-        *,
-        role: str,
-        text: str | None,
-        invocation_id: str | None = None,
-        tool_name: str | None = None,
-    ) -> dict[str, Any] | None:
-        if text is None:
-            return None
-
-        message: dict[str, Any] = {
-            "role": role,
-            "text": text,
-        }
-        if invocation_id is not None:
-            message["invocationId"] = invocation_id
-        if tool_name is not None:
-            message["toolName"] = tool_name
-        return message
-
-    @staticmethod
-    def _should_agent_speak_first(
-        initial_messages: list[dict[str, Any]] | None,
-    ) -> bool:
-        if not initial_messages:
-            return True
-        return initial_messages[-1].get("role") in {
-            "MESSAGE_ROLE_USER",
-            "MESSAGE_ROLE_TOOL_RESULT",
-        }
-
-    @staticmethod
-    def _should_add_resumption_user_message(
-        initial_messages: list[dict[str, Any]] | None,
-    ) -> bool:
-        if not initial_messages:
-            return False
-        return initial_messages[-1].get("role") == "MESSAGE_ROLE_TOOL_RESULT"
-
     @staticmethod
     def _is_benign_websocket_close(exc: ConnectionClosed) -> bool:
         return any(
@@ -636,18 +512,3 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
                 parts.append(text)
             return "\n".join(parts) if parts else None
         return None
-
-    @staticmethod
-    def _stringify_tool_result(content: Any) -> str:
-        if isinstance(content, str):
-            return content
-        if isinstance(content, list):
-            parts: list[str] = []
-            for part in content:
-                if isinstance(part, dict):
-                    text = part.get("text")
-                    if isinstance(text, str):
-                        parts.append(text)
-            if parts:
-                return "".join(parts)
-        return json.dumps(content, ensure_ascii=True, default=str)

api/services/pipecat/realtime_feedback_events.py

@@ -30,6 +30,7 @@ def build_user_transcription_event(
     text: str,
     final: bool,
     timestamp: str | None = None,
+    end_timestamp: str | None = None,
     user_id: str | None = None,
 ) -> dict[str, Any]:
     payload: dict[str, Any] = {
@@ -38,6 +39,8 @@ def build_user_transcription_event(
     }
     if timestamp is not None:
         payload["timestamp"] = timestamp
+    if end_timestamp is not None:
+        payload["end_timestamp"] = end_timestamp
     if user_id is not None:
         payload["user_id"] = user_id
     return {
@@ -50,10 +53,13 @@ def build_bot_text_event(
     *,
     text: str,
     timestamp: str | None = None,
+    end_timestamp: str | None = None,
 ) -> dict[str, Any]:
     payload: dict[str, Any] = {"text": text}
     if timestamp is not None:
         payload["timestamp"] = timestamp
+    if end_timestamp is not None:
+        payload["end_timestamp"] = end_timestamp
     return {
         "type": RealtimeFeedbackType.BOT_TEXT.value,
         "payload": payload,
@@ -160,4 +166,4 @@ def stamp_realtime_feedback_event(
 
 def realtime_feedback_event_sort_key(event: dict[str, Any]) -> str:
     payload_timestamp = (event.get("payload") or {}).get("timestamp")
-    return payload_timestamp or event.get("timestamp") or ""
+    return event.get("timestamp") or payload_timestamp or ""

api/services/pipecat/realtime_feedback_observer.py

@@ -36,6 +36,9 @@ from api.services.pipecat.realtime_feedback_events import (
 
 if TYPE_CHECKING:
     from api.services.pipecat.in_memory_buffers import InMemoryLogsBuffer
+    from api.services.pipecat.transcript_log_coordinator import (
+        TranscriptLogCoordinator,
+    )
 
 from pipecat.frames.frames import (
     BotStartedSpeakingFrame,
@@ -72,7 +75,7 @@ class RealtimeFeedbackObserver(BaseObserver):
     - TTFB metrics (LLM generation time only)
 
     Logs buffer persistence (only final data for post-call analysis):
-    - Complete user transcripts per turn (via on_user_turn_stopped)
+    - Complete user transcripts per turn (via on_user_turn_message_added)
     - Complete assistant transcripts per turn (via on_assistant_turn_stopped)
     - Function calls and TTFB metrics
 
@@ -294,40 +297,36 @@ class RealtimeFeedbackObserver(BaseObserver):
 
 
 def register_turn_log_handlers(
-    logs_buffer: "InMemoryLogsBuffer",
+    transcript_coordinator: "TranscriptLogCoordinator",
     user_aggregator,
     assistant_aggregator,
 ):
     """Register event handlers on aggregators to persist final turn transcripts.
 
-    Hooks into on_user_turn_stopped and on_assistant_turn_stopped to store
-    complete turn text in the logs buffer. Works for both WebRTC and telephony
-    calls — independent of WebSocket availability.
+    Hooks into on_user_turn_message_added and on_assistant_turn_stopped to store
+    complete turn text through the turn-aware coordinator. Works for both
+    WebRTC and telephony calls — independent of WebSocket availability.
     """
 
-    @user_aggregator.event_handler("on_user_turn_stopped")
-    async def on_user_turn_stopped(aggregator, strategy, message):
-        logs_buffer.increment_turn()
+    @user_aggregator.event_handler("on_user_turn_message_added")
+    async def on_user_turn_message_added(aggregator, message):
         try:
-            await logs_buffer.append(
-                build_user_transcription_event(
-                    text=message.content,
-                    final=True,
-                    timestamp=message.timestamp,
-                )
+            await transcript_coordinator.record_user_transcript(
+                text=message.content,
+                timestamp=message.timestamp,
+                end_timestamp=getattr(message, "end_timestamp", None),
             )
         except Exception as e:
-            logger.error(f"Failed to append user turn to logs buffer: {e}")
+            logger.error(f"Failed to coordinate user turn transcript: {e}")
 
     @assistant_aggregator.event_handler("on_assistant_turn_stopped")
     async def on_assistant_turn_stopped(aggregator, message):
         if message.content:
             try:
-                await logs_buffer.append(
-                    build_bot_text_event(
-                        text=message.content,
-                        timestamp=message.timestamp,
-                    )
+                await transcript_coordinator.record_assistant_transcript(
+                    text=message.content,
+                    timestamp=message.timestamp,
+                    end_timestamp=getattr(message, "end_timestamp", None),
                 )
             except Exception as e:
-                logger.error(f"Failed to append assistant turn to logs buffer: {e}")
+                logger.error(f"Failed to coordinate assistant turn transcript: {e}")

api/services/pipecat/run_pipeline.py

@@ -6,12 +6,26 @@ from loguru import logger
 
 from api.db import db_client
 from api.enums import WorkflowRunMode
-from api.services.configuration.options import DEEPGRAM_FLUX_MODELS
+from api.schemas.workflow_configurations import (
+    DEFAULT_MAX_CALL_DURATION_SECONDS,
+    DEFAULT_MAX_USER_IDLE_TIMEOUT_SECONDS,
+    DEFAULT_PROVISIONAL_VAD_PAUSE_SECS,
+    DEFAULT_SMART_TURN_STOP_SECS,
+    DEFAULT_TURN_START_MIN_WORDS,
+    DEFAULT_TURN_START_STRATEGY,
+)
+from api.services.call_concurrency import call_concurrency
 from api.services.configuration.registry import ServiceProviders
 from api.services.integrations import (
     IntegrationRuntimeContext,
     create_runtime_sessions,
 )
+from api.services.pipecat.active_calls import (
+    register_active_call as register_worker_active_call,
+)
+from api.services.pipecat.active_calls import (
+    unregister_active_call as unregister_worker_active_call,
+)
 from api.services.pipecat.audio_config import AudioConfig, create_audio_config
 from api.services.pipecat.event_handlers import (
     register_audio_data_handler,
@@ -47,10 +61,12 @@ from api.services.pipecat.service_factory import (
     create_realtime_llm_service,
     create_stt_service,
     create_tts_service,
+    stt_uses_external_turns,
 )
 from api.services.pipecat.tracing_config import (
     ensure_tracing,
 )
+from api.services.pipecat.transcript_log_coordinator import TranscriptLogCoordinator
 from api.services.pipecat.transport_setup import create_webrtc_transport
 from api.services.pipecat.worker_runner import run_pipeline_worker
 from api.services.pipecat.ws_sender_registry import get_ws_sender
@@ -76,7 +92,8 @@ from pipecat.turns.user_mute import (
 )
 from pipecat.turns.user_start import (
     ExternalUserTurnStartStrategy,
-    TranscriptionUserTurnStartStrategy,
+    MinWordsUserTurnStartStrategy,
+    ProvisionalVADUserTurnStartStrategy,
 )
 from pipecat.turns.user_start.vad_user_turn_start_strategy import (
     VADUserTurnStartStrategy,
@@ -93,52 +110,143 @@ from pipecat.utils.run_context import set_current_org_id, set_current_run_id
 # Setup tracing if enabled
 ensure_tracing()
 
+DEFAULT_USER_TURN_STOP_TIMEOUT = 5.0
+EXTERNAL_TURN_USER_STOP_TIMEOUT = 30.0
+
+
+def _resolve_user_turn_stop_timeout(
+    run_configs: dict, *, uses_external_turns: bool
+) -> float:
+    if "user_turn_stop_timeout" in run_configs:
+        return float(run_configs["user_turn_stop_timeout"])
+    if uses_external_turns:
+        return EXTERNAL_TURN_USER_STOP_TIMEOUT
+    return DEFAULT_USER_TURN_STOP_TIMEOUT
+
+
+def _resolve_turn_start_min_words(run_configs: dict) -> int:
+    return max(
+        1,
+        int(run_configs.get("turn_start_min_words", DEFAULT_TURN_START_MIN_WORDS)),
+    )
+
+
+def _resolve_provisional_vad_pause_secs(run_configs: dict) -> float:
+    return max(
+        0.1,
+        float(
+            run_configs.get(
+                "provisional_vad_pause_secs", DEFAULT_PROVISIONAL_VAD_PAUSE_SECS
+            )
+        ),
+    )
+
+
+def _create_non_realtime_user_turn_start_strategies(
+    run_configs: dict, *, uses_external_turns: bool
+):
+    """Return user turn start strategies for non-realtime pipelines."""
+
+    turn_start_strategy = run_configs.get(
+        "turn_start_strategy", DEFAULT_TURN_START_STRATEGY
+    )
+
+    if turn_start_strategy == "min_words":
+        return [
+            MinWordsUserTurnStartStrategy(
+                min_words=_resolve_turn_start_min_words(run_configs)
+            )
+        ]
+
+    if turn_start_strategy == "provisional_vad":
+        return [
+            ProvisionalVADUserTurnStartStrategy(
+                pause_secs=_resolve_provisional_vad_pause_secs(run_configs)
+            )
+        ]
+
+    if uses_external_turns:
+        # The STT emits its own turn boundaries and owns interruptions. Local
+        # VAD is deliberately kept out of the default start strategies: it would
+        # win the race on raw voice activity and start the turn before the STT
+        # confirms a real turn.
+        return [ExternalUserTurnStartStrategy(enable_interruptions=True)]
+
+    return [VADUserTurnStartStrategy()]
+
+
+def _create_non_realtime_user_turn_stop_strategies(
+    run_configs: dict, *, uses_external_turns: bool
+):
+    """Return user turn stop strategies for non-realtime pipelines."""
+
+    if uses_external_turns:
+        return [ExternalUserTurnStopStrategy()]
+
+    if run_configs.get("turn_stop_strategy") == "turn_analyzer":
+        smart_turn_params = SmartTurnParams(
+            stop_secs=run_configs.get(
+                "smart_turn_stop_secs", DEFAULT_SMART_TURN_STOP_SECS
+            )
+        )
+        return [
+            TurnAnalyzerUserTurnStopStrategy(
+                turn_analyzer=LocalSmartTurnAnalyzerV3(params=smart_turn_params)
+            )
+        ]
+
+    return [SpeechTimeoutUserTurnStopStrategy()]
+
 
 def _create_realtime_user_turn_config(provider: str):
     """Return user turn strategies and optional local VAD for realtime providers."""
+
+    def external_provider_turn_config():
+        return (
+            UserTurnStrategies(
+                start=[ExternalUserTurnStartStrategy()],
+                stop=[ExternalUserTurnStopStrategy(wait_for_transcript=False)],
+            ),
+            None,
+        )
+
+    def local_vad_turn_config(*, enable_interruptions: bool):
+        return (
+            UserTurnStrategies(
+                start=[
+                    VADUserTurnStartStrategy(enable_interruptions=enable_interruptions)
+                ],
+                stop=[SpeechTimeoutUserTurnStopStrategy(wait_for_transcript=False)],
+            ),
+            SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+        )
+
     if provider in {
         ServiceProviders.GOOGLE_REALTIME.value,
         ServiceProviders.GOOGLE_VERTEX_REALTIME.value,
     }:
         # Let Gemini Live own barge-in via its server-side VAD, but keep local
         # Silero VAD for early user-turn start and speaking-state tracking.
-        return (
-            UserTurnStrategies(
-                start=[VADUserTurnStartStrategy(enable_interruptions=False)],
-                stop=[SpeechTimeoutUserTurnStopStrategy()],
-            ),
-            SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
-        )
+        return local_vad_turn_config(enable_interruptions=False)
 
-    if provider == ServiceProviders.OPENAI_REALTIME.value:
-        # OpenAI Realtime already emits speaking-state frames and interruption
-        # events from the provider, so the aggregator should follow those
-        # external signals rather than run its own local VAD.
-        return (
-            UserTurnStrategies(
-                start=[ExternalUserTurnStartStrategy()],
-                stop=[ExternalUserTurnStopStrategy()],
-            ),
-            None,
-        )
+    if provider in {
+        ServiceProviders.OPENAI_REALTIME.value,
+        ServiceProviders.AZURE_REALTIME.value,
+    }:
+        # OpenAI-compatible Realtime services already emit speaking-state frames
+        # and interruption events from the provider, so the aggregator should
+        # follow those external signals rather than run its own local VAD.
+        return external_provider_turn_config()
     if provider == ServiceProviders.GROK_REALTIME.value:
         # Grok Voice Agent emits server-side speech-start/stop and
         # interruption signals, so local VAD should stay out of the way.
-        return (
-            UserTurnStrategies(
-                start=[ExternalUserTurnStartStrategy()],
-                stop=[ExternalUserTurnStopStrategy()],
-            ),
-            None,
-        )
+        return external_provider_turn_config()
+    if provider == ServiceProviders.ULTRAVOX_REALTIME.value:
+        # Ultravox does not emit user-turn frames, so local VAD supplies
+        # lifecycle signals for Dograh observers/controllers.
+        return local_vad_turn_config(enable_interruptions=True)
 
-    return (
-        UserTurnStrategies(
-            start=[VADUserTurnStartStrategy()],
-            stop=[SpeechTimeoutUserTurnStopStrategy()],
-        ),
-        SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
-    )
+    return local_vad_turn_config(enable_interruptions=True)
 
 
 async def run_pipeline_telephony(
@@ -147,7 +255,38 @@ async def run_pipeline_telephony(
     provider_name: str,
     workflow_id: int,
     workflow_run_id: int,
-    user_id: int,
+    organization_id: int,
+    call_id: str,
+    transport_kwargs: dict,
+) -> None:
+    """Run a pipeline for any telephony provider."""
+    # Register before any async setup so deploy drains see calls that are still
+    # resolving DB/config/transport state.
+    register_worker_active_call(workflow_run_id)
+    try:
+        await _run_pipeline_telephony_impl(
+            websocket,
+            provider_name=provider_name,
+            workflow_id=workflow_id,
+            workflow_run_id=workflow_run_id,
+            organization_id=organization_id,
+            call_id=call_id,
+            transport_kwargs=transport_kwargs,
+        )
+    finally:
+        try:
+            await call_concurrency.unregister_active_call(workflow_run_id)
+        finally:
+            unregister_worker_active_call(workflow_run_id)
+
+
+async def _run_pipeline_telephony_impl(
+    websocket,
+    *,
+    provider_name: str,
+    workflow_id: int,
+    workflow_run_id: int,
+    organization_id: int,
     call_id: str,
     transport_kwargs: dict,
 ) -> None:
@@ -162,7 +301,9 @@ async def run_pipeline_telephony(
         provider_name: Stable identifier of the provider (registry key).
         workflow_id: Workflow being executed.
         workflow_run_id: Workflow run row.
-        user_id: Owner of the workflow.
+        organization_id: Tenant owning the workflow and the run. Every lookup
+            below is scoped by it; the workflow owner is read off the workflow
+            row and used only for attribution.
         call_id: Provider call identifier.
         transport_kwargs: Provider-specific kwargs forwarded to the transport
             factory (e.g. stream_sid + call_sid for Twilio).
@@ -170,12 +311,15 @@ async def run_pipeline_telephony(
     logger.debug(f"Running {provider_name} pipeline for workflow_run {workflow_run_id}")
     set_current_run_id(workflow_run_id)
 
-    workflow = await db_client.get_workflow(workflow_id, user_id)
-    if workflow:
-        set_current_org_id(workflow.organization_id)
+    workflow = await db_client.get_workflow(
+        workflow_id, organization_id=organization_id
+    )
+    if not workflow:
+        raise HTTPException(status_code=404, detail="Workflow not found")
+    set_current_org_id(workflow.organization_id)
 
     ambient_noise_config = None
-    if workflow and workflow.workflow_configurations:
+    if workflow.workflow_configurations:
         ambient_noise_config = workflow.workflow_configurations.get(
             "ambient_noise_configuration"
         )
@@ -184,26 +328,30 @@ async def run_pipeline_telephony(
     # (test call, campaign dispatch, inbound). Transports use it to load creds
     # from the right config row. Falls back to None for legacy runs (transports
     # then resolve the org's default config).
-    workflow_run = await db_client.get_workflow_run(workflow_run_id)
+    workflow_run = await db_client.get_workflow_run(
+        workflow_run_id, organization_id=organization_id
+    )
+    if not workflow_run:
+        raise HTTPException(status_code=404, detail="Workflow run not found")
+    if workflow_run.workflow_id != workflow_id:
+        raise HTTPException(status_code=400, detail="workflow_run_workflow_mismatch")
+
     telephony_configuration_id = None
-    if workflow_run and workflow_run.initial_context:
+    if workflow_run.initial_context:
         telephony_configuration_id = workflow_run.initial_context.get(
             "telephony_configuration_id"
         )
 
-    # Resolve effective user config here so the transport can tune its
+    # Resolve effective org config here so the transport can tune its
     # bot-stopped-speaking fallback based on is_realtime; pass the resolved
     # values into _run_pipeline so it doesn't fetch them again.
     from api.services.configuration.ai_model_configuration import (
         get_effective_ai_model_configuration_for_workflow,
     )
 
-    run_configs = (
-        (workflow_run.definition.workflow_configurations or {}) if workflow_run else {}
-    )
+    run_configs = workflow_run.definition.workflow_configurations or {}
     user_config = await get_effective_ai_model_configuration_for_workflow(
-        user_id=user_id,
-        organization_id=workflow.organization_id if workflow else None,
+        organization_id=workflow.organization_id,
         workflow_configurations=run_configs,
     )
     is_realtime = bool(user_config.is_realtime and user_config.realtime is not None)
@@ -223,14 +371,16 @@ async def run_pipeline_telephony(
     )
 
     try:
-        await _run_pipeline(
+        await _run_pipeline_impl(
             transport,
             workflow_id,
             workflow_run_id,
-            user_id,
+            # Attribution only — scoping is driven by organization_id below.
+            workflow.user_id,
             audio_config=audio_config,
             workflow_run=workflow_run,
             resolved_user_config=user_config,
+            organization_id=organization_id,
         )
     except Exception as e:
         logger.error(
@@ -247,6 +397,37 @@ async def run_pipeline_smallwebrtc(
     user_id: int,
     call_context_vars: dict = {},
     user_provider_id: str | None = None,
+    organization_id: int | None = None,
+) -> None:
+    """Run pipeline for WebRTC connections."""
+    # Register before any async setup so deploy drains see calls that are still
+    # resolving DB/config/transport state.
+    register_worker_active_call(workflow_run_id)
+    try:
+        await _run_pipeline_smallwebrtc_impl(
+            webrtc_connection,
+            workflow_id,
+            workflow_run_id,
+            user_id,
+            call_context_vars=call_context_vars,
+            user_provider_id=user_provider_id,
+            organization_id=organization_id,
+        )
+    finally:
+        try:
+            await call_concurrency.unregister_active_call(workflow_run_id)
+        finally:
+            unregister_worker_active_call(workflow_run_id)
+
+
+async def _run_pipeline_smallwebrtc_impl(
+    webrtc_connection: SmallWebRTCConnection,
+    workflow_id: int,
+    workflow_run_id: int,
+    user_id: int,
+    call_context_vars: dict = {},
+    user_provider_id: str | None = None,
+    organization_id: int | None = None,
 ) -> None:
     """Run pipeline for WebRTC connections"""
     logger.debug(
@@ -254,8 +435,14 @@ async def run_pipeline_smallwebrtc(
     )
     set_current_run_id(workflow_run_id)
 
-    # Get workflow to extract all pipeline configurations
-    workflow = await db_client.get_workflow(workflow_id, user_id)
+    workflow_scope = (
+        {"organization_id": organization_id}
+        if organization_id is not None
+        else {"user_id": user_id}
+    )
+
+    # Get workflow to extract all pipeline configurations.
+    workflow = await db_client.get_workflow(workflow_id, **workflow_scope)
 
     # Set org context early so tasks created by the transport inherit it
     if workflow:
@@ -271,19 +458,26 @@ async def run_pipeline_smallwebrtc(
     # Create audio configuration for WebRTC
     audio_config = create_audio_config(WorkflowRunMode.SMALLWEBRTC.value)
 
-    # Resolve workflow_run + effective user_config here so the transport can
+    # Resolve workflow_run + effective org config here so the transport can
     # tune its bot-stopped-speaking fallback based on is_realtime. _run_pipeline
     # reuses these via kwargs so we don't fetch twice.
     from api.services.configuration.ai_model_configuration import (
         get_effective_ai_model_configuration_for_workflow,
     )
 
-    workflow_run = await db_client.get_workflow_run(workflow_run_id, user_id)
+    workflow_run = await db_client.get_workflow_run(workflow_run_id, **workflow_scope)
+    if not workflow_run:
+        raise HTTPException(status_code=404, detail="Workflow run not found")
+    if workflow_run.workflow_id != workflow_id:
+        raise HTTPException(
+            status_code=400,
+            detail="workflow_run_workflow_mismatch",
+        )
+
     run_configs = (
         (workflow_run.definition.workflow_configurations or {}) if workflow_run else {}
     )
     user_config = await get_effective_ai_model_configuration_for_workflow(
-        user_id=user_id,
         organization_id=workflow.organization_id if workflow else None,
         workflow_configurations=run_configs,
     )
@@ -296,7 +490,7 @@ async def run_pipeline_smallwebrtc(
         ambient_noise_config,
         is_realtime=is_realtime,
     )
-    await _run_pipeline(
+    await _run_pipeline_impl(
         transport,
         workflow_id,
         workflow_run_id,
@@ -306,6 +500,7 @@ async def run_pipeline_smallwebrtc(
         user_provider_id=user_provider_id,
         workflow_run=workflow_run,
         resolved_user_config=user_config,
+        organization_id=organization_id,
     )
 
 
@@ -319,6 +514,41 @@ async def _run_pipeline(
     user_provider_id: str | None = None,
     workflow_run=None,
     resolved_user_config=None,
+    organization_id: int | None = None,
+) -> None:
+    """Run the pipeline with active-call drain accounting."""
+    register_worker_active_call(workflow_run_id)
+    try:
+        await _run_pipeline_impl(
+            transport,
+            workflow_id,
+            workflow_run_id,
+            user_id,
+            call_context_vars=call_context_vars,
+            audio_config=audio_config,
+            user_provider_id=user_provider_id,
+            workflow_run=workflow_run,
+            resolved_user_config=resolved_user_config,
+            organization_id=organization_id,
+        )
+    finally:
+        try:
+            await call_concurrency.unregister_active_call(workflow_run_id)
+        finally:
+            unregister_worker_active_call(workflow_run_id)
+
+
+async def _run_pipeline_impl(
+    transport,
+    workflow_id: int,
+    workflow_run_id: int,
+    user_id: int,
+    call_context_vars: dict = {},
+    audio_config: AudioConfig = None,
+    user_provider_id: str | None = None,
+    workflow_run=None,
+    resolved_user_config=None,
+    organization_id: int | None = None,
 ) -> None:
     """
     Run the pipeline with the given transport and configuration
@@ -329,11 +559,26 @@ async def _run_pipeline(
         workflow_run_id: The ID of the workflow run
         user_id: The ID of the user
         workflow_run: Pre-fetched workflow run row. Fetched here if None.
-        resolved_user_config: User configuration with model_overrides already
-            applied. Fetched and resolved here if None.
+        resolved_user_config: Organization model configuration with workflow
+            model_overrides already applied. Fetched and resolved here if None.
     """
+    workflow_scope = (
+        {"organization_id": organization_id}
+        if organization_id is not None
+        else {"user_id": user_id}
+    )
+
     if workflow_run is None:
-        workflow_run = await db_client.get_workflow_run(workflow_run_id, user_id)
+        workflow_run = await db_client.get_workflow_run(
+            workflow_run_id, **workflow_scope
+        )
+    if not workflow_run:
+        raise HTTPException(status_code=404, detail="Workflow run not found")
+    if workflow_run.workflow_id != workflow_id:
+        raise HTTPException(
+            status_code=400,
+            detail="workflow_run_workflow_mismatch",
+        )
 
     # If the workflow run is already completed, we don't need to run it again
     if workflow_run.is_completed:
@@ -346,7 +591,7 @@ async def _run_pipeline(
         merged_call_context_vars = {**merged_call_context_vars, **call_context_vars}
 
     # Get workflow for metadata (name, organization_id, call_disposition_codes)
-    workflow = await db_client.get_workflow(workflow_id, user_id)
+    workflow = await db_client.get_workflow(workflow_id, **workflow_scope)
     if not workflow:
         raise HTTPException(status_code=404, detail="Workflow not found")
 
@@ -356,11 +601,13 @@ async def _run_pipeline(
     run_configs = run_definition.workflow_configurations or {}
 
     # Extract configurations from the version's workflow_configurations
-    max_call_duration_seconds = 300  # Default 5 minutes
-    max_user_idle_timeout = 10.0  # Default 10 seconds
-    smart_turn_stop_secs = 2.0  # Default 2 seconds for incomplete turn timeout
-    turn_stop_strategy = "transcription"  # Default to transcription-based detection
+    max_call_duration_seconds = DEFAULT_MAX_CALL_DURATION_SECONDS
+    max_user_idle_timeout = DEFAULT_MAX_USER_IDLE_TIMEOUT_SECONDS
     keyterms = None  # Dictionary words for STT boosting
+    transcript_config = run_configs.get("transcript_configuration") or {}
+    include_transcript_end_timestamps = bool(
+        transcript_config.get("include_end_timestamps", False)
+    )
 
     if run_configs:
         if "max_call_duration" in run_configs:
@@ -369,12 +616,6 @@ async def _run_pipeline(
         if "max_user_idle_timeout" in run_configs:
             max_user_idle_timeout = run_configs["max_user_idle_timeout"]
 
-        if "smart_turn_stop_secs" in run_configs:
-            smart_turn_stop_secs = run_configs["smart_turn_stop_secs"]
-
-        if "turn_stop_strategy" in run_configs:
-            turn_stop_strategy = run_configs["turn_stop_strategy"]
-
         if "dictionary" in run_configs:
             dictionary = run_configs["dictionary"]
             if dictionary and isinstance(dictionary, str):
@@ -382,7 +623,7 @@ async def _run_pipeline(
                     term.strip() for term in dictionary.split(",") if term.strip()
                 ]
 
-    # Resolve model overrides from the version onto global user config (skip
+    # Resolve model overrides from the version onto global org config (skip
     # when the caller already resolved it).
     if resolved_user_config is None:
         from api.services.configuration.ai_model_configuration import (
@@ -390,7 +631,6 @@ async def _run_pipeline(
         )
 
         user_config = await get_effective_ai_model_configuration_for_workflow(
-            user_id=user_id,
             organization_id=workflow.organization_id,
             workflow_configurations=run_configs,
         )
@@ -620,59 +860,56 @@ async def _run_pipeline(
 
     # Configure turn strategies based on STT provider, model, and workflow configuration
     if is_realtime:
+        uses_external_turns = False
         # Realtime services still need user-turn tracking even when the model
         # itself owns speech generation and interruption behavior.
         user_turn_strategies, user_vad_analyzer = _create_realtime_user_turn_config(
             user_config.realtime.provider
         )
     else:
-        # Deepgram Flux uses external turn detection (VAD + External start/stop)
-        # Other models use configurable turn detection strategy
-        is_deepgram_flux = (
-            user_config.stt.provider == ServiceProviders.DEEPGRAM.value
-            and user_config.stt.model in DEEPGRAM_FLUX_MODELS
+        # Some STT services emit their own turn boundaries, so the aggregator
+        # follows those external signals. Other models use configurable turn
+        # detection.
+        uses_external_turns = stt_uses_external_turns(user_config)
+        user_turn_start_strategies = _create_non_realtime_user_turn_start_strategies(
+            run_configs,
+            uses_external_turns=uses_external_turns,
+        )
+        turn_start_strategy = run_configs.get(
+            "turn_start_strategy", DEFAULT_TURN_START_STRATEGY
+        )
+        logger.info(
+            f"[run {workflow_run_id}] Non-realtime interrupt strategy "
+            f"requested={turn_start_strategy} "
+            f"uses_external_turns={uses_external_turns}"
         )
 
-        if is_deepgram_flux:
-            user_turn_strategies = UserTurnStrategies(
-                start=[
-                    VADUserTurnStartStrategy(),
-                    ExternalUserTurnStartStrategy(enable_interruptions=True),
-                ],
-                stop=[ExternalUserTurnStopStrategy()],
-            )
-        elif turn_stop_strategy == "turn_analyzer":
-            # Smart Turn Analyzer: best for longer responses with natural pauses
-            smart_turn_params = SmartTurnParams(stop_secs=smart_turn_stop_secs)
-            user_turn_strategies = UserTurnStrategies(
-                start=[
-                    VADUserTurnStartStrategy(),
-                    TranscriptionUserTurnStartStrategy(),
-                ],
-                stop=[
-                    TurnAnalyzerUserTurnStopStrategy(
-                        turn_analyzer=LocalSmartTurnAnalyzerV3(params=smart_turn_params)
-                    )
-                ],
-            )
-        else:
-            # Transcription-based (default): best for short 1-2 word responses
-            user_turn_strategies = UserTurnStrategies(
-                start=[
-                    VADUserTurnStartStrategy(),
-                    TranscriptionUserTurnStartStrategy(),
-                ],
-                stop=[SpeechTimeoutUserTurnStopStrategy()],
-            )
+        user_turn_stop_strategies = _create_non_realtime_user_turn_stop_strategies(
+            run_configs,
+            uses_external_turns=uses_external_turns,
+        )
+        user_turn_strategies = UserTurnStrategies(
+            start=user_turn_start_strategies,
+            stop=user_turn_stop_strategies,
+        )
+
+    user_turn_stop_timeout = _resolve_user_turn_stop_timeout(
+        run_configs,
+        uses_external_turns=uses_external_turns,
+    )
 
     user_params = LLMUserAggregatorParams(
         user_turn_strategies=user_turn_strategies,
         user_mute_strategies=user_mute_strategies,
+        user_turn_stop_timeout=user_turn_stop_timeout,
         user_idle_timeout=max_user_idle_timeout,
         vad_analyzer=user_vad_analyzer,
     )
     context_aggregator = LLMContextAggregatorPair(
-        context, assistant_params=assistant_params, user_params=user_params
+        context,
+        assistant_params=assistant_params,
+        user_params=user_params,
+        realtime_service_mode=is_realtime,
     )
 
     # Create usage metrics aggregator with engine's callback
@@ -797,6 +1034,12 @@ async def _run_pipeline(
 
     # Create pipeline task with audio configuration
     task = create_pipeline_task(pipeline, workflow_run_id, audio_config)
+    transcript_log_coordinator = TranscriptLogCoordinator(in_memory_logs_buffer)
+    if task.turn_tracking_observer is None:
+        raise RuntimeError("Transcript logging requires turn tracking to be enabled")
+    transcript_log_coordinator.attach_turn_tracking_observer(
+        task.turn_tracking_observer
+    )
 
     for runtime_session in integration_runtime_sessions:
         runtime_session.attach(task)
@@ -851,7 +1094,9 @@ async def _run_pipeline(
 
     # Register turn log handlers for all call types (WebRTC and telephony)
     register_turn_log_handlers(
-        in_memory_logs_buffer, user_context_aggregator, assistant_context_aggregator
+        transcript_log_coordinator,
+        user_context_aggregator,
+        assistant_context_aggregator,
     )
 
     # Register event handlers — resolve provider_id for PostHog tracking
@@ -865,11 +1110,13 @@ async def _run_pipeline(
         engine=engine,
         audio_buffer=audio_buffer,
         in_memory_logs_buffer=in_memory_logs_buffer,
+        transcript_log_coordinator=transcript_log_coordinator,
         pipeline_metrics_aggregator=pipeline_metrics_aggregator,
         audio_config=audio_config,
         pre_call_fetch_task=pre_call_fetch_task,
         user_provider_id=user_provider_id,
         integration_runtime_sessions=integration_runtime_sessions,
+        include_transcript_end_timestamps=include_transcript_end_timestamps,
     )
 
     register_audio_data_handler(audio_buffer, workflow_run_id, in_memory_audio_buffer)

api/services/pipecat/service_factory.py

@@ -6,8 +6,14 @@ from fastapi import HTTPException
 from loguru import logger
 
 from api.constants import MPS_API_URL
-from api.services.configuration.options import DEEPGRAM_FLUX_MODELS
+from api.services.configuration.options import (
+    DEEPGRAM_FLUX_MODELS,
+    DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGE_OPTIONS,
+)
 from api.services.configuration.registry import ServiceProviders
+from api.services.pipecat.gemini_json_schema_adapter import (
+    DograhGeminiJSONSchemaAdapter,
+)
 from api.services.pipecat.minimax_tts import MiniMaxOwnedSessionTTSService
 from api.utils.url_security import validate_user_configured_service_url
 from pipecat.services.assemblyai.stt import AssemblyAISTTService, AssemblyAISTTSettings
@@ -15,21 +21,28 @@ from pipecat.services.aws.llm import AWSBedrockLLMService, AWSBedrockLLMSettings
 from pipecat.services.azure.llm import AzureLLMService, AzureLLMSettings
 from pipecat.services.azure.stt import AzureSTTService, AzureSTTSettings
 from pipecat.services.azure.tts import AzureTTSService, AzureTTSSettings
-from pipecat.services.cartesia.stt import CartesiaSTTService
+from pipecat.services.cartesia.stt import CartesiaSTTService, CartesiaSTTSettings
 from pipecat.services.cartesia.tts import (
     CartesiaTTSService,
     CartesiaTTSSettings,
     GenerationConfig,
 )
+from pipecat.services.cartesia.turns.stt import CartesiaTurnsSTTService
 from pipecat.services.deepgram.flux.stt import (
     DeepgramFluxSTTService,
     DeepgramFluxSTTSettings,
 )
 from pipecat.services.deepgram.stt import DeepgramSTTService, DeepgramSTTSettings
 from pipecat.services.deepgram.tts import DeepgramTTSService, DeepgramTTSSettings
+from pipecat.services.dograh.flux.stt import DograhFluxSTTService
 from pipecat.services.dograh.llm import DograhLLMService
 from pipecat.services.dograh.stt import DograhSTTService, DograhSTTSettings
 from pipecat.services.dograh.tts import DograhTTSService, DograhTTSSettings
+from pipecat.services.elevenlabs.stt import (
+    CommitStrategy,
+    ElevenLabsRealtimeSTTService,
+    ElevenLabsRealtimeSTTSettings,
+)
 from pipecat.services.elevenlabs.tts import ElevenLabsTTSService, ElevenLabsTTSSettings
 from pipecat.services.gladia.stt import GladiaSTTService, GladiaSTTSettings
 from pipecat.services.google.llm import GoogleLLMService, GoogleLLMSettings
@@ -73,6 +86,7 @@ from pipecat.services.speechmatics.stt import (
     SpeechmaticsSTTService,
     SpeechmaticsSTTSettings,
 )
+from pipecat.services.xai.tts import XAIHttpTTSService, XAITTSSettings
 from pipecat.transcriptions.language import Language
 from pipecat.utils.text.xml_function_tag_filter import XMLFunctionTagFilter
 
@@ -94,6 +108,75 @@ DEEPGRAM_FLUX_LANGUAGE_HINTS = {
 }
 
 
+def dograh_stt_uses_flux_language(language: str | None) -> bool:
+    language = language or "multi"
+    return language in DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGE_OPTIONS
+
+
+def _resolve_elevenlabs_stt_language(
+    language_code: str | None,
+) -> Language | str | None:
+    if not language_code or language_code == "auto":
+        return None
+    try:
+        return Language(language_code)
+    except ValueError:
+        return language_code
+
+
+def _elevenlabs_websocket_url(base_url: str) -> str:
+    """Normalize an ElevenLabs API base URL for WebSocket clients."""
+    base_url = base_url.strip()
+    parsed = urlparse(base_url)
+    if not parsed.netloc:
+        return base_url.rstrip("/")
+
+    websocket_scheme = {
+        "http": "ws",
+        "https": "wss",
+    }.get(parsed.scheme, parsed.scheme)
+    return urlunparse(
+        parsed._replace(
+            scheme=websocket_scheme,
+            path=parsed.path.rstrip("/"),
+        )
+    )
+
+
+def _elevenlabs_realtime_stt_host(base_url: str) -> str:
+    """Return the host/path prefix Pipecat's ElevenLabs realtime STT expects.
+
+    Pipecat's realtime STT service builds
+    ``wss://{host}/v1/speech-to-text/realtime`` internally, so remove the scheme
+    from the same normalized WebSocket URL used by ElevenLabs TTS. Preserve
+    netloc (including optional ports) and any path prefix used by BYOK proxies.
+    """
+    websocket_url = _elevenlabs_websocket_url(base_url)
+    parsed = urlparse(websocket_url)
+    if parsed.netloc:
+        path = parsed.path
+        return f"{parsed.netloc}{path}" if path else parsed.netloc
+    return websocket_url
+
+
+def stt_uses_external_turns(user_config) -> bool:
+    if user_config.stt.provider == ServiceProviders.DEEPGRAM.value:
+        return user_config.stt.model in DEEPGRAM_FLUX_MODELS
+    if user_config.stt.provider == ServiceProviders.DOGRAH.value:
+        return dograh_stt_uses_flux_language(getattr(user_config.stt, "language", None))
+    if user_config.stt.provider == ServiceProviders.CARTESIA.value:
+        return user_config.stt.model == "ink-2"
+    return False
+
+
+class DograhGoogleLLMService(GoogleLLMService):
+    adapter_class = DograhGeminiJSONSchemaAdapter
+
+
+class DograhGoogleVertexLLMService(GoogleVertexLLMService):
+    adapter_class = DograhGeminiJSONSchemaAdapter
+
+
 def _validate_runtime_service_url(url: str, field_name: str) -> None:
     try:
         validate_user_configured_service_url(
@@ -166,6 +249,7 @@ def create_stt_service(
         return OpenAISTTService(
             api_key=user_config.stt.api_key,
             settings=OpenAISTTSettings(model=user_config.stt.model),
+            should_interrupt=False,  # Let UserAggregator own interruption confirmation.
             **kwargs,
         )
     elif user_config.stt.provider == ServiceProviders.GOOGLE.value:
@@ -186,13 +270,48 @@ def create_stt_service(
             sample_rate=audio_config.transport_in_sample_rate,
         )
     elif user_config.stt.provider == ServiceProviders.CARTESIA.value:
+        if user_config.stt.model == "ink-2":
+            return CartesiaTurnsSTTService(
+                api_key=user_config.stt.api_key,
+                should_interrupt=False,  # Let UserAggregator emit interruption frames.
+                sample_rate=audio_config.transport_in_sample_rate,
+            )
+
+        language = getattr(user_config.stt, "language", None) or "en"
         return CartesiaSTTService(
             api_key=user_config.stt.api_key,
+            settings=CartesiaSTTSettings(
+                model=user_config.stt.model,
+                language=language,
+            ),
             sample_rate=audio_config.transport_in_sample_rate,
         )
     elif user_config.stt.provider == ServiceProviders.DOGRAH.value:
         base_url = MPS_API_URL.replace("http://", "ws://").replace("https://", "wss://")
         language = getattr(user_config.stt, "language", None) or "multi"
+
+        if dograh_stt_uses_flux_language(language):
+            # Dograh's Flux proxy only supports multilingual auto-detect and the
+            # same language hint subset as Deepgram Flux multilingual.
+            settings_kwargs = {
+                "model": "flux-general-multi",
+                "eot_timeout_ms": 3000,
+                "eot_threshold": 0.7,
+                "eager_eot_threshold": 0.5,
+                "keyterm": keyterms or [],
+            }
+            language_hint = DEEPGRAM_FLUX_LANGUAGE_HINTS.get(language)
+            if language_hint:
+                settings_kwargs["language_hints"] = [language_hint]
+            return DograhFluxSTTService(
+                base_url=base_url,
+                api_key=user_config.stt.api_key,
+                correlation_id=correlation_id,
+                settings=DeepgramFluxSTTSettings(**settings_kwargs),
+                should_interrupt=False,  # external turn strategies own interruption
+                sample_rate=audio_config.transport_in_sample_rate,
+            )
+
         return DograhSTTService(
             base_url=base_url,
             api_key=user_config.stt.api_key,
@@ -347,6 +466,24 @@ def create_stt_service(
             ),
             sample_rate=audio_config.transport_in_sample_rate,
         )
+    elif user_config.stt.provider == ServiceProviders.ELEVENLABS.value:
+        language_code = getattr(user_config.stt, "language", None)
+        pipecat_language = _resolve_elevenlabs_stt_language(language_code)
+
+        _validate_runtime_service_url(user_config.stt.base_url, "base_url")
+        elevenlabs_host = _elevenlabs_realtime_stt_host(user_config.stt.base_url)
+
+        return ElevenLabsRealtimeSTTService(
+            api_key=user_config.stt.api_key,
+            base_url=elevenlabs_host,
+            commit_strategy=CommitStrategy.VAD,
+            settings=ElevenLabsRealtimeSTTSettings(
+                model=user_config.stt.model,
+                language=pipecat_language,
+            ),
+            should_interrupt=False,
+            sample_rate=audio_config.transport_in_sample_rate,
+        )
     else:
         raise HTTPException(
             status_code=400, detail=f"Invalid STT provider {user_config.stt.provider}"
@@ -420,13 +557,11 @@ def create_tts_service(
             voice_id = user_config.tts.voice.split(" - ")[1]
         except IndexError:
             voice_id = user_config.tts.voice
-        # ElevenLabs TTS uses WebSocket. Users configure base_url with an HTTP
-        # scheme (matching ElevenLabs documentation, e.g.
-        # https://api.eu.residency.elevenlabs.io); rewrite it to the WS scheme.
+        # ElevenLabs TTS consumes the full normalized WebSocket URL. Realtime
+        # STT uses the same normalization before adapting it to Pipecat's
+        # scheme-less base_url contract.
         _validate_runtime_service_url(user_config.tts.base_url, "base_url")
-        elevenlabs_url = user_config.tts.base_url.replace("https://", "wss://").replace(
-            "http://", "ws://"
-        )
+        elevenlabs_url = _elevenlabs_websocket_url(user_config.tts.base_url)
         return ElevenLabsTTSService(
             reconnect_on_error=False,
             api_key=user_config.tts.api_key,
@@ -673,12 +808,47 @@ def create_tts_service(
             skip_aggregator_types=["recording_router", "recording"],
             silence_time_s=1.0,
         )
+    elif user_config.tts.provider == ServiceProviders.XAI.value:
+        voice = getattr(user_config.tts, "voice", None) or "eve"
+        language_code = getattr(user_config.tts, "language", None) or "en"
+        if language_code.lower() == "auto":
+            pipecat_language = "auto"
+        else:
+            try:
+                pipecat_language = Language(language_code)
+            except ValueError:
+                pipecat_language = Language.EN
+        return XAIHttpTTSService(
+            api_key=user_config.tts.api_key,
+            sample_rate=audio_config.transport_out_sample_rate,
+            encoding="pcm",
+            settings=XAITTSSettings(
+                voice=voice,
+                language=pipecat_language,
+            ),
+            text_filters=[xml_function_tag_filter],
+            skip_aggregator_types=["recording_router", "recording"],
+            silence_time_s=1.0,
+        )
     else:
         raise HTTPException(
             status_code=400, detail=f"Invalid TTS provider {user_config.tts.provider}"
         )
 
 
+def _migrate_deprecated_google_model(model: str) -> str:
+    """Google removed the ``gemini-2.0-flash*`` models. Transparently upgrade
+    any stored config that still references them to the 2.5 equivalent so old
+    user configurations keep working instead of failing at runtime."""
+    if model and model.startswith("gemini-2.0-flash"):
+        migrated = model.replace("gemini-2.0-", "gemini-2.5-", 1)
+        logger.warning(
+            f"Google model '{model}' is no longer supported; using '{migrated}' instead"
+        )
+        return migrated
+    return model
+
+
 def create_llm_service_from_provider(
     provider: str,
     model: str,
@@ -736,12 +906,13 @@ def create_llm_service_from_provider(
             **kwargs,
         )
     elif provider == ServiceProviders.GOOGLE.value:
-        return GoogleLLMService(
+        model = _migrate_deprecated_google_model(model)
+        return DograhGoogleLLMService(
             api_key=api_key,
             settings=GoogleLLMSettings(model=model, temperature=0.1),
         )
     elif provider == ServiceProviders.GOOGLE_VERTEX.value:
-        return GoogleVertexLLMService(
+        return DograhGoogleVertexLLMService(
             credentials=credentials,
             project_id=project_id,
             location=location or "us-east4",
@@ -858,14 +1029,28 @@ def create_realtime_llm_service(user_config, audio_config: "AudioConfig"):
         from api.services.pipecat.realtime.grok_realtime import (
             DograhGrokRealtimeLLMService,
         )
-        from pipecat.services.xai.realtime.events import SessionProperties
+        from pipecat.services.xai.realtime.events import (
+            AudioConfiguration,
+            AudioInput,
+            InputAudioTranscription,
+            SessionProperties,
+        )
+
+        grok_voice = voice or "ara"
+        if grok_voice.lower() in {"ara", "rex", "sal", "eve", "leo"}:
+            grok_voice = grok_voice.lower()
 
         return DograhGrokRealtimeLLMService(
             api_key=api_key,
             settings=DograhGrokRealtimeLLMService.Settings(
                 model=model,
                 session_properties=SessionProperties(
-                    voice=voice or "Ara",
+                    voice=grok_voice,
+                    audio=AudioConfiguration(
+                        input=AudioInput(
+                            transcription=InputAudioTranscription(),
+                        ),
+                    ),
                 ),
             ),
         )
@@ -944,19 +1129,25 @@ def create_realtime_llm_service(user_config, audio_config: "AudioConfig"):
                 detail="Azure Realtime requires an endpoint.",
             )
         _validate_runtime_service_url(endpoint, "endpoint")
-        api_version = (
-            getattr(realtime_config, "api_version", None) or "2025-04-01-preview"
-        )
-        # Construct the Azure Realtime WebSocket URL
-        # https://<resource>.openai.azure.com/openai/realtime?api-version=<ver>&deployment=<model>
+        api_version = getattr(realtime_config, "api_version", None) or "v1"
         parsed_endpoint = urlparse(endpoint)
+        if api_version == "v1":
+            # Azure's GA Realtime API uses the deployment name as `model` and
+            # deliberately has no date-based api-version query parameter.
+            path = "/openai/v1/realtime"
+            query = urlencode({"model": model})
+        else:
+            # Preserve explicitly configured preview deployments while users
+            # migrate. Microsoft deprecated this protocol on April 30, 2026.
+            path = "/openai/realtime"
+            query = urlencode({"api-version": api_version, "deployment": model})
         wss_url = urlunparse(
             (
                 "wss",
                 parsed_endpoint.netloc,
-                "/openai/realtime",
+                path,
                 "",
-                urlencode({"api-version": api_version, "deployment": model}),
+                query,
                 "",
             )
         )

api/services/pipecat/transcript_log_coordinator.py

@@ -0,0 +1,298 @@
+"""Turn-aware coordination for immutable persisted transcript events.
+
+The transcript text, speech timing, and logical turn lifecycle are produced by
+different parts of the pipeline and can arrive in either order. This module is
+the single place where those facts are joined. It emits a transcript event only
+after the owning logical turn has ended (or during a final flush), and never
+mutates an event after it has been appended to the logs buffer.
+"""
+
+import asyncio
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+from api.services.pipecat.realtime_feedback_events import (
+    build_bot_text_event,
+    build_user_transcription_event,
+)
+
+if TYPE_CHECKING:
+    from api.services.pipecat.in_memory_buffers import InMemoryLogsBuffer
+    from pipecat.observers.turn_tracking_observer import TurnTrackingObserver
+
+
+def _now_iso() -> str:
+    return datetime.now(UTC).isoformat(timespec="milliseconds")
+
+
+@dataclass
+class _TranscriptSide:
+    text: str | None = None
+    transcript_timestamp: str | None = None
+    event_timestamp: str | None = None
+    speech_start_timestamp: str | None = None
+    speech_end_timestamp: str | None = None
+    speaking: bool = False
+    emitted: bool = False
+    node_id: str | None = None
+    node_name: str | None = None
+
+
+@dataclass
+class _TurnTranscriptState:
+    turn_id: int
+    ended: bool = False
+    interrupted: bool = False
+    user: _TranscriptSide = field(default_factory=_TranscriptSide)
+    assistant: _TranscriptSide = field(default_factory=_TranscriptSide)
+
+
+class TranscriptLogCoordinator:
+    """Join turn, transcript, and speech facts before appending log events."""
+
+    def __init__(self, logs_buffer: "InMemoryLogsBuffer"):
+        self._logs_buffer = logs_buffer
+        self._states: dict[int, _TurnTranscriptState] = {}
+        self._active_turn_id: int | None = None
+        self._lock = asyncio.Lock()
+
+    def attach_turn_tracking_observer(self, observer: "TurnTrackingObserver") -> None:
+        """Subscribe to the canonical turn owner's correlated lifecycle events."""
+
+        @observer.event_handler("on_turn_started")
+        async def on_turn_started(_observer, turn_number: int):
+            await self.record_turn_started(turn_number)
+
+        @observer.event_handler("on_turn_ended")
+        async def on_turn_ended(
+            _observer,
+            turn_number: int,
+            _duration: float,
+            was_interrupted: bool,
+        ):
+            await self.record_turn_ended(turn_number, interrupted=was_interrupted)
+
+        @observer.event_handler("on_user_speech_started_for_turn")
+        async def on_user_speech_started_for_turn(_observer, turn_number: int, _data):
+            callback_timestamp = _now_iso()
+            await self.record_user_started_speaking(turn_number, callback_timestamp)
+
+        @observer.event_handler("on_user_speech_stopped_for_turn")
+        async def on_user_speech_stopped_for_turn(_observer, turn_number: int, _data):
+            callback_timestamp = _now_iso()
+            await self.record_user_stopped_speaking(turn_number, callback_timestamp)
+
+        @observer.event_handler("on_bot_started_speaking")
+        async def on_bot_started_speaking(_observer, turn_number: int, _data):
+            await self.record_bot_started_speaking(turn_number)
+
+        @observer.event_handler("on_bot_stopped_speaking")
+        async def on_bot_stopped_speaking(_observer, turn_number: int, _data):
+            await self.record_bot_stopped_speaking(turn_number)
+
+    def _state(self, turn_id: int) -> _TurnTranscriptState:
+        state = self._states.get(turn_id)
+        if state is None:
+            state = _TurnTranscriptState(turn_id=turn_id)
+            self._states[turn_id] = state
+        return state
+
+    async def record_turn_started(self, turn_id: int) -> None:
+        async with self._lock:
+            self._state(turn_id)
+            if self._active_turn_id is None or turn_id >= self._active_turn_id:
+                self._active_turn_id = turn_id
+                self._logs_buffer.set_current_turn(turn_id)
+
+    async def record_turn_ended(self, turn_id: int, *, interrupted: bool) -> None:
+        async with self._lock:
+            state = self._state(turn_id)
+            state.ended = True
+            state.interrupted = interrupted
+            if self._active_turn_id == turn_id:
+                self._active_turn_id = None
+            await self._emit_ready_sides(state)
+
+    async def record_user_started_speaking(
+        self, turn_id: int, timestamp: str | None = None
+    ) -> None:
+        async with self._lock:
+            state = self._state(turn_id)
+            side = state.user
+            previous_start = side.speech_start_timestamp
+            candidate_start = timestamp or _now_iso()
+            side.speech_start_timestamp = (
+                min(previous_start, candidate_start)
+                if previous_start is not None
+                else candidate_start
+            )
+            side.speaking = True
+
+    async def record_user_stopped_speaking(
+        self, turn_id: int, timestamp: str | None = None
+    ) -> None:
+        async with self._lock:
+            side = self._state(turn_id).user
+            previous_end = side.speech_end_timestamp
+            candidate_end = timestamp or _now_iso()
+            side.speech_end_timestamp = (
+                max(previous_end, candidate_end)
+                if previous_end is not None
+                else candidate_end
+            )
+            side.speaking = False
+
+    async def record_bot_started_speaking(
+        self, turn_id: int, timestamp: str | None = None
+    ) -> None:
+        async with self._lock:
+            side = self._state(turn_id).assistant
+            if side.speech_start_timestamp is None:
+                side.speech_start_timestamp = timestamp or _now_iso()
+            side.speaking = True
+
+    async def record_bot_stopped_speaking(
+        self, turn_id: int, timestamp: str | None = None
+    ) -> None:
+        async with self._lock:
+            state = self._state(turn_id)
+            side = state.assistant
+            side.speech_end_timestamp = timestamp or _now_iso()
+            side.speaking = False
+            await self._emit_ready_sides(state)
+
+    async def record_user_transcript(
+        self,
+        *,
+        text: str,
+        timestamp: str | None,
+        end_timestamp: str | None = None,
+        event_timestamp: str | None = None,
+    ) -> None:
+        async with self._lock:
+            state = self._select_user_turn()
+            side = state.user
+            first_text = side.text is None
+            side.text = text if first_text else f"{side.text}\n{text}"
+            if first_text:
+                side.transcript_timestamp = timestamp
+                self._capture_node(side)
+            side.event_timestamp = event_timestamp or _now_iso()
+            if end_timestamp and not side.speech_end_timestamp:
+                side.speech_end_timestamp = end_timestamp
+            await self._emit_ready_sides(state)
+
+    async def record_assistant_transcript(
+        self,
+        *,
+        text: str,
+        timestamp: str | None,
+        end_timestamp: str | None = None,
+        event_timestamp: str | None = None,
+    ) -> None:
+        async with self._lock:
+            state = self._select_assistant_turn()
+            side = state.assistant
+            first_text = side.text is None
+            side.text = text if first_text else f"{side.text}\n{text}"
+            if first_text:
+                side.transcript_timestamp = timestamp
+                self._capture_node(side)
+            side.event_timestamp = event_timestamp or _now_iso()
+            if end_timestamp and not side.speech_end_timestamp:
+                side.speech_end_timestamp = end_timestamp
+            await self._emit_ready_sides(state)
+
+    def _select_user_turn(self) -> _TurnTranscriptState:
+        if self._active_turn_id is not None:
+            active = self._state(self._active_turn_id)
+            if active.user.text is None:
+                return active
+        candidates = [
+            state
+            for state in self._states.values()
+            if state.user.speech_start_timestamp and state.user.text is None
+        ]
+        if candidates:
+            return min(candidates, key=lambda state: state.turn_id)
+        if self._states:
+            return max(self._states.values(), key=lambda state: state.turn_id)
+        return self._state(1)
+
+    def _select_assistant_turn(self) -> _TurnTranscriptState:
+        candidates = [
+            state
+            for state in self._states.values()
+            if state.assistant.speech_start_timestamp and state.assistant.text is None
+        ]
+        interrupted = [
+            state for state in candidates if state.ended and state.interrupted
+        ]
+        if interrupted:
+            return min(interrupted, key=lambda state: state.turn_id)
+        if self._active_turn_id is not None:
+            active = self._state(self._active_turn_id)
+            if active.assistant.text is None:
+                return active
+        if candidates:
+            return min(candidates, key=lambda state: state.turn_id)
+        if self._states:
+            return max(self._states.values(), key=lambda state: state.turn_id)
+        return self._state(1)
+
+    def _capture_node(self, side: _TranscriptSide) -> None:
+        side.node_id = self._logs_buffer.current_node_id
+        side.node_name = self._logs_buffer.current_node_name
+
+    async def _emit_ready_sides(self, state: _TurnTranscriptState) -> None:
+        if not state.ended:
+            return
+        await self._emit_user(state)
+        if not state.assistant.speaking:
+            await self._emit_assistant(state)
+
+    async def _emit_user(self, state: _TurnTranscriptState) -> None:
+        side = state.user
+        if side.emitted or not side.text:
+            return
+        event = build_user_transcription_event(
+            text=side.text,
+            final=True,
+            timestamp=side.speech_start_timestamp or side.transcript_timestamp,
+            end_timestamp=side.speech_end_timestamp,
+        )
+        await self._append(state, side, event)
+
+    async def _emit_assistant(self, state: _TurnTranscriptState) -> None:
+        side = state.assistant
+        if side.emitted or not side.text:
+            return
+        event = build_bot_text_event(
+            text=side.text,
+            timestamp=side.speech_start_timestamp or side.transcript_timestamp,
+            end_timestamp=side.speech_end_timestamp,
+        )
+        await self._append(state, side, event)
+
+    async def _append(
+        self, state: _TurnTranscriptState, side: _TranscriptSide, event: dict
+    ) -> None:
+        await self._logs_buffer.append(
+            event,
+            timestamp=side.event_timestamp,
+            turn=state.turn_id,
+            node_id=side.node_id,
+            node_name=side.node_name,
+            use_current_node=False,
+        )
+        side.emitted = True
+
+    async def flush(self) -> None:
+        """Emit any remaining transcript text without inventing missing timing."""
+        async with self._lock:
+            for state in sorted(self._states.values(), key=lambda item: item.turn_id):
+                state.ended = True
+                state.user.speaking = False
+                state.assistant.speaking = False
+                await self._emit_ready_sides(state)

api/services/posthog_client.py

@@ -7,6 +7,7 @@ from api.constants import POSTHOG_API_KEY, POSTHOG_HOST
 
 _posthog_client: Posthog | None = None
 POSTHOG_SERVER_GROUP_IDENTIFY_DISTINCT_ID = "server-group-identify"
+POSTHOG_ORGANIZATION_GROUP_TYPE = "organization"
 
 
 def get_posthog() -> Posthog | None:

api/services/quota_service.py

@@ -7,6 +7,7 @@ across different endpoints (WebRTC signaling, telephony, public API triggers).
 from dataclasses import dataclass
 from typing import Any
 
+import httpx
 from loguru import logger
 
 from api.constants import DEPLOYMENT_MODE
@@ -25,13 +26,20 @@ from api.services.mps_service_key_client import mps_service_key_client
 
 MINIMUM_DOGRAH_CREDITS_FOR_CALL = 0.10
 
-LEGACY_QUOTA_EXCEEDED_MESSAGE = (
-    "You have exhausted your trial credits. "
-    "Please email founders@dograh.com for additional Dograh credits "
-    "or change providers in Models configurations."
+_MPS_UNREACHABLE_ERRORS = (
+    httpx.TimeoutException,
+    httpx.NetworkError,
+    httpx.RemoteProtocolError,
+    httpx.ProxyError,
 )
 
-BILLING_V2_QUOTA_EXCEEDED_MESSAGE = (
+OSS_QUOTA_EXCEEDED_MESSAGE = (
+    "You have exhausted your trial credits. "
+    "Please sign up on app.dograh.com to create a "
+    "new service key and set up in your model configurations."
+)
+
+HOSTED_QUOTA_EXCEEDED_MESSAGE = (
     "You have exhausted your Dograh credits. "
     "Please purchase more credits from /billing "
     "or change providers in Models configurations."
@@ -60,22 +68,35 @@ def _safe_float(value: Any, default: float = 0.0) -> float:
         return default
 
 
-def _insufficient_billing_v2_quota_result() -> QuotaCheckResult:
+def _insufficient_hosted_quota_result() -> QuotaCheckResult:
     return QuotaCheckResult(
         has_quota=False,
         error_code="insufficient_credits",
-        error_message=BILLING_V2_QUOTA_EXCEEDED_MESSAGE,
+        error_message=HOSTED_QUOTA_EXCEEDED_MESSAGE,
     )
 
 
-def _insufficient_legacy_quota_result() -> QuotaCheckResult:
+def _insufficient_oss_quota_result() -> QuotaCheckResult:
     return QuotaCheckResult(
         has_quota=False,
         error_code="quota_exceeded",
-        error_message=LEGACY_QUOTA_EXCEEDED_MESSAGE,
+        error_message=OSS_QUOTA_EXCEEDED_MESSAGE,
     )
 
 
+def _mps_unreachable_result(
+    operation: str,
+    error: httpx.RequestError,
+) -> QuotaCheckResult:
+    logger.warning(
+        "MPS unreachable during {}; allowing workflow run to proceed without "
+        "quota verification: {}",
+        operation,
+        error,
+    )
+    return QuotaCheckResult(has_quota=True)
+
+
 def _service_uses_dograh(service: Any) -> bool:
     provider = getattr(service, "provider", None)
     return (
@@ -157,10 +178,10 @@ async def _authorize_hosted_workflow_run_start(
     workflow_id: int | None,
     workflow_run_id: int | None,
     user_config: Any,
-) -> tuple[QuotaCheckResult, bool]:
-    """Authorize hosted v2 billing and return whether MPS handled enforcement."""
-    if DEPLOYMENT_MODE == "oss" or organization_id is None:
-        return QuotaCheckResult(has_quota=True), False
+) -> QuotaCheckResult:
+    """Authorize a hosted workflow run against the org's MPS billing account."""
+    if organization_id is None:
+        return QuotaCheckResult(has_quota=True)
 
     requires_correlation = bool(
         workflow_run_id and uses_managed_model_services_v2(user_config)
@@ -169,16 +190,13 @@ async def _authorize_hosted_workflow_run_start(
         get_dograh_service_api_key(user_config) if requires_correlation else None
     )
     if requires_correlation and not service_key:
-        return (
-            QuotaCheckResult(
-                has_quota=False,
-                error_code="invalid_service_key",
-                error_message=(
-                    "You have invalid keys in your model configuration. "
-                    "Please validate the service keys."
-                ),
+        return QuotaCheckResult(
+            has_quota=False,
+            error_code="invalid_service_key",
+            error_message=(
+                "You have invalid keys in your model configuration. "
+                "Please validate the service keys."
             ),
-            True,
         )
 
     try:
@@ -198,6 +216,8 @@ async def _authorize_hosted_workflow_run_start(
                 "workflow_id": workflow_id,
             },
         )
+    except _MPS_UNREACHABLE_ERRORS as e:
+        return _mps_unreachable_result("hosted run authorization", e)
     except Exception as e:
         logger.warning(
             "Failed to authorize workflow start with MPS for org {}: {}",
@@ -205,38 +225,28 @@ async def _authorize_hosted_workflow_run_start(
             e,
         )
         if _is_service_key_org_mismatch_error(e):
-            return (
-                QuotaCheckResult(
-                    has_quota=False,
-                    error_code="service_key_org_mismatch",
-                    error_message=SERVICE_TOKEN_ORG_MISMATCH_MESSAGE,
-                ),
-                True,
-            )
-        return (
-            QuotaCheckResult(
+            return QuotaCheckResult(
                 has_quota=False,
-                error_code="quota_check_failed",
-                error_message="Could not verify Dograh credits. Please try again.",
-            ),
-            True,
+                error_code="service_key_org_mismatch",
+                error_message=SERVICE_TOKEN_ORG_MISMATCH_MESSAGE,
+            )
+        return QuotaCheckResult(
+            has_quota=False,
+            error_code="quota_check_failed",
+            error_message="Could not verify Dograh credits. Please try again.",
         )
 
-    billing_mode = authorization.get("billing_mode")
-    if billing_mode != "v2":
-        return QuotaCheckResult(has_quota=True), False
-
     remaining = _safe_float(authorization.get("remaining_credits"))
     if (
         not authorization.get("allowed", False)
         or remaining < MINIMUM_DOGRAH_CREDITS_FOR_CALL
     ):
         logger.warning(
-            "Insufficient Dograh billing v2 credits for org {}: {:.2f} credits remaining",
+            "Insufficient Dograh credits for org {}: {:.2f} credits remaining",
             organization_id,
             remaining,
         )
-        return _insufficient_billing_v2_quota_result(), True
+        return _insufficient_hosted_quota_result()
 
     try:
         await _store_run_correlation_id(
@@ -249,35 +259,27 @@ async def _authorize_hosted_workflow_run_start(
             workflow_run_id,
             e,
         )
-        return (
-            QuotaCheckResult(
-                has_quota=False,
-                error_code="quota_check_failed",
-                error_message="Could not verify Dograh credits. Please try again.",
-            ),
-            True,
+        return QuotaCheckResult(
+            has_quota=False,
+            error_code="quota_check_failed",
+            error_message="Could not verify Dograh credits. Please try again.",
         )
     logger.info(
-        "Dograh billing v2 run authorization passed for org {}: {:.2f} credits remaining",
+        "Dograh run authorization passed for org {}: {:.2f} credits remaining",
         organization_id,
         remaining,
     )
-    return QuotaCheckResult(has_quota=True), True
+    return QuotaCheckResult(has_quota=True)
 
 
-async def _authorize_legacy_dograh_keys(
+async def _authorize_oss_dograh_keys(
     *,
     dograh_api_keys: set[str],
-    organization_id: int | None,
-    workflow_owner: UserModel,
 ) -> QuotaCheckResult:
+    """Check per-key MPS credits for OSS deployments before a run starts."""
     for api_key in dograh_api_keys:
         try:
-            usage = await mps_service_key_client.check_service_key_usage(
-                api_key,
-                organization_id=organization_id,
-                created_by=workflow_owner.provider_id,
-            )
+            usage = await mps_service_key_client.check_service_key_usage(api_key)
             remaining = usage.get("remaining_credits", 0.0)
 
             # Require at least $0.10 for a short call
@@ -286,12 +288,14 @@ async def _authorize_legacy_dograh_keys(
                     f"Insufficient Dograh credits for key ...{api_key[-8:]}: "
                     f"${remaining:.2f} remaining"
                 )
-                return _insufficient_legacy_quota_result()
+                return _insufficient_oss_quota_result()
 
             logger.info(
                 f"Dograh quota check passed for key ...{api_key[-8:]}: "
                 f"{remaining:.2f} credits remaining"
             )
+        except _MPS_UNREACHABLE_ERRORS as e:
+            return _mps_unreachable_result("OSS service-key quota check", e)
         except Exception as e:
             logger.error(f"Failed to check quota for Dograh key: {str(e)}")
             error_str = str(e)
@@ -339,6 +343,8 @@ async def _authorize_oss_managed_v2_correlation(
             workflow_run_id,
             response.get("correlation_id"),
         )
+    except _MPS_UNREACHABLE_ERRORS as e:
+        return _mps_unreachable_result("OSS correlation creation", e)
     except Exception as e:
         logger.error(
             "Failed to authorize OSS managed v2 workflow start for workflow {} run {}: {}",
@@ -358,31 +364,64 @@ async def _authorize_oss_managed_v2_correlation(
 async def authorize_workflow_run_start(
     *,
     workflow_id: int,
+    organization_id: int,
     workflow_run_id: int | None = None,
     actor_user: UserModel | None = None,
 ) -> QuotaCheckResult:
     """Authorize a workflow run before any billable call/text runtime starts.
 
-    The workflow organization is the billing subject for hosted v2. The workflow
-    owner is used only to resolve the effective model configuration and legacy
-    service-key metadata.
+    The workflow organization is the billing subject for hosted deployments.
+    OSS deployments are billed per service key instead. The workflow owner is
+    used only as billing metadata.
     """
+    if organization_id is None:
+        logger.warning(
+            "Workflow start authorization denied: missing organization scope for workflow {}",
+            workflow_id,
+        )
+        return QuotaCheckResult(
+            has_quota=False,
+            error_code="workflow_not_found",
+            error_message="Workflow not found",
+        )
+
     try:
-        workflow = await db_client.get_workflow_by_id(workflow_id)
-        if not workflow:
-            return QuotaCheckResult(
-                has_quota=False,
-                error_code="workflow_not_found",
-                error_message="Workflow not found",
-            )
+        workflow = await db_client.get_workflow(
+            workflow_id,
+            organization_id=organization_id,
+        )
+    except Exception as e:
+        logger.error(
+            "Workflow start authorization denied: failed to load workflow {} for org {}: {}",
+            workflow_id,
+            organization_id,
+            e,
+        )
+        return QuotaCheckResult(
+            has_quota=False,
+            error_code="workflow_not_found",
+            error_message="Workflow not found",
+        )
 
-        actor_org_id = getattr(actor_user, "selected_organization_id", None)
-        if actor_org_id is not None and actor_org_id != workflow.organization_id:
+    if not workflow:
+        logger.warning(
+            "Workflow start authorization denied: workflow {} not found for org {}",
+            workflow_id,
+            organization_id,
+        )
+        return QuotaCheckResult(
+            has_quota=False,
+            error_code="workflow_not_found",
+            error_message="Workflow not found",
+        )
+
+    try:
+        actor_id = getattr(actor_user, "id", None) if actor_user is not None else None
+        if actor_user is not None and actor_id is None:
             logger.warning(
-                "Workflow start authorization denied: actor org {} does not match workflow {} org {}",
-                actor_org_id,
+                "Workflow start authorization denied: actor is missing id for workflow {} org {}",
                 workflow_id,
-                workflow.organization_id,
+                organization_id,
             )
             return QuotaCheckResult(
                 has_quota=False,
@@ -390,6 +429,42 @@ async def authorize_workflow_run_start(
                 error_message="Workflow not found",
             )
 
+        if actor_id is not None:
+            try:
+                is_member = await db_client.is_user_member_of_organization(
+                    user_id=actor_id,
+                    organization_id=organization_id,
+                )
+            except Exception as e:
+                logger.error(
+                    "Workflow start authorization denied: failed to validate actor {} membership for workflow {} org {}: {}",
+                    actor_id,
+                    workflow_id,
+                    organization_id,
+                    e,
+                )
+                return QuotaCheckResult(
+                    has_quota=False,
+                    error_code="workflow_not_found",
+                    error_message="Workflow not found",
+                )
+            if not is_member:
+                logger.warning(
+                    "Workflow start authorization denied: actor {} is not a member of workflow {} org {}",
+                    actor_id,
+                    workflow_id,
+                    organization_id,
+                )
+                return QuotaCheckResult(
+                    has_quota=False,
+                    error_code="workflow_not_found",
+                    error_message="Workflow not found",
+                )
+
+        # A DB read failure here is a "cannot verify" condition, not a
+        # definitive "not found": let it fall through to the outer handler so
+        # it fails closed. The None case below is a genuine missing row and keeps
+        # its specific code.
         workflow_owner = await db_client.get_user_by_id(workflow.user_id)
         if not workflow_owner:
             return QuotaCheckResult(
@@ -398,47 +473,72 @@ async def authorize_workflow_run_start(
                 error_message="User not found",
             )
 
+        # The run executes its pinned definition's configuration, so the MPS
+        # correlation must be minted for the service key in that snapshot.
+        # workflow.workflow_configurations is a legacy column synced to the
+        # draft on every save, which can carry a different service key than
+        # the definition the run will actually use.
+        workflow_configurations = workflow.workflow_configurations
+        if workflow_run_id is not None:
+            # As with the owner lookup, a DB read failure falls through to the
+            # outer fail-closed handler; only a genuinely missing/mismatched run
+            # returns the specific code below.
+            workflow_run = await db_client.get_workflow_run(
+                workflow_run_id, organization_id=organization_id
+            )
+            if workflow_run is None or workflow_run.workflow_id != workflow.id:
+                logger.warning(
+                    "Workflow start authorization denied: workflow run {} not found for workflow {} org {}",
+                    workflow_run_id,
+                    workflow_id,
+                    organization_id,
+                )
+                return QuotaCheckResult(
+                    has_quota=False,
+                    error_code="workflow_run_not_found",
+                    error_message="Workflow run not found",
+                )
+            if workflow_run.definition is not None:
+                workflow_configurations = (
+                    workflow_run.definition.workflow_configurations
+                )
+
         user_config = await get_effective_ai_model_configuration_for_workflow(
-            user_id=workflow_owner.id,
-            organization_id=workflow.organization_id,
-            workflow_configurations=workflow.workflow_configurations,
+            organization_id=organization_id,
+            workflow_configurations=workflow_configurations,
         )
 
         if DEPLOYMENT_MODE != "oss":
-            hosted_result, hosted_enforced = await _authorize_hosted_workflow_run_start(
+            return await _authorize_hosted_workflow_run_start(
                 workflow_owner=workflow_owner,
-                organization_id=workflow.organization_id,
+                organization_id=organization_id,
                 workflow_id=workflow.id,
                 workflow_run_id=workflow_run_id,
                 user_config=user_config,
             )
-            if hosted_enforced or not hosted_result.has_quota:
-                return hosted_result
 
         dograh_api_keys = _dograh_api_keys(user_config)
-        if not dograh_api_keys:
-            return QuotaCheckResult(has_quota=True)
-
-        legacy_result = await _authorize_legacy_dograh_keys(
-            dograh_api_keys=dograh_api_keys,
-            organization_id=(
-                None if DEPLOYMENT_MODE == "oss" else workflow.organization_id
-            ),
-            workflow_owner=workflow_owner,
-        )
-        if not legacy_result.has_quota:
-            return legacy_result
-
-        if DEPLOYMENT_MODE == "oss":
-            return await _authorize_oss_managed_v2_correlation(
-                workflow_id=workflow.id,
-                workflow_run_id=workflow_run_id,
-                user_config=user_config,
+        if dograh_api_keys:
+            oss_result = await _authorize_oss_dograh_keys(
+                dograh_api_keys=dograh_api_keys,
             )
+            if not oss_result.has_quota:
+                return oss_result
 
-        return QuotaCheckResult(has_quota=True)
+        return await _authorize_oss_managed_v2_correlation(
+            workflow_id=workflow.id,
+            workflow_run_id=workflow_run_id,
+            user_config=user_config,
+        )
 
     except Exception as e:
         logger.error(f"Error during quota check: {str(e)}")
-        # On unexpected error, allow the call to proceed
-        return QuotaCheckResult(has_quota=True)
+        # Only an httpx transport failure raised while calling MPS is allowed to
+        # fail open, and those failures are handled at the MPS call sites above.
+        # Database, configuration, response-validation, and programming errors
+        # all reach this handler and fail closed.
+        return QuotaCheckResult(
+            has_quota=False,
+            error_code="quota_check_failed",
+            error_message="Could not verify Dograh credits. Please try again.",
+        )

api/services/storage.py

@@ -9,8 +9,11 @@ from api.constants import (
     MINIO_PUBLIC_ENDPOINT,
     MINIO_SECRET_KEY,
     MINIO_SECURE,
+    S3_ADDRESSING_STYLE,
     S3_BUCKET,
+    S3_ENDPOINT_URL,
     S3_REGION,
+    S3_SIGNATURE_VERSION,
 )
 from api.enums import Environment, StorageBackend
 
@@ -57,7 +60,13 @@ def get_storage_for_backend(backend: str) -> BaseFileSystem:
         logger.info(
             f"Initializing {backend} storage with bucket '{bucket}' in region '{region}'"
         )
-        return S3FileSystem(bucket, region)
+        return S3FileSystem(
+            bucket_name=bucket,
+            region_name=region,
+            endpoint_url=S3_ENDPOINT_URL,
+            signature_version=S3_SIGNATURE_VERSION,
+            addressing_style=S3_ADDRESSING_STYLE,
+        )
 
     # Future backend implementations can be added here:
     # elif backend == StorageBackend.GCS:  # Code 3

api/services/telephony/README.md

@@ -77,7 +77,7 @@ class TelephonyProvider(ABC):
     async def verify_webhook_signature(self, url: str, params: Dict[str, Any], signature: str) -> bool
     
     @abstractmethod
-    async def get_webhook_response(self, workflow_id: int, user_id: int, workflow_run_id: int) -> str
+    async def get_webhook_response(self, workflow_id: int, organization_id: int, workflow_run_id: int) -> str
 ```
 
 ## Configuration Loading

api/services/telephony/ari_manager.py

@@ -26,6 +26,10 @@ from loguru import logger
 from api.constants import REDIS_URL
 from api.db import db_client
 from api.enums import CallType, WorkflowRunMode
+from api.services.call_concurrency import (
+    CallConcurrencyLimitError,
+    call_concurrency,
+)
 from api.services.quota_service import authorize_workflow_run_start
 from api.services.telephony.call_transfer_manager import get_call_transfer_manager
 from api.services.telephony.transfer_event_protocol import (
@@ -119,11 +123,20 @@ class ARIConnection:
         r = await self._get_redis()
         return await r.exists(f"{_EXT_CHANNEL_KEY_PREFIX}{channel_id}") > 0
 
-    async def _delete_ext_channel(self, channel_id: str):
+    async def _delete_ext_channel(self, channel_id: Optional[str]):
         """Remove the external media channel marker."""
+        if not channel_id:
+            return
         r = await self._get_redis()
         await r.delete(f"{_EXT_CHANNEL_KEY_PREFIX}{channel_id}")
 
+    async def _delete_transfer_channel_mapping(self, channel_id: Optional[str]):
+        """Remove transfer destination channel correlation marker."""
+        if not channel_id:
+            return
+        r = await self._get_redis()
+        await r.delete(f"ari:transfer_channel:{channel_id}")
+
     async def _set_pending_bridge(
         self,
         ext_channel_id: str,
@@ -340,19 +353,18 @@ class ARIConnection:
 
                 workflow_run_id = args_dict.get("workflow_run_id")
                 workflow_id = args_dict.get("workflow_id")
-                user_id = args_dict.get("user_id")
 
-                if not workflow_run_id or not workflow_id or not user_id:
+                if not workflow_run_id or not workflow_id:
                     logger.warning(
                         f"[ARI org={self.organization_id}] StasisStart missing required args: "
-                        f"workflow_run_id={workflow_run_id}, workflow_id={workflow_id}, user_id={user_id}"
+                        f"workflow_run_id={workflow_run_id}, workflow_id={workflow_id}"
                     )
                     return
 
                 # Start pipeline connection in background task
                 asyncio.create_task(
                     self._handle_stasis_start(
-                        channel_id, channel_state, workflow_run_id, workflow_id, user_id
+                        channel_id, channel_state, workflow_run_id, workflow_id
                     )
                 )
 
@@ -521,7 +533,6 @@ class ARIConnection:
     async def _create_external_media(
         self,
         workflow_id: str,
-        user_id: str,
         workflow_run_id: str,
         channel_id: Optional[str] = None,
     ) -> str:
@@ -537,10 +548,10 @@ class ARIConnection:
         the POST and avoid racing against the StasisStart event.
         """
         # v() appends URI query params to the websocket_client.conf URL
-        # e.g. wss://api.dograh.com/ws/ari?workflow_id=1&user_id=2&workflow_run_id=3
+        # e.g. wss://api.dograh.com/ws/ari?workflow_id=1&organization_id=2&workflow_run_id=3
         transport_data = (
             f"v(workflow_id={workflow_id},"
-            f"user_id={user_id},"
+            f"organization_id={self.organization_id},"
             f"workflow_run_id={workflow_run_id})"
         )
 
@@ -604,6 +615,8 @@ class ARIConnection:
         channel = event.get("channel", {})
         caller_number = channel.get("caller", {}).get("number", "unknown")
         called_number = channel.get("dialplan", {}).get("exten", "unknown")
+        concurrency_slot = None
+        workflow_run = None
 
         try:
             # 1. Resolve the workflow from the called extension via the
@@ -650,6 +663,20 @@ class ARIConnection:
 
             user_id = workflow.user_id
 
+            try:
+                concurrency_slot = await call_concurrency.acquire_org_slot(
+                    self.organization_id,
+                    source="ari_inbound",
+                    timeout=0,
+                )
+            except CallConcurrencyLimitError:
+                logger.warning(
+                    f"[ARI org={self.organization_id}] Concurrent call limit "
+                    f"reached; hanging up inbound channel {channel_id}"
+                )
+                await self._delete_channel(channel_id)
+                return
+
             # 3. Create workflow run
             call_id = channel_id
             # Capture the upstream-PBX (VICIdial) identity off the SIP headers so
@@ -675,7 +702,9 @@ class ARIConnection:
                 gathered_context={
                     "call_id": call_id,
                 },
+                organization_id=self.organization_id,
             )
+            await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run.id)
 
             logger.info(
                 f"[ARI org={self.organization_id}] Created inbound workflow run "
@@ -687,6 +716,7 @@ class ARIConnection:
             # store the MPS correlation id before the pipeline starts.
             quota_result = await authorize_workflow_run_start(
                 workflow_id=inbound_workflow_id,
+                organization_id=self.organization_id,
                 workflow_run_id=workflow_run.id,
             )
             if not quota_result.has_quota:
@@ -694,6 +724,7 @@ class ARIConnection:
                     f"[ARI org={self.organization_id}] Quota exceeded for user {user_id} "
                     f"— hanging up inbound call {channel_id}"
                 )
+                await call_concurrency.release_workflow_run_slot(workflow_run.id)
                 await self._delete_channel(channel_id)
                 return
 
@@ -706,9 +737,12 @@ class ARIConnection:
                 channel_state,
                 str(workflow_run.id),
                 str(inbound_workflow_id),
-                str(user_id),
             )
         except Exception as e:
+            if workflow_run:
+                await call_concurrency.release_workflow_run_slot(workflow_run.id)
+            elif concurrency_slot:
+                await call_concurrency.release_slot(concurrency_slot)
             logger.error(
                 f"[ARI org={self.organization_id}] Error handling inbound StasisStart "
                 f"for channel {channel_id}: {e}"
@@ -724,7 +758,6 @@ class ARIConnection:
         channel_state: str,
         workflow_run_id: str,
         workflow_id: str,
-        user_id: str,
     ):
         """Set up external media for a caller channel that has entered Stasis.
 
@@ -768,7 +801,6 @@ class ARIConnection:
             # 3. Create the ext media channel with the id we just registered.
             created_id = await self._create_external_media(
                 workflow_id,
-                user_id,
                 workflow_run_id,
                 channel_id=ext_channel_id,
             )
@@ -844,6 +876,14 @@ class ARIConnection:
         the bridge and both channels — like endConferenceOnExit.
         """
         try:
+            # Release the org concurrency slot. Normally the pipeline's own
+            # teardown does this when the ext media websocket closes, but if
+            # the pipeline never started (caller hung up before external
+            # media connected, ext media creation failed, ...) this is the
+            # only cleanup that runs before the Redis stale timeout. No-op
+            # when the slot was already released.
+            await call_concurrency.unregister_active_call(int(workflow_run_id))
+
             workflow_run = await db_client.get_workflow_run_by_id(int(workflow_run_id))
             if not workflow_run or not workflow_run.gathered_context:
                 logger.warning(
@@ -859,6 +899,11 @@ class ARIConnection:
             ext_channel_id = ctx.get("ext_channel_id")
             bridge_id = ctx.get("bridge_id")
             transfer_state = ctx.get("transfer_state")
+            transfer_bridge_id = ctx.get("transfer_bridge_id") or bridge_id
+            transfer_caller_channel_id = (
+                ctx.get("transfer_caller_channel_id") or call_id
+            )
+            transfer_destination_channel_id = ctx.get("transfer_destination_channel_id")
 
             # Check if this is a call transfer scenario external channel. Skip full teardown if
             # transfer is in progress and this is the external media channel
@@ -889,6 +934,64 @@ class ARIConnection:
                 )
                 return
 
+            if (
+                transfer_state == "complete"
+                and transfer_bridge_id
+                and transfer_caller_channel_id
+                and transfer_destination_channel_id
+                and channel_id
+                in (
+                    transfer_caller_channel_id,
+                    transfer_destination_channel_id,
+                )
+            ):
+                peer_channel_id = (
+                    transfer_destination_channel_id
+                    if channel_id == transfer_caller_channel_id
+                    else transfer_caller_channel_id
+                )
+                logger.info(
+                    f"[ARI org={self.organization_id}] Completed transfer participant "
+                    f"{channel_id} left Stasis; tearing down peer {peer_channel_id} "
+                    f"and bridge {transfer_bridge_id}"
+                )
+
+                # Mark terminal state before issuing ARI deletes so duplicate
+                # StasisEnd events run through the idempotent normal cleanup path.
+                ctx["transfer_state"] = "terminated"
+                await db_client.update_workflow_run(
+                    run_id=int(workflow_run_id), gathered_context=ctx
+                )
+
+                await self._delete_bridge(transfer_bridge_id)
+                if peer_channel_id and peer_channel_id != channel_id:
+                    await self._delete_channel(peer_channel_id)
+
+                keys_to_delete = [
+                    cid
+                    for cid in (
+                        transfer_caller_channel_id,
+                        transfer_destination_channel_id,
+                        ext_channel_id,
+                        channel_id,
+                    )
+                    if cid
+                ]
+                if keys_to_delete:
+                    await self._delete_channel_run(*keys_to_delete)
+
+                await self._delete_ext_channel(ext_channel_id)
+                await self._delete_transfer_channel_mapping(
+                    transfer_destination_channel_id
+                )
+
+                logger.info(
+                    f"[ARI org={self.organization_id}] Completed transfer teardown "
+                    f"finished for channel={channel_id}, peer={peer_channel_id}, "
+                    f"bridge={transfer_bridge_id}"
+                )
+                return
+
             # Normal full teardown for non-transfer scenarios (transfer_state is None or not in-progress)
             # Delete the bridge first (removes channels from it)
             if bridge_id:
@@ -908,6 +1011,7 @@ class ARIConnection:
 
             # Clean up the Redis marker for external channel
             await self._delete_ext_channel(ext_channel_id)
+            await self._delete_transfer_channel_mapping(transfer_destination_channel_id)
 
             logger.info(
                 f"[ARI org={self.organization_id}] StasisEnd full teardown for "

api/services/telephony/base.py

@@ -56,6 +56,15 @@ class NormalizedInboundData:
     raw_data: Dict[str, Any] = field(default_factory=dict)  # Original webhook data
 
 
+@dataclass
+class AnsweringMachineDetectionResult:
+    """Standardized answering-machine detection result across providers."""
+
+    call_id: str
+    answered_by: str
+    raw_data: Dict[str, Any] = field(default_factory=dict)
+
+
 class TelephonyProvider(ABC):
     """
     Abstract base class for telephony providers.
@@ -141,14 +150,15 @@ class TelephonyProvider(ABC):
 
     @abstractmethod
     async def get_webhook_response(
-        self, workflow_id: int, user_id: int, workflow_run_id: int
+        self, workflow_id: int, organization_id: int, workflow_run_id: int
     ) -> str:
         """
         Generate the initial webhook response for starting a call session.
 
         Args:
             workflow_id: The workflow ID
-            user_id: The user ID
+            organization_id: The organization owning the workflow; providers
+                embed it in the media websocket URL they hand back
             workflow_run_id: The workflow run ID
 
         Returns:
@@ -192,12 +202,29 @@ class TelephonyProvider(ABC):
         """
         pass
 
+    def supports_answering_machine_detection(self) -> bool:
+        """Return whether this provider can request answering-machine detection."""
+        return False
+
+    def apply_answering_machine_detection_call_params(
+        self,
+        data: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        """Add provider-specific AMD parameters to an outbound call request."""
+        return data
+
+    def parse_answering_machine_detection_result(
+        self, data: Dict[str, Any]
+    ) -> Optional[AnsweringMachineDetectionResult]:
+        """Parse provider-specific callback data into a normalized AMD result."""
+        return None
+
     @abstractmethod
     async def handle_websocket(
         self,
         websocket: "WebSocket",
         workflow_id: int,
-        user_id: int,
+        organization_id: int,
         workflow_run_id: int,
     ) -> None:
         """
@@ -206,10 +233,14 @@ class TelephonyProvider(ABC):
         This method encapsulates all provider-specific WebSocket handshake and
         message routing logic, keeping the main websocket endpoint clean.
 
+        ``organization_id`` is the tenant that every workflow/run lookup must be
+        scoped by. The workflow owner is deliberately not passed in — derive it
+        from the workflow row where it's needed for attribution.
+
         Args:
             websocket: The WebSocket connection
             workflow_id: The workflow ID
-            user_id: The user ID
+            organization_id: The organization owning the workflow and run
             workflow_run_id: The workflow run ID
         """
         pass
@@ -329,17 +360,19 @@ class TelephonyProvider(ABC):
         *,
         organization_id: int,
         workflow_id: int,
-        user_id: int,
         workflow_run_id: int,
         params: Dict[str, str],
     ) -> None:
-        """Handle the agent-stream WebSocket where credentials are passed inline.
+        """Handle the provider-specific agent-stream WebSocket.
 
-        Used by ``/api/v1/agent-stream/{workflow_uuid}`` when the caller carries
-        provider credentials in the query string (no stored
-        ``TelephonyConfigurationModel`` row required). ``organization_id`` is
-        passed so providers can scope any config lookups to the workflow's
-        org. Default raises so providers that haven't opted in fail loudly.
+        Used by ``/api/v1/agent-stream/{provider_name}/{workflow_uuid}`` when
+        the caller carries a provider stream protocol. ``organization_id`` is
+        passed so providers can scope any config lookups to the workflow's org.
+        Default raises so providers that haven't opted in fail loudly.
+
+        The route holds an org concurrency slot while this runs, so
+        implementations must bound their pre-pipeline handshake reads with a
+        timeout — an idle socket must not hold the slot indefinitely.
         """
         raise NotImplementedError(
             f"Agent-stream not supported for provider {self.PROVIDER_NAME}"