apunkt/dograh
1
0
Fork 0
mirror of https://github.com/dograh-hq/dograh.git synced 2026-07-10 11:12:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix: increase concurrency limit an handle it across all call paths (#508)

Browse source 
* fix: increase concurrency limit an handle it across all call paths

* fix: fix review comments and test

* fix: address concurrency review findings (campaign-scoped counter, cleanup hardening, webrtc run validation)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat: emit usage_concurrent_call_limit_reached PostHog event

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix: align usage event with MPS org-event convention (per-member fan-out)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat: cap max_call_duration at 20 min via typed workflow_configurations request

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Abhishek 2026-07-09 18:29:01 +05:30 • committed by GitHub
parent f3bcf24370
commit 041c31a613
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
40 changed files with 2369 additions and 467 deletions
Download patch file Download diff file Expand all files Collapse all files

6
api/constants.py
View file

@ -150,7 +150,11 @@ COUNTRY_CODES = {
"IE": "353", # Ireland
}
DEFAULT_ORG_CONCURRENCY_LIMIT = os.getenv("DEFAULT_ORG_CONCURRENCY_LIMIT", 2)
# Floor at 1 so a misconfigured env var (0 or negative) can't silently block
# every call in the deployment.
DEFAULT_ORG_CONCURRENCY_LIMIT = max(
1, int(os.getenv("DEFAULT_ORG_CONCURRENCY_LIMIT", "10"))
)
DEFAULT_CAMPAIGN_RETRY_CONFIG = {
"enabled": True,
"max_retries": 1,

17
api/db/organization_client.py
View file

@ -9,6 +9,7 @@ from api.db.base_client import BaseDBClient
from api.db.models import (
APIKeyModel,
OrganizationModel,
UserModel,
organization_users_association,
)
from api.utils.api_key import generate_api_key
@ -25,6 +26,22 @@ class OrganizationClient(BaseDBClient):
)
return result.scalars().first()
async def get_organization_users(self, organization_id: int) -> list[UserModel]:
"""Get all users linked to an organization (many-to-many)."""
async with self.async_session() as session:
result = await session.execute(
select(UserModel)
.join(
organization_users_association,
organization_users_association.c.user_id == UserModel.id,
)
.where(
organization_users_association.c.organization_id == organization_id
)
.order_by(UserModel.id)
)
return list(result.scalars().all())
async def get_or_create_organization_by_provider_id(
self, org_provider_id: str, user_id: int
) -> tuple[OrganizationModel, bool]:

2
api/enums.py
View file

@ -200,3 +200,5 @@ class PostHogEvent(str, Enum):
SIGNED_IN = "signed_in"
ORGANIZATION_CREATED = "organization_created"
ORGANIZATION_USER_ASSOCIATED = "organization_user_associated"
# usage_* events track orgs hitting capacity/limit boundaries
USAGE_CONCURRENT_CALL_LIMIT_REACHED = "usage_concurrent_call_limit_reached"

2
api/errors/telephony_errors.py
View file

@ -14,6 +14,7 @@ class TelephonyError(Enum):
ACCOUNT_VALIDATION_FAILED = "ACCOUNT_VALIDATION_FAILED"
PHONE_NUMBER_NOT_CONFIGURED = "PHONE_NUMBER_NOT_CONFIGURED"
SIGNATURE_VALIDATION_FAILED = "SIGNATURE_VALIDATION_FAILED"
CONCURRENT_CALL_LIMIT = "CONCURRENT_CALL_LIMIT"
QUOTA_EXCEEDED = "QUOTA_EXCEEDED"
GENERAL_AUTH_FAILED = "GENERAL_AUTH_FAILED"
VALID = "VALID"
@ -26,6 +27,7 @@ TELEPHONY_ERROR_MESSAGES = {
TelephonyError.ACCOUNT_VALIDATION_FAILED: "Authentication error: Account credentials do not match. Please verify your account SID configuration in the dashboard matches your telephony provider settings.",
TelephonyError.PHONE_NUMBER_NOT_CONFIGURED: "Phone number not configured: This number is not set up for inbound calls in your account. Please add this number to your telephony configuration.",
TelephonyError.SIGNATURE_VALIDATION_FAILED: "Security error: Webhook signature validation failed. Please verify your auth token configuration and ensure requests are coming from your telephony provider.",
TelephonyError.CONCURRENT_CALL_LIMIT: "Service temporarily unavailable: Your account has reached its concurrent call limit. Please try again later.",
TelephonyError.QUOTA_EXCEEDED: "Service temporarily unavailable: Your account has exceeded usage limits. Please contact your administrator or upgrade your plan to continue receiving calls.",
TelephonyError.GENERAL_AUTH_FAILED: "Authentication failed: Please check your webhook URL configuration and ensure your telephony provider settings match your dashboard configuration.",
}

90
api/routes/agent_stream.py
View file

@ -18,6 +18,10 @@ from starlette.websockets import WebSocketDisconnect
from api.db import db_client
from api.enums import CallType, WorkflowRunState
from api.services.call_concurrency import (
CallConcurrencyLimitError,
call_concurrency,
)
from api.services.quota_service import authorize_workflow_run_start
from api.services.telephony import registry as telephony_registry
@ -51,6 +55,16 @@ async def agent_stream_websocket(
await websocket.close(code=1008, reason="Workflow not found")
return
try:
concurrency_slot = await call_concurrency.acquire_org_slot(
workflow.organization_id,
source=f"agent_stream:{provider_name}",
timeout=0,
)
except CallConcurrencyLimitError:
await websocket.close(code=1008, reason="Concurrent call limit reached")
return
numeric_suffix = int(str(uuid.uuid4()).replace("-", "")[:8], 16) % 100000000
workflow_run_name = f"WR-AGS-{numeric_suffix:08d}"
initial_context = {
@ -58,14 +72,20 @@ async def agent_stream_websocket(
"provider": provider_name,
"direction": "inbound",
}
workflow_run = await db_client.create_workflow_run(
workflow_run_name,
workflow.id,
provider_name,
user_id=workflow.user_id,
call_type=CallType.INBOUND,
initial_context=initial_context,
)
try:
workflow_run = await db_client.create_workflow_run(
workflow_run_name,
workflow.id,
provider_name,
user_id=workflow.user_id,
call_type=CallType.INBOUND,
initial_context=initial_context,
organization_id=workflow.organization_id,
)
await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run.id)
except Exception:
await call_concurrency.release_slot(concurrency_slot)
raise
set_current_run_id(workflow_run.id)
set_current_org_id(workflow.organization_id)
@ -79,36 +99,40 @@ async def agent_stream_websocket(
f"agent-stream quota exceeded for user {workflow.user_id}: "
f"{quota_result.error_message}"
)
await call_concurrency.release_workflow_run_slot(workflow_run.id)
await websocket.close(
code=1008, reason=quota_result.error_message or "Quota exceeded"
)
return
await db_client.update_workflow_run(
run_id=workflow_run.id, state=WorkflowRunState.RUNNING.value
)
provider_instance = spec.provider_cls({})
try:
await provider_instance.handle_external_websocket(
websocket,
organization_id=workflow.organization_id,
workflow_id=workflow.id,
user_id=workflow.user_id,
workflow_run_id=workflow_run.id,
params=params,
await db_client.update_workflow_run(
run_id=workflow_run.id, state=WorkflowRunState.RUNNING.value
)
except NotImplementedError as e:
logger.warning(f"agent-stream provider {provider_name} not supported: {e}")
provider_instance = spec.provider_cls({})
try:
await websocket.close(code=1011, reason=str(e))
except RuntimeError:
pass
except WebSocketDisconnect as e:
logger.info(f"agent-stream disconnected: code={e.code} reason={e.reason}")
except Exception as e:
logger.error(f"agent-stream error for run {workflow_run.id}: {e}")
try:
await websocket.close(1011, "Internal server error")
except RuntimeError:
pass
await provider_instance.handle_external_websocket(
websocket,
organization_id=workflow.organization_id,
workflow_id=workflow.id,
user_id=workflow.user_id,
workflow_run_id=workflow_run.id,
params=params,
)
except NotImplementedError as e:
logger.warning(f"agent-stream provider {provider_name} not supported: {e}")
try:
await websocket.close(code=1011, reason=str(e))
except RuntimeError:
pass
except WebSocketDisconnect as e:
logger.info(f"agent-stream disconnected: code={e.code} reason={e.reason}")
except Exception as e:
logger.error(f"agent-stream error for run {workflow_run.id}: {e}")
try:
await websocket.close(1011, "Internal server error")
except RuntimeError:
pass
finally:
await call_concurrency.unregister_active_call(workflow_run.id)

47
api/routes/public_agent.py
View file

@ -14,6 +14,10 @@ from pydantic import BaseModel
from api.db import db_client
from api.enums import TriggerState, WorkflowStatus
from api.services.call_concurrency import (
CallConcurrencyLimitError,
call_concurrency,
)
from api.services.quota_service import authorize_workflow_run_start
from api.services.telephony.factory import (
get_default_telephony_provider,
@ -243,15 +247,32 @@ async def _execute_resolved_target(
initial_context["api_key_created_by"] = api_key_created_by
initial_context.update(request.initial_context or {})
workflow_run = await db_client.create_workflow_run(
name=workflow_run_name,
workflow_id=target.workflow.id,
mode=workflow_run_mode,
initial_context=initial_context,
user_id=execution_user_id,
use_draft=use_draft,
organization_id=target.organization_id,
)
try:
concurrency_slot = await call_concurrency.acquire_org_slot(
target.organization_id,
source="public_agent",
timeout=0,
)
except CallConcurrencyLimitError:
raise HTTPException(
status_code=429,
detail="Concurrent call limit reached",
)
try:
workflow_run = await db_client.create_workflow_run(
name=workflow_run_name,
workflow_id=target.workflow.id,
mode=workflow_run_mode,
initial_context=initial_context,
user_id=execution_user_id,
use_draft=use_draft,
organization_id=target.organization_id,
)
await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run.id)
except Exception:
await call_concurrency.release_slot(concurrency_slot)
raise
logger.info(
f"Created workflow run {workflow_run.id} for public agent "
@ -267,10 +288,15 @@ async def _execute_resolved_target(
workflow_run_id=workflow_run.id,
)
if not quota_result.has_quota:
await call_concurrency.release_workflow_run_slot(workflow_run.id)
raise HTTPException(status_code=402, detail=quota_result.error_message)
# 9. Construct webhook URL for telephony provider callback
backend_endpoint, _ = await get_backend_endpoints()
try:
backend_endpoint, _ = await get_backend_endpoints()
except Exception:
await call_concurrency.release_workflow_run_slot(workflow_run.id)
raise
webhook_endpoint = provider.WEBHOOK_ENDPOINT
webhook_url = (
@ -297,6 +323,7 @@ async def _execute_resolved_target(
logger.warning(
f"Failed to initiate call for workflow run {workflow_run.id}: {e}"
)
await call_concurrency.release_workflow_run_slot(workflow_run.id)
raise HTTPException(
status_code=400,
detail=f"Failed to initiate call: {e}",

1
api/routes/public_embed.py
View file

@ -313,6 +313,7 @@ async def initialize_embed_session(
**(init_request.context_variables or {}),
"provider": WorkflowRunMode.SMALLWEBRTC.value,
},
organization_id=embed_token.organization_id,
)
except Exception as e:
logger.error(f"Failed to create workflow run: {e}")

342
api/routes/telephony.py
View file

@ -25,6 +25,11 @@ from api.enums import CallType, WorkflowRunMode, WorkflowRunState
from api.errors.telephony_errors import TelephonyError
from api.sdk_expose import sdk_expose
from api.services.auth.depends import get_user
from api.services.call_concurrency import (
CallConcurrencyLimitError,
WorkflowRunSlotAlreadyBoundError,
call_concurrency,
)
from api.services.quota_service import authorize_workflow_run_start
from api.services.telephony.call_transfer_manager import get_call_transfer_manager
from api.services.telephony.factory import (
@ -139,70 +144,6 @@ async def initiate_call(
# Determine the workflow run mode based on provider type
workflow_run_mode = provider.PROVIDER_NAME
workflow_run_id = request.workflow_run_id
if not workflow_run_id:
# Merge template context variables (e.g. caller_number, called_number
# set in workflow settings for testing pre-call data fetch).
template_vars = workflow.template_context_variables or {}
numeric_suffix = int(str(uuid.uuid4()).replace("-", "")[:8], 16) % 100000000
workflow_run_name = f"WR-TEL-OUT-{numeric_suffix:08d}"
workflow_run = await db_client.create_workflow_run(
workflow_run_name,
workflow.id,
workflow_run_mode,
user_id=execution_user_id,
call_type=CallType.OUTBOUND,
initial_context={
**template_vars,
"phone_number": phone_number,
"called_number": phone_number,
"provider": provider.PROVIDER_NAME,
"telephony_configuration_id": telephony_configuration_id,
},
use_draft=True,
organization_id=user.selected_organization_id,
)
workflow_run_id = workflow_run.id
else:
workflow_run = await db_client.get_workflow_run(
workflow_run_id, organization_id=user.selected_organization_id
)
if not workflow_run:
raise HTTPException(status_code=400, detail="Workflow run not found")
if workflow_run.workflow_id != workflow.id:
raise HTTPException(
status_code=400,
detail="workflow_run_workflow_mismatch",
)
workflow_run_name = workflow_run.name
# Check Dograh quota after the run exists so hosted v2 can mint and store
# the MPS correlation id before initiating the call.
quota_result = await authorize_workflow_run_start(
workflow_id=workflow.id,
workflow_run_id=workflow_run_id,
actor_user=user,
)
if not quota_result.has_quota:
raise HTTPException(status_code=402, detail=quota_result.error_message)
# Construct webhook URL based on provider type
backend_endpoint, _ = await get_backend_endpoints()
webhook_endpoint = provider.WEBHOOK_ENDPOINT
webhook_url = (
f"{backend_endpoint}/api/v1/telephony/{webhook_endpoint}"
f"?workflow_id={workflow.id}"
f"&user_id={execution_user_id}"
f"&workflow_run_id={workflow_run_id}"
f"&organization_id={user.selected_organization_id}"
)
keywords = {"workflow_id": workflow.id, "user_id": execution_user_id}
# Resolve optional caller-ID. The config has already been validated against
# the user's organization, so filtering by config_id is sufficient for
# tenant isolation.
@ -220,14 +161,102 @@ async def initiate_call(
raise HTTPException(status_code=400, detail="from_phone_number_not_found")
from_number = phone_row.address_normalized
# Initiate call via provider
result = await provider.initiate_call(
to_number=phone_number,
webhook_url=webhook_url,
workflow_run_id = request.workflow_run_id
try:
concurrency_slot = await call_concurrency.acquire_org_slot(
user.selected_organization_id,
source="telephony_outbound",
timeout=0,
)
except CallConcurrencyLimitError:
raise HTTPException(status_code=429, detail="Concurrent call limit reached")
try:
if not workflow_run_id:
# Merge template context variables (e.g. caller_number, called_number
# set in workflow settings for testing pre-call data fetch).
template_vars = workflow.template_context_variables or {}
numeric_suffix = int(str(uuid.uuid4()).replace("-", "")[:8], 16) % 100000000
workflow_run_name = f"WR-TEL-OUT-{numeric_suffix:08d}"
workflow_run = await db_client.create_workflow_run(
workflow_run_name,
workflow.id,
workflow_run_mode,
user_id=execution_user_id,
call_type=CallType.OUTBOUND,
initial_context={
**template_vars,
"phone_number": phone_number,
"called_number": phone_number,
"provider": provider.PROVIDER_NAME,
"telephony_configuration_id": telephony_configuration_id,
},
use_draft=True,
organization_id=user.selected_organization_id,
)
workflow_run_id = workflow_run.id
else:
workflow_run = await db_client.get_workflow_run(
workflow_run_id, organization_id=user.selected_organization_id
)
if not workflow_run:
raise HTTPException(status_code=400, detail="Workflow run not found")
if workflow_run.workflow_id != workflow.id:
raise HTTPException(
status_code=400,
detail="workflow_run_workflow_mismatch",
)
workflow_run_name = workflow_run.name
await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run_id)
except WorkflowRunSlotAlreadyBoundError:
raise HTTPException(
status_code=409,
detail="Workflow run already has an active call",
)
except Exception:
await call_concurrency.release_slot(concurrency_slot)
raise
# Check Dograh quota after the run exists so hosted v2 can mint and store
# the MPS correlation id before initiating the call.
quota_result = await authorize_workflow_run_start(
workflow_id=workflow.id,
workflow_run_id=workflow_run_id,
from_number=from_number,
**keywords,
actor_user=user,
)
if not quota_result.has_quota:
await call_concurrency.release_workflow_run_slot(workflow_run_id)
raise HTTPException(status_code=402, detail=quota_result.error_message)
try:
# Construct webhook URL based on provider type
backend_endpoint, _ = await get_backend_endpoints()
webhook_endpoint = provider.WEBHOOK_ENDPOINT
webhook_url = (
f"{backend_endpoint}/api/v1/telephony/{webhook_endpoint}"
f"?workflow_id={workflow.id}"
f"&user_id={execution_user_id}"
f"&workflow_run_id={workflow_run_id}"
f"&organization_id={user.selected_organization_id}"
)
keywords = {"workflow_id": workflow.id, "user_id": execution_user_id}
# Initiate call via provider
result = await provider.initiate_call(
to_number=phone_number,
webhook_url=webhook_url,
workflow_run_id=workflow_run_id,
from_number=from_number,
**keywords,
)
except Exception:
await call_concurrency.release_workflow_run_slot(workflow_run_id)
raise
# Store provider metadata and caller_number in workflow run context
gathered_context = {
@ -425,6 +454,7 @@ async def _create_inbound_workflow_run(
normalized_data,
telephony_configuration_id: int,
from_phone_number_id: Optional[int] = None,
organization_id: int | None = None,
) -> int:
"""Create workflow run for inbound call and return run ID"""
call_id = normalized_data.call_id
@ -456,6 +486,7 @@ async def _create_inbound_workflow_run(
"raw_webhook_data": normalized_data.raw_data,
},
},
organization_id=organization_id,
)
logger.info(
@ -765,40 +796,67 @@ async def handle_inbound_run(request: Request):
TelephonyError.SIGNATURE_VALIDATION_FAILED
)
# 5. Create workflow run + authorize quota before returning provider
# stream instructions.
workflow_run_id = await _create_inbound_workflow_run(
workflow_id,
user_id,
provider_class.PROVIDER_NAME,
normalized_data,
telephony_configuration_id=telephony_configuration_id,
from_phone_number_id=phone_row.id,
)
quota_result = await authorize_workflow_run_start(
workflow_id=workflow_id,
workflow_run_id=workflow_run_id,
)
if not quota_result.has_quota:
logger.warning(
f"User {user_id} has exceeded quota: {quota_result.error_message}"
try:
concurrency_slot = await call_concurrency.acquire_org_slot(
config.organization_id,
source=f"inbound:{provider_class.PROVIDER_NAME}",
timeout=0,
)
except CallConcurrencyLimitError:
return provider_class.generate_validation_error_response(
TelephonyError.QUOTA_EXCEEDED
TelephonyError.CONCURRENT_CALL_LIMIT
)
backend_endpoint, wss_backend_endpoint = await get_backend_endpoints()
websocket_url = (
f"{wss_backend_endpoint}/api/v1/telephony/ws/"
f"{workflow_id}/{user_id}/{workflow_run_id}"
)
workflow_run_id = None
try:
# 5. Create workflow run + authorize quota before returning provider
# stream instructions.
workflow_run_id = await _create_inbound_workflow_run(
workflow_id,
user_id,
provider_class.PROVIDER_NAME,
normalized_data,
telephony_configuration_id=telephony_configuration_id,
from_phone_number_id=phone_row.id,
organization_id=config.organization_id,
)
await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run_id)
return await provider_instance.start_inbound_stream(
websocket_url=websocket_url,
workflow_run_id=workflow_run_id,
normalized_data=normalized_data,
backend_endpoint=backend_endpoint,
)
quota_result = await authorize_workflow_run_start(
workflow_id=workflow_id,
workflow_run_id=workflow_run_id,
)
if not quota_result.has_quota:
logger.warning(
f"User {user_id} has exceeded quota: {quota_result.error_message}"
)
await call_concurrency.release_workflow_run_slot(workflow_run_id)
return provider_class.generate_validation_error_response(
TelephonyError.QUOTA_EXCEEDED
)
backend_endpoint, wss_backend_endpoint = await get_backend_endpoints()
websocket_url = (
f"{wss_backend_endpoint}/api/v1/telephony/ws/"
f"{workflow_id}/{user_id}/{workflow_run_id}"
)
return await provider_instance.start_inbound_stream(
websocket_url=websocket_url,
workflow_run_id=workflow_run_id,
normalized_data=normalized_data,
backend_endpoint=backend_endpoint,
)
except WorkflowRunSlotAlreadyBoundError:
return provider_class.generate_validation_error_response(
TelephonyError.CONCURRENT_CALL_LIMIT
)
except Exception:
if workflow_run_id:
await call_concurrency.release_workflow_run_slot(workflow_run_id)
else:
await call_concurrency.release_slot(concurrency_slot)
raise
except ValueError as e:
logger.error(f"/inbound/run request parsing error: {e}")
@ -900,38 +958,72 @@ async def handle_inbound_telephony(
logger.error(f"Request validation failed: {error_type}")
return provider_class.generate_validation_error_response(error_type)
# Create workflow run.
user_id = workflow_context["user_id"]
workflow_run_id = await _create_inbound_workflow_run(
workflow_id,
workflow_context["user_id"],
workflow_context["provider"],
normalized_data,
telephony_configuration_id=workflow_context["telephony_configuration_id"],
from_phone_number_id=workflow_context.get("from_phone_number_id"),
)
quota_result = await authorize_workflow_run_start(
workflow_id=workflow_id,
workflow_run_id=workflow_run_id,
)
if not quota_result.has_quota:
logger.warning(
f"User {user_id} has exceeded quota for inbound calls: {quota_result.error_message}"
organization_id = workflow_context["organization_id"]
try:
concurrency_slot = await call_concurrency.acquire_org_slot(
organization_id,
source=f"inbound_legacy:{workflow_context['provider']}",
timeout=0,
)
except CallConcurrencyLimitError:
return provider_class.generate_validation_error_response(
TelephonyError.QUOTA_EXCEEDED
TelephonyError.CONCURRENT_CALL_LIMIT
)
# Generate response URLs
backend_endpoint, wss_backend_endpoint = await get_backend_endpoints()
websocket_url = f"{wss_backend_endpoint}/api/v1/telephony/ws/{workflow_id}/{workflow_context['user_id']}/{workflow_run_id}"
workflow_run_id = None
try:
# Create workflow run.
workflow_run_id = await _create_inbound_workflow_run(
workflow_id,
workflow_context["user_id"],
workflow_context["provider"],
normalized_data,
telephony_configuration_id=workflow_context[
"telephony_configuration_id"
],
from_phone_number_id=workflow_context.get("from_phone_number_id"),
organization_id=organization_id,
)
await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run_id)
response = await provider_instance.start_inbound_stream(
websocket_url=websocket_url,
workflow_run_id=workflow_run_id,
normalized_data=normalized_data,
backend_endpoint=backend_endpoint,
)
quota_result = await authorize_workflow_run_start(
workflow_id=workflow_id,
workflow_run_id=workflow_run_id,
)
if not quota_result.has_quota:
logger.warning(
f"User {user_id} has exceeded quota for inbound calls: "
f"{quota_result.error_message}"
)
await call_concurrency.release_workflow_run_slot(workflow_run_id)
return provider_class.generate_validation_error_response(
TelephonyError.QUOTA_EXCEEDED
)
# Generate response URLs
backend_endpoint, wss_backend_endpoint = await get_backend_endpoints()
websocket_url = (
f"{wss_backend_endpoint}/api/v1/telephony/ws/"
f"{workflow_id}/{workflow_context['user_id']}/{workflow_run_id}"
)
response = await provider_instance.start_inbound_stream(
websocket_url=websocket_url,
workflow_run_id=workflow_run_id,
normalized_data=normalized_data,
backend_endpoint=backend_endpoint,
)
except WorkflowRunSlotAlreadyBoundError:
return provider_class.generate_validation_error_response(
TelephonyError.CONCURRENT_CALL_LIMIT
)
except Exception:
if workflow_run_id:
await call_concurrency.release_workflow_run_slot(workflow_run_id)
else:
await call_concurrency.release_slot(concurrency_slot)
raise
logger.info(
f"Generated {normalized_data.provider} response for call {normalized_data.call_id}"

222
api/routes/webrtc_signaling.py
View file

@ -40,6 +40,11 @@ from api.routes.turn_credentials import (
generate_turn_credentials,
)
from api.services.auth.depends import get_user_ws
from api.services.call_concurrency import (
CallConcurrencyLimitError,
WorkflowRunSlotAlreadyBoundError,
call_concurrency,
)
from api.services.pipecat.run_pipeline import run_pipeline_smallwebrtc
from api.services.pipecat.ws_sender_registry import (
register_ws_sender,
@ -321,6 +326,8 @@ class SignalingManager:
workflow_id: int,
workflow_run_id: int,
user: UserModel,
enforce_call_concurrency: bool = False,
call_concurrency_source: str = "webrtc",
):
"""Handle WebSocket connection for signaling."""
await websocket.accept()
@ -338,6 +345,8 @@ class SignalingManager:
workflow_run_id,
user,
connection_key,
enforce_call_concurrency,
call_concurrency_source,
)
except WebSocketDisconnect:
logger.info(f"WebSocket disconnected for {connection_id}")
@ -379,6 +388,8 @@ class SignalingManager:
workflow_run_id: int,
user: UserModel,
connection_key: str,
enforce_call_concurrency: bool,
call_concurrency_source: str = "webrtc",
):
"""Handle incoming WebSocket messages."""
msg_type = message.get("type")
@ -386,7 +397,14 @@ class SignalingManager:
if msg_type == "offer":
await self._handle_offer(
ws, payload, workflow_id, workflow_run_id, user, connection_key
ws,
payload,
workflow_id,
workflow_run_id,
user,
connection_key,
enforce_call_concurrency,
call_concurrency_source,
)
elif msg_type == "ice-candidate":
await self._handle_ice_candidate(payload, connection_key)
@ -401,6 +419,8 @@ class SignalingManager:
workflow_run_id: int,
user: UserModel,
connection_key: str,
enforce_call_concurrency: bool,
call_concurrency_source: str = "webrtc",
):
"""Handle offer message and create answer with ICE trickling."""
pc_id = payload.get("pc_id")
@ -472,69 +492,135 @@ class SignalingManager:
}
)
else:
concurrency_slot = None
concurrency_bound = False
pipeline_started = False
pc = None
if enforce_call_concurrency:
if org_id is None:
await ws.send_json(
{
"type": "error",
"payload": {
"error_type": "organization_not_found",
"message": "Workflow organization not found",
},
}
)
return
try:
concurrency_slot = await call_concurrency.acquire_org_slot(
org_id,
source=call_concurrency_source,
timeout=0,
)
await call_concurrency.bind_workflow_run(
concurrency_slot,
workflow_run_id,
)
concurrency_bound = True
except CallConcurrencyLimitError:
await ws.send_json(
{
"type": "error",
"payload": {
"error_type": "concurrency_limit_exceeded",
"message": "Concurrent call limit reached",
},
}
)
return
except WorkflowRunSlotAlreadyBoundError:
await ws.send_json(
{
"type": "error",
"payload": {
"error_type": "workflow_run_already_active",
"message": "Workflow run already has an active call",
},
}
)
return
# Create new connection using correct SmallWebRTC API
# Generate ICE servers with time-limited TURN credentials for this user
user_ice_servers = get_ice_servers(user_id=str(user.id))
pc = SmallWebRTCConnection(
ice_servers=user_ice_servers, connection_timeout_secs=60
)
# Set the pc_id before initialization so it's available in get_answer()
pc._pc_id = pc_id
# Initialize connection with offer
await pc.initialize(sdp=sdp, type=type_)
# Store peer connection using client's pc_id
self._track_peer_connection(connection_key, pc_id, pc)
# Register WebSocket sender for real-time feedback
async def ws_sender(message: dict):
if ws.application_state == WebSocketState.CONNECTED:
await ws.send_json(message)
register_ws_sender(workflow_run_id, ws_sender)
# Setup closed handler
@pc.event_handler("closed")
async def handle_disconnected(webrtc_connection: SmallWebRTCConnection):
logger.info(f"PeerConnection closed: {webrtc_connection.pc_id}")
owner_connection_id = self._forget_peer_connection(
webrtc_connection.pc_id
try:
user_ice_servers = get_ice_servers(user_id=str(user.id))
pc = SmallWebRTCConnection(
ice_servers=user_ice_servers, connection_timeout_secs=60
)
if owner_connection_id == connection_key:
await self._notify_call_ended_and_close_websocket(
ws,
workflow_run_id,
webrtc_connection.pc_id,
reason="peer_connection_closed",
# Set the pc_id before initialization so it's available in get_answer()
pc._pc_id = pc_id
# Initialize connection with offer
await pc.initialize(sdp=sdp, type=type_)
# Store peer connection using client's pc_id
self._track_peer_connection(connection_key, pc_id, pc)
# Register WebSocket sender for real-time feedback
async def ws_sender(message: dict):
if ws.application_state == WebSocketState.CONNECTED:
await ws.send_json(message)
register_ws_sender(workflow_run_id, ws_sender)
# Setup closed handler
@pc.event_handler("closed")
async def handle_disconnected(
webrtc_connection: SmallWebRTCConnection,
):
logger.info(f"PeerConnection closed: {webrtc_connection.pc_id}")
owner_connection_id = self._forget_peer_connection(
webrtc_connection.pc_id
)
if owner_connection_id == connection_key:
await self._notify_call_ended_and_close_websocket(
ws,
workflow_run_id,
webrtc_connection.pc_id,
reason="peer_connection_closed",
)
# Start pipeline in background
asyncio.create_task(
run_pipeline_smallwebrtc(
pc,
workflow_id,
workflow_run_id,
user.id,
call_context_vars,
user_provider_id=str(user.provider_id),
# Start pipeline in background
asyncio.create_task(
run_pipeline_smallwebrtc(
pc,
workflow_id,
workflow_run_id,
user.id,
call_context_vars,
user_provider_id=str(user.provider_id),
)
)
)
pipeline_started = True
# Get answer after initialization
answer = pc.get_answer()
# Get answer after initialization
answer = pc.get_answer()
# Send answer immediately (ICE candidates will be sent separately via trickling)
await ws.send_json(
{
"type": "answer",
"payload": {
"sdp": filter_outbound_sdp(answer["sdp"]),
"type": answer["type"],
"pc_id": answer["pc_id"],
},
}
)
# Send answer immediately (ICE candidates will be sent separately via trickling)
await ws.send_json(
{
"type": "answer",
"payload": {
"sdp": filter_outbound_sdp(answer["sdp"]),
"type": answer["type"],
"pc_id": answer["pc_id"],
},
}
)
except Exception:
if pipeline_started and pc is not None:
try:
await pc.disconnect()
except Exception as e:
logger.debug(f"Failed to disconnect failed offer pc: {e}")
elif concurrency_bound:
await call_concurrency.release_workflow_run_slot(workflow_run_id)
elif concurrency_slot is not None:
await call_concurrency.release_slot(concurrency_slot)
raise
async def _handle_ice_candidate(self, payload: dict, connection_key: str):
"""Handle incoming ICE candidate from client.
@ -635,8 +721,23 @@ async def signaling_websocket(
logger.warning(f"workflow run {workflow_run_id} not found for user {user.id}")
raise HTTPException(status_code=400, detail="Bad workflow_run_id")
# The URL's workflow_id drives org resolution, quota, and concurrency
# accounting downstream — reject a workflow_id that doesn't own this run
# so a caller can't charge their call to an unrelated workflow/org.
if workflow_run.workflow_id != workflow_id:
logger.warning(
f"workflow run {workflow_run_id} does not belong to workflow "
f"{workflow_id} (belongs to {workflow_run.workflow_id})"
)
raise HTTPException(status_code=400, detail="Bad workflow_run_id")
await signaling_manager.handle_websocket(
websocket, workflow_id, workflow_run_id, user
websocket,
workflow_id,
workflow_run_id,
user,
enforce_call_concurrency=True,
call_concurrency_source="webrtc",
)
@ -693,5 +794,10 @@ async def public_signaling_websocket(
# Handle the WebSocket connection using the existing signaling manager
await signaling_manager.handle_websocket(
websocket, embed_token.workflow_id, embed_session.workflow_run_id, user
websocket,
embed_token.workflow_id,
embed_session.workflow_run_id,
user,
enforce_call_concurrency=True,
call_concurrency_source="public_embed",
)

14
api/routes/workflow.py
View file

@ -18,6 +18,7 @@ from api.db.workflow_template_client import WorkflowTemplateClient
from api.enums import CallType, PostHogEvent, StorageBackend, WorkflowStatus
from api.schemas.ai_model_configuration import OrganizationAIModelConfigurationV2
from api.schemas.workflow import WorkflowRunResponseSchema
from api.schemas.workflow_configurations import WorkflowConfigurationDefaults
from api.sdk_expose import sdk_expose
from api.services.auth.depends import get_user
from api.services.configuration.ai_model_configuration import (
@ -284,7 +285,10 @@ class UpdateWorkflowRequest(BaseModel):
name: str | None = None
workflow_definition: dict | None = None
template_context_variables: dict | None = None
workflow_configurations: dict | None = None
# Typed so field constraints (e.g. the max_call_duration cap) are
# enforced by FastAPI; extra="allow" keeps passthrough keys like
# model_configuration_v2_override intact.
workflow_configurations: WorkflowConfigurationDefaults | None = None
class WorkflowVersionResponse(BaseModel):
@ -1039,7 +1043,13 @@ async def update_workflow(
# Validate model overrides. v2 uses a complete workflow-level model
# configuration; legacy v1 uses partial service overlays.
workflow_configurations = request.workflow_configurations
# exclude_unset keeps stored configs sparse: keys the request didn't
# send stay absent so runtime defaults keep applying to them.
workflow_configurations = (
request.workflow_configurations.model_dump(exclude_unset=True)
if request.workflow_configurations is not None
else None
)
if workflow_configurations and workflow_configurations.get(
WORKFLOW_MODEL_CONFIGURATION_V2_OVERRIDE_KEY
):

22
api/schemas/workflow_configurations.py
View file

@ -1,8 +1,12 @@
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
from pydantic import BaseModel, ConfigDict, Field, model_validator
DEFAULT_MAX_CALL_DURATION_SECONDS = 300
# Hard ceiling on configurable call duration. Must stay <= the concurrency
# rate limiter's stale_call_timeout (20 min): a call running past that has
# its slot purged as stale and the org concurrency limit under-counts.
MAX_CALL_DURATION_SECONDS = 1200
DEFAULT_MAX_USER_IDLE_TIMEOUT_SECONDS = 10.0
DEFAULT_SMART_TURN_STOP_SECS = 2.0
DEFAULT_TURN_START_STRATEGY = "default"
@ -22,10 +26,24 @@ class AmbientNoiseConfigurationDefaults(BaseModel):
class WorkflowConfigurationDefaults(BaseModel):
model_config = ConfigDict(extra="allow")
@model_validator(mode="before")
@classmethod
def _treat_null_as_unset(cls, data):
# Stored configs (and older clients) carry explicit JSON nulls for
# keys the user never configured; dropping them lets the field
# defaults apply instead of failing validation.
if isinstance(data, dict):
return {k: v for k, v in data.items() if v is not None}
return data
ambient_noise_configuration: AmbientNoiseConfigurationDefaults = Field(
default_factory=AmbientNoiseConfigurationDefaults
)
max_call_duration: int = DEFAULT_MAX_CALL_DURATION_SECONDS
max_call_duration: int = Field(
default=DEFAULT_MAX_CALL_DURATION_SECONDS,
gt=0,
le=MAX_CALL_DURATION_SECONDS,
)
max_user_idle_timeout: float = DEFAULT_MAX_USER_IDLE_TIMEOUT_SECONDS
smart_turn_stop_secs: float = DEFAULT_SMART_TURN_STOP_SECS
turn_start_strategy: Literal["default", "min_words", "provisional_vad"] = (

4
api/services/auth/depends.py
View file

@ -14,6 +14,7 @@ from api.services.auth.stack_auth import stackauth
from api.services.configuration.registry import ServiceProviders
from api.services.mps_billing import ensure_hosted_mps_billing_account_v2
from api.services.posthog_client import (
POSTHOG_ORGANIZATION_GROUP_TYPE,
capture_event,
group_identify,
set_person_properties,
@ -33,9 +34,6 @@ async def require_local_auth() -> None:
raise HTTPException(status_code=404, detail="Not found")
POSTHOG_ORGANIZATION_GROUP_TYPE = "organization"
async def get_user(
authorization: Annotated[str | None, Header()] = None,
x_api_key: Annotated[str | None, Header(alias="X-API-Key")] = None,

282
api/services/call_concurrency.py Normal file
View file

@ -0,0 +1,282 @@
import asyncio
import time
from dataclasses import dataclass
from loguru import logger
from api.constants import DEFAULT_ORG_CONCURRENCY_LIMIT
from api.db import db_client
from api.enums import OrganizationConfigurationKey, PostHogEvent
from api.services.campaign.rate_limiter import rate_limiter
from api.services.posthog_client import capture_event
@dataclass(frozen=True)
class CallConcurrencySlot:
organization_id: int
slot_id: str
max_concurrent: int
source: str
scope_key: str | None = None
class CallConcurrencyLimitError(Exception):
"""Raised when an org has no available concurrent call slots."""
def __init__(
self,
*,
organization_id: int,
source: str,
wait_time: float,
max_concurrent: int,
):
self.organization_id = organization_id
self.source = source
self.wait_time = wait_time
self.max_concurrent = max_concurrent
super().__init__(
f"Concurrent call limit reached for org {organization_id} "
f"(source={source}, limit={max_concurrent}, waited={wait_time:.1f}s)"
)
class WorkflowRunSlotAlreadyBoundError(Exception):
"""Raised when a workflow run already owns a concurrent call slot."""
def __init__(self, workflow_run_id: int):
self.workflow_run_id = workflow_run_id
super().__init__(
f"Workflow run {workflow_run_id} already has an active call slot"
)
class CallConcurrencyService:
def __init__(self):
self.default_concurrent_limit = int(DEFAULT_ORG_CONCURRENCY_LIMIT)
async def get_org_concurrent_limit(self, organization_id: int) -> int:
"""Get the concurrent call limit for an organization."""
try:
config = await db_client.get_configuration(
organization_id,
OrganizationConfigurationKey.CONCURRENT_CALL_LIMIT.value,
)
if config and config.value:
value = config.value.get("value")
if value is not None:
return int(value)
except Exception as e:
logger.warning(
f"Error getting concurrent limit for org {organization_id}: {e}"
)
return self.default_concurrent_limit
async def acquire_org_slot(
self,
organization_id: int,
*,
source: str,
timeout: float = 0,
scope_key: str | None = None,
scope_max_concurrent: int | None = None,
retry_interval: float = 1,
) -> CallConcurrencySlot:
"""Acquire a slot in the org-wide concurrency counter.
``scope_key``/``scope_max_concurrent`` additionally bound a secondary
counter (e.g. ``campaign:<id>``) so a source can cap its own
concurrency without measuring — or being starved by — unrelated calls
in the same org.
"""
max_concurrent = await self.get_org_concurrent_limit(organization_id)
if scope_max_concurrent is not None:
scope_max_concurrent = int(scope_max_concurrent)
wait_start = time.time()
while True:
acquisition = await rate_limiter.try_acquire_concurrent_slot_details(
organization_id,
max_concurrent,
scope_key=scope_key,
scope_max_concurrent=scope_max_concurrent,
)
if acquisition:
logger.info(
f"Acquired concurrent call slot for org {organization_id}: "
f"source={source}, active_calls="
f"{acquisition.active_count}/{max_concurrent}, "
f"slot_id={acquisition.slot_id}"
)
return CallConcurrencySlot(
organization_id=organization_id,
slot_id=acquisition.slot_id,
max_concurrent=max_concurrent,
source=source,
scope_key=scope_key,
)
wait_time = time.time() - wait_start
if wait_time >= timeout:
current_count = await rate_limiter.get_concurrent_count(organization_id)
scope_note = (
f", scope={scope_key} (limit={scope_max_concurrent})"
if scope_key
else ""
)
logger.warning(
f"Concurrent call limit reached for org {organization_id}: "
f"source={source}, active_calls={current_count}/{max_concurrent}"
f"{scope_note}, waited={wait_time:.1f}s"
)
properties = {
"event_source": "dograh",
"organization_id": organization_id,
"source": source,
"max_concurrent": max_concurrent,
"active_calls": current_count,
"waited_seconds": round(wait_time, 1),
}
if scope_key:
properties["scope_key"] = scope_key
properties["scope_max_concurrent"] = scope_max_concurrent
await self._notify_limit_reached(organization_id, properties)
raise CallConcurrencyLimitError(
organization_id=organization_id,
source=source,
wait_time=wait_time,
max_concurrent=max_concurrent,
)
logger.debug(
f"Waiting for concurrent call slot for org {organization_id}, "
f"source={source}, waited {wait_time:.1f}s"
)
await asyncio.sleep(min(retry_interval, max(0, timeout - wait_time)))
async def _notify_limit_reached(
self, organization_id: int, properties: dict
) -> None:
"""Fan the usage event out to every org member's provider_id, matching
how MPS emits org-scoped billing events (billing_posthog_service.py)
into the shared PostHog project. Never raises.
NOTE: intentionally NOT attaching ``$groups`` (organization) to this
event. PostHog evaluates a $groups event at both person and group
scope, which double-triggers person-scoped workflows enrolled on the
event. The org is still available as the ``organization_id`` property.
"""
try:
members = await db_client.get_organization_users(organization_id)
if not members:
logger.debug(
f"No users found for org {organization_id}; skipping "
"concurrent-call-limit PostHog event"
)
return
for member in members:
capture_event(
distinct_id=str(member.provider_id),
event=PostHogEvent.USAGE_CONCURRENT_CALL_LIMIT_REACHED,
properties=properties,
)
except Exception:
logger.exception(
"Failed to send concurrent-call-limit PostHog event for org "
f"{organization_id}"
)
async def bind_workflow_run(
self, slot: CallConcurrencySlot, workflow_run_id: int
) -> None:
stored = await rate_limiter.store_workflow_slot_mapping_if_absent(
workflow_run_id,
slot.organization_id,
slot.slot_id,
scope_key=slot.scope_key,
)
if stored:
return
await self.release_slot(slot)
raise WorkflowRunSlotAlreadyBoundError(workflow_run_id)
async def register_active_call(
self,
organization_id: int,
workflow_run_id: int,
*,
source: str,
timeout: float = 0,
scope_key: str | None = None,
scope_max_concurrent: int | None = None,
retry_interval: float = 1,
) -> CallConcurrencySlot:
slot = await self.acquire_org_slot(
organization_id,
source=source,
timeout=timeout,
scope_key=scope_key,
scope_max_concurrent=scope_max_concurrent,
retry_interval=retry_interval,
)
await self.bind_workflow_run(slot, workflow_run_id)
return slot
async def unregister_active_call(self, workflow_run_id: int) -> bool:
"""Release the run's slot without ever raising.
Callers invoke this from ``finally`` blocks during pipeline/socket
teardown; a cleanup failure must not mask the original exception.
The slot mapping survives a failed release, so a later cleanup path
(status callback, StasisEnd) or the Redis stale timeout recovers it.
"""
try:
return await self.release_workflow_run_slot(workflow_run_id)
except asyncio.CancelledError:
raise
except Exception as e:
logger.warning(
f"Failed to release concurrent call slot for workflow run "
f"{workflow_run_id}: {e}"
)
return False
async def release_slot(self, slot: CallConcurrencySlot | None) -> bool:
if slot is None:
return False
released = await rate_limiter.release_concurrent_slot(
slot.organization_id, slot.slot_id, scope_key=slot.scope_key
)
return bool(released)
async def release_workflow_run_slot(self, workflow_run_id: int) -> bool:
mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run_id)
if not mapping:
return False
org_id, slot_id, scope_key = mapping
released = await rate_limiter.release_concurrent_slot(
org_id, slot_id, scope_key=scope_key
)
if released is None:
# Redis error while releasing — keep the mapping so a later
# cleanup path can retry instead of orphaning a live slot until
# the stale timeout.
logger.warning(
f"Failed to release concurrent slot for workflow run "
f"{workflow_run_id}; keeping mapping for retry"
)
return False
await rate_limiter.delete_workflow_slot_mapping(workflow_run_id)
if released:
logger.info(f"Released concurrent slot for workflow run {workflow_run_id}")
else:
logger.debug(
f"Concurrent slot mapping for workflow run {workflow_run_id} "
"had no live slot; deleted stale mapping"
)
return released
call_concurrency = CallConcurrencyService()

264
api/services/campaign/campaign_call_dispatcher.py
View file

@ -5,10 +5,14 @@ from typing import TYPE_CHECKING, Optional
from loguru import logger
from api.constants import DEFAULT_ORG_CONCURRENCY_LIMIT
from api.db import db_client
from api.db.models import QueuedRunModel, WorkflowRunModel
from api.enums import OrganizationConfigurationKey, WorkflowRunState
from api.enums import WorkflowRunState
from api.services.call_concurrency import (
CallConcurrencyLimitError,
CallConcurrencySlot,
call_concurrency,
)
from api.services.campaign.circuit_breaker import circuit_breaker
from api.services.campaign.errors import (
ConcurrentSlotAcquisitionError,
@ -29,9 +33,6 @@ if TYPE_CHECKING:
class CampaignCallDispatcher:
"""Manages rate-limited and concurrent-limited call dispatching"""
def __init__(self):
self.default_concurrent_limit = int(DEFAULT_ORG_CONCURRENCY_LIMIT)
async def get_provider_for_campaign(self, campaign) -> "TelephonyProvider":
"""Get the telephony provider pinned to this campaign's config. Falls back
to the org's default config for legacy campaigns whose
@ -53,18 +54,7 @@ class CampaignCallDispatcher:
async def get_org_concurrent_limit(self, organization_id: int) -> int:
"""Get the concurrent call limit for an organization."""
try:
config = await db_client.get_configuration(
organization_id,
OrganizationConfigurationKey.CONCURRENT_CALL_LIMIT.value,
)
if config and config.value:
return int(config.value["value"])
except Exception as e:
logger.warning(
f"Error getting concurrent limit for org {organization_id}: {e}"
)
return self.default_concurrent_limit
return await call_concurrency.get_org_concurrent_limit(organization_id)
async def process_batch(self, campaign_id: int, batch_size: int = 10) -> int:
"""
@ -119,12 +109,14 @@ class CampaignCallDispatcher:
)
# Acquire concurrent slot - waits until a slot is available
slot_id = await self.acquire_concurrent_slot(
concurrency_slot = await self.acquire_concurrent_slot(
campaign.organization_id, campaign
)
# Dispatch the call
workflow_run = await self.dispatch_call(queued_run, campaign, slot_id)
workflow_run = await self.dispatch_call(
queued_run, campaign, concurrency_slot
)
# Update queued run as processed
await db_client.update_queued_run(
@ -233,68 +225,61 @@ class CampaignCallDispatcher:
)
async def dispatch_call(
self, queued_run: QueuedRunModel, campaign: any, slot_id: str
self,
queued_run: QueuedRunModel,
campaign: any,
concurrency_slot: CallConcurrencySlot,
) -> Optional[WorkflowRunModel]:
"""Creates workflow run and initiates call. Requires a pre-acquired slot_id."""
"""Creates workflow run and initiates call. Requires a pre-acquired slot."""
from_number = None
workflow_run = None
slot_bound = False
# Get workflow details
workflow = await db_client.get_workflow_by_id(campaign.workflow_id)
if not workflow:
# Release slot before raising
await rate_limiter.release_concurrent_slot(
campaign.organization_id, slot_id
)
raise ValueError(f"Workflow {campaign.workflow_id} not found")
# Extract phone number
phone_number = queued_run.context_variables.get("phone_number")
if not phone_number:
# Release slot before raising
await rate_limiter.release_concurrent_slot(
campaign.organization_id, slot_id
)
raise ValueError(f"No phone number in queued run {queued_run.id}")
# Get provider for this campaign's pinned telephony config.
provider = await self.get_provider_for_campaign(campaign)
workflow_run_mode = provider.PROVIDER_NAME
# Acquire a unique from_number from the pool scoped to this campaign's
# telephony configuration so orgs with multiple configs don't leak
# caller IDs across configs.
from_number = await self.acquire_from_number(
campaign.organization_id,
telephony_configuration_id=campaign.telephony_configuration_id,
)
if from_number is None:
# Release concurrent slot before raising
await rate_limiter.release_concurrent_slot(
campaign.organization_id, slot_id
)
raise PhoneNumberPoolExhaustedError(
organization_id=campaign.organization_id
)
logger.info(f"Provider name: {provider.PROVIDER_NAME}")
logger.info(f"Queued run context: {queued_run.context_variables}")
# Merge context variables (queued_run context already includes retry info if applicable)
initial_context = {
**queued_run.context_variables,
"campaign_id": campaign.id,
"provider": provider.PROVIDER_NAME,
"source_uuid": queued_run.source_uuid,
"caller_number": from_number,
"called_number": phone_number,
"telephony_configuration_id": campaign.telephony_configuration_id,
}
logger.info(f"Final initial_context: {initial_context}")
# Create workflow run with queued_run_id tracking
workflow_run_name = f"WR-CAMPAIGN-{campaign.id}-{queued_run.id}"
try:
# Get workflow details
workflow = await db_client.get_workflow_by_id(campaign.workflow_id)
if not workflow:
raise ValueError(f"Workflow {campaign.workflow_id} not found")
# Extract phone number
phone_number = queued_run.context_variables.get("phone_number")
if not phone_number:
raise ValueError(f"No phone number in queued run {queued_run.id}")
# Get provider for this campaign's pinned telephony config.
provider = await self.get_provider_for_campaign(campaign)
workflow_run_mode = provider.PROVIDER_NAME
# Acquire a unique from_number from the pool scoped to this campaign's
# telephony configuration so orgs with multiple configs don't leak
# caller IDs across configs.
from_number = await self.acquire_from_number(
campaign.organization_id,
telephony_configuration_id=campaign.telephony_configuration_id,
)
if from_number is None:
raise PhoneNumberPoolExhaustedError(
organization_id=campaign.organization_id
)
logger.info(f"Provider name: {provider.PROVIDER_NAME}")
logger.info(f"Queued run context: {queued_run.context_variables}")
# Merge context variables (queued_run context already includes retry info if applicable)
initial_context = {
**queued_run.context_variables,
"campaign_id": campaign.id,
"provider": provider.PROVIDER_NAME,
"source_uuid": queued_run.source_uuid,
"caller_number": from_number,
"called_number": phone_number,
"telephony_configuration_id": campaign.telephony_configuration_id,
}
logger.info(f"Final initial_context: {initial_context}")
# Create workflow run with queued_run_id tracking
workflow_run_name = f"WR-CAMPAIGN-{campaign.id}-{queued_run.id}"
workflow_run = await db_client.create_workflow_run(
name=workflow_run_name,
workflow_id=campaign.workflow_id,
@ -304,11 +289,8 @@ class CampaignCallDispatcher:
campaign_id=campaign.id,
queued_run_id=queued_run.id, # Link to queued run for retry tracking
)
# Store slot_id mapping in Redis for cleanup later
await rate_limiter.store_workflow_slot_mapping(
workflow_run.id, campaign.organization_id, slot_id
)
await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run.id)
slot_bound = True
# Store from_number mapping for cleanup on call completion
await rate_limiter.store_workflow_from_number_mapping(
@ -319,9 +301,10 @@ class CampaignCallDispatcher:
)
except Exception as e:
# Release slot and from_number on error
await rate_limiter.release_concurrent_slot(
campaign.organization_id, slot_id
)
if slot_bound and workflow_run:
await call_concurrency.release_workflow_run_slot(workflow_run.id)
else:
await call_concurrency.release_slot(concurrency_slot)
if from_number:
await rate_limiter.release_from_number(
campaign.organization_id,
@ -357,21 +340,7 @@ class CampaignCallDispatcher:
gathered_context={"error": error_message},
)
mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run.id)
if mapping:
org_id, mapped_slot_id = mapping
await rate_limiter.release_concurrent_slot(org_id, mapped_slot_id)
await rate_limiter.delete_workflow_slot_mapping(workflow_run.id)
from_number_mapping = await rate_limiter.get_workflow_from_number_mapping(
workflow_run.id
)
if from_number_mapping:
fn_org_id, fn_number, fn_tcid = from_number_mapping
await rate_limiter.release_from_number(
fn_org_id, fn_number, telephony_configuration_id=fn_tcid
)
await rate_limiter.delete_workflow_from_number_mapping(workflow_run.id)
await self.release_call_slot(workflow_run.id)
raise ValueError(error_message)
@ -446,23 +415,7 @@ class CampaignCallDispatcher:
reason="call_initiation_failed",
)
# Release concurrent slot on failure
mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run.id)
if mapping:
org_id, slot_id = mapping
await rate_limiter.release_concurrent_slot(org_id, slot_id)
await rate_limiter.delete_workflow_slot_mapping(workflow_run.id)
# Release from_number on failure
from_number_mapping = await rate_limiter.get_workflow_from_number_mapping(
workflow_run.id
)
if from_number_mapping:
fn_org_id, fn_number, fn_tcid = from_number_mapping
await rate_limiter.release_from_number(
fn_org_id, fn_number, telephony_configuration_id=fn_tcid
)
await rate_limiter.delete_workflow_from_number_mapping(workflow_run.id)
await self.release_call_slot(workflow_run.id)
raise
@ -501,7 +454,7 @@ class CampaignCallDispatcher:
async def acquire_concurrent_slot(
self, organization_id: int, campaign: any, timeout: float = 600
) -> str:
) -> CallConcurrencySlot:
"""
Acquires a concurrent call slot - waits if necessary until a slot is available.
@ -510,54 +463,41 @@ class CampaignCallDispatcher:
campaign: The campaign object
timeout: Maximum time to wait for a slot (default 10 minutes)
Returns the slot_id which must be released when the call completes.
Returns the slot which must be released when the call completes.
Raises:
ConcurrentSlotAcquisitionError: If slot cannot be acquired within timeout
"""
# Get concurrent limit for organization
org_concurrent_limit = await self.get_org_concurrent_limit(organization_id)
# Check for campaign-level max_concurrency in orchestrator_metadata
# Check for campaign-level max_concurrency in orchestrator_metadata.
# It caps this campaign's own concurrent calls via a campaign-scoped
# counter — the org-wide limit still applies on top, but calls from
# other sources (WebRTC, inbound, other campaigns) don't count
# against the campaign's cap.
campaign_max_concurrency = None
if campaign.orchestrator_metadata:
campaign_max_concurrency = campaign.orchestrator_metadata.get(
"max_concurrency"
)
# Use the lower of campaign limit and org limit
if campaign_max_concurrency is not None:
max_concurrent = min(campaign_max_concurrency, org_concurrent_limit)
else:
max_concurrent = org_concurrent_limit
# Track wait time for alerting
wait_start = time.time()
# Wait until we can acquire a concurrent slot
while True:
slot_id = await rate_limiter.try_acquire_concurrent_slot(
organization_id, max_concurrent
try:
return await call_concurrency.acquire_org_slot(
organization_id,
source=f"campaign:{campaign.id}",
timeout=timeout,
scope_key=(
f"campaign:{campaign.id}"
if campaign_max_concurrency is not None
else None
),
scope_max_concurrent=campaign_max_concurrency,
retry_interval=1,
)
if slot_id:
return slot_id
# Check if we've been waiting too long
wait_time = time.time() - wait_start
if wait_time > timeout:
raise ConcurrentSlotAcquisitionError(
organization_id=organization_id,
campaign_id=campaign.id,
wait_time=wait_time,
)
logger.debug(
f"Attempting to get a slot for {organization_id} {campaign.id}, "
f"waited {wait_time:.1f}s"
)
# Wait before retrying
await asyncio.sleep(1)
except CallConcurrencyLimitError as e:
raise ConcurrentSlotAcquisitionError(
organization_id=organization_id,
campaign_id=campaign.id,
wait_time=e.wait_time,
) from e
async def acquire_from_number(
self,
@ -602,17 +542,9 @@ class CampaignCallDispatcher:
Release concurrent slot and from_number when a call completes.
Called by Twilio webhooks or workflow completion handlers.
"""
slot_released = False
mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run_id)
if mapping:
org_id, slot_id = mapping
success = await rate_limiter.release_concurrent_slot(org_id, slot_id)
if success:
await rate_limiter.delete_workflow_slot_mapping(workflow_run_id)
logger.info(
f"Released concurrent slot for workflow run {workflow_run_id}"
)
slot_released = True
slot_released = await call_concurrency.release_workflow_run_slot(
workflow_run_id
)
# Release from_number back to its (org, telephony config) pool
from_number_mapping = await rate_limiter.get_workflow_from_number_mapping(

156
api/services/campaign/rate_limiter.py
View file

@ -1,5 +1,6 @@
import time
import uuid
from dataclasses import dataclass
from typing import Optional
import redis.asyncio as aioredis
@ -8,6 +9,12 @@ from loguru import logger
from api.constants import REDIS_URL
@dataclass(frozen=True)
class ConcurrentSlotAcquisition:
slot_id: str
active_count: int
class RateLimiter:
"""Sliding window rate limiter to enforce strict per-second limits and concurrent call limits"""
@ -100,34 +107,71 @@ class RateLimiter:
Try to acquire a concurrent call slot.
Returns a unique slot_id if successful, None if limit reached.
"""
acquisition = await self.try_acquire_concurrent_slot_details(
organization_id, max_concurrent
)
return acquisition.slot_id if acquisition else None
async def try_acquire_concurrent_slot_details(
self,
organization_id: int,
max_concurrent: int = 20,
*,
scope_key: str | None = None,
scope_max_concurrent: int | None = None,
) -> Optional[ConcurrentSlotAcquisition]:
"""
Try to acquire a concurrent call slot.
Returns the slot_id and post-acquire active count if successful,
or None if the limit is reached.
When ``scope_key``/``scope_max_concurrent`` are provided, the slot is
also registered in a secondary counter (``concurrent_calls:<scope_key>``,
e.g. ``campaign:<id>``) and acquisition additionally requires that
counter to be below ``scope_max_concurrent``. Both counters are
updated atomically. The scope-scoped slot must be released with the
same ``scope_key``.
"""
redis_client = await self._get_redis()
concurrent_key = f"concurrent_calls:{organization_id}"
scope_concurrent_key = f"concurrent_calls:{scope_key}" if scope_key else ""
now = time.time()
stale_cutoff = now - self.stale_call_timeout
# Lua script for atomic operation
# Lua script for atomic operation across the org counter and the
# optional scope counter (empty scope key = org-only acquisition).
lua_script = """
local key = KEYS[1]
local scope_key = KEYS[2]
local now = tonumber(ARGV[1])
local max_concurrent = tonumber(ARGV[2])
local stale_cutoff = tonumber(ARGV[3])
local slot_id = ARGV[4]
-- Remove stale entries (older than 30 minutes)
local scope_max_concurrent = tonumber(ARGV[5])
-- Remove stale entries (older than the stale-call timeout)
redis.call('ZREMRANGEBYSCORE', key, 0, stale_cutoff)
-- Get current count
local current_count = redis.call('ZCARD', key)
if current_count < max_concurrent then
-- Add new slot
redis.call('ZADD', key, now, slot_id)
redis.call('EXPIRE', key, 3600) -- Expire after 1 hour
return slot_id
else
if current_count >= max_concurrent then
return nil
end
if scope_key ~= '' then
redis.call('ZREMRANGEBYSCORE', scope_key, 0, stale_cutoff)
if redis.call('ZCARD', scope_key) >= scope_max_concurrent then
return nil
end
redis.call('ZADD', scope_key, now, slot_id)
redis.call('EXPIRE', scope_key, 3600)
end
redis.call('ZADD', key, now, slot_id)
redis.call('EXPIRE', key, 3600) -- Expire after 1 hour
return {slot_id, current_count + 1}
"""
# Generate unique slot ID (timestamp + random component)
@ -136,22 +180,38 @@ class RateLimiter:
try:
result = await redis_client.eval(
lua_script,
1,
2,
concurrent_key,
scope_concurrent_key,
now,
max_concurrent,
stale_cutoff,
slot_id,
scope_max_concurrent if scope_max_concurrent is not None else 0,
)
if not result:
return None
acquired_slot_id, active_count = result
return ConcurrentSlotAcquisition(
slot_id=str(acquired_slot_id),
active_count=int(active_count),
)
return result
except Exception as e:
logger.error(f"Concurrent limiter error: {e}")
return None
async def release_concurrent_slot(self, organization_id: int, slot_id: str) -> bool:
async def release_concurrent_slot(
self,
organization_id: int,
slot_id: str,
scope_key: str | None = None,
) -> bool | None:
"""
Release a concurrent call slot.
Returns True if slot was released, False otherwise.
Release a concurrent call slot (and its scope counter entry, if any).
Returns True if the slot was released, False if it was already gone
(released/stale-expired), or None on a Redis error — callers that
track cleanup state should keep it around for retry when None.
"""
if not slot_id:
return False
@ -161,6 +221,8 @@ class RateLimiter:
try:
removed = await redis_client.zrem(concurrent_key, slot_id)
if scope_key:
await redis_client.zrem(f"concurrent_calls:{scope_key}", slot_id)
if removed:
logger.debug(
f"Released concurrent slot {slot_id} for org {organization_id}"
@ -168,7 +230,7 @@ class RateLimiter:
return bool(removed)
except Exception as e:
logger.error(f"Error releasing concurrent slot: {e}")
return False
return None
async def get_concurrent_count(self, organization_id: int) -> int:
"""
@ -212,12 +274,62 @@ class RateLimiter:
logger.error(f"Error storing workflow slot mapping: {e}")
return False
async def store_workflow_slot_mapping_if_absent(
self,
workflow_run_id: int,
organization_id: int,
slot_id: str,
scope_key: str | None = None,
) -> bool:
"""
Store the workflow_run_id -> concurrent slot mapping only if no mapping
already exists. This prevents duplicate public/WebRTC starts for the
same workflow run from overwriting the cleanup pointer.
"""
redis_client = await self._get_redis()
mapping_key = f"workflow_slot_mapping:{workflow_run_id}"
lua_script = """
local key = KEYS[1]
local org_id = ARGV[1]
local slot_id = ARGV[2]
local ttl = tonumber(ARGV[3])
local scope_key = ARGV[4]
if redis.call('EXISTS', key) == 1 then
return 0
end
redis.call('HSET', key, 'org_id', org_id, 'slot_id', slot_id)
if scope_key ~= '' then
redis.call('HSET', key, 'scope_key', scope_key)
end
redis.call('EXPIRE', key, ttl)
return 1
"""
try:
stored = await redis_client.eval(
lua_script,
1,
mapping_key,
organization_id,
slot_id,
self.stale_call_timeout,
scope_key or "",
)
return bool(stored)
except Exception as e:
logger.error(f"Error storing workflow slot mapping if absent: {e}")
return False
async def get_workflow_slot_mapping(
self, workflow_run_id: int
) -> Optional[tuple[int, str]]:
) -> Optional[tuple[int, str, str | None]]:
"""
Get the concurrent slot mapping for a workflow run.
Returns (organization_id, slot_id) tuple or None if not found.
Returns (organization_id, slot_id, scope_key) or None if not found;
scope_key is None for slots acquired without a scope counter.
"""
redis_client = await self._get_redis()
mapping_key = f"workflow_slot_mapping:{workflow_run_id}"
@ -225,7 +337,11 @@ class RateLimiter:
try:
mapping = await redis_client.hgetall(mapping_key)
if mapping and "org_id" in mapping and "slot_id" in mapping:
return (int(mapping["org_id"]), mapping["slot_id"])
return (
int(mapping["org_id"]),
mapping["slot_id"],
mapping.get("scope_key") or None,
)
return None
except Exception as e:
logger.error(f"Error getting workflow slot mapping: {e}")

28
api/services/pipecat/run_pipeline.py
View file

@ -14,14 +14,17 @@ from api.schemas.workflow_configurations import (
DEFAULT_TURN_START_MIN_WORDS,
DEFAULT_TURN_START_STRATEGY,
)
from api.services.call_concurrency import call_concurrency
from api.services.configuration.registry import ServiceProviders
from api.services.integrations import (
IntegrationRuntimeContext,
create_runtime_sessions,
)
from api.services.pipecat.active_calls import (
register_active_call,
unregister_active_call,
register_active_call as register_worker_active_call,
)
from api.services.pipecat.active_calls import (
unregister_active_call as unregister_worker_active_call,
)
from api.services.pipecat.audio_config import AudioConfig, create_audio_config
from api.services.pipecat.event_handlers import (
@ -258,7 +261,7 @@ async def run_pipeline_telephony(
"""Run a pipeline for any telephony provider."""
# Register before any async setup so deploy drains see calls that are still
# resolving DB/config/transport state.
register_active_call(workflow_run_id)
register_worker_active_call(workflow_run_id)
try:
await _run_pipeline_telephony_impl(
websocket,
@ -270,7 +273,10 @@ async def run_pipeline_telephony(
transport_kwargs=transport_kwargs,
)
finally:
unregister_active_call(workflow_run_id)
try:
await call_concurrency.unregister_active_call(workflow_run_id)
finally:
unregister_worker_active_call(workflow_run_id)
async def _run_pipeline_telephony_impl(
@ -383,7 +389,7 @@ async def run_pipeline_smallwebrtc(
"""Run pipeline for WebRTC connections."""
# Register before any async setup so deploy drains see calls that are still
# resolving DB/config/transport state.
register_active_call(workflow_run_id)
register_worker_active_call(workflow_run_id)
try:
await _run_pipeline_smallwebrtc_impl(
webrtc_connection,
@ -394,7 +400,10 @@ async def run_pipeline_smallwebrtc(
user_provider_id=user_provider_id,
)
finally:
unregister_active_call(workflow_run_id)
try:
await call_concurrency.unregister_active_call(workflow_run_id)
finally:
unregister_worker_active_call(workflow_run_id)
async def _run_pipeline_smallwebrtc_impl(
@ -478,7 +487,7 @@ async def _run_pipeline(
resolved_user_config=None,
) -> None:
"""Run the pipeline with active-call drain accounting."""
register_active_call(workflow_run_id)
register_worker_active_call(workflow_run_id)
try:
await _run_pipeline_impl(
transport,
@ -492,7 +501,10 @@ async def _run_pipeline(
resolved_user_config=resolved_user_config,
)
finally:
unregister_active_call(workflow_run_id)
try:
await call_concurrency.unregister_active_call(workflow_run_id)
finally:
unregister_worker_active_call(workflow_run_id)
async def _run_pipeline_impl(

1
api/services/posthog_client.py
View file

@ -7,6 +7,7 @@ from api.constants import POSTHOG_API_KEY, POSTHOG_HOST
_posthog_client: Posthog | None = None
POSTHOG_SERVER_GROUP_IDENTIFY_DISTINCT_ID = "server-group-identify"
POSTHOG_ORGANIZATION_GROUP_TYPE = "organization"
def get_posthog() -> Posthog | None:

35
api/services/telephony/ari_manager.py
View file

@ -26,6 +26,10 @@ from loguru import logger
from api.constants import REDIS_URL
from api.db import db_client
from api.enums import CallType, WorkflowRunMode
from api.services.call_concurrency import (
CallConcurrencyLimitError,
call_concurrency,
)
from api.services.quota_service import authorize_workflow_run_start
from api.services.telephony.call_transfer_manager import get_call_transfer_manager
from api.services.telephony.transfer_event_protocol import (
@ -527,6 +531,8 @@ class ARIConnection:
channel = event.get("channel", {})
caller_number = channel.get("caller", {}).get("number", "unknown")
called_number = channel.get("dialplan", {}).get("exten", "unknown")
concurrency_slot = None
workflow_run = None
try:
# 1. Resolve the workflow from the called extension via the
@ -573,6 +579,20 @@ class ARIConnection:
user_id = workflow.user_id
try:
concurrency_slot = await call_concurrency.acquire_org_slot(
self.organization_id,
source="ari_inbound",
timeout=0,
)
except CallConcurrencyLimitError:
logger.warning(
f"[ARI org={self.organization_id}] Concurrent call limit "
f"reached; hanging up inbound channel {channel_id}"
)
await self._delete_channel(channel_id)
return
# 3. Create workflow run
call_id = channel_id
workflow_run = await db_client.create_workflow_run(
@ -591,7 +611,9 @@ class ARIConnection:
gathered_context={
"call_id": call_id,
},
organization_id=self.organization_id,
)
await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run.id)
logger.info(
f"[ARI org={self.organization_id}] Created inbound workflow run "
@ -610,6 +632,7 @@ class ARIConnection:
f"[ARI org={self.organization_id}] Quota exceeded for user {user_id} "
f"— hanging up inbound call {channel_id}"
)
await call_concurrency.release_workflow_run_slot(workflow_run.id)
await self._delete_channel(channel_id)
return
@ -625,6 +648,10 @@ class ARIConnection:
str(user_id),
)
except Exception as e:
if workflow_run:
await call_concurrency.release_workflow_run_slot(workflow_run.id)
elif concurrency_slot:
await call_concurrency.release_slot(concurrency_slot)
logger.error(
f"[ARI org={self.organization_id}] Error handling inbound StasisStart "
f"for channel {channel_id}: {e}"
@ -760,6 +787,14 @@ class ARIConnection:
the bridge and both channels — like endConferenceOnExit.
"""
try:
# Release the org concurrency slot. Normally the pipeline's own
# teardown does this when the ext media websocket closes, but if
# the pipeline never started (caller hung up before external
# media connected, ext media creation failed, ...) this is the
# only cleanup that runs before the Redis stale timeout. No-op
# when the slot was already released.
await call_concurrency.unregister_active_call(int(workflow_run_id))
workflow_run = await db_client.get_workflow_run_by_id(int(workflow_run_id))
if not workflow_run or not workflow_run.gathered_context:
logger.warning(

4
api/services/telephony/base.py
View file

@ -365,6 +365,10 @@ class TelephonyProvider(ABC):
the caller carries a provider stream protocol. ``organization_id`` is
passed so providers can scope any config lookups to the workflow's org.
Default raises so providers that haven't opted in fail loudly.
The route holds an org concurrency slot while this runs, so
implementations must bound their pre-pipeline handshake reads with a
timeout — an idle socket must not hold the slot indefinitely.
"""
raise NotImplementedError(
f"Agent-stream not supported for provider {self.PROVIDER_NAME}"

35
api/services/telephony/providers/cloudonix/provider.py
View file

@ -2,6 +2,7 @@
Cloudonix implementation of the TelephonyProvider interface.
"""
import asyncio
import json
import random
from typing import TYPE_CHECKING, Any, Dict, List, Optional
@ -25,6 +26,11 @@ if TYPE_CHECKING:
CLOUDONIX_API_BASE_URL = "https://api.cloudonix.io"
# Cloudonix sends the connected/start handshake immediately after the media
# stream opens. The agent-stream route holds an org concurrency slot while we
# wait, so an idle socket must not be able to hold it indefinitely.
AGENT_STREAM_HANDSHAKE_TIMEOUT_S = 10
class CloudonixProvider(TelephonyProvider):
"""
@ -533,14 +539,31 @@ class CloudonixProvider(TelephonyProvider):
from api.services.pipecat.run_pipeline import run_pipeline_telephony
try:
first_msg = await websocket.receive_text()
msg = json.loads(first_msg)
if msg.get("event") != "connected":
logger.error(f"Expected 'connected' event, got: {msg.get('event')}")
await websocket.close(code=4400, reason="Expected connected event")
try:
first_msg = await asyncio.wait_for(
websocket.receive_text(),
timeout=AGENT_STREAM_HANDSHAKE_TIMEOUT_S,
)
msg = json.loads(first_msg)
if msg.get("event") != "connected":
logger.error(f"Expected 'connected' event, got: {msg.get('event')}")
await websocket.close(code=4400, reason="Expected connected event")
return
start_msg = json.loads(
await asyncio.wait_for(
websocket.receive_text(),
timeout=AGENT_STREAM_HANDSHAKE_TIMEOUT_S,
)
)
except asyncio.TimeoutError:
logger.warning(
f"Cloudonix agent-stream handshake timed out for workflow_run "
f"{workflow_run_id}"
)
await websocket.close(code=4408, reason="Handshake timeout")
return
start_msg = json.loads(await websocket.receive_text())
if start_msg.get("event") != "start":
logger.error("Expected 'start' event second")
await websocket.close(code=4400, reason="Expected start event")

6
api/services/telephony/status_processor.py
View file

@ -144,8 +144,9 @@ async def _process_status_update(workflow_run_id: int, status: StatusCallbackReq
f"[run {workflow_run_id}] Call completed with duration: {status.duration}s"
)
await campaign_call_dispatcher.release_call_slot(workflow_run_id)
if workflow_run.campaign_id:
await campaign_call_dispatcher.release_call_slot(workflow_run_id)
await circuit_breaker.record_and_evaluate(
workflow_run.campaign_id, is_failure=False
)
@ -162,8 +163,9 @@ async def _process_status_update(workflow_run_id: int, status: StatusCallbackReq
f"[run {workflow_run_id}] Call failed with status: {normalized_status.value}"
)
await campaign_call_dispatcher.release_call_slot(workflow_run_id)
if workflow_run.campaign_id:
await campaign_call_dispatcher.release_call_slot(workflow_run_id)
is_failure = normalized_status in FAILURE_NOT_CONNECTED_STATUSES
await circuit_breaker.record_and_evaluate(
workflow_run.campaign_id,

31
api/tests/telephony/cloudonix/test_routes.py
View file

@ -244,3 +244,34 @@ def test_parse_cloudonix_cdr_preserves_zero_billsec():
)
assert req["duration"] == "0"
@pytest.mark.asyncio
async def test_agent_stream_handshake_timeout_closes_socket(monkeypatch):
"""An idle agent-stream socket holds an org concurrency slot, so the
handshake read must be bounded rather than waiting forever."""
import asyncio
from api.services.telephony.providers.cloudonix import provider as provider_module
monkeypatch.setattr(provider_module, "AGENT_STREAM_HANDSHAKE_TIMEOUT_S", 0.05)
provider = CloudonixProvider({})
async def never_returns():
await asyncio.Event().wait()
websocket = SimpleNamespace(receive_text=never_returns, close=AsyncMock())
await asyncio.wait_for(
provider.handle_external_websocket(
websocket,
organization_id=1,
workflow_id=2,
user_id=3,
workflow_run_id=4,
params={},
),
timeout=5,
)
websocket.close.assert_awaited_once_with(code=4408, reason="Handshake timeout")

36
api/tests/telephony/test_ari_manager_transfer_teardown.py
View file

@ -1,5 +1,6 @@
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
@ -7,6 +8,13 @@ from api.services.telephony import ari_manager
from api.services.telephony.ari_manager import ARIConnection
@pytest.fixture
def fake_call_concurrency(monkeypatch):
fake = SimpleNamespace(unregister_active_call=AsyncMock(return_value=True))
monkeypatch.setattr(ari_manager, "call_concurrency", fake)
return fake
class _FakeDbClient:
def __init__(self, gathered_context):
self.workflow_run = SimpleNamespace(gathered_context=gathered_context)
@ -77,6 +85,7 @@ def _completed_transfer_context():
@pytest.mark.asyncio
async def test_completed_transfer_tears_down_destination_when_caller_leaves(
monkeypatch,
fake_call_concurrency,
):
fake_db = _FakeDbClient(_completed_transfer_context())
monkeypatch.setattr(ari_manager, "db_client", fake_db)
@ -90,11 +99,13 @@ async def test_completed_transfer_tears_down_destination_when_caller_leaves(
assert "dest-chan" in conn.deleted_channel_runs
assert conn.deleted_transfer_channel_mappings == ["dest-chan"]
assert fake_db.workflow_run.gathered_context["transfer_state"] == "terminated"
fake_call_concurrency.unregister_active_call.assert_awaited_with(123)
@pytest.mark.asyncio
async def test_completed_transfer_tears_down_caller_when_destination_leaves(
monkeypatch,
fake_call_concurrency,
):
fake_db = _FakeDbClient(_completed_transfer_context())
monkeypatch.setattr(ari_manager, "db_client", fake_db)
@ -108,6 +119,31 @@ async def test_completed_transfer_tears_down_caller_when_destination_leaves(
assert "dest-chan" in conn.deleted_channel_runs
assert conn.deleted_transfer_channel_mappings == ["dest-chan"]
assert fake_db.workflow_run.gathered_context["transfer_state"] == "terminated"
fake_call_concurrency.unregister_active_call.assert_awaited_with(123)
@pytest.mark.asyncio
async def test_stasis_end_releases_concurrency_slot_on_normal_teardown(
monkeypatch,
fake_call_concurrency,
):
"""StasisEnd must release the org slot even when the pipeline never ran
(e.g. the caller hung up before external media connected)."""
fake_db = _FakeDbClient(
{
"call_id": "caller-chan",
"ext_channel_id": "ext-chan",
"bridge_id": "bridge-1",
}
)
monkeypatch.setattr(ari_manager, "db_client", fake_db)
conn = _RecordingARIConnection()
await conn._handle_stasis_end("caller-chan", "123")
fake_call_concurrency.unregister_active_call.assert_awaited_once_with(123)
assert conn.deleted_bridges == ["bridge-1"]
assert conn.deleted_channels == ["ext-chan"]
@pytest.mark.asyncio

10
api/tests/telephony/test_status_processor.py
View file

@ -29,6 +29,9 @@ async def test_initialized_no_answer_enqueues_workflow_completion():
with (
patch("api.services.telephony.status_processor.db_client") as mock_db,
patch(
"api.services.telephony.status_processor.campaign_call_dispatcher"
) as mock_dispatcher,
patch(
"api.services.telephony.status_processor.enqueue_job",
new_callable=AsyncMock,
@ -36,6 +39,7 @@ async def test_initialized_no_answer_enqueues_workflow_completion():
):
mock_db.get_workflow_run_by_id = AsyncMock(return_value=workflow_run)
mock_db.update_workflow_run = AsyncMock()
mock_dispatcher.release_call_slot = AsyncMock(return_value=True)
await _process_status_update(123, status)
@ -58,6 +62,7 @@ async def test_initialized_no_answer_enqueues_workflow_completion():
mock_enqueue.assert_awaited_once_with(
FunctionNames.RUN_INTEGRATIONS_POST_WORKFLOW_RUN, 123
)
mock_dispatcher.release_call_slot.assert_awaited_once_with(123)
@pytest.mark.asyncio
@ -79,6 +84,9 @@ async def test_running_terminal_status_does_not_enqueue_workflow_completion():
with (
patch("api.services.telephony.status_processor.db_client") as mock_db,
patch(
"api.services.telephony.status_processor.campaign_call_dispatcher"
) as mock_dispatcher,
patch(
"api.services.telephony.status_processor.enqueue_job",
new_callable=AsyncMock,
@ -86,6 +94,7 @@ async def test_running_terminal_status_does_not_enqueue_workflow_completion():
):
mock_db.get_workflow_run_by_id = AsyncMock(return_value=workflow_run)
mock_db.update_workflow_run = AsyncMock()
mock_dispatcher.release_call_slot = AsyncMock(return_value=True)
await _process_status_update(456, status)
@ -96,3 +105,4 @@ async def test_running_terminal_status_does_not_enqueue_workflow_completion():
"telephony_failed",
]
mock_enqueue.assert_not_awaited()
mock_dispatcher.release_call_slot.assert_awaited_once_with(456)

22
api/tests/test_active_calls.py
View file

@ -7,6 +7,7 @@ idempotent, and the count only reaches zero when every registered run is gone.
"""
import asyncio
from unittest.mock import AsyncMock
import pytest
from fastapi import FastAPI
@ -91,6 +92,12 @@ async def test_run_pipeline_counts_call_during_setup(monkeypatch):
"get_workflow_run",
fake_get_workflow_run,
)
unregister_concurrency = AsyncMock()
monkeypatch.setattr(
run_pipeline_module.call_concurrency,
"unregister_active_call",
unregister_concurrency,
)
task = asyncio.create_task(
run_pipeline_module._run_pipeline(
@ -108,6 +115,7 @@ async def test_run_pipeline_counts_call_during_setup(monkeypatch):
with pytest.raises(RuntimeError, match="setup failed"):
await asyncio.wait_for(task, timeout=1.0)
assert active_calls.active_call_count() == 0
unregister_concurrency.assert_awaited_once_with(42)
@pytest.mark.asyncio
@ -123,6 +131,12 @@ async def test_webrtc_entrypoint_counts_call_during_setup(monkeypatch):
monkeypatch.setattr(
run_pipeline_module.db_client, "get_workflow", fake_get_workflow
)
unregister_concurrency = AsyncMock()
monkeypatch.setattr(
run_pipeline_module.call_concurrency,
"unregister_active_call",
unregister_concurrency,
)
task = asyncio.create_task(
run_pipeline_module.run_pipeline_smallwebrtc(
@ -140,6 +154,7 @@ async def test_webrtc_entrypoint_counts_call_during_setup(monkeypatch):
with pytest.raises(RuntimeError, match="setup failed"):
await asyncio.wait_for(task, timeout=1.0)
assert active_calls.active_call_count() == 0
unregister_concurrency.assert_awaited_once_with(43)
@pytest.mark.asyncio
@ -155,6 +170,12 @@ async def test_telephony_entrypoint_counts_call_during_setup(monkeypatch):
monkeypatch.setattr(
run_pipeline_module.db_client, "get_workflow", fake_get_workflow
)
unregister_concurrency = AsyncMock()
monkeypatch.setattr(
run_pipeline_module.call_concurrency,
"unregister_active_call",
unregister_concurrency,
)
task = asyncio.create_task(
run_pipeline_module.run_pipeline_telephony(
@ -175,6 +196,7 @@ async def test_telephony_entrypoint_counts_call_during_setup(monkeypatch):
with pytest.raises(RuntimeError, match="setup failed"):
await asyncio.wait_for(task, timeout=1.0)
assert active_calls.active_call_count() == 0
unregister_concurrency.assert_awaited_once_with(44)
def test_active_calls_route_requires_configured_secret(monkeypatch):

57
api/tests/test_agent_stream_route.py
View file

@ -3,6 +3,8 @@ from unittest.mock import AsyncMock, patch
import pytest
from api.services.call_concurrency import CallConcurrencyLimitError
class _FakeWebSocket:
def __init__(self, query_params: dict[str, str] | None = None):
@ -34,6 +36,7 @@ async def test_agent_stream_uses_provider_path_param_not_query_param():
with (
patch("api.routes.agent_stream.telephony_registry") as registry,
patch("api.routes.agent_stream.db_client") as db_client,
patch("api.routes.agent_stream.call_concurrency") as mock_concurrency,
patch(
"api.routes.agent_stream.authorize_workflow_run_start",
new=AsyncMock(
@ -41,6 +44,13 @@ async def test_agent_stream_uses_provider_path_param_not_query_param():
),
),
):
slot = object()
mock_concurrency.acquire_org_slot = AsyncMock(return_value=slot)
mock_concurrency.bind_workflow_run = AsyncMock()
mock_concurrency.release_slot = AsyncMock()
mock_concurrency.release_workflow_run_slot = AsyncMock()
mock_concurrency.unregister_active_call = AsyncMock()
registry.get_optional.return_value = spec
db_client.get_workflow_by_uuid_unscoped = AsyncMock(return_value=workflow)
db_client.create_workflow_run = AsyncMock(return_value=workflow_run)
@ -50,9 +60,17 @@ async def test_agent_stream_uses_provider_path_param_not_query_param():
registry.get_optional.assert_called_once_with("cloudonix")
db_client.create_workflow_run.assert_awaited_once()
mock_concurrency.acquire_org_slot.assert_awaited_once_with(
workflow.organization_id,
source="agent_stream:cloudonix",
timeout=0,
)
mock_concurrency.bind_workflow_run.assert_awaited_once_with(slot, workflow_run.id)
mock_concurrency.unregister_active_call.assert_awaited_once_with(workflow_run.id)
create_args = db_client.create_workflow_run.await_args.args
create_kwargs = db_client.create_workflow_run.await_args.kwargs
assert create_args[2] == "cloudonix"
assert create_kwargs["organization_id"] == workflow.organization_id
assert create_kwargs["initial_context"] == {
"existing": "context",
"provider": "cloudonix",
@ -62,3 +80,42 @@ async def test_agent_stream_uses_provider_path_param_not_query_param():
_, provider_kwargs = provider.handle_external_websocket.await_args
assert provider_kwargs["params"] == {"custom": "value"}
websocket.close.assert_not_awaited()
@pytest.mark.asyncio
async def test_agent_stream_rejects_when_concurrency_limit_reached():
from api.routes.agent_stream import agent_stream_websocket
websocket = _FakeWebSocket()
workflow = SimpleNamespace(
id=11,
user_id=22,
organization_id=33,
template_context_variables={},
)
spec = SimpleNamespace(provider_cls=lambda _config: object())
with (
patch("api.routes.agent_stream.telephony_registry") as registry,
patch("api.routes.agent_stream.db_client") as db_client,
patch("api.routes.agent_stream.call_concurrency") as mock_concurrency,
):
registry.get_optional.return_value = spec
db_client.get_workflow_by_uuid_unscoped = AsyncMock(return_value=workflow)
db_client.create_workflow_run = AsyncMock()
mock_concurrency.acquire_org_slot = AsyncMock(
side_effect=CallConcurrencyLimitError(
organization_id=workflow.organization_id,
source="agent_stream:cloudonix",
wait_time=0,
max_concurrent=1,
)
)
await agent_stream_websocket(websocket, "cloudonix", "agent-uuid")
websocket.close.assert_awaited_once_with(
code=1008,
reason="Concurrent call limit reached",
)
db_client.create_workflow_run.assert_not_awaited()

333
api/tests/test_call_concurrency.py Normal file
View file

@ -0,0 +1,333 @@
from unittest.mock import AsyncMock, patch
import pytest
from api.services.call_concurrency import (
CallConcurrencyLimitError,
CallConcurrencyService,
)
from api.services.campaign.rate_limiter import ConcurrentSlotAcquisition
@pytest.mark.asyncio
async def test_acquire_org_slot_logs_post_acquire_count_and_limit():
service = CallConcurrencyService()
with (
patch("api.services.call_concurrency.db_client") as mock_db,
patch("api.services.call_concurrency.rate_limiter") as mock_rate_limiter,
patch("api.services.call_concurrency.logger") as mock_logger,
):
mock_db.get_configuration = AsyncMock(return_value=None)
mock_rate_limiter.try_acquire_concurrent_slot_details = AsyncMock(
return_value=ConcurrentSlotAcquisition(
slot_id="slot-123",
active_count=7,
)
)
slot = await service.acquire_org_slot(199, source="test_source")
assert slot.organization_id == 199
assert slot.slot_id == "slot-123"
assert slot.max_concurrent == 10
assert slot.source == "test_source"
mock_rate_limiter.try_acquire_concurrent_slot_details.assert_awaited_once_with(
199, 10, scope_key=None, scope_max_concurrent=None
)
mock_logger.info.assert_called_once()
log_message = mock_logger.info.call_args.args[0]
assert "org 199" in log_message
assert "source=test_source" in log_message
assert "active_calls=7/10" in log_message
assert "slot_id=slot-123" in log_message
@pytest.mark.asyncio
async def test_acquire_org_slot_logs_warning_when_limit_reached():
service = CallConcurrencyService()
with (
patch("api.services.call_concurrency.db_client") as mock_db,
patch("api.services.call_concurrency.rate_limiter") as mock_rate_limiter,
patch("api.services.call_concurrency.logger") as mock_logger,
):
mock_db.get_configuration = AsyncMock(return_value=None)
mock_rate_limiter.try_acquire_concurrent_slot_details = AsyncMock(
return_value=None
)
mock_rate_limiter.get_concurrent_count = AsyncMock(return_value=12)
with pytest.raises(CallConcurrencyLimitError):
await service.acquire_org_slot(199, source="test_source", timeout=0)
mock_rate_limiter.get_concurrent_count.assert_awaited_once_with(199)
mock_logger.warning.assert_called_once()
log_message = mock_logger.warning.call_args.args[0]
assert "Concurrent call limit reached for org 199" in log_message
assert "source=test_source" in log_message
assert "active_calls=12/10" in log_message
@pytest.mark.asyncio
async def test_acquire_org_slot_fires_usage_event_per_org_member_when_limit_reached():
"""Mirrors the MPS org-event convention: one event per org member with the
member's provider_id as distinct_id, event_source property, no $groups."""
from types import SimpleNamespace
from api.enums import PostHogEvent
service = CallConcurrencyService()
members = [
SimpleNamespace(provider_id="user-a"),
SimpleNamespace(provider_id="user-b"),
]
with (
patch("api.services.call_concurrency.db_client") as mock_db,
patch("api.services.call_concurrency.rate_limiter") as mock_rate_limiter,
patch("api.services.call_concurrency.capture_event") as mock_capture,
):
mock_db.get_configuration = AsyncMock(return_value=None)
mock_db.get_organization_users = AsyncMock(return_value=members)
mock_rate_limiter.try_acquire_concurrent_slot_details = AsyncMock(
return_value=None
)
mock_rate_limiter.get_concurrent_count = AsyncMock(return_value=10)
with pytest.raises(CallConcurrencyLimitError):
await service.acquire_org_slot(199, source="webrtc", timeout=0)
mock_db.get_organization_users.assert_awaited_once_with(199)
assert mock_capture.call_count == 2
distinct_ids = [c.kwargs["distinct_id"] for c in mock_capture.call_args_list]
assert distinct_ids == ["user-a", "user-b"]
for call in mock_capture.call_args_list:
kwargs = call.kwargs
assert kwargs["event"] == PostHogEvent.USAGE_CONCURRENT_CALL_LIMIT_REACHED
assert "groups" not in kwargs
assert kwargs["properties"]["event_source"] == "dograh"
assert kwargs["properties"]["organization_id"] == 199
assert kwargs["properties"]["source"] == "webrtc"
assert kwargs["properties"]["active_calls"] == 10
assert kwargs["properties"]["max_concurrent"] == 10
assert "scope_key" not in kwargs["properties"]
@pytest.mark.asyncio
async def test_acquire_org_slot_passes_scope_to_rate_limiter():
service = CallConcurrencyService()
with (
patch("api.services.call_concurrency.db_client") as mock_db,
patch("api.services.call_concurrency.rate_limiter") as mock_rate_limiter,
):
mock_db.get_configuration = AsyncMock(return_value=None)
mock_rate_limiter.try_acquire_concurrent_slot_details = AsyncMock(
return_value=ConcurrentSlotAcquisition(slot_id="slot-123", active_count=1)
)
mock_rate_limiter.store_workflow_slot_mapping_if_absent = AsyncMock(
return_value=True
)
slot = await service.acquire_org_slot(
199,
source="campaign:42",
scope_key="campaign:42",
scope_max_concurrent=3,
)
await service.bind_workflow_run(slot, 501)
assert slot.scope_key == "campaign:42"
mock_rate_limiter.try_acquire_concurrent_slot_details.assert_awaited_once_with(
199, 10, scope_key="campaign:42", scope_max_concurrent=3
)
mock_rate_limiter.store_workflow_slot_mapping_if_absent.assert_awaited_once_with(
501, 199, "slot-123", scope_key="campaign:42"
)
@pytest.mark.asyncio
async def test_release_workflow_run_slot_keeps_mapping_on_redis_error():
service = CallConcurrencyService()
with patch("api.services.call_concurrency.rate_limiter") as mock_rate_limiter:
mock_rate_limiter.get_workflow_slot_mapping = AsyncMock(
return_value=(11, "slot-1", None)
)
# None = Redis error during release (vs False = slot already gone)
mock_rate_limiter.release_concurrent_slot = AsyncMock(return_value=None)
mock_rate_limiter.delete_workflow_slot_mapping = AsyncMock()
released = await service.release_workflow_run_slot(501)
assert released is False
mock_rate_limiter.release_concurrent_slot.assert_awaited_once_with(
11, "slot-1", scope_key=None
)
mock_rate_limiter.delete_workflow_slot_mapping.assert_not_awaited()
@pytest.mark.asyncio
async def test_release_workflow_run_slot_deletes_mapping_when_slot_already_gone():
service = CallConcurrencyService()
with patch("api.services.call_concurrency.rate_limiter") as mock_rate_limiter:
mock_rate_limiter.get_workflow_slot_mapping = AsyncMock(
return_value=(11, "slot-1", "campaign:42")
)
mock_rate_limiter.release_concurrent_slot = AsyncMock(return_value=False)
mock_rate_limiter.delete_workflow_slot_mapping = AsyncMock(return_value=True)
released = await service.release_workflow_run_slot(501)
assert released is False
mock_rate_limiter.release_concurrent_slot.assert_awaited_once_with(
11, "slot-1", scope_key="campaign:42"
)
mock_rate_limiter.delete_workflow_slot_mapping.assert_awaited_once_with(501)
@pytest.mark.asyncio
async def test_unregister_active_call_never_raises():
service = CallConcurrencyService()
with patch("api.services.call_concurrency.rate_limiter") as mock_rate_limiter:
mock_rate_limiter.get_workflow_slot_mapping = AsyncMock(
side_effect=RuntimeError("redis down")
)
released = await service.unregister_active_call(501)
assert released is False
# ---------------------------------------------------------------------------
# Redis integration tests for scoped (campaign-level) slot acquisition
# ---------------------------------------------------------------------------
import os # noqa: E402
import uuid # noqa: E402
from api.services.campaign.rate_limiter import RateLimiter # noqa: E402
requires_redis = pytest.mark.skipif(
"REDIS_URL" not in os.environ,
reason="Requires Redis (set REDIS_URL via .env.test)",
)
def _unique_org_id() -> int:
return uuid.uuid4().int % 10_000_000
@requires_redis
@pytest.mark.asyncio
async def test_scoped_acquisition_enforces_scope_limit_independently_of_org():
"""A campaign scope caps its own calls without measuring — or being
starved by — other calls in the same org counter."""
rl = RateLimiter()
org_id = _unique_org_id()
scope = f"campaign:{org_id}"
org_key = f"concurrent_calls:{org_id}"
scope_key_full = f"concurrent_calls:{scope}"
redis_client = await rl._get_redis()
try:
# Unscoped (e.g. WebRTC) calls fill part of the org counter.
for _ in range(3):
assert await rl.try_acquire_concurrent_slot_details(org_id, 10)
# Scope limit 2: two scoped acquisitions succeed...
first = await rl.try_acquire_concurrent_slot_details(
org_id, 10, scope_key=scope, scope_max_concurrent=2
)
second = await rl.try_acquire_concurrent_slot_details(
org_id, 10, scope_key=scope, scope_max_concurrent=2
)
assert first and second
# ...the third is rejected by the scope even though the org has room.
third = await rl.try_acquire_concurrent_slot_details(
org_id, 10, scope_key=scope, scope_max_concurrent=2
)
assert third is None
# Unscoped calls are unaffected by the scope being full.
assert await rl.try_acquire_concurrent_slot_details(org_id, 10)
# Releasing with the scope key frees both counters.
released = await rl.release_concurrent_slot(
org_id, first.slot_id, scope_key=scope
)
assert released is True
assert await redis_client.zscore(org_key, first.slot_id) is None
assert await redis_client.zscore(scope_key_full, first.slot_id) is None
# And the scope accepts a new call again.
assert await rl.try_acquire_concurrent_slot_details(
org_id, 10, scope_key=scope, scope_max_concurrent=2
)
finally:
await redis_client.delete(org_key, scope_key_full)
await rl.close()
@requires_redis
@pytest.mark.asyncio
async def test_org_limit_still_binds_scoped_acquisition():
rl = RateLimiter()
org_id = _unique_org_id()
scope = f"campaign:{org_id}"
org_key = f"concurrent_calls:{org_id}"
scope_key_full = f"concurrent_calls:{scope}"
redis_client = await rl._get_redis()
try:
assert await rl.try_acquire_concurrent_slot_details(org_id, 1)
# Org counter is full, so the scoped acquire fails and must not
# leave a phantom entry in the scope counter.
rejected = await rl.try_acquire_concurrent_slot_details(
org_id, 1, scope_key=scope, scope_max_concurrent=5
)
assert rejected is None
assert await redis_client.zcard(scope_key_full) == 0
finally:
await redis_client.delete(org_key, scope_key_full)
await rl.close()
@requires_redis
@pytest.mark.asyncio
async def test_workflow_slot_mapping_round_trips_scope_key():
rl = RateLimiter()
run_id = _unique_org_id()
mapping_key = f"workflow_slot_mapping:{run_id}"
redis_client = await rl._get_redis()
try:
stored = await rl.store_workflow_slot_mapping_if_absent(
run_id, 11, "slot-1", scope_key="campaign:42"
)
assert stored is True
assert await rl.get_workflow_slot_mapping(run_id) == (
11,
"slot-1",
"campaign:42",
)
# Unscoped mappings surface scope_key=None.
run_id_2 = _unique_org_id()
try:
await rl.store_workflow_slot_mapping_if_absent(run_id_2, 11, "slot-2")
assert await rl.get_workflow_slot_mapping(run_id_2) == (
11,
"slot-2",
None,
)
finally:
await redis_client.delete(f"workflow_slot_mapping:{run_id_2}")
finally:
await redis_client.delete(mapping_key)
await rl.close()

66
api/tests/test_campaign_call_dispatcher.py
View file

@ -25,6 +25,7 @@ from api.db.models import (
WorkflowModel,
WorkflowRunModel,
)
from api.services.call_concurrency import CallConcurrencySlot
from api.services.campaign.campaign_call_dispatcher import CampaignCallDispatcher
# =============================================================================
@ -259,6 +260,27 @@ def mock_rate_limiter():
}
@pytest.fixture(autouse=True)
def mock_call_concurrency():
async def acquire_slot(organization_id, *, source, **kwargs):
return CallConcurrencySlot(
organization_id=organization_id,
slot_id=f"slot-{uuid.uuid4().hex[:8]}",
max_concurrent=20,
source=source,
scope_key=kwargs.get("scope_key"),
)
with patch(
"api.services.campaign.campaign_call_dispatcher.call_concurrency"
) as mock_concurrency:
mock_concurrency.acquire_org_slot = AsyncMock(side_effect=acquire_slot)
mock_concurrency.bind_workflow_run = AsyncMock()
mock_concurrency.release_slot = AsyncMock(return_value=True)
mock_concurrency.release_workflow_run_slot = AsyncMock(return_value=True)
yield mock_concurrency
# =============================================================================
# Tests
# =============================================================================
@ -793,3 +815,47 @@ class TestProcessBatchEdgeCases:
{"campaign_id": campaign_test_data.campaign_id},
)
await session.commit()
class TestAcquireConcurrentSlotScoping:
"""Campaign max_concurrency must scope to the campaign, not the org counter."""
def _campaign(self, orchestrator_metadata):
campaign = MagicMock()
campaign.id = 42
campaign.orchestrator_metadata = orchestrator_metadata
return campaign
@pytest.mark.asyncio
async def test_campaign_max_concurrency_uses_campaign_scope(
self, mock_call_concurrency
):
dispatcher = CampaignCallDispatcher()
campaign = self._campaign({"max_concurrency": 3})
await dispatcher.acquire_concurrent_slot(7, campaign, timeout=5)
mock_call_concurrency.acquire_org_slot.assert_awaited_once_with(
7,
source="campaign:42",
timeout=5,
scope_key="campaign:42",
scope_max_concurrent=3,
retry_interval=1,
)
@pytest.mark.asyncio
async def test_no_campaign_max_concurrency_skips_scope(self, mock_call_concurrency):
dispatcher = CampaignCallDispatcher()
campaign = self._campaign({})
await dispatcher.acquire_concurrent_slot(7, campaign, timeout=5)
mock_call_concurrency.acquire_org_slot.assert_awaited_once_with(
7,
source="campaign:42",
timeout=5,
scope_key=None,
scope_max_concurrent=None,
retry_interval=1,
)

5
api/tests/test_circuit_breaker.py
View file

@ -677,15 +677,20 @@ class TestProcessStatusUpdateCircuitBreaker:
with (
patch("api.services.telephony.status_processor.db_client") as mock_db,
patch(
"api.services.telephony.status_processor.campaign_call_dispatcher"
) as mock_dispatcher,
patch("api.services.telephony.status_processor.circuit_breaker") as mock_cb,
):
mock_db.get_workflow_run_by_id = AsyncMock(return_value=mock_workflow_run)
mock_db.update_workflow_run = AsyncMock()
mock_dispatcher.release_call_slot = AsyncMock(return_value=True)
await _process_status_update(100, status)
# Circuit breaker should NOT be called for non-campaign calls
mock_cb.record_and_evaluate.assert_not_called()
mock_dispatcher.release_call_slot.assert_awaited_once_with(100)
# =============================================================================

30
api/tests/test_from_number_pool_isolation.py
View file

@ -20,6 +20,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from api.services.call_concurrency import CallConcurrencySlot
from api.services.campaign.campaign_call_dispatcher import CampaignCallDispatcher
from api.services.campaign.rate_limiter import RateLimiter
@ -266,6 +267,9 @@ class TestDispatcherThreadsTelephonyConfig:
patch(
"api.services.campaign.campaign_call_dispatcher.rate_limiter"
) as mock_rl,
patch(
"api.services.campaign.campaign_call_dispatcher.call_concurrency"
) as mock_concurrency,
patch(
"api.services.campaign.campaign_call_dispatcher.get_backend_endpoints",
AsyncMock(return_value=("https://example.com", None)),
@ -280,14 +284,21 @@ class TestDispatcherThreadsTelephonyConfig:
mock_db.get_workflow_by_id = AsyncMock(return_value=SimpleNamespace(id=1))
mock_db.create_workflow_run = AsyncMock(return_value=workflow_run)
mock_db.update_workflow_run = AsyncMock()
mock_concurrency.bind_workflow_run = AsyncMock()
mock_concurrency.release_slot = AsyncMock()
mock_concurrency.release_workflow_run_slot = AsyncMock()
mock_rl.acquire_from_number = AsyncMock(return_value="+15551110001")
mock_rl.release_from_number = AsyncMock()
mock_rl.release_concurrent_slot = AsyncMock()
mock_rl.store_workflow_slot_mapping = AsyncMock()
mock_rl.store_workflow_from_number_mapping = AsyncMock()
await dispatcher.dispatch_call(queued_run, campaign, slot_id="slot-1")
slot = CallConcurrencySlot(
organization_id=org_id,
slot_id="slot-1",
max_concurrent=1,
source="test",
)
await dispatcher.dispatch_call(queued_run, campaign, slot)
# acquire_from_number on rate_limiter must be called with the
# campaign's telephony_configuration_id.
@ -337,10 +348,15 @@ class TestDispatcherThreadsTelephonyConfig:
dispatcher = CampaignCallDispatcher()
with patch(
"api.services.campaign.campaign_call_dispatcher.rate_limiter"
) as mock_rl:
mock_rl.get_workflow_slot_mapping = AsyncMock(return_value=None)
with (
patch(
"api.services.campaign.campaign_call_dispatcher.rate_limiter"
) as mock_rl,
patch(
"api.services.campaign.campaign_call_dispatcher.call_concurrency"
) as mock_concurrency,
):
mock_concurrency.release_workflow_run_slot = AsyncMock(return_value=False)
mock_rl.get_workflow_from_number_mapping = AsyncMock(
return_value=(org_id, from_number, config_id)
)

131
api/tests/test_public_agent_routes.py
View file

@ -5,6 +5,7 @@ from fastapi import FastAPI
from fastapi.testclient import TestClient
from api.routes.public_agent import router
from api.services.call_concurrency import CallConcurrencyLimitError
def _make_test_app() -> FastAPI:
@ -56,6 +57,7 @@ def test_trigger_route_executes_as_workflow_owner():
with (
patch("api.routes.public_agent.db_client") as mock_db,
patch("api.routes.public_agent.call_concurrency") as mock_concurrency,
patch(
"api.routes.public_agent.authorize_workflow_run_start",
new=quota_mock,
@ -69,6 +71,12 @@ def test_trigger_route_executes_as_workflow_owner():
new=AsyncMock(return_value=("https://api.example.com", "wss://ignored")),
),
):
slot = object()
mock_concurrency.acquire_org_slot = AsyncMock(return_value=slot)
mock_concurrency.bind_workflow_run = AsyncMock()
mock_concurrency.release_workflow_run_slot = AsyncMock()
mock_concurrency.release_slot = AsyncMock()
mock_db.validate_api_key = AsyncMock(
return_value=SimpleNamespace(id=7, organization_id=11, created_by=22)
)
@ -96,6 +104,12 @@ def test_trigger_route_executes_as_workflow_owner():
workflow_id=workflow.id,
workflow_run_id=501,
)
mock_concurrency.acquire_org_slot.assert_awaited_once_with(
workflow.organization_id,
source="public_agent",
timeout=0,
)
mock_concurrency.bind_workflow_run.assert_awaited_once_with(slot, 501)
mock_db.get_workflow.assert_awaited_once_with(workflow.id, organization_id=11)
create_kwargs = mock_db.create_workflow_run.await_args.kwargs
@ -126,6 +140,7 @@ def test_workflow_uuid_route_uses_scoped_lookup_and_shared_execution():
with (
patch("api.routes.public_agent.db_client") as mock_db,
patch("api.routes.public_agent.call_concurrency") as mock_concurrency,
patch(
"api.routes.public_agent.authorize_workflow_run_start",
new=quota_mock,
@ -139,6 +154,12 @@ def test_workflow_uuid_route_uses_scoped_lookup_and_shared_execution():
new=AsyncMock(return_value=("https://api.example.com", "wss://ignored")),
),
):
slot = object()
mock_concurrency.acquire_org_slot = AsyncMock(return_value=slot)
mock_concurrency.bind_workflow_run = AsyncMock()
mock_concurrency.release_workflow_run_slot = AsyncMock()
mock_concurrency.release_slot = AsyncMock()
mock_db.validate_api_key = AsyncMock(
return_value=SimpleNamespace(id=8, organization_id=11, created_by=22)
)
@ -160,6 +181,12 @@ def test_workflow_uuid_route_uses_scoped_lookup_and_shared_execution():
11,
)
assert not mock_db.get_agent_trigger_by_path.called
mock_concurrency.acquire_org_slot.assert_awaited_once_with(
workflow.organization_id,
source="public_agent",
timeout=0,
)
mock_concurrency.bind_workflow_run.assert_awaited_once_with(slot, 601)
create_kwargs = mock_db.create_workflow_run.await_args.kwargs
assert create_kwargs["user_id"] == workflow.user_id
@ -170,6 +197,110 @@ def test_workflow_uuid_route_uses_scoped_lookup_and_shared_execution():
assert "agent_uuid" not in create_kwargs["initial_context"]
def test_trigger_route_rejects_when_concurrency_limit_reached():
app = _make_test_app()
client = TestClient(app)
workflow = _active_workflow(trigger_path="trigger-uuid-123")
provider = _provider()
with (
patch("api.routes.public_agent.db_client") as mock_db,
patch("api.routes.public_agent.call_concurrency") as mock_concurrency,
patch(
"api.routes.public_agent.get_default_telephony_provider",
new=AsyncMock(return_value=provider),
),
):
mock_concurrency.acquire_org_slot = AsyncMock(
side_effect=CallConcurrencyLimitError(
organization_id=11,
source="public_agent",
wait_time=0,
max_concurrent=2,
)
)
mock_db.validate_api_key = AsyncMock(
return_value=SimpleNamespace(id=7, organization_id=11, created_by=22)
)
mock_db.get_agent_trigger_by_path = AsyncMock(
return_value=SimpleNamespace(
workflow_id=workflow.id,
organization_id=11,
state="active",
)
)
mock_db.get_workflow = AsyncMock(return_value=workflow)
mock_db.get_default_telephony_configuration = AsyncMock(
return_value=SimpleNamespace(id=55)
)
mock_db.create_workflow_run = AsyncMock()
response = client.post(
"/public/agent/trigger-uuid-123",
headers={"X-API-Key": "test-api-key"},
json={"phone_number": "+15551234567"},
)
assert response.status_code == 429
assert response.json()["detail"] == "Concurrent call limit reached"
mock_db.create_workflow_run.assert_not_called()
def test_trigger_route_releases_concurrency_slot_when_quota_fails():
app = _make_test_app()
client = TestClient(app)
workflow = _active_workflow(trigger_path="trigger-uuid-123")
provider = _provider()
quota_mock = AsyncMock(
return_value=SimpleNamespace(has_quota=False, error_message="Quota exceeded")
)
with (
patch("api.routes.public_agent.db_client") as mock_db,
patch("api.routes.public_agent.call_concurrency") as mock_concurrency,
patch(
"api.routes.public_agent.authorize_workflow_run_start",
new=quota_mock,
),
patch(
"api.routes.public_agent.get_default_telephony_provider",
new=AsyncMock(return_value=provider),
),
):
mock_concurrency.acquire_org_slot = AsyncMock(return_value=object())
mock_concurrency.bind_workflow_run = AsyncMock()
mock_concurrency.release_workflow_run_slot = AsyncMock()
mock_concurrency.release_slot = AsyncMock()
mock_db.validate_api_key = AsyncMock(
return_value=SimpleNamespace(id=7, organization_id=11, created_by=22)
)
mock_db.get_agent_trigger_by_path = AsyncMock(
return_value=SimpleNamespace(
workflow_id=workflow.id,
organization_id=11,
state="active",
)
)
mock_db.get_workflow = AsyncMock(return_value=workflow)
mock_db.get_default_telephony_configuration = AsyncMock(
return_value=SimpleNamespace(id=55)
)
mock_db.create_workflow_run = AsyncMock(return_value=SimpleNamespace(id=501))
response = client.post(
"/public/agent/trigger-uuid-123",
headers={"X-API-Key": "test-api-key"},
json={"phone_number": "+15551234567"},
)
assert response.status_code == 402
mock_concurrency.release_workflow_run_slot.assert_awaited_once_with(501)
provider.initiate_call.assert_not_awaited()
def test_workflow_uuid_route_rejects_archived_workflows():
app = _make_test_app()
client = TestClient(app)

3
api/tests/test_public_embed_cors.py
View file

@ -25,6 +25,7 @@ _ACTIVE_TOKEN = SimpleNamespace(
expires_at=None,
allowed_domains=[],
workflow_id=1,
organization_id=11,
created_by=7,
usage_limit=None,
usage_count=0,
@ -37,6 +38,7 @@ _RESTRICTED_TOKEN = SimpleNamespace(
expires_at=None,
allowed_domains=["allowed.example.com"],
workflow_id=2,
organization_id=11,
created_by=7,
usage_limit=None,
usage_count=0,
@ -49,6 +51,7 @@ _LOCALHOST_TOKEN = SimpleNamespace(
expires_at=None,
allowed_domains=["localhost:3000", "localhost:3020"],
workflow_id=3,
organization_id=11,
created_by=7,
usage_limit=None,
usage_count=0,

141
api/tests/test_telephony_routes.py
View file

@ -6,8 +6,10 @@ from fastapi import FastAPI
from fastapi.testclient import TestClient
from api.enums import WorkflowRunMode, WorkflowRunState
from api.routes.telephony import _handle_telephony_websocket, router
from api.errors.telephony_errors import TelephonyError
from api.routes.telephony import _handle_telephony_websocket, handle_inbound_run, router
from api.services.auth.depends import get_user
from api.services.call_concurrency import CallConcurrencyLimitError
def _make_test_app() -> FastAPI:
@ -55,6 +57,7 @@ def test_initiate_call_executes_as_workflow_owner_for_shared_org_workflow():
with (
patch("api.routes.telephony.db_client") as mock_db,
patch("api.routes.telephony.call_concurrency") as mock_concurrency,
patch(
"api.routes.telephony.authorize_workflow_run_start",
new=quota_mock,
@ -68,6 +71,12 @@ def test_initiate_call_executes_as_workflow_owner_for_shared_org_workflow():
new=AsyncMock(return_value=("https://api.example.com", "wss://ignored")),
),
):
slot = object()
mock_concurrency.acquire_org_slot = AsyncMock(return_value=slot)
mock_concurrency.bind_workflow_run = AsyncMock()
mock_concurrency.release_slot = AsyncMock()
mock_concurrency.release_workflow_run_slot = AsyncMock()
mock_db.get_user_configurations = AsyncMock(
return_value=SimpleNamespace(test_phone_number=None)
)
@ -104,6 +113,12 @@ def test_initiate_call_executes_as_workflow_owner_for_shared_org_workflow():
assert create_kwargs["user_id"] == workflow.user_id
assert create_kwargs["organization_id"] == workflow.organization_id
assert create_kwargs["initial_context"]["template_key"] == "template-value"
mock_concurrency.acquire_org_slot.assert_awaited_once_with(
workflow.organization_id,
source="telephony_outbound",
timeout=0,
)
mock_concurrency.bind_workflow_run.assert_awaited_once_with(slot, 501)
initiate_kwargs = provider.initiate_call.await_args.kwargs
assert initiate_kwargs["workflow_id"] == workflow.id
@ -124,6 +139,7 @@ def test_initiate_call_uses_organization_preference_phone_number():
with (
patch("api.routes.telephony.db_client") as mock_db,
patch("api.routes.telephony.call_concurrency") as mock_concurrency,
patch(
"api.routes.telephony.authorize_workflow_run_start",
new=quota_mock,
@ -137,6 +153,11 @@ def test_initiate_call_uses_organization_preference_phone_number():
new=AsyncMock(return_value=("https://api.example.com", "wss://ignored")),
),
):
mock_concurrency.acquire_org_slot = AsyncMock(return_value=object())
mock_concurrency.bind_workflow_run = AsyncMock()
mock_concurrency.release_slot = AsyncMock()
mock_concurrency.release_workflow_run_slot = AsyncMock()
mock_db.get_user_configurations = AsyncMock(
return_value=SimpleNamespace(test_phone_number="+15550000000")
)
@ -178,6 +199,7 @@ def test_initiate_call_rejects_existing_run_for_different_workflow():
with (
patch("api.routes.telephony.db_client") as mock_db,
patch("api.routes.telephony.call_concurrency") as mock_concurrency,
patch(
"api.routes.telephony.authorize_workflow_run_start",
new=quota_mock,
@ -187,6 +209,11 @@ def test_initiate_call_rejects_existing_run_for_different_workflow():
new=AsyncMock(return_value=provider),
),
):
mock_concurrency.acquire_org_slot = AsyncMock(return_value=object())
mock_concurrency.bind_workflow_run = AsyncMock()
mock_concurrency.release_slot = AsyncMock()
mock_concurrency.release_workflow_run_slot = AsyncMock()
mock_db.get_user_configurations = AsyncMock(
return_value=SimpleNamespace(test_phone_number=None)
)
@ -215,10 +242,119 @@ def test_initiate_call_rejects_existing_run_for_different_workflow():
assert response.status_code == 400
assert response.json()["detail"] == "workflow_run_workflow_mismatch"
mock_db.get_workflow_run.assert_awaited_once_with(501, organization_id=11)
mock_concurrency.release_slot.assert_awaited_once()
assert not mock_db.create_workflow_run.called
assert provider.initiate_call.await_count == 0
def test_initiate_call_rejects_when_concurrency_limit_reached():
app = _make_test_app()
client = TestClient(app)
workflow = _workflow()
provider = _provider()
with (
patch("api.routes.telephony.db_client") as mock_db,
patch("api.routes.telephony.call_concurrency") as mock_concurrency,
patch(
"api.routes.telephony.get_default_telephony_provider",
new=AsyncMock(return_value=provider),
),
):
mock_concurrency.acquire_org_slot = AsyncMock(
side_effect=CallConcurrencyLimitError(
organization_id=workflow.organization_id,
source="telephony_outbound",
wait_time=0,
max_concurrent=1,
)
)
mock_db.get_default_telephony_configuration = AsyncMock(
return_value=SimpleNamespace(id=55)
)
mock_db.get_workflow = AsyncMock(return_value=workflow)
mock_db.create_workflow_run = AsyncMock()
response = client.post(
"/telephony/initiate-call",
json={"workflow_id": workflow.id, "phone_number": "+15551234567"},
)
assert response.status_code == 429
assert response.json()["detail"] == "Concurrent call limit reached"
mock_db.create_workflow_run.assert_not_called()
provider.initiate_call.assert_not_awaited()
@pytest.mark.asyncio
async def test_inbound_run_rejects_when_concurrency_limit_reached():
request = SimpleNamespace(headers={}, url="https://api.example.com/inbound/run")
provider_class = SimpleNamespace(
PROVIDER_NAME="twilio",
generate_validation_error_response=Mock(return_value="limit-response"),
)
normalized_data = SimpleNamespace(
provider="twilio",
direction="inbound",
to_number="+15551230000",
from_number="+15557650000",
to_country="US",
from_country="US",
account_id="acct-1",
call_id="call-1",
raw_data={},
)
config = SimpleNamespace(id=55, organization_id=11)
phone_row = SimpleNamespace(id=77, inbound_workflow_id=33)
workflow = SimpleNamespace(id=33, user_id=99)
provider_instance = SimpleNamespace(
verify_inbound_signature=AsyncMock(return_value=True)
)
with (
patch(
"api.routes.telephony.parse_webhook_request",
new=AsyncMock(return_value=({}, "raw-body")),
),
patch(
"api.routes.telephony._detect_provider",
new=AsyncMock(return_value=provider_class),
),
patch(
"api.routes.telephony.normalize_webhook_data",
return_value=normalized_data,
),
patch("api.routes.telephony.db_client") as mock_db,
patch(
"api.routes.telephony.get_telephony_provider_by_id",
new=AsyncMock(return_value=provider_instance),
),
patch("api.routes.telephony.call_concurrency") as mock_concurrency,
):
mock_db.find_inbound_route_by_account = AsyncMock(
return_value=(config, phone_row)
)
mock_db.get_workflow = AsyncMock(return_value=workflow)
mock_db.create_workflow_run = AsyncMock()
mock_concurrency.acquire_org_slot = AsyncMock(
side_effect=CallConcurrencyLimitError(
organization_id=config.organization_id,
source="inbound:twilio",
wait_time=0,
max_concurrent=1,
)
)
response = await handle_inbound_run(request)
assert response == "limit-response"
provider_class.generate_validation_error_response.assert_called_once_with(
TelephonyError.CONCURRENT_CALL_LIMIT
)
mock_db.create_workflow_run.assert_not_awaited()
@pytest.mark.asyncio
async def test_smallwebrtc_run_reaching_telephony_websocket_closes_without_running():
websocket = AsyncMock()
@ -235,11 +371,13 @@ async def test_smallwebrtc_run_reaching_telephony_websocket_closes_without_runni
with (
patch("api.routes.telephony.db_client") as mock_db,
patch("api.routes.telephony.call_concurrency") as mock_concurrency,
patch(
"api.routes.telephony.get_telephony_provider_for_run",
new=provider_lookup,
),
):
mock_concurrency.unregister_active_call = AsyncMock()
mock_db.get_workflow_run = AsyncMock(return_value=workflow_run)
mock_db.get_workflow_by_id = AsyncMock(return_value=workflow)
mock_db.update_workflow_run = AsyncMock()
@ -255,3 +393,4 @@ async def test_smallwebrtc_run_reaching_telephony_websocket_closes_without_runni
)
assert mock_db.update_workflow_run.await_count == 0
assert provider_lookup.await_count == 0
mock_concurrency.unregister_active_call.assert_not_awaited()

167
api/tests/test_webrtc_signaling_concurrency.py Normal file
View file

@ -0,0 +1,167 @@
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import pytest
from api.routes.webrtc_signaling import SignalingManager
from api.services.call_concurrency import CallConcurrencyLimitError
class _FakeWebSocket:
def __init__(self):
self.send_json = AsyncMock()
class _FakePeerConnection:
def __init__(self):
self.renegotiate = AsyncMock()
def get_answer(self):
return {"sdp": "v=0\r\n", "type": "answer", "pc_id": "pc-1"}
def _offer_payload(pc_id: str = "pc-1") -> dict:
return {
"pc_id": pc_id,
"sdp": "v=0\r\n",
"type": "offer",
}
@pytest.mark.asyncio
async def test_public_embed_offer_rejects_when_org_concurrency_limit_reached():
manager = SignalingManager()
ws = _FakeWebSocket()
user = SimpleNamespace(id=7)
with (
patch("api.routes.webrtc_signaling.db_client") as mock_db,
patch(
"api.routes.webrtc_signaling.authorize_workflow_run_start",
new=AsyncMock(
return_value=SimpleNamespace(has_quota=True, error_message="")
),
),
patch("api.routes.webrtc_signaling.call_concurrency") as mock_concurrency,
):
mock_db.get_workflow_organization_id = AsyncMock(return_value=11)
mock_concurrency.acquire_org_slot = AsyncMock(
side_effect=CallConcurrencyLimitError(
organization_id=11,
source="public_embed",
wait_time=0,
max_concurrent=2,
)
)
mock_concurrency.bind_workflow_run = AsyncMock()
await manager._handle_offer(
ws,
_offer_payload(),
workflow_id=33,
workflow_run_id=501,
user=user,
connection_key="conn-1",
enforce_call_concurrency=True,
call_concurrency_source="public_embed",
)
ws.send_json.assert_awaited_once_with(
{
"type": "error",
"payload": {
"error_type": "concurrency_limit_exceeded",
"message": "Concurrent call limit reached",
},
}
)
mock_concurrency.bind_workflow_run.assert_not_called()
@pytest.mark.asyncio
async def test_public_embed_renegotiation_does_not_acquire_another_slot():
manager = SignalingManager()
ws = _FakeWebSocket()
user = SimpleNamespace(id=7)
connection_key = "conn-1"
pc = _FakePeerConnection()
manager._peer_connections["pc-1"] = pc
manager._peer_connection_owners["pc-1"] = connection_key
with (
patch("api.routes.webrtc_signaling.db_client") as mock_db,
patch(
"api.routes.webrtc_signaling.authorize_workflow_run_start",
new=AsyncMock(
return_value=SimpleNamespace(has_quota=True, error_message="")
),
),
patch("api.routes.webrtc_signaling.call_concurrency") as mock_concurrency,
):
mock_db.get_workflow_organization_id = AsyncMock(return_value=11)
mock_concurrency.acquire_org_slot = AsyncMock()
await manager._handle_offer(
ws,
_offer_payload(),
workflow_id=33,
workflow_run_id=501,
user=user,
connection_key=connection_key,
enforce_call_concurrency=True,
call_concurrency_source="public_embed",
)
mock_concurrency.acquire_org_slot.assert_not_called()
pc.renegotiate.assert_awaited_once()
assert ws.send_json.await_args.args[0]["type"] == "answer"
@pytest.mark.asyncio
async def test_signaling_websocket_rejects_run_not_owned_by_workflow():
"""The URL workflow_id drives org/quota/concurrency accounting, so a
workflow_id that doesn't own the run must be rejected before signaling."""
from fastapi import HTTPException
from api.routes.webrtc_signaling import signaling_websocket
ws = _FakeWebSocket()
user = SimpleNamespace(id=7)
with (
patch("api.routes.webrtc_signaling.db_client") as mock_db,
patch("api.routes.webrtc_signaling.signaling_manager") as mock_manager,
):
mock_db.get_workflow_run = AsyncMock(
return_value=SimpleNamespace(id=501, workflow_id=99)
)
mock_manager.handle_websocket = AsyncMock()
with pytest.raises(HTTPException) as exc_info:
await signaling_websocket(
ws, workflow_id=33, workflow_run_id=501, user=user
)
assert exc_info.value.status_code == 400
mock_manager.handle_websocket.assert_not_called()
@pytest.mark.asyncio
async def test_signaling_websocket_accepts_matching_workflow_and_run():
from api.routes.webrtc_signaling import signaling_websocket
ws = _FakeWebSocket()
user = SimpleNamespace(id=7)
with (
patch("api.routes.webrtc_signaling.db_client") as mock_db,
patch("api.routes.webrtc_signaling.signaling_manager") as mock_manager,
):
mock_db.get_workflow_run = AsyncMock(
return_value=SimpleNamespace(id=501, workflow_id=33)
)
mock_manager.handle_websocket = AsyncMock()
await signaling_websocket(ws, workflow_id=33, workflow_run_id=501, user=user)
mock_manager.handle_websocket.assert_awaited_once()

61
api/tests/test_workflow_configurations_schema.py Normal file
View file

@ -0,0 +1,61 @@
import pytest
from pydantic import ValidationError
from api.schemas.workflow_configurations import (
DEFAULT_MAX_CALL_DURATION_SECONDS,
MAX_CALL_DURATION_SECONDS,
WorkflowConfigurationDefaults,
)
def test_max_call_duration_default_within_bounds():
config = WorkflowConfigurationDefaults()
assert config.max_call_duration == DEFAULT_MAX_CALL_DURATION_SECONDS
def test_max_call_duration_accepts_cap():
config = WorkflowConfigurationDefaults(max_call_duration=MAX_CALL_DURATION_SECONDS)
assert config.max_call_duration == MAX_CALL_DURATION_SECONDS
def test_max_call_duration_rejects_over_cap():
with pytest.raises(ValidationError):
WorkflowConfigurationDefaults(max_call_duration=MAX_CALL_DURATION_SECONDS + 1)
def test_max_call_duration_rejects_non_positive():
with pytest.raises(ValidationError):
WorkflowConfigurationDefaults(max_call_duration=0)
def test_null_values_treated_as_unset():
"""Stored configs / older clients send explicit JSON nulls for keys the
user never configured; they must validate as defaults, not fail."""
config = WorkflowConfigurationDefaults.model_validate(
{
"max_call_duration": None,
"turn_start_strategy": None,
"turn_start_min_words": None,
}
)
assert config.max_call_duration == DEFAULT_MAX_CALL_DURATION_SECONDS
# Nulls count as unset, so a sparse round-trip drops them entirely.
assert config.model_dump(exclude_unset=True) == {}
def test_exclude_unset_round_trip_stays_sparse():
config = WorkflowConfigurationDefaults.model_validate(
{"max_call_duration": 600, "custom_extra_key": {"a": 1}}
)
assert config.model_dump(exclude_unset=True) == {
"max_call_duration": 600,
"custom_extra_key": {"a": 1},
}
def test_cap_stays_within_concurrency_stale_timeout():
"""A call outliving the rate limiter's stale window has its concurrency
slot purged mid-call, so the cap must never exceed it."""
from api.services.campaign.rate_limiter import rate_limiter
assert MAX_CALL_DURATION_SECONDS <= rate_limiter.stale_call_timeout

2
docs/api-reference/openapi.json
View file

File diff suppressed because one or more lines are too long

2
docs/developer/environment-variables.mdx
View file

@ -179,7 +179,7 @@ Controls concurrency for [Campaigns](/core-concepts/campaigns), Dograh's bulk ou
| Variable | Default | Description |
|---|---|---|
| `DEFAULT_ORG_CONCURRENCY_LIMIT` | `2` | Maximum concurrent outbound calls per organization |
| `DEFAULT_ORG_CONCURRENCY_LIMIT` | `10` | Maximum concurrent active calls per organization (values below 1 are clamped to 1) |
---

77
sdk/python/src/dograh_sdk/_generated_models.py
View file

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: dograh-openapi-XXXXXX.json.qHm1SwtOq5
# timestamp: 2026-07-07T12:20:41+00:00
# filename: dograh-openapi-XXXXXX.json.c3Lmi0soSL
# timestamp: 2026-07-09T12:55:26+00:00
from __future__ import annotations
@ -10,6 +10,14 @@ from typing import Annotated, Any, Literal
from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, RootModel
class AmbientNoiseConfigurationDefaults(BaseModel):
model_config = ConfigDict(
extra='allow',
)
enabled: Annotated[bool | None, Field(title='Enabled')] = False
volume: Annotated[float | None, Field(title='Volume')] = 0.3
class CalculatorToolDefinition(BaseModel):
"""
Tool definition for Calculator tools.
@ -582,19 +590,6 @@ class TransferCallToolDefinition(BaseModel):
"""
class UpdateWorkflowRequest(BaseModel):
name: Annotated[str | None, Field(title='Name')] = None
workflow_definition: Annotated[
dict[str, Any] | None, Field(title='Workflow Definition')
] = None
template_context_variables: Annotated[
dict[str, Any] | None, Field(title='Template Context Variables')
] = None
workflow_configurations: Annotated[
dict[str, Any] | None, Field(title='Workflow Configurations')
] = None
class ValidationError(BaseModel):
loc: Annotated[list[str | int], Field(title='Location')]
msg: Annotated[str, Field(title='Message')]
@ -603,6 +598,47 @@ class ValidationError(BaseModel):
ctx: Annotated[dict[str, Any] | None, Field(title='Context')] = None
class TurnStartStrategy(Enum):
default = 'default'
min_words = 'min_words'
provisional_vad = 'provisional_vad'
class TurnStopStrategy(Enum):
transcription = 'transcription'
turn_analyzer = 'turn_analyzer'
class WorkflowConfigurationDefaults(BaseModel):
model_config = ConfigDict(
extra='allow',
)
ambient_noise_configuration: AmbientNoiseConfigurationDefaults | None = None
max_call_duration: Annotated[
int | None, Field(gt=0, le=1200, title='Max Call Duration')
] = 300
max_user_idle_timeout: Annotated[
float | None, Field(title='Max User Idle Timeout')
] = 10.0
smart_turn_stop_secs: Annotated[
float | None, Field(title='Smart Turn Stop Secs')
] = 2.0
turn_start_strategy: Annotated[
TurnStartStrategy | None, Field(title='Turn Start Strategy')
] = 'default'
turn_start_min_words: Annotated[int | None, Field(title='Turn Start Min Words')] = 3
provisional_vad_pause_secs: Annotated[
float | None, Field(title='Provisional Vad Pause Secs')
] = 1.5
turn_stop_strategy: Annotated[
TurnStopStrategy | None, Field(title='Turn Stop Strategy')
] = 'transcription'
dictionary: Annotated[str | None, Field(title='Dictionary')] = ''
context_compaction_enabled: Annotated[
bool | None, Field(title='Context Compaction Enabled')
] = False
class WorkflowListResponse(BaseModel):
"""
Lightweight response for workflow listings (excludes large fields).
@ -778,6 +814,17 @@ class RecordingListResponseSchema(BaseModel):
total: Annotated[int, Field(title='Total')]
class UpdateWorkflowRequest(BaseModel):
name: Annotated[str | None, Field(title='Name')] = None
workflow_definition: Annotated[
dict[str, Any] | None, Field(title='Workflow Definition')
] = None
template_context_variables: Annotated[
dict[str, Any] | None, Field(title='Template Context Variables')
] = None
workflow_configurations: WorkflowConfigurationDefaults | None = None
class CreateToolRequest(BaseModel):
"""
Request schema for creating a reusable tool.

75
sdk/typescript/src/_generated_models.ts
View file

@ -272,6 +272,21 @@ export interface paths {
export type webhooks = Record<string, never>;
export interface components {
schemas: {
/** AmbientNoiseConfigurationDefaults */
AmbientNoiseConfigurationDefaults: {
/**
* Enabled
* @default false
*/
enabled: boolean;
/**
* Volume
* @default 0.3
*/
volume: number;
} & {
[key: string]: unknown;
};
/**
* CalculatorToolDefinition
* @description Tool definition for Calculator tools.
@ -1079,10 +1094,7 @@ export interface components {
template_context_variables?: {
[key: string]: unknown;
} | null;
/** Workflow Configurations */
workflow_configurations?: {
[key: string]: unknown;
} | null;
workflow_configurations?: components["schemas"]["WorkflowConfigurationDefaults"] | null;
};
/** ValidationError */
ValidationError: {
@ -1097,6 +1109,59 @@ export interface components {
/** Context */
ctx?: Record<string, never>;
};
/** WorkflowConfigurationDefaults */
WorkflowConfigurationDefaults: {
ambient_noise_configuration?: components["schemas"]["AmbientNoiseConfigurationDefaults"];
/**
* Max Call Duration
* @default 300
*/
max_call_duration: number;
/**
* Max User Idle Timeout
* @default 10
*/
max_user_idle_timeout: number;
/**
* Smart Turn Stop Secs
* @default 2
*/
smart_turn_stop_secs: number;
/**
* Turn Start Strategy
* @default default
* @enum {string}
*/
turn_start_strategy: "default" | "min_words" | "provisional_vad";
/**
* Turn Start Min Words
* @default 3
*/
turn_start_min_words: number;
/**
* Provisional Vad Pause Secs
* @default 1.5
*/
provisional_vad_pause_secs: number;
/**
* Turn Stop Strategy
* @default transcription
* @enum {string}
*/
turn_stop_strategy: "transcription" | "turn_analyzer";
/**
* Dictionary
* @default
*/
dictionary: string;
/**
* Context Compaction Enabled
* @default false
*/
context_compaction_enabled: boolean;
} & {
[key: string]: unknown;
};
/**
* WorkflowListResponse
* @description Lightweight response for workflow listings (excludes large fields).
@ -1164,6 +1229,7 @@ export interface components {
headers: never;
pathItems: never;
}
export type AmbientNoiseConfigurationDefaults = components['schemas']['AmbientNoiseConfigurationDefaults'];
export type CalculatorToolDefinition = components['schemas']['CalculatorToolDefinition'];
export type CallDispositionCodes = components['schemas']['CallDispositionCodes'];
export type CreateToolRequest = components['schemas']['CreateToolRequest'];
@ -1201,6 +1267,7 @@ export type TransferCallConfig = components['schemas']['TransferCallConfig'];
export type TransferCallToolDefinition = components['schemas']['TransferCallToolDefinition'];
export type UpdateWorkflowRequest = components['schemas']['UpdateWorkflowRequest'];
export type ValidationError = components['schemas']['ValidationError'];
export type WorkflowConfigurationDefaults = components['schemas']['WorkflowConfigurationDefaults'];
export type WorkflowListResponse = components['schemas']['WorkflowListResponse'];
export type WorkflowResponse = components['schemas']['WorkflowResponse'];
export type $defs = Record<string, never>;

7
ui/src/app/workflow/[workflowId]/run/[runId]/hooks/useWebSocketRTC.tsx
View file

@ -393,8 +393,10 @@ export const useWebSocketRTC = ({ workflowId, workflowRunId, accessToken, initia
// Stop the connection and surface the handled service error.
cleanupConnection({ graceful: false, status: 'failed' });
} else {
// Log other errors as actual errors
const serverErrorMessage = message.payload?.message || 'Server error';
logger.error('Server error:', message.payload);
setPermissionError(serverErrorMessage);
cleanupConnection({ graceful: false, status: 'failed' });
}
break;
@ -606,7 +608,7 @@ export const useWebSocketRTC = ({ workflowId, workflowRunId, accessToken, initia
}
};
});
}, [getWebSocketUrl, cleanupConnection]);
}, [getWebSocketUrl, cleanupConnection, setPermissionError]);
const negotiate = async () => {
const pc = pcRef.current;
@ -661,6 +663,7 @@ export const useWebSocketRTC = ({ workflowId, workflowRunId, accessToken, initia
setIsStarting(true);
setConnectionActive(false);
setIsCompleted(false);
setPermissionError(null);
setConnectionStatus('connecting');
try {