apunkt/dograh
1
0
Fork 0
mirror of https://github.com/dograh-hq/dograh.git synced 2026-07-10 11:12:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix: increase concurrency limit an handle it across all call paths (#508)

Browse source 
* fix: increase concurrency limit an handle it across all call paths

* fix: fix review comments and test

* fix: address concurrency review findings (campaign-scoped counter, cleanup hardening, webrtc run validation)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat: emit usage_concurrent_call_limit_reached PostHog event

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix: align usage event with MPS org-event convention (per-member fan-out)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat: cap max_call_duration at 20 min via typed workflow_configurations request

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Abhishek 2026-07-09 18:29:01 +05:30 • committed by GitHub
parent f3bcf24370
commit 041c31a613
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
40 changed files with 2369 additions and 467 deletions
Download patch file Download diff file Expand all files Collapse all files

4
api/services/auth/depends.py
View file

@ -14,6 +14,7 @@ from api.services.auth.stack_auth import stackauth
from api.services.configuration.registry import ServiceProviders
from api.services.mps_billing import ensure_hosted_mps_billing_account_v2
from api.services.posthog_client import (
POSTHOG_ORGANIZATION_GROUP_TYPE,
capture_event,
group_identify,
set_person_properties,
@ -33,9 +34,6 @@ async def require_local_auth() -> None:
raise HTTPException(status_code=404, detail="Not found")
POSTHOG_ORGANIZATION_GROUP_TYPE = "organization"
async def get_user(
authorization: Annotated[str | None, Header()] = None,
x_api_key: Annotated[str | None, Header(alias="X-API-Key")] = None,

282
api/services/call_concurrency.py Normal file
View file

@ -0,0 +1,282 @@
import asyncio
import time
from dataclasses import dataclass
from loguru import logger
from api.constants import DEFAULT_ORG_CONCURRENCY_LIMIT
from api.db import db_client
from api.enums import OrganizationConfigurationKey, PostHogEvent
from api.services.campaign.rate_limiter import rate_limiter
from api.services.posthog_client import capture_event
@dataclass(frozen=True)
class CallConcurrencySlot:
organization_id: int
slot_id: str
max_concurrent: int
source: str
scope_key: str | None = None
class CallConcurrencyLimitError(Exception):
"""Raised when an org has no available concurrent call slots."""
def __init__(
self,
*,
organization_id: int,
source: str,
wait_time: float,
max_concurrent: int,
):
self.organization_id = organization_id
self.source = source
self.wait_time = wait_time
self.max_concurrent = max_concurrent
super().__init__(
f"Concurrent call limit reached for org {organization_id} "
f"(source={source}, limit={max_concurrent}, waited={wait_time:.1f}s)"
)
class WorkflowRunSlotAlreadyBoundError(Exception):
"""Raised when a workflow run already owns a concurrent call slot."""
def __init__(self, workflow_run_id: int):
self.workflow_run_id = workflow_run_id
super().__init__(
f"Workflow run {workflow_run_id} already has an active call slot"
)
class CallConcurrencyService:
def __init__(self):
self.default_concurrent_limit = int(DEFAULT_ORG_CONCURRENCY_LIMIT)
async def get_org_concurrent_limit(self, organization_id: int) -> int:
"""Get the concurrent call limit for an organization."""
try:
config = await db_client.get_configuration(
organization_id,
OrganizationConfigurationKey.CONCURRENT_CALL_LIMIT.value,
)
if config and config.value:
value = config.value.get("value")
if value is not None:
return int(value)
except Exception as e:
logger.warning(
f"Error getting concurrent limit for org {organization_id}: {e}"
)
return self.default_concurrent_limit
async def acquire_org_slot(
self,
organization_id: int,
*,
source: str,
timeout: float = 0,
scope_key: str | None = None,
scope_max_concurrent: int | None = None,
retry_interval: float = 1,
) -> CallConcurrencySlot:
"""Acquire a slot in the org-wide concurrency counter.
``scope_key``/``scope_max_concurrent`` additionally bound a secondary
counter (e.g. ``campaign:<id>``) so a source can cap its own
concurrency without measuring — or being starved by — unrelated calls
in the same org.
"""
max_concurrent = await self.get_org_concurrent_limit(organization_id)
if scope_max_concurrent is not None:
scope_max_concurrent = int(scope_max_concurrent)
wait_start = time.time()
while True:
acquisition = await rate_limiter.try_acquire_concurrent_slot_details(
organization_id,
max_concurrent,
scope_key=scope_key,
scope_max_concurrent=scope_max_concurrent,
)
if acquisition:
logger.info(
f"Acquired concurrent call slot for org {organization_id}: "
f"source={source}, active_calls="
f"{acquisition.active_count}/{max_concurrent}, "
f"slot_id={acquisition.slot_id}"
)
return CallConcurrencySlot(
organization_id=organization_id,
slot_id=acquisition.slot_id,
max_concurrent=max_concurrent,
source=source,
scope_key=scope_key,
)
wait_time = time.time() - wait_start
if wait_time >= timeout:
current_count = await rate_limiter.get_concurrent_count(organization_id)
scope_note = (
f", scope={scope_key} (limit={scope_max_concurrent})"
if scope_key
else ""
)
logger.warning(
f"Concurrent call limit reached for org {organization_id}: "
f"source={source}, active_calls={current_count}/{max_concurrent}"
f"{scope_note}, waited={wait_time:.1f}s"
)
properties = {
"event_source": "dograh",
"organization_id": organization_id,
"source": source,
"max_concurrent": max_concurrent,
"active_calls": current_count,
"waited_seconds": round(wait_time, 1),
}
if scope_key:
properties["scope_key"] = scope_key
properties["scope_max_concurrent"] = scope_max_concurrent
await self._notify_limit_reached(organization_id, properties)
raise CallConcurrencyLimitError(
organization_id=organization_id,
source=source,
wait_time=wait_time,
max_concurrent=max_concurrent,
)
logger.debug(
f"Waiting for concurrent call slot for org {organization_id}, "
f"source={source}, waited {wait_time:.1f}s"
)
await asyncio.sleep(min(retry_interval, max(0, timeout - wait_time)))
async def _notify_limit_reached(
self, organization_id: int, properties: dict
) -> None:
"""Fan the usage event out to every org member's provider_id, matching
how MPS emits org-scoped billing events (billing_posthog_service.py)
into the shared PostHog project. Never raises.
NOTE: intentionally NOT attaching ``$groups`` (organization) to this
event. PostHog evaluates a $groups event at both person and group
scope, which double-triggers person-scoped workflows enrolled on the
event. The org is still available as the ``organization_id`` property.
"""
try:
members = await db_client.get_organization_users(organization_id)
if not members:
logger.debug(
f"No users found for org {organization_id}; skipping "
"concurrent-call-limit PostHog event"
)
return
for member in members:
capture_event(
distinct_id=str(member.provider_id),
event=PostHogEvent.USAGE_CONCURRENT_CALL_LIMIT_REACHED,
properties=properties,
)
except Exception:
logger.exception(
"Failed to send concurrent-call-limit PostHog event for org "
f"{organization_id}"
)
async def bind_workflow_run(
self, slot: CallConcurrencySlot, workflow_run_id: int
) -> None:
stored = await rate_limiter.store_workflow_slot_mapping_if_absent(
workflow_run_id,
slot.organization_id,
slot.slot_id,
scope_key=slot.scope_key,
)
if stored:
return
await self.release_slot(slot)
raise WorkflowRunSlotAlreadyBoundError(workflow_run_id)
async def register_active_call(
self,
organization_id: int,
workflow_run_id: int,
*,
source: str,
timeout: float = 0,
scope_key: str | None = None,
scope_max_concurrent: int | None = None,
retry_interval: float = 1,
) -> CallConcurrencySlot:
slot = await self.acquire_org_slot(
organization_id,
source=source,
timeout=timeout,
scope_key=scope_key,
scope_max_concurrent=scope_max_concurrent,
retry_interval=retry_interval,
)
await self.bind_workflow_run(slot, workflow_run_id)
return slot
async def unregister_active_call(self, workflow_run_id: int) -> bool:
"""Release the run's slot without ever raising.
Callers invoke this from ``finally`` blocks during pipeline/socket
teardown; a cleanup failure must not mask the original exception.
The slot mapping survives a failed release, so a later cleanup path
(status callback, StasisEnd) or the Redis stale timeout recovers it.
"""
try:
return await self.release_workflow_run_slot(workflow_run_id)
except asyncio.CancelledError:
raise
except Exception as e:
logger.warning(
f"Failed to release concurrent call slot for workflow run "
f"{workflow_run_id}: {e}"
)
return False
async def release_slot(self, slot: CallConcurrencySlot | None) -> bool:
if slot is None:
return False
released = await rate_limiter.release_concurrent_slot(
slot.organization_id, slot.slot_id, scope_key=slot.scope_key
)
return bool(released)
async def release_workflow_run_slot(self, workflow_run_id: int) -> bool:
mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run_id)
if not mapping:
return False
org_id, slot_id, scope_key = mapping
released = await rate_limiter.release_concurrent_slot(
org_id, slot_id, scope_key=scope_key
)
if released is None:
# Redis error while releasing — keep the mapping so a later
# cleanup path can retry instead of orphaning a live slot until
# the stale timeout.
logger.warning(
f"Failed to release concurrent slot for workflow run "
f"{workflow_run_id}; keeping mapping for retry"
)
return False
await rate_limiter.delete_workflow_slot_mapping(workflow_run_id)
if released:
logger.info(f"Released concurrent slot for workflow run {workflow_run_id}")
else:
logger.debug(
f"Concurrent slot mapping for workflow run {workflow_run_id} "
"had no live slot; deleted stale mapping"
)
return released
call_concurrency = CallConcurrencyService()

264
api/services/campaign/campaign_call_dispatcher.py
View file

@ -5,10 +5,14 @@ from typing import TYPE_CHECKING, Optional
from loguru import logger
from api.constants import DEFAULT_ORG_CONCURRENCY_LIMIT
from api.db import db_client
from api.db.models import QueuedRunModel, WorkflowRunModel
from api.enums import OrganizationConfigurationKey, WorkflowRunState
from api.enums import WorkflowRunState
from api.services.call_concurrency import (
CallConcurrencyLimitError,
CallConcurrencySlot,
call_concurrency,
)
from api.services.campaign.circuit_breaker import circuit_breaker
from api.services.campaign.errors import (
ConcurrentSlotAcquisitionError,
@ -29,9 +33,6 @@ if TYPE_CHECKING:
class CampaignCallDispatcher:
"""Manages rate-limited and concurrent-limited call dispatching"""
def __init__(self):
self.default_concurrent_limit = int(DEFAULT_ORG_CONCURRENCY_LIMIT)
async def get_provider_for_campaign(self, campaign) -> "TelephonyProvider":
"""Get the telephony provider pinned to this campaign's config. Falls back
to the org's default config for legacy campaigns whose
@ -53,18 +54,7 @@ class CampaignCallDispatcher:
async def get_org_concurrent_limit(self, organization_id: int) -> int:
"""Get the concurrent call limit for an organization."""
try:
config = await db_client.get_configuration(
organization_id,
OrganizationConfigurationKey.CONCURRENT_CALL_LIMIT.value,
)
if config and config.value:
return int(config.value["value"])
except Exception as e:
logger.warning(
f"Error getting concurrent limit for org {organization_id}: {e}"
)
return self.default_concurrent_limit
return await call_concurrency.get_org_concurrent_limit(organization_id)
async def process_batch(self, campaign_id: int, batch_size: int = 10) -> int:
"""
@ -119,12 +109,14 @@ class CampaignCallDispatcher:
)
# Acquire concurrent slot - waits until a slot is available
slot_id = await self.acquire_concurrent_slot(
concurrency_slot = await self.acquire_concurrent_slot(
campaign.organization_id, campaign
)
# Dispatch the call
workflow_run = await self.dispatch_call(queued_run, campaign, slot_id)
workflow_run = await self.dispatch_call(
queued_run, campaign, concurrency_slot
)
# Update queued run as processed
await db_client.update_queued_run(
@ -233,68 +225,61 @@ class CampaignCallDispatcher:
)
async def dispatch_call(
self, queued_run: QueuedRunModel, campaign: any, slot_id: str
self,
queued_run: QueuedRunModel,
campaign: any,
concurrency_slot: CallConcurrencySlot,
) -> Optional[WorkflowRunModel]:
"""Creates workflow run and initiates call. Requires a pre-acquired slot_id."""
"""Creates workflow run and initiates call. Requires a pre-acquired slot."""
from_number = None
workflow_run = None
slot_bound = False
# Get workflow details
workflow = await db_client.get_workflow_by_id(campaign.workflow_id)
if not workflow:
# Release slot before raising
await rate_limiter.release_concurrent_slot(
campaign.organization_id, slot_id
)
raise ValueError(f"Workflow {campaign.workflow_id} not found")
# Extract phone number
phone_number = queued_run.context_variables.get("phone_number")
if not phone_number:
# Release slot before raising
await rate_limiter.release_concurrent_slot(
campaign.organization_id, slot_id
)
raise ValueError(f"No phone number in queued run {queued_run.id}")
# Get provider for this campaign's pinned telephony config.
provider = await self.get_provider_for_campaign(campaign)
workflow_run_mode = provider.PROVIDER_NAME
# Acquire a unique from_number from the pool scoped to this campaign's
# telephony configuration so orgs with multiple configs don't leak
# caller IDs across configs.
from_number = await self.acquire_from_number(
campaign.organization_id,
telephony_configuration_id=campaign.telephony_configuration_id,
)
if from_number is None:
# Release concurrent slot before raising
await rate_limiter.release_concurrent_slot(
campaign.organization_id, slot_id
)
raise PhoneNumberPoolExhaustedError(
organization_id=campaign.organization_id
)
logger.info(f"Provider name: {provider.PROVIDER_NAME}")
logger.info(f"Queued run context: {queued_run.context_variables}")
# Merge context variables (queued_run context already includes retry info if applicable)
initial_context = {
**queued_run.context_variables,
"campaign_id": campaign.id,
"provider": provider.PROVIDER_NAME,
"source_uuid": queued_run.source_uuid,
"caller_number": from_number,
"called_number": phone_number,
"telephony_configuration_id": campaign.telephony_configuration_id,
}
logger.info(f"Final initial_context: {initial_context}")
# Create workflow run with queued_run_id tracking
workflow_run_name = f"WR-CAMPAIGN-{campaign.id}-{queued_run.id}"
try:
# Get workflow details
workflow = await db_client.get_workflow_by_id(campaign.workflow_id)
if not workflow:
raise ValueError(f"Workflow {campaign.workflow_id} not found")
# Extract phone number
phone_number = queued_run.context_variables.get("phone_number")
if not phone_number:
raise ValueError(f"No phone number in queued run {queued_run.id}")
# Get provider for this campaign's pinned telephony config.
provider = await self.get_provider_for_campaign(campaign)
workflow_run_mode = provider.PROVIDER_NAME
# Acquire a unique from_number from the pool scoped to this campaign's
# telephony configuration so orgs with multiple configs don't leak
# caller IDs across configs.
from_number = await self.acquire_from_number(
campaign.organization_id,
telephony_configuration_id=campaign.telephony_configuration_id,
)
if from_number is None:
raise PhoneNumberPoolExhaustedError(
organization_id=campaign.organization_id
)
logger.info(f"Provider name: {provider.PROVIDER_NAME}")
logger.info(f"Queued run context: {queued_run.context_variables}")
# Merge context variables (queued_run context already includes retry info if applicable)
initial_context = {
**queued_run.context_variables,
"campaign_id": campaign.id,
"provider": provider.PROVIDER_NAME,
"source_uuid": queued_run.source_uuid,
"caller_number": from_number,
"called_number": phone_number,
"telephony_configuration_id": campaign.telephony_configuration_id,
}
logger.info(f"Final initial_context: {initial_context}")
# Create workflow run with queued_run_id tracking
workflow_run_name = f"WR-CAMPAIGN-{campaign.id}-{queued_run.id}"
workflow_run = await db_client.create_workflow_run(
name=workflow_run_name,
workflow_id=campaign.workflow_id,
@ -304,11 +289,8 @@ class CampaignCallDispatcher:
campaign_id=campaign.id,
queued_run_id=queued_run.id, # Link to queued run for retry tracking
)
# Store slot_id mapping in Redis for cleanup later
await rate_limiter.store_workflow_slot_mapping(
workflow_run.id, campaign.organization_id, slot_id
)
await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run.id)
slot_bound = True
# Store from_number mapping for cleanup on call completion
await rate_limiter.store_workflow_from_number_mapping(
@ -319,9 +301,10 @@ class CampaignCallDispatcher:
)
except Exception as e:
# Release slot and from_number on error
await rate_limiter.release_concurrent_slot(
campaign.organization_id, slot_id
)
if slot_bound and workflow_run:
await call_concurrency.release_workflow_run_slot(workflow_run.id)
else:
await call_concurrency.release_slot(concurrency_slot)
if from_number:
await rate_limiter.release_from_number(
campaign.organization_id,
@ -357,21 +340,7 @@ class CampaignCallDispatcher:
gathered_context={"error": error_message},
)
mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run.id)
if mapping:
org_id, mapped_slot_id = mapping
await rate_limiter.release_concurrent_slot(org_id, mapped_slot_id)
await rate_limiter.delete_workflow_slot_mapping(workflow_run.id)
from_number_mapping = await rate_limiter.get_workflow_from_number_mapping(
workflow_run.id
)
if from_number_mapping:
fn_org_id, fn_number, fn_tcid = from_number_mapping
await rate_limiter.release_from_number(
fn_org_id, fn_number, telephony_configuration_id=fn_tcid
)
await rate_limiter.delete_workflow_from_number_mapping(workflow_run.id)
await self.release_call_slot(workflow_run.id)
raise ValueError(error_message)
@ -446,23 +415,7 @@ class CampaignCallDispatcher:
reason="call_initiation_failed",
)
# Release concurrent slot on failure
mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run.id)
if mapping:
org_id, slot_id = mapping
await rate_limiter.release_concurrent_slot(org_id, slot_id)
await rate_limiter.delete_workflow_slot_mapping(workflow_run.id)
# Release from_number on failure
from_number_mapping = await rate_limiter.get_workflow_from_number_mapping(
workflow_run.id
)
if from_number_mapping:
fn_org_id, fn_number, fn_tcid = from_number_mapping
await rate_limiter.release_from_number(
fn_org_id, fn_number, telephony_configuration_id=fn_tcid
)
await rate_limiter.delete_workflow_from_number_mapping(workflow_run.id)
await self.release_call_slot(workflow_run.id)
raise
@ -501,7 +454,7 @@ class CampaignCallDispatcher:
async def acquire_concurrent_slot(
self, organization_id: int, campaign: any, timeout: float = 600
) -> str:
) -> CallConcurrencySlot:
"""
Acquires a concurrent call slot - waits if necessary until a slot is available.
@ -510,54 +463,41 @@ class CampaignCallDispatcher:
campaign: The campaign object
timeout: Maximum time to wait for a slot (default 10 minutes)
Returns the slot_id which must be released when the call completes.
Returns the slot which must be released when the call completes.
Raises:
ConcurrentSlotAcquisitionError: If slot cannot be acquired within timeout
"""
# Get concurrent limit for organization
org_concurrent_limit = await self.get_org_concurrent_limit(organization_id)
# Check for campaign-level max_concurrency in orchestrator_metadata
# Check for campaign-level max_concurrency in orchestrator_metadata.
# It caps this campaign's own concurrent calls via a campaign-scoped
# counter — the org-wide limit still applies on top, but calls from
# other sources (WebRTC, inbound, other campaigns) don't count
# against the campaign's cap.
campaign_max_concurrency = None
if campaign.orchestrator_metadata:
campaign_max_concurrency = campaign.orchestrator_metadata.get(
"max_concurrency"
)
# Use the lower of campaign limit and org limit
if campaign_max_concurrency is not None:
max_concurrent = min(campaign_max_concurrency, org_concurrent_limit)
else:
max_concurrent = org_concurrent_limit
# Track wait time for alerting
wait_start = time.time()
# Wait until we can acquire a concurrent slot
while True:
slot_id = await rate_limiter.try_acquire_concurrent_slot(
organization_id, max_concurrent
try:
return await call_concurrency.acquire_org_slot(
organization_id,
source=f"campaign:{campaign.id}",
timeout=timeout,
scope_key=(
f"campaign:{campaign.id}"
if campaign_max_concurrency is not None
else None
),
scope_max_concurrent=campaign_max_concurrency,
retry_interval=1,
)
if slot_id:
return slot_id
# Check if we've been waiting too long
wait_time = time.time() - wait_start
if wait_time > timeout:
raise ConcurrentSlotAcquisitionError(
organization_id=organization_id,
campaign_id=campaign.id,
wait_time=wait_time,
)
logger.debug(
f"Attempting to get a slot for {organization_id} {campaign.id}, "
f"waited {wait_time:.1f}s"
)
# Wait before retrying
await asyncio.sleep(1)
except CallConcurrencyLimitError as e:
raise ConcurrentSlotAcquisitionError(
organization_id=organization_id,
campaign_id=campaign.id,
wait_time=e.wait_time,
) from e
async def acquire_from_number(
self,
@ -602,17 +542,9 @@ class CampaignCallDispatcher:
Release concurrent slot and from_number when a call completes.
Called by Twilio webhooks or workflow completion handlers.
"""
slot_released = False
mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run_id)
if mapping:
org_id, slot_id = mapping
success = await rate_limiter.release_concurrent_slot(org_id, slot_id)
if success:
await rate_limiter.delete_workflow_slot_mapping(workflow_run_id)
logger.info(
f"Released concurrent slot for workflow run {workflow_run_id}"
)
slot_released = True
slot_released = await call_concurrency.release_workflow_run_slot(
workflow_run_id
)
# Release from_number back to its (org, telephony config) pool
from_number_mapping = await rate_limiter.get_workflow_from_number_mapping(

156
api/services/campaign/rate_limiter.py
View file

@ -1,5 +1,6 @@
import time
import uuid
from dataclasses import dataclass
from typing import Optional
import redis.asyncio as aioredis
@ -8,6 +9,12 @@ from loguru import logger
from api.constants import REDIS_URL
@dataclass(frozen=True)
class ConcurrentSlotAcquisition:
slot_id: str
active_count: int
class RateLimiter:
"""Sliding window rate limiter to enforce strict per-second limits and concurrent call limits"""
@ -100,34 +107,71 @@ class RateLimiter:
Try to acquire a concurrent call slot.
Returns a unique slot_id if successful, None if limit reached.
"""
acquisition = await self.try_acquire_concurrent_slot_details(
organization_id, max_concurrent
)
return acquisition.slot_id if acquisition else None
async def try_acquire_concurrent_slot_details(
self,
organization_id: int,
max_concurrent: int = 20,
*,
scope_key: str | None = None,
scope_max_concurrent: int | None = None,
) -> Optional[ConcurrentSlotAcquisition]:
"""
Try to acquire a concurrent call slot.
Returns the slot_id and post-acquire active count if successful,
or None if the limit is reached.
When ``scope_key``/``scope_max_concurrent`` are provided, the slot is
also registered in a secondary counter (``concurrent_calls:<scope_key>``,
e.g. ``campaign:<id>``) and acquisition additionally requires that
counter to be below ``scope_max_concurrent``. Both counters are
updated atomically. The scope-scoped slot must be released with the
same ``scope_key``.
"""
redis_client = await self._get_redis()
concurrent_key = f"concurrent_calls:{organization_id}"
scope_concurrent_key = f"concurrent_calls:{scope_key}" if scope_key else ""
now = time.time()
stale_cutoff = now - self.stale_call_timeout
# Lua script for atomic operation
# Lua script for atomic operation across the org counter and the
# optional scope counter (empty scope key = org-only acquisition).
lua_script = """
local key = KEYS[1]
local scope_key = KEYS[2]
local now = tonumber(ARGV[1])
local max_concurrent = tonumber(ARGV[2])
local stale_cutoff = tonumber(ARGV[3])
local slot_id = ARGV[4]
-- Remove stale entries (older than 30 minutes)
local scope_max_concurrent = tonumber(ARGV[5])
-- Remove stale entries (older than the stale-call timeout)
redis.call('ZREMRANGEBYSCORE', key, 0, stale_cutoff)
-- Get current count
local current_count = redis.call('ZCARD', key)
if current_count < max_concurrent then
-- Add new slot
redis.call('ZADD', key, now, slot_id)
redis.call('EXPIRE', key, 3600) -- Expire after 1 hour
return slot_id
else
if current_count >= max_concurrent then
return nil
end
if scope_key ~= '' then
redis.call('ZREMRANGEBYSCORE', scope_key, 0, stale_cutoff)
if redis.call('ZCARD', scope_key) >= scope_max_concurrent then
return nil
end
redis.call('ZADD', scope_key, now, slot_id)
redis.call('EXPIRE', scope_key, 3600)
end
redis.call('ZADD', key, now, slot_id)
redis.call('EXPIRE', key, 3600) -- Expire after 1 hour
return {slot_id, current_count + 1}
"""
# Generate unique slot ID (timestamp + random component)
@ -136,22 +180,38 @@ class RateLimiter:
try:
result = await redis_client.eval(
lua_script,
1,
2,
concurrent_key,
scope_concurrent_key,
now,
max_concurrent,
stale_cutoff,
slot_id,
scope_max_concurrent if scope_max_concurrent is not None else 0,
)
if not result:
return None
acquired_slot_id, active_count = result
return ConcurrentSlotAcquisition(
slot_id=str(acquired_slot_id),
active_count=int(active_count),
)
return result
except Exception as e:
logger.error(f"Concurrent limiter error: {e}")
return None
async def release_concurrent_slot(self, organization_id: int, slot_id: str) -> bool:
async def release_concurrent_slot(
self,
organization_id: int,
slot_id: str,
scope_key: str | None = None,
) -> bool | None:
"""
Release a concurrent call slot.
Returns True if slot was released, False otherwise.
Release a concurrent call slot (and its scope counter entry, if any).
Returns True if the slot was released, False if it was already gone
(released/stale-expired), or None on a Redis error — callers that
track cleanup state should keep it around for retry when None.
"""
if not slot_id:
return False
@ -161,6 +221,8 @@ class RateLimiter:
try:
removed = await redis_client.zrem(concurrent_key, slot_id)
if scope_key:
await redis_client.zrem(f"concurrent_calls:{scope_key}", slot_id)
if removed:
logger.debug(
f"Released concurrent slot {slot_id} for org {organization_id}"
@ -168,7 +230,7 @@ class RateLimiter:
return bool(removed)
except Exception as e:
logger.error(f"Error releasing concurrent slot: {e}")
return False
return None
async def get_concurrent_count(self, organization_id: int) -> int:
"""
@ -212,12 +274,62 @@ class RateLimiter:
logger.error(f"Error storing workflow slot mapping: {e}")
return False
async def store_workflow_slot_mapping_if_absent(
self,
workflow_run_id: int,
organization_id: int,
slot_id: str,
scope_key: str | None = None,
) -> bool:
"""
Store the workflow_run_id -> concurrent slot mapping only if no mapping
already exists. This prevents duplicate public/WebRTC starts for the
same workflow run from overwriting the cleanup pointer.
"""
redis_client = await self._get_redis()
mapping_key = f"workflow_slot_mapping:{workflow_run_id}"
lua_script = """
local key = KEYS[1]
local org_id = ARGV[1]
local slot_id = ARGV[2]
local ttl = tonumber(ARGV[3])
local scope_key = ARGV[4]
if redis.call('EXISTS', key) == 1 then
return 0
end
redis.call('HSET', key, 'org_id', org_id, 'slot_id', slot_id)
if scope_key ~= '' then
redis.call('HSET', key, 'scope_key', scope_key)
end
redis.call('EXPIRE', key, ttl)
return 1
"""
try:
stored = await redis_client.eval(
lua_script,
1,
mapping_key,
organization_id,
slot_id,
self.stale_call_timeout,
scope_key or "",
)
return bool(stored)
except Exception as e:
logger.error(f"Error storing workflow slot mapping if absent: {e}")
return False
async def get_workflow_slot_mapping(
self, workflow_run_id: int
) -> Optional[tuple[int, str]]:
) -> Optional[tuple[int, str, str | None]]:
"""
Get the concurrent slot mapping for a workflow run.
Returns (organization_id, slot_id) tuple or None if not found.
Returns (organization_id, slot_id, scope_key) or None if not found;
scope_key is None for slots acquired without a scope counter.
"""
redis_client = await self._get_redis()
mapping_key = f"workflow_slot_mapping:{workflow_run_id}"
@ -225,7 +337,11 @@ class RateLimiter:
try:
mapping = await redis_client.hgetall(mapping_key)
if mapping and "org_id" in mapping and "slot_id" in mapping:
return (int(mapping["org_id"]), mapping["slot_id"])
return (
int(mapping["org_id"]),
mapping["slot_id"],
mapping.get("scope_key") or None,
)
return None
except Exception as e:
logger.error(f"Error getting workflow slot mapping: {e}")

28
api/services/pipecat/run_pipeline.py
View file

@ -14,14 +14,17 @@ from api.schemas.workflow_configurations import (
DEFAULT_TURN_START_MIN_WORDS,
DEFAULT_TURN_START_STRATEGY,
)
from api.services.call_concurrency import call_concurrency
from api.services.configuration.registry import ServiceProviders
from api.services.integrations import (
IntegrationRuntimeContext,
create_runtime_sessions,
)
from api.services.pipecat.active_calls import (
register_active_call,
unregister_active_call,
register_active_call as register_worker_active_call,
)
from api.services.pipecat.active_calls import (
unregister_active_call as unregister_worker_active_call,
)
from api.services.pipecat.audio_config import AudioConfig, create_audio_config
from api.services.pipecat.event_handlers import (
@ -258,7 +261,7 @@ async def run_pipeline_telephony(
"""Run a pipeline for any telephony provider."""
# Register before any async setup so deploy drains see calls that are still
# resolving DB/config/transport state.
register_active_call(workflow_run_id)
register_worker_active_call(workflow_run_id)
try:
await _run_pipeline_telephony_impl(
websocket,
@ -270,7 +273,10 @@ async def run_pipeline_telephony(
transport_kwargs=transport_kwargs,
)
finally:
unregister_active_call(workflow_run_id)
try:
await call_concurrency.unregister_active_call(workflow_run_id)
finally:
unregister_worker_active_call(workflow_run_id)
async def _run_pipeline_telephony_impl(
@ -383,7 +389,7 @@ async def run_pipeline_smallwebrtc(
"""Run pipeline for WebRTC connections."""
# Register before any async setup so deploy drains see calls that are still
# resolving DB/config/transport state.
register_active_call(workflow_run_id)
register_worker_active_call(workflow_run_id)
try:
await _run_pipeline_smallwebrtc_impl(
webrtc_connection,
@ -394,7 +400,10 @@ async def run_pipeline_smallwebrtc(
user_provider_id=user_provider_id,
)
finally:
unregister_active_call(workflow_run_id)
try:
await call_concurrency.unregister_active_call(workflow_run_id)
finally:
unregister_worker_active_call(workflow_run_id)
async def _run_pipeline_smallwebrtc_impl(
@ -478,7 +487,7 @@ async def _run_pipeline(
resolved_user_config=None,
) -> None:
"""Run the pipeline with active-call drain accounting."""
register_active_call(workflow_run_id)
register_worker_active_call(workflow_run_id)
try:
await _run_pipeline_impl(
transport,
@ -492,7 +501,10 @@ async def _run_pipeline(
resolved_user_config=resolved_user_config,
)
finally:
unregister_active_call(workflow_run_id)
try:
await call_concurrency.unregister_active_call(workflow_run_id)
finally:
unregister_worker_active_call(workflow_run_id)
async def _run_pipeline_impl(

1
api/services/posthog_client.py
View file

@ -7,6 +7,7 @@ from api.constants import POSTHOG_API_KEY, POSTHOG_HOST
_posthog_client: Posthog | None = None
POSTHOG_SERVER_GROUP_IDENTIFY_DISTINCT_ID = "server-group-identify"
POSTHOG_ORGANIZATION_GROUP_TYPE = "organization"
def get_posthog() -> Posthog | None:

35
api/services/telephony/ari_manager.py
View file

@ -26,6 +26,10 @@ from loguru import logger
from api.constants import REDIS_URL
from api.db import db_client
from api.enums import CallType, WorkflowRunMode
from api.services.call_concurrency import (
CallConcurrencyLimitError,
call_concurrency,
)
from api.services.quota_service import authorize_workflow_run_start
from api.services.telephony.call_transfer_manager import get_call_transfer_manager
from api.services.telephony.transfer_event_protocol import (
@ -527,6 +531,8 @@ class ARIConnection:
channel = event.get("channel", {})
caller_number = channel.get("caller", {}).get("number", "unknown")
called_number = channel.get("dialplan", {}).get("exten", "unknown")
concurrency_slot = None
workflow_run = None
try:
# 1. Resolve the workflow from the called extension via the
@ -573,6 +579,20 @@ class ARIConnection:
user_id = workflow.user_id
try:
concurrency_slot = await call_concurrency.acquire_org_slot(
self.organization_id,
source="ari_inbound",
timeout=0,
)
except CallConcurrencyLimitError:
logger.warning(
f"[ARI org={self.organization_id}] Concurrent call limit "
f"reached; hanging up inbound channel {channel_id}"
)
await self._delete_channel(channel_id)
return
# 3. Create workflow run
call_id = channel_id
workflow_run = await db_client.create_workflow_run(
@ -591,7 +611,9 @@ class ARIConnection:
gathered_context={
"call_id": call_id,
},
organization_id=self.organization_id,
)
await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run.id)
logger.info(
f"[ARI org={self.organization_id}] Created inbound workflow run "
@ -610,6 +632,7 @@ class ARIConnection:
f"[ARI org={self.organization_id}] Quota exceeded for user {user_id} "
f"— hanging up inbound call {channel_id}"
)
await call_concurrency.release_workflow_run_slot(workflow_run.id)
await self._delete_channel(channel_id)
return
@ -625,6 +648,10 @@ class ARIConnection:
str(user_id),
)
except Exception as e:
if workflow_run:
await call_concurrency.release_workflow_run_slot(workflow_run.id)
elif concurrency_slot:
await call_concurrency.release_slot(concurrency_slot)
logger.error(
f"[ARI org={self.organization_id}] Error handling inbound StasisStart "
f"for channel {channel_id}: {e}"
@ -760,6 +787,14 @@ class ARIConnection:
the bridge and both channels — like endConferenceOnExit.
"""
try:
# Release the org concurrency slot. Normally the pipeline's own
# teardown does this when the ext media websocket closes, but if
# the pipeline never started (caller hung up before external
# media connected, ext media creation failed, ...) this is the
# only cleanup that runs before the Redis stale timeout. No-op
# when the slot was already released.
await call_concurrency.unregister_active_call(int(workflow_run_id))
workflow_run = await db_client.get_workflow_run_by_id(int(workflow_run_id))
if not workflow_run or not workflow_run.gathered_context:
logger.warning(

4
api/services/telephony/base.py
View file

@ -365,6 +365,10 @@ class TelephonyProvider(ABC):
the caller carries a provider stream protocol. ``organization_id`` is
passed so providers can scope any config lookups to the workflow's org.
Default raises so providers that haven't opted in fail loudly.
The route holds an org concurrency slot while this runs, so
implementations must bound their pre-pipeline handshake reads with a
timeout — an idle socket must not hold the slot indefinitely.
"""
raise NotImplementedError(
f"Agent-stream not supported for provider {self.PROVIDER_NAME}"

35
api/services/telephony/providers/cloudonix/provider.py
View file

@ -2,6 +2,7 @@
Cloudonix implementation of the TelephonyProvider interface.
"""
import asyncio
import json
import random
from typing import TYPE_CHECKING, Any, Dict, List, Optional
@ -25,6 +26,11 @@ if TYPE_CHECKING:
CLOUDONIX_API_BASE_URL = "https://api.cloudonix.io"
# Cloudonix sends the connected/start handshake immediately after the media
# stream opens. The agent-stream route holds an org concurrency slot while we
# wait, so an idle socket must not be able to hold it indefinitely.
AGENT_STREAM_HANDSHAKE_TIMEOUT_S = 10
class CloudonixProvider(TelephonyProvider):
"""
@ -533,14 +539,31 @@ class CloudonixProvider(TelephonyProvider):
from api.services.pipecat.run_pipeline import run_pipeline_telephony
try:
first_msg = await websocket.receive_text()
msg = json.loads(first_msg)
if msg.get("event") != "connected":
logger.error(f"Expected 'connected' event, got: {msg.get('event')}")
await websocket.close(code=4400, reason="Expected connected event")
try:
first_msg = await asyncio.wait_for(
websocket.receive_text(),
timeout=AGENT_STREAM_HANDSHAKE_TIMEOUT_S,
)
msg = json.loads(first_msg)
if msg.get("event") != "connected":
logger.error(f"Expected 'connected' event, got: {msg.get('event')}")
await websocket.close(code=4400, reason="Expected connected event")
return
start_msg = json.loads(
await asyncio.wait_for(
websocket.receive_text(),
timeout=AGENT_STREAM_HANDSHAKE_TIMEOUT_S,
)
)
except asyncio.TimeoutError:
logger.warning(
f"Cloudonix agent-stream handshake timed out for workflow_run "
f"{workflow_run_id}"
)
await websocket.close(code=4408, reason="Handshake timeout")
return
start_msg = json.loads(await websocket.receive_text())
if start_msg.get("event") != "start":
logger.error("Expected 'start' event second")
await websocket.close(code=4400, reason="Expected start event")

6
api/services/telephony/status_processor.py
View file

@ -144,8 +144,9 @@ async def _process_status_update(workflow_run_id: int, status: StatusCallbackReq
f"[run {workflow_run_id}] Call completed with duration: {status.duration}s"
)
await campaign_call_dispatcher.release_call_slot(workflow_run_id)
if workflow_run.campaign_id:
await campaign_call_dispatcher.release_call_slot(workflow_run_id)
await circuit_breaker.record_and_evaluate(
workflow_run.campaign_id, is_failure=False
)
@ -162,8 +163,9 @@ async def _process_status_update(workflow_run_id: int, status: StatusCallbackReq
f"[run {workflow_run_id}] Call failed with status: {normalized_status.value}"
)
await campaign_call_dispatcher.release_call_slot(workflow_run_id)
if workflow_run.campaign_id:
await campaign_call_dispatcher.release_call_slot(workflow_run_id)
is_failure = normalized_status in FAILURE_NOT_CONNECTED_STATUSES
await circuit_breaker.record_and_evaluate(
workflow_run.campaign_id,