mirror of
https://github.com/dograh-hq/dograh.git
synced 2026-07-25 12:01:04 +02:00
fix: increase concurrency limit an handle it across all call paths
This commit is contained in:
Abhishek Kumar
parent
d9b9a1efc8
commit
1a3f6ac09a
26 changed files with 1466 additions and 415 deletions
190
api/services/call_concurrency.py
Normal file
190
api/services/call_concurrency.py
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
import asyncio
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from api.constants import DEFAULT_ORG_CONCURRENCY_LIMIT
|
||||
from api.db import db_client
|
||||
from api.enums import OrganizationConfigurationKey
|
||||
from api.services.campaign.rate_limiter import rate_limiter
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CallConcurrencySlot:
|
||||
organization_id: int
|
||||
slot_id: str
|
||||
max_concurrent: int
|
||||
source: str
|
||||
|
||||
|
||||
class CallConcurrencyLimitError(Exception):
|
||||
"""Raised when an org has no available concurrent call slots."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
organization_id: int,
|
||||
source: str,
|
||||
wait_time: float,
|
||||
max_concurrent: int,
|
||||
):
|
||||
self.organization_id = organization_id
|
||||
self.source = source
|
||||
self.wait_time = wait_time
|
||||
self.max_concurrent = max_concurrent
|
||||
super().__init__(
|
||||
f"Concurrent call limit reached for org {organization_id} "
|
||||
f"(source={source}, limit={max_concurrent}, waited={wait_time:.1f}s)"
|
||||
)
|
||||
|
||||
|
||||
class WorkflowRunSlotAlreadyBoundError(Exception):
|
||||
"""Raised when a workflow run already owns a concurrent call slot."""
|
||||
|
||||
def __init__(self, workflow_run_id: int):
|
||||
self.workflow_run_id = workflow_run_id
|
||||
super().__init__(
|
||||
f"Workflow run {workflow_run_id} already has an active call slot"
|
||||
)
|
||||
|
||||
|
||||
class CallConcurrencyService:
|
||||
def __init__(self):
|
||||
self.default_concurrent_limit = int(DEFAULT_ORG_CONCURRENCY_LIMIT)
|
||||
|
||||
async def get_org_concurrent_limit(self, organization_id: int) -> int:
|
||||
"""Get the concurrent call limit for an organization."""
|
||||
try:
|
||||
config = await db_client.get_configuration(
|
||||
organization_id,
|
||||
OrganizationConfigurationKey.CONCURRENT_CALL_LIMIT.value,
|
||||
)
|
||||
if config and config.value:
|
||||
value = config.value.get("value")
|
||||
if value is not None:
|
||||
return int(value)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Error getting concurrent limit for org {organization_id}: {e}"
|
||||
)
|
||||
return self.default_concurrent_limit
|
||||
|
||||
async def acquire_org_slot(
|
||||
self,
|
||||
organization_id: int,
|
||||
*,
|
||||
source: str,
|
||||
timeout: float = 0,
|
||||
max_concurrent_override: int | None = None,
|
||||
retry_interval: float = 1,
|
||||
) -> CallConcurrencySlot:
|
||||
org_concurrent_limit = await self.get_org_concurrent_limit(organization_id)
|
||||
if max_concurrent_override is None:
|
||||
max_concurrent = org_concurrent_limit
|
||||
else:
|
||||
max_concurrent = min(int(max_concurrent_override), org_concurrent_limit)
|
||||
|
||||
wait_start = time.time()
|
||||
while True:
|
||||
acquisition = await rate_limiter.try_acquire_concurrent_slot_details(
|
||||
organization_id, max_concurrent
|
||||
)
|
||||
if acquisition:
|
||||
logger.info(
|
||||
f"Acquired concurrent call slot for org {organization_id}: "
|
||||
f"source={source}, active_calls="
|
||||
f"{acquisition.active_count}/{max_concurrent}, "
|
||||
f"slot_id={acquisition.slot_id}"
|
||||
)
|
||||
return CallConcurrencySlot(
|
||||
organization_id=organization_id,
|
||||
slot_id=acquisition.slot_id,
|
||||
max_concurrent=max_concurrent,
|
||||
source=source,
|
||||
)
|
||||
|
||||
wait_time = time.time() - wait_start
|
||||
if wait_time >= timeout:
|
||||
current_count = await rate_limiter.get_concurrent_count(organization_id)
|
||||
logger.warning(
|
||||
f"Concurrent call limit reached for org {organization_id}: "
|
||||
f"source={source}, active_calls={current_count}/{max_concurrent}, "
|
||||
f"waited={wait_time:.1f}s"
|
||||
)
|
||||
raise CallConcurrencyLimitError(
|
||||
organization_id=organization_id,
|
||||
source=source,
|
||||
wait_time=wait_time,
|
||||
max_concurrent=max_concurrent,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"Waiting for concurrent call slot for org {organization_id}, "
|
||||
f"source={source}, waited {wait_time:.1f}s"
|
||||
)
|
||||
await asyncio.sleep(min(retry_interval, max(0, timeout - wait_time)))
|
||||
|
||||
async def bind_workflow_run(
|
||||
self, slot: CallConcurrencySlot, workflow_run_id: int
|
||||
) -> None:
|
||||
stored = await rate_limiter.store_workflow_slot_mapping_if_absent(
|
||||
workflow_run_id,
|
||||
slot.organization_id,
|
||||
slot.slot_id,
|
||||
)
|
||||
if stored:
|
||||
return
|
||||
|
||||
await self.release_slot(slot)
|
||||
raise WorkflowRunSlotAlreadyBoundError(workflow_run_id)
|
||||
|
||||
async def register_active_call(
|
||||
self,
|
||||
organization_id: int,
|
||||
workflow_run_id: int,
|
||||
*,
|
||||
source: str,
|
||||
timeout: float = 0,
|
||||
max_concurrent_override: int | None = None,
|
||||
retry_interval: float = 1,
|
||||
) -> CallConcurrencySlot:
|
||||
slot = await self.acquire_org_slot(
|
||||
organization_id,
|
||||
source=source,
|
||||
timeout=timeout,
|
||||
max_concurrent_override=max_concurrent_override,
|
||||
retry_interval=retry_interval,
|
||||
)
|
||||
await self.bind_workflow_run(slot, workflow_run_id)
|
||||
return slot
|
||||
|
||||
async def unregister_active_call(self, workflow_run_id: int) -> bool:
|
||||
return await self.release_workflow_run_slot(workflow_run_id)
|
||||
|
||||
async def release_slot(self, slot: CallConcurrencySlot | None) -> bool:
|
||||
if slot is None:
|
||||
return False
|
||||
return await rate_limiter.release_concurrent_slot(
|
||||
slot.organization_id, slot.slot_id
|
||||
)
|
||||
|
||||
async def release_workflow_run_slot(self, workflow_run_id: int) -> bool:
|
||||
mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run_id)
|
||||
if not mapping:
|
||||
return False
|
||||
|
||||
org_id, slot_id = mapping
|
||||
released = await rate_limiter.release_concurrent_slot(org_id, slot_id)
|
||||
await rate_limiter.delete_workflow_slot_mapping(workflow_run_id)
|
||||
if released:
|
||||
logger.info(f"Released concurrent slot for workflow run {workflow_run_id}")
|
||||
else:
|
||||
logger.debug(
|
||||
f"Concurrent slot mapping for workflow run {workflow_run_id} "
|
||||
"had no live slot; deleted stale mapping"
|
||||
)
|
||||
return released
|
||||
|
||||
|
||||
call_concurrency = CallConcurrencyService()
|
||||
|
|
@ -5,10 +5,14 @@ from typing import TYPE_CHECKING, Optional
|
|||
|
||||
from loguru import logger
|
||||
|
||||
from api.constants import DEFAULT_ORG_CONCURRENCY_LIMIT
|
||||
from api.db import db_client
|
||||
from api.db.models import QueuedRunModel, WorkflowRunModel
|
||||
from api.enums import OrganizationConfigurationKey, WorkflowRunState
|
||||
from api.enums import WorkflowRunState
|
||||
from api.services.call_concurrency import (
|
||||
CallConcurrencyLimitError,
|
||||
CallConcurrencySlot,
|
||||
call_concurrency,
|
||||
)
|
||||
from api.services.campaign.circuit_breaker import circuit_breaker
|
||||
from api.services.campaign.errors import (
|
||||
ConcurrentSlotAcquisitionError,
|
||||
|
|
@ -29,9 +33,6 @@ if TYPE_CHECKING:
|
|||
class CampaignCallDispatcher:
|
||||
"""Manages rate-limited and concurrent-limited call dispatching"""
|
||||
|
||||
def __init__(self):
|
||||
self.default_concurrent_limit = int(DEFAULT_ORG_CONCURRENCY_LIMIT)
|
||||
|
||||
async def get_provider_for_campaign(self, campaign) -> "TelephonyProvider":
|
||||
"""Get the telephony provider pinned to this campaign's config. Falls back
|
||||
to the org's default config for legacy campaigns whose
|
||||
|
|
@ -53,18 +54,7 @@ class CampaignCallDispatcher:
|
|||
|
||||
async def get_org_concurrent_limit(self, organization_id: int) -> int:
|
||||
"""Get the concurrent call limit for an organization."""
|
||||
try:
|
||||
config = await db_client.get_configuration(
|
||||
organization_id,
|
||||
OrganizationConfigurationKey.CONCURRENT_CALL_LIMIT.value,
|
||||
)
|
||||
if config and config.value:
|
||||
return int(config.value["value"])
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Error getting concurrent limit for org {organization_id}: {e}"
|
||||
)
|
||||
return self.default_concurrent_limit
|
||||
return await call_concurrency.get_org_concurrent_limit(organization_id)
|
||||
|
||||
async def process_batch(self, campaign_id: int, batch_size: int = 10) -> int:
|
||||
"""
|
||||
|
|
@ -119,12 +109,14 @@ class CampaignCallDispatcher:
|
|||
)
|
||||
|
||||
# Acquire concurrent slot - waits until a slot is available
|
||||
slot_id = await self.acquire_concurrent_slot(
|
||||
concurrency_slot = await self.acquire_concurrent_slot(
|
||||
campaign.organization_id, campaign
|
||||
)
|
||||
|
||||
# Dispatch the call
|
||||
workflow_run = await self.dispatch_call(queued_run, campaign, slot_id)
|
||||
workflow_run = await self.dispatch_call(
|
||||
queued_run, campaign, concurrency_slot
|
||||
)
|
||||
|
||||
# Update queued run as processed
|
||||
await db_client.update_queued_run(
|
||||
|
|
@ -233,68 +225,61 @@ class CampaignCallDispatcher:
|
|||
)
|
||||
|
||||
async def dispatch_call(
|
||||
self, queued_run: QueuedRunModel, campaign: any, slot_id: str
|
||||
self,
|
||||
queued_run: QueuedRunModel,
|
||||
campaign: any,
|
||||
concurrency_slot: CallConcurrencySlot,
|
||||
) -> Optional[WorkflowRunModel]:
|
||||
"""Creates workflow run and initiates call. Requires a pre-acquired slot_id."""
|
||||
"""Creates workflow run and initiates call. Requires a pre-acquired slot."""
|
||||
from_number = None
|
||||
workflow_run = None
|
||||
slot_bound = False
|
||||
|
||||
# Get workflow details
|
||||
workflow = await db_client.get_workflow_by_id(campaign.workflow_id)
|
||||
if not workflow:
|
||||
# Release slot before raising
|
||||
await rate_limiter.release_concurrent_slot(
|
||||
campaign.organization_id, slot_id
|
||||
)
|
||||
raise ValueError(f"Workflow {campaign.workflow_id} not found")
|
||||
|
||||
# Extract phone number
|
||||
phone_number = queued_run.context_variables.get("phone_number")
|
||||
if not phone_number:
|
||||
# Release slot before raising
|
||||
await rate_limiter.release_concurrent_slot(
|
||||
campaign.organization_id, slot_id
|
||||
)
|
||||
raise ValueError(f"No phone number in queued run {queued_run.id}")
|
||||
|
||||
# Get provider for this campaign's pinned telephony config.
|
||||
provider = await self.get_provider_for_campaign(campaign)
|
||||
workflow_run_mode = provider.PROVIDER_NAME
|
||||
|
||||
# Acquire a unique from_number from the pool scoped to this campaign's
|
||||
# telephony configuration so orgs with multiple configs don't leak
|
||||
# caller IDs across configs.
|
||||
from_number = await self.acquire_from_number(
|
||||
campaign.organization_id,
|
||||
telephony_configuration_id=campaign.telephony_configuration_id,
|
||||
)
|
||||
if from_number is None:
|
||||
# Release concurrent slot before raising
|
||||
await rate_limiter.release_concurrent_slot(
|
||||
campaign.organization_id, slot_id
|
||||
)
|
||||
raise PhoneNumberPoolExhaustedError(
|
||||
organization_id=campaign.organization_id
|
||||
)
|
||||
|
||||
logger.info(f"Provider name: {provider.PROVIDER_NAME}")
|
||||
logger.info(f"Queued run context: {queued_run.context_variables}")
|
||||
|
||||
# Merge context variables (queued_run context already includes retry info if applicable)
|
||||
initial_context = {
|
||||
**queued_run.context_variables,
|
||||
"campaign_id": campaign.id,
|
||||
"provider": provider.PROVIDER_NAME,
|
||||
"source_uuid": queued_run.source_uuid,
|
||||
"caller_number": from_number,
|
||||
"called_number": phone_number,
|
||||
"telephony_configuration_id": campaign.telephony_configuration_id,
|
||||
}
|
||||
|
||||
logger.info(f"Final initial_context: {initial_context}")
|
||||
|
||||
# Create workflow run with queued_run_id tracking
|
||||
workflow_run_name = f"WR-CAMPAIGN-{campaign.id}-{queued_run.id}"
|
||||
try:
|
||||
# Get workflow details
|
||||
workflow = await db_client.get_workflow_by_id(campaign.workflow_id)
|
||||
if not workflow:
|
||||
raise ValueError(f"Workflow {campaign.workflow_id} not found")
|
||||
|
||||
# Extract phone number
|
||||
phone_number = queued_run.context_variables.get("phone_number")
|
||||
if not phone_number:
|
||||
raise ValueError(f"No phone number in queued run {queued_run.id}")
|
||||
|
||||
# Get provider for this campaign's pinned telephony config.
|
||||
provider = await self.get_provider_for_campaign(campaign)
|
||||
workflow_run_mode = provider.PROVIDER_NAME
|
||||
|
||||
# Acquire a unique from_number from the pool scoped to this campaign's
|
||||
# telephony configuration so orgs with multiple configs don't leak
|
||||
# caller IDs across configs.
|
||||
from_number = await self.acquire_from_number(
|
||||
campaign.organization_id,
|
||||
telephony_configuration_id=campaign.telephony_configuration_id,
|
||||
)
|
||||
if from_number is None:
|
||||
raise PhoneNumberPoolExhaustedError(
|
||||
organization_id=campaign.organization_id
|
||||
)
|
||||
|
||||
logger.info(f"Provider name: {provider.PROVIDER_NAME}")
|
||||
logger.info(f"Queued run context: {queued_run.context_variables}")
|
||||
|
||||
# Merge context variables (queued_run context already includes retry info if applicable)
|
||||
initial_context = {
|
||||
**queued_run.context_variables,
|
||||
"campaign_id": campaign.id,
|
||||
"provider": provider.PROVIDER_NAME,
|
||||
"source_uuid": queued_run.source_uuid,
|
||||
"caller_number": from_number,
|
||||
"called_number": phone_number,
|
||||
"telephony_configuration_id": campaign.telephony_configuration_id,
|
||||
}
|
||||
|
||||
logger.info(f"Final initial_context: {initial_context}")
|
||||
|
||||
# Create workflow run with queued_run_id tracking
|
||||
workflow_run_name = f"WR-CAMPAIGN-{campaign.id}-{queued_run.id}"
|
||||
workflow_run = await db_client.create_workflow_run(
|
||||
name=workflow_run_name,
|
||||
workflow_id=campaign.workflow_id,
|
||||
|
|
@ -304,11 +289,8 @@ class CampaignCallDispatcher:
|
|||
campaign_id=campaign.id,
|
||||
queued_run_id=queued_run.id, # Link to queued run for retry tracking
|
||||
)
|
||||
|
||||
# Store slot_id mapping in Redis for cleanup later
|
||||
await rate_limiter.store_workflow_slot_mapping(
|
||||
workflow_run.id, campaign.organization_id, slot_id
|
||||
)
|
||||
await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run.id)
|
||||
slot_bound = True
|
||||
|
||||
# Store from_number mapping for cleanup on call completion
|
||||
await rate_limiter.store_workflow_from_number_mapping(
|
||||
|
|
@ -319,9 +301,10 @@ class CampaignCallDispatcher:
|
|||
)
|
||||
except Exception as e:
|
||||
# Release slot and from_number on error
|
||||
await rate_limiter.release_concurrent_slot(
|
||||
campaign.organization_id, slot_id
|
||||
)
|
||||
if slot_bound and workflow_run:
|
||||
await call_concurrency.release_workflow_run_slot(workflow_run.id)
|
||||
else:
|
||||
await call_concurrency.release_slot(concurrency_slot)
|
||||
if from_number:
|
||||
await rate_limiter.release_from_number(
|
||||
campaign.organization_id,
|
||||
|
|
@ -357,21 +340,7 @@ class CampaignCallDispatcher:
|
|||
gathered_context={"error": error_message},
|
||||
)
|
||||
|
||||
mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run.id)
|
||||
if mapping:
|
||||
org_id, mapped_slot_id = mapping
|
||||
await rate_limiter.release_concurrent_slot(org_id, mapped_slot_id)
|
||||
await rate_limiter.delete_workflow_slot_mapping(workflow_run.id)
|
||||
|
||||
from_number_mapping = await rate_limiter.get_workflow_from_number_mapping(
|
||||
workflow_run.id
|
||||
)
|
||||
if from_number_mapping:
|
||||
fn_org_id, fn_number, fn_tcid = from_number_mapping
|
||||
await rate_limiter.release_from_number(
|
||||
fn_org_id, fn_number, telephony_configuration_id=fn_tcid
|
||||
)
|
||||
await rate_limiter.delete_workflow_from_number_mapping(workflow_run.id)
|
||||
await self.release_call_slot(workflow_run.id)
|
||||
|
||||
raise ValueError(error_message)
|
||||
|
||||
|
|
@ -446,23 +415,7 @@ class CampaignCallDispatcher:
|
|||
reason="call_initiation_failed",
|
||||
)
|
||||
|
||||
# Release concurrent slot on failure
|
||||
mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run.id)
|
||||
if mapping:
|
||||
org_id, slot_id = mapping
|
||||
await rate_limiter.release_concurrent_slot(org_id, slot_id)
|
||||
await rate_limiter.delete_workflow_slot_mapping(workflow_run.id)
|
||||
|
||||
# Release from_number on failure
|
||||
from_number_mapping = await rate_limiter.get_workflow_from_number_mapping(
|
||||
workflow_run.id
|
||||
)
|
||||
if from_number_mapping:
|
||||
fn_org_id, fn_number, fn_tcid = from_number_mapping
|
||||
await rate_limiter.release_from_number(
|
||||
fn_org_id, fn_number, telephony_configuration_id=fn_tcid
|
||||
)
|
||||
await rate_limiter.delete_workflow_from_number_mapping(workflow_run.id)
|
||||
await self.release_call_slot(workflow_run.id)
|
||||
|
||||
raise
|
||||
|
||||
|
|
@ -501,7 +454,7 @@ class CampaignCallDispatcher:
|
|||
|
||||
async def acquire_concurrent_slot(
|
||||
self, organization_id: int, campaign: any, timeout: float = 600
|
||||
) -> str:
|
||||
) -> CallConcurrencySlot:
|
||||
"""
|
||||
Acquires a concurrent call slot - waits if necessary until a slot is available.
|
||||
|
||||
|
|
@ -510,14 +463,11 @@ class CampaignCallDispatcher:
|
|||
campaign: The campaign object
|
||||
timeout: Maximum time to wait for a slot (default 10 minutes)
|
||||
|
||||
Returns the slot_id which must be released when the call completes.
|
||||
Returns the slot which must be released when the call completes.
|
||||
|
||||
Raises:
|
||||
ConcurrentSlotAcquisitionError: If slot cannot be acquired within timeout
|
||||
"""
|
||||
# Get concurrent limit for organization
|
||||
org_concurrent_limit = await self.get_org_concurrent_limit(organization_id)
|
||||
|
||||
# Check for campaign-level max_concurrency in orchestrator_metadata
|
||||
campaign_max_concurrency = None
|
||||
if campaign.orchestrator_metadata:
|
||||
|
|
@ -525,39 +475,20 @@ class CampaignCallDispatcher:
|
|||
"max_concurrency"
|
||||
)
|
||||
|
||||
# Use the lower of campaign limit and org limit
|
||||
if campaign_max_concurrency is not None:
|
||||
max_concurrent = min(campaign_max_concurrency, org_concurrent_limit)
|
||||
else:
|
||||
max_concurrent = org_concurrent_limit
|
||||
|
||||
# Track wait time for alerting
|
||||
wait_start = time.time()
|
||||
|
||||
# Wait until we can acquire a concurrent slot
|
||||
while True:
|
||||
slot_id = await rate_limiter.try_acquire_concurrent_slot(
|
||||
organization_id, max_concurrent
|
||||
try:
|
||||
return await call_concurrency.acquire_org_slot(
|
||||
organization_id,
|
||||
source=f"campaign:{campaign.id}",
|
||||
timeout=timeout,
|
||||
max_concurrent_override=campaign_max_concurrency,
|
||||
retry_interval=1,
|
||||
)
|
||||
if slot_id:
|
||||
return slot_id
|
||||
|
||||
# Check if we've been waiting too long
|
||||
wait_time = time.time() - wait_start
|
||||
if wait_time > timeout:
|
||||
raise ConcurrentSlotAcquisitionError(
|
||||
organization_id=organization_id,
|
||||
campaign_id=campaign.id,
|
||||
wait_time=wait_time,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"Attempting to get a slot for {organization_id} {campaign.id}, "
|
||||
f"waited {wait_time:.1f}s"
|
||||
)
|
||||
|
||||
# Wait before retrying
|
||||
await asyncio.sleep(1)
|
||||
except CallConcurrencyLimitError as e:
|
||||
raise ConcurrentSlotAcquisitionError(
|
||||
organization_id=organization_id,
|
||||
campaign_id=campaign.id,
|
||||
wait_time=e.wait_time,
|
||||
) from e
|
||||
|
||||
async def acquire_from_number(
|
||||
self,
|
||||
|
|
@ -602,17 +533,9 @@ class CampaignCallDispatcher:
|
|||
Release concurrent slot and from_number when a call completes.
|
||||
Called by Twilio webhooks or workflow completion handlers.
|
||||
"""
|
||||
slot_released = False
|
||||
mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run_id)
|
||||
if mapping:
|
||||
org_id, slot_id = mapping
|
||||
success = await rate_limiter.release_concurrent_slot(org_id, slot_id)
|
||||
if success:
|
||||
await rate_limiter.delete_workflow_slot_mapping(workflow_run_id)
|
||||
logger.info(
|
||||
f"Released concurrent slot for workflow run {workflow_run_id}"
|
||||
)
|
||||
slot_released = True
|
||||
slot_released = await call_concurrency.release_workflow_run_slot(
|
||||
workflow_run_id
|
||||
)
|
||||
|
||||
# Release from_number back to its (org, telephony config) pool
|
||||
from_number_mapping = await rate_limiter.get_workflow_from_number_mapping(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
import redis.asyncio as aioredis
|
||||
|
|
@ -8,6 +9,12 @@ from loguru import logger
|
|||
from api.constants import REDIS_URL
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConcurrentSlotAcquisition:
|
||||
slot_id: str
|
||||
active_count: int
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""Sliding window rate limiter to enforce strict per-second limits and concurrent call limits"""
|
||||
|
||||
|
|
@ -100,6 +107,19 @@ class RateLimiter:
|
|||
Try to acquire a concurrent call slot.
|
||||
Returns a unique slot_id if successful, None if limit reached.
|
||||
"""
|
||||
acquisition = await self.try_acquire_concurrent_slot_details(
|
||||
organization_id, max_concurrent
|
||||
)
|
||||
return acquisition.slot_id if acquisition else None
|
||||
|
||||
async def try_acquire_concurrent_slot_details(
|
||||
self, organization_id: int, max_concurrent: int = 20
|
||||
) -> Optional[ConcurrentSlotAcquisition]:
|
||||
"""
|
||||
Try to acquire a concurrent call slot.
|
||||
Returns the slot_id and post-acquire active count if successful,
|
||||
or None if the limit is reached.
|
||||
"""
|
||||
redis_client = await self._get_redis()
|
||||
|
||||
concurrent_key = f"concurrent_calls:{organization_id}"
|
||||
|
|
@ -124,7 +144,7 @@ class RateLimiter:
|
|||
-- Add new slot
|
||||
redis.call('ZADD', key, now, slot_id)
|
||||
redis.call('EXPIRE', key, 3600) -- Expire after 1 hour
|
||||
return slot_id
|
||||
return {slot_id, current_count + 1}
|
||||
else
|
||||
return nil
|
||||
end
|
||||
|
|
@ -143,7 +163,14 @@ class RateLimiter:
|
|||
stale_cutoff,
|
||||
slot_id,
|
||||
)
|
||||
return result
|
||||
if not result:
|
||||
return None
|
||||
|
||||
acquired_slot_id, active_count = result
|
||||
return ConcurrentSlotAcquisition(
|
||||
slot_id=str(acquired_slot_id),
|
||||
active_count=int(active_count),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Concurrent limiter error: {e}")
|
||||
return None
|
||||
|
|
@ -212,6 +239,46 @@ class RateLimiter:
|
|||
logger.error(f"Error storing workflow slot mapping: {e}")
|
||||
return False
|
||||
|
||||
async def store_workflow_slot_mapping_if_absent(
|
||||
self, workflow_run_id: int, organization_id: int, slot_id: str
|
||||
) -> bool:
|
||||
"""
|
||||
Store the workflow_run_id -> concurrent slot mapping only if no mapping
|
||||
already exists. This prevents duplicate public/WebRTC starts for the
|
||||
same workflow run from overwriting the cleanup pointer.
|
||||
"""
|
||||
redis_client = await self._get_redis()
|
||||
mapping_key = f"workflow_slot_mapping:{workflow_run_id}"
|
||||
|
||||
lua_script = """
|
||||
local key = KEYS[1]
|
||||
local org_id = ARGV[1]
|
||||
local slot_id = ARGV[2]
|
||||
local ttl = tonumber(ARGV[3])
|
||||
|
||||
if redis.call('EXISTS', key) == 1 then
|
||||
return 0
|
||||
end
|
||||
|
||||
redis.call('HSET', key, 'org_id', org_id, 'slot_id', slot_id)
|
||||
redis.call('EXPIRE', key, ttl)
|
||||
return 1
|
||||
"""
|
||||
|
||||
try:
|
||||
stored = await redis_client.eval(
|
||||
lua_script,
|
||||
1,
|
||||
mapping_key,
|
||||
organization_id,
|
||||
slot_id,
|
||||
self.stale_call_timeout,
|
||||
)
|
||||
return bool(stored)
|
||||
except Exception as e:
|
||||
logger.error(f"Error storing workflow slot mapping if absent: {e}")
|
||||
return False
|
||||
|
||||
async def get_workflow_slot_mapping(
|
||||
self, workflow_run_id: int
|
||||
) -> Optional[tuple[int, str]]:
|
||||
|
|
|
|||
|
|
@ -14,14 +14,17 @@ from api.schemas.workflow_configurations import (
|
|||
DEFAULT_TURN_START_MIN_WORDS,
|
||||
DEFAULT_TURN_START_STRATEGY,
|
||||
)
|
||||
from api.services.call_concurrency import call_concurrency
|
||||
from api.services.configuration.registry import ServiceProviders
|
||||
from api.services.integrations import (
|
||||
IntegrationRuntimeContext,
|
||||
create_runtime_sessions,
|
||||
)
|
||||
from api.services.pipecat.active_calls import (
|
||||
register_active_call,
|
||||
unregister_active_call,
|
||||
register_active_call as register_worker_active_call,
|
||||
)
|
||||
from api.services.pipecat.active_calls import (
|
||||
unregister_active_call as unregister_worker_active_call,
|
||||
)
|
||||
from api.services.pipecat.audio_config import AudioConfig, create_audio_config
|
||||
from api.services.pipecat.event_handlers import (
|
||||
|
|
@ -258,7 +261,7 @@ async def run_pipeline_telephony(
|
|||
"""Run a pipeline for any telephony provider."""
|
||||
# Register before any async setup so deploy drains see calls that are still
|
||||
# resolving DB/config/transport state.
|
||||
register_active_call(workflow_run_id)
|
||||
register_worker_active_call(workflow_run_id)
|
||||
try:
|
||||
await _run_pipeline_telephony_impl(
|
||||
websocket,
|
||||
|
|
@ -270,7 +273,10 @@ async def run_pipeline_telephony(
|
|||
transport_kwargs=transport_kwargs,
|
||||
)
|
||||
finally:
|
||||
unregister_active_call(workflow_run_id)
|
||||
try:
|
||||
await call_concurrency.unregister_active_call(workflow_run_id)
|
||||
finally:
|
||||
unregister_worker_active_call(workflow_run_id)
|
||||
|
||||
|
||||
async def _run_pipeline_telephony_impl(
|
||||
|
|
@ -383,7 +389,7 @@ async def run_pipeline_smallwebrtc(
|
|||
"""Run pipeline for WebRTC connections."""
|
||||
# Register before any async setup so deploy drains see calls that are still
|
||||
# resolving DB/config/transport state.
|
||||
register_active_call(workflow_run_id)
|
||||
register_worker_active_call(workflow_run_id)
|
||||
try:
|
||||
await _run_pipeline_smallwebrtc_impl(
|
||||
webrtc_connection,
|
||||
|
|
@ -394,7 +400,10 @@ async def run_pipeline_smallwebrtc(
|
|||
user_provider_id=user_provider_id,
|
||||
)
|
||||
finally:
|
||||
unregister_active_call(workflow_run_id)
|
||||
try:
|
||||
await call_concurrency.unregister_active_call(workflow_run_id)
|
||||
finally:
|
||||
unregister_worker_active_call(workflow_run_id)
|
||||
|
||||
|
||||
async def _run_pipeline_smallwebrtc_impl(
|
||||
|
|
@ -478,7 +487,7 @@ async def _run_pipeline(
|
|||
resolved_user_config=None,
|
||||
) -> None:
|
||||
"""Run the pipeline with active-call drain accounting."""
|
||||
register_active_call(workflow_run_id)
|
||||
register_worker_active_call(workflow_run_id)
|
||||
try:
|
||||
await _run_pipeline_impl(
|
||||
transport,
|
||||
|
|
@ -492,7 +501,10 @@ async def _run_pipeline(
|
|||
resolved_user_config=resolved_user_config,
|
||||
)
|
||||
finally:
|
||||
unregister_active_call(workflow_run_id)
|
||||
try:
|
||||
await call_concurrency.unregister_active_call(workflow_run_id)
|
||||
finally:
|
||||
unregister_worker_active_call(workflow_run_id)
|
||||
|
||||
|
||||
async def _run_pipeline_impl(
|
||||
|
|
|
|||
|
|
@ -26,6 +26,10 @@ from loguru import logger
|
|||
from api.constants import REDIS_URL
|
||||
from api.db import db_client
|
||||
from api.enums import CallType, WorkflowRunMode
|
||||
from api.services.call_concurrency import (
|
||||
CallConcurrencyLimitError,
|
||||
call_concurrency,
|
||||
)
|
||||
from api.services.quota_service import authorize_workflow_run_start
|
||||
from api.services.telephony.call_transfer_manager import get_call_transfer_manager
|
||||
from api.services.telephony.transfer_event_protocol import (
|
||||
|
|
@ -527,6 +531,8 @@ class ARIConnection:
|
|||
channel = event.get("channel", {})
|
||||
caller_number = channel.get("caller", {}).get("number", "unknown")
|
||||
called_number = channel.get("dialplan", {}).get("exten", "unknown")
|
||||
concurrency_slot = None
|
||||
workflow_run = None
|
||||
|
||||
try:
|
||||
# 1. Resolve the workflow from the called extension via the
|
||||
|
|
@ -573,6 +579,20 @@ class ARIConnection:
|
|||
|
||||
user_id = workflow.user_id
|
||||
|
||||
try:
|
||||
concurrency_slot = await call_concurrency.acquire_org_slot(
|
||||
self.organization_id,
|
||||
source="ari_inbound",
|
||||
timeout=0,
|
||||
)
|
||||
except CallConcurrencyLimitError:
|
||||
logger.warning(
|
||||
f"[ARI org={self.organization_id}] Concurrent call limit "
|
||||
f"reached; hanging up inbound channel {channel_id}"
|
||||
)
|
||||
await self._delete_channel(channel_id)
|
||||
return
|
||||
|
||||
# 3. Create workflow run
|
||||
call_id = channel_id
|
||||
workflow_run = await db_client.create_workflow_run(
|
||||
|
|
@ -591,7 +611,9 @@ class ARIConnection:
|
|||
gathered_context={
|
||||
"call_id": call_id,
|
||||
},
|
||||
organization_id=self.organization_id,
|
||||
)
|
||||
await call_concurrency.bind_workflow_run(concurrency_slot, workflow_run.id)
|
||||
|
||||
logger.info(
|
||||
f"[ARI org={self.organization_id}] Created inbound workflow run "
|
||||
|
|
@ -610,6 +632,7 @@ class ARIConnection:
|
|||
f"[ARI org={self.organization_id}] Quota exceeded for user {user_id} "
|
||||
f"— hanging up inbound call {channel_id}"
|
||||
)
|
||||
await call_concurrency.release_workflow_run_slot(workflow_run.id)
|
||||
await self._delete_channel(channel_id)
|
||||
return
|
||||
|
||||
|
|
@ -625,6 +648,10 @@ class ARIConnection:
|
|||
str(user_id),
|
||||
)
|
||||
except Exception as e:
|
||||
if workflow_run:
|
||||
await call_concurrency.release_workflow_run_slot(workflow_run.id)
|
||||
elif concurrency_slot:
|
||||
await call_concurrency.release_slot(concurrency_slot)
|
||||
logger.error(
|
||||
f"[ARI org={self.organization_id}] Error handling inbound StasisStart "
|
||||
f"for channel {channel_id}: {e}"
|
||||
|
|
|
|||
|
|
@ -144,8 +144,9 @@ async def _process_status_update(workflow_run_id: int, status: StatusCallbackReq
|
|||
f"[run {workflow_run_id}] Call completed with duration: {status.duration}s"
|
||||
)
|
||||
|
||||
await campaign_call_dispatcher.release_call_slot(workflow_run_id)
|
||||
|
||||
if workflow_run.campaign_id:
|
||||
await campaign_call_dispatcher.release_call_slot(workflow_run_id)
|
||||
await circuit_breaker.record_and_evaluate(
|
||||
workflow_run.campaign_id, is_failure=False
|
||||
)
|
||||
|
|
@ -162,8 +163,9 @@ async def _process_status_update(workflow_run_id: int, status: StatusCallbackReq
|
|||
f"[run {workflow_run_id}] Call failed with status: {normalized_status.value}"
|
||||
)
|
||||
|
||||
await campaign_call_dispatcher.release_call_slot(workflow_run_id)
|
||||
|
||||
if workflow_run.campaign_id:
|
||||
await campaign_call_dispatcher.release_call_slot(workflow_run_id)
|
||||
is_failure = normalized_status in FAILURE_NOT_CONNECTED_STATUSES
|
||||
await circuit_breaker.record_and_evaluate(
|
||||
workflow_run.campaign_id,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue