fix: address concurrency review findings (campaign-scoped counter, cleanup hardening, webrtc run validation)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

api/services/call_concurrency.py

@@ -16,6 +16,7 @@ class CallConcurrencySlot:
     slot_id: str
     max_concurrent: int
     source: str
+    scope_key: str | None = None
 
 
 class CallConcurrencyLimitError(Exception):
@@ -76,19 +77,28 @@ class CallConcurrencyService:
         *,
         source: str,
         timeout: float = 0,
-        max_concurrent_override: int | None = None,
+        scope_key: str | None = None,
+        scope_max_concurrent: int | None = None,
         retry_interval: float = 1,
     ) -> CallConcurrencySlot:
-        org_concurrent_limit = await self.get_org_concurrent_limit(organization_id)
-        if max_concurrent_override is None:
-            max_concurrent = org_concurrent_limit
-        else:
-            max_concurrent = min(int(max_concurrent_override), org_concurrent_limit)
+        """Acquire a slot in the org-wide concurrency counter.
+
+        ``scope_key``/``scope_max_concurrent`` additionally bound a secondary
+        counter (e.g. ``campaign:<id>``) so a source can cap its own
+        concurrency without measuring — or being starved by — unrelated calls
+        in the same org.
+        """
+        max_concurrent = await self.get_org_concurrent_limit(organization_id)
+        if scope_max_concurrent is not None:
+            scope_max_concurrent = int(scope_max_concurrent)
 
         wait_start = time.time()
         while True:
             acquisition = await rate_limiter.try_acquire_concurrent_slot_details(
-                organization_id, max_concurrent
+                organization_id,
+                max_concurrent,
+                scope_key=scope_key,
+                scope_max_concurrent=scope_max_concurrent,
             )
             if acquisition:
                 logger.info(
@@ -102,15 +112,21 @@ class CallConcurrencyService:
                     slot_id=acquisition.slot_id,
                     max_concurrent=max_concurrent,
                     source=source,
+                    scope_key=scope_key,
                 )
 
             wait_time = time.time() - wait_start
             if wait_time >= timeout:
                 current_count = await rate_limiter.get_concurrent_count(organization_id)
+                scope_note = (
+                    f", scope={scope_key} (limit={scope_max_concurrent})"
+                    if scope_key
+                    else ""
+                )
                 logger.warning(
                     f"Concurrent call limit reached for org {organization_id}: "
-                    f"source={source}, active_calls={current_count}/{max_concurrent}, "
-                    f"waited={wait_time:.1f}s"
+                    f"source={source}, active_calls={current_count}/{max_concurrent}"
+                    f"{scope_note}, waited={wait_time:.1f}s"
                 )
                 raise CallConcurrencyLimitError(
                     organization_id=organization_id,
@@ -132,6 +148,7 @@ class CallConcurrencyService:
             workflow_run_id,
             slot.organization_id,
             slot.slot_id,
+            scope_key=slot.scope_key,
         )
         if stored:
             return
@@ -146,36 +163,66 @@ class CallConcurrencyService:
         *,
         source: str,
         timeout: float = 0,
-        max_concurrent_override: int | None = None,
+        scope_key: str | None = None,
+        scope_max_concurrent: int | None = None,
         retry_interval: float = 1,
     ) -> CallConcurrencySlot:
         slot = await self.acquire_org_slot(
             organization_id,
             source=source,
             timeout=timeout,
-            max_concurrent_override=max_concurrent_override,
+            scope_key=scope_key,
+            scope_max_concurrent=scope_max_concurrent,
             retry_interval=retry_interval,
         )
         await self.bind_workflow_run(slot, workflow_run_id)
         return slot
 
     async def unregister_active_call(self, workflow_run_id: int) -> bool:
-        return await self.release_workflow_run_slot(workflow_run_id)
+        """Release the run's slot without ever raising.
+
+        Callers invoke this from ``finally`` blocks during pipeline/socket
+        teardown; a cleanup failure must not mask the original exception.
+        The slot mapping survives a failed release, so a later cleanup path
+        (status callback, StasisEnd) or the Redis stale timeout recovers it.
+        """
+        try:
+            return await self.release_workflow_run_slot(workflow_run_id)
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            logger.warning(
+                f"Failed to release concurrent call slot for workflow run "
+                f"{workflow_run_id}: {e}"
+            )
+            return False
 
     async def release_slot(self, slot: CallConcurrencySlot | None) -> bool:
         if slot is None:
             return False
-        return await rate_limiter.release_concurrent_slot(
-            slot.organization_id, slot.slot_id
+        released = await rate_limiter.release_concurrent_slot(
+            slot.organization_id, slot.slot_id, scope_key=slot.scope_key
         )
+        return bool(released)
 
     async def release_workflow_run_slot(self, workflow_run_id: int) -> bool:
         mapping = await rate_limiter.get_workflow_slot_mapping(workflow_run_id)
         if not mapping:
             return False
 
-        org_id, slot_id = mapping
-        released = await rate_limiter.release_concurrent_slot(org_id, slot_id)
+        org_id, slot_id, scope_key = mapping
+        released = await rate_limiter.release_concurrent_slot(
+            org_id, slot_id, scope_key=scope_key
+        )
+        if released is None:
+            # Redis error while releasing — keep the mapping so a later
+            # cleanup path can retry instead of orphaning a live slot until
+            # the stale timeout.
+            logger.warning(
+                f"Failed to release concurrent slot for workflow run "
+                f"{workflow_run_id}; keeping mapping for retry"
+            )
+            return False
         await rate_limiter.delete_workflow_slot_mapping(workflow_run_id)
         if released:
             logger.info(f"Released concurrent slot for workflow run {workflow_run_id}")

api/services/campaign/campaign_call_dispatcher.py

@@ -468,7 +468,11 @@ class CampaignCallDispatcher:
         Raises:
             ConcurrentSlotAcquisitionError: If slot cannot be acquired within timeout
         """
-        # Check for campaign-level max_concurrency in orchestrator_metadata
+        # Check for campaign-level max_concurrency in orchestrator_metadata.
+        # It caps this campaign's own concurrent calls via a campaign-scoped
+        # counter — the org-wide limit still applies on top, but calls from
+        # other sources (WebRTC, inbound, other campaigns) don't count
+        # against the campaign's cap.
         campaign_max_concurrency = None
         if campaign.orchestrator_metadata:
             campaign_max_concurrency = campaign.orchestrator_metadata.get(
@@ -480,7 +484,12 @@ class CampaignCallDispatcher:
                 organization_id,
                 source=f"campaign:{campaign.id}",
                 timeout=timeout,
-                max_concurrent_override=campaign_max_concurrency,
+                scope_key=(
+                    f"campaign:{campaign.id}"
+                    if campaign_max_concurrency is not None
+                    else None
+                ),
+                scope_max_concurrent=campaign_max_concurrency,
                 retry_interval=1,
             )
         except CallConcurrencyLimitError as e:

api/services/campaign/rate_limiter.py

@@ -113,41 +113,65 @@ class RateLimiter:
         return acquisition.slot_id if acquisition else None
 
     async def try_acquire_concurrent_slot_details(
-        self, organization_id: int, max_concurrent: int = 20
+        self,
+        organization_id: int,
+        max_concurrent: int = 20,
+        *,
+        scope_key: str | None = None,
+        scope_max_concurrent: int | None = None,
     ) -> Optional[ConcurrentSlotAcquisition]:
         """
         Try to acquire a concurrent call slot.
         Returns the slot_id and post-acquire active count if successful,
         or None if the limit is reached.
+
+        When ``scope_key``/``scope_max_concurrent`` are provided, the slot is
+        also registered in a secondary counter (``concurrent_calls:<scope_key>``,
+        e.g. ``campaign:<id>``) and acquisition additionally requires that
+        counter to be below ``scope_max_concurrent``. Both counters are
+        updated atomically. The scope-scoped slot must be released with the
+        same ``scope_key``.
         """
         redis_client = await self._get_redis()
 
         concurrent_key = f"concurrent_calls:{organization_id}"
+        scope_concurrent_key = f"concurrent_calls:{scope_key}" if scope_key else ""
         now = time.time()
         stale_cutoff = now - self.stale_call_timeout
 
-        # Lua script for atomic operation
+        # Lua script for atomic operation across the org counter and the
+        # optional scope counter (empty scope key = org-only acquisition).
         lua_script = """
         local key = KEYS[1]
+        local scope_key = KEYS[2]
         local now = tonumber(ARGV[1])
         local max_concurrent = tonumber(ARGV[2])
         local stale_cutoff = tonumber(ARGV[3])
         local slot_id = ARGV[4]
-        
-        -- Remove stale entries (older than 30 minutes)
+        local scope_max_concurrent = tonumber(ARGV[5])
+
+        -- Remove stale entries (older than the stale-call timeout)
         redis.call('ZREMRANGEBYSCORE', key, 0, stale_cutoff)
-        
+
         -- Get current count
         local current_count = redis.call('ZCARD', key)
-        
-        if current_count < max_concurrent then
-            -- Add new slot
-            redis.call('ZADD', key, now, slot_id)
-            redis.call('EXPIRE', key, 3600)  -- Expire after 1 hour
-            return {slot_id, current_count + 1}
-        else
+
+        if current_count >= max_concurrent then
             return nil
         end
+
+        if scope_key ~= '' then
+            redis.call('ZREMRANGEBYSCORE', scope_key, 0, stale_cutoff)
+            if redis.call('ZCARD', scope_key) >= scope_max_concurrent then
+                return nil
+            end
+            redis.call('ZADD', scope_key, now, slot_id)
+            redis.call('EXPIRE', scope_key, 3600)
+        end
+
+        redis.call('ZADD', key, now, slot_id)
+        redis.call('EXPIRE', key, 3600)  -- Expire after 1 hour
+        return {slot_id, current_count + 1}
         """
 
         # Generate unique slot ID (timestamp + random component)
@@ -156,12 +180,14 @@ class RateLimiter:
         try:
             result = await redis_client.eval(
                 lua_script,
-                1,
+                2,
                 concurrent_key,
+                scope_concurrent_key,
                 now,
                 max_concurrent,
                 stale_cutoff,
                 slot_id,
+                scope_max_concurrent if scope_max_concurrent is not None else 0,
             )
             if not result:
                 return None
@@ -175,10 +201,17 @@ class RateLimiter:
             logger.error(f"Concurrent limiter error: {e}")
             return None
 
-    async def release_concurrent_slot(self, organization_id: int, slot_id: str) -> bool:
+    async def release_concurrent_slot(
+        self,
+        organization_id: int,
+        slot_id: str,
+        scope_key: str | None = None,
+    ) -> bool | None:
         """
-        Release a concurrent call slot.
-        Returns True if slot was released, False otherwise.
+        Release a concurrent call slot (and its scope counter entry, if any).
+        Returns True if the slot was released, False if it was already gone
+        (released/stale-expired), or None on a Redis error — callers that
+        track cleanup state should keep it around for retry when None.
         """
         if not slot_id:
             return False
@@ -188,6 +221,8 @@ class RateLimiter:
 
         try:
             removed = await redis_client.zrem(concurrent_key, slot_id)
+            if scope_key:
+                await redis_client.zrem(f"concurrent_calls:{scope_key}", slot_id)
             if removed:
                 logger.debug(
                     f"Released concurrent slot {slot_id} for org {organization_id}"
@@ -195,7 +230,7 @@ class RateLimiter:
             return bool(removed)
         except Exception as e:
             logger.error(f"Error releasing concurrent slot: {e}")
-            return False
+            return None
 
     async def get_concurrent_count(self, organization_id: int) -> int:
         """
@@ -240,7 +275,11 @@ class RateLimiter:
             return False
 
     async def store_workflow_slot_mapping_if_absent(
-        self, workflow_run_id: int, organization_id: int, slot_id: str
+        self,
+        workflow_run_id: int,
+        organization_id: int,
+        slot_id: str,
+        scope_key: str | None = None,
     ) -> bool:
         """
         Store the workflow_run_id -> concurrent slot mapping only if no mapping
@@ -255,12 +294,16 @@ class RateLimiter:
         local org_id = ARGV[1]
         local slot_id = ARGV[2]
         local ttl = tonumber(ARGV[3])
+        local scope_key = ARGV[4]
 
         if redis.call('EXISTS', key) == 1 then
             return 0
         end
 
         redis.call('HSET', key, 'org_id', org_id, 'slot_id', slot_id)
+        if scope_key ~= '' then
+            redis.call('HSET', key, 'scope_key', scope_key)
+        end
         redis.call('EXPIRE', key, ttl)
         return 1
         """
@@ -273,6 +316,7 @@ class RateLimiter:
                 organization_id,
                 slot_id,
                 self.stale_call_timeout,
+                scope_key or "",
             )
             return bool(stored)
         except Exception as e:
@@ -281,10 +325,11 @@ class RateLimiter:
 
     async def get_workflow_slot_mapping(
         self, workflow_run_id: int
-    ) -> Optional[tuple[int, str]]:
+    ) -> Optional[tuple[int, str, str | None]]:
         """
         Get the concurrent slot mapping for a workflow run.
-        Returns (organization_id, slot_id) tuple or None if not found.
+        Returns (organization_id, slot_id, scope_key) or None if not found;
+        scope_key is None for slots acquired without a scope counter.
         """
         redis_client = await self._get_redis()
         mapping_key = f"workflow_slot_mapping:{workflow_run_id}"
@@ -292,7 +337,11 @@ class RateLimiter:
         try:
             mapping = await redis_client.hgetall(mapping_key)
             if mapping and "org_id" in mapping and "slot_id" in mapping:
-                return (int(mapping["org_id"]), mapping["slot_id"])
+                return (
+                    int(mapping["org_id"]),
+                    mapping["slot_id"],
+                    mapping.get("scope_key") or None,
+                )
             return None
         except Exception as e:
             logger.error(f"Error getting workflow slot mapping: {e}")

api/services/telephony/ari_manager.py

@@ -787,6 +787,14 @@ class ARIConnection:
         the bridge and both channels — like endConferenceOnExit.
         """
         try:
+            # Release the org concurrency slot. Normally the pipeline's own
+            # teardown does this when the ext media websocket closes, but if
+            # the pipeline never started (caller hung up before external
+            # media connected, ext media creation failed, ...) this is the
+            # only cleanup that runs before the Redis stale timeout. No-op
+            # when the slot was already released.
+            await call_concurrency.unregister_active_call(int(workflow_run_id))
+
             workflow_run = await db_client.get_workflow_run_by_id(int(workflow_run_id))
             if not workflow_run or not workflow_run.gathered_context:
                 logger.warning(

api/services/telephony/base.py

@@ -365,6 +365,10 @@ class TelephonyProvider(ABC):
         the caller carries a provider stream protocol. ``organization_id`` is
         passed so providers can scope any config lookups to the workflow's org.
         Default raises so providers that haven't opted in fail loudly.
+
+        The route holds an org concurrency slot while this runs, so
+        implementations must bound their pre-pipeline handshake reads with a
+        timeout — an idle socket must not hold the slot indefinitely.
         """
         raise NotImplementedError(
             f"Agent-stream not supported for provider {self.PROVIDER_NAME}"

api/services/telephony/providers/cloudonix/provider.py

@@ -2,6 +2,7 @@
 Cloudonix implementation of the TelephonyProvider interface.
 """
 
+import asyncio
 import json
 import random
 from typing import TYPE_CHECKING, Any, Dict, List, Optional
@@ -25,6 +26,11 @@ if TYPE_CHECKING:
 
 CLOUDONIX_API_BASE_URL = "https://api.cloudonix.io"
 
+# Cloudonix sends the connected/start handshake immediately after the media
+# stream opens. The agent-stream route holds an org concurrency slot while we
+# wait, so an idle socket must not be able to hold it indefinitely.
+AGENT_STREAM_HANDSHAKE_TIMEOUT_S = 10
+
 
 class CloudonixProvider(TelephonyProvider):
     """
@@ -533,14 +539,31 @@ class CloudonixProvider(TelephonyProvider):
         from api.services.pipecat.run_pipeline import run_pipeline_telephony
 
         try:
-            first_msg = await websocket.receive_text()
-            msg = json.loads(first_msg)
-            if msg.get("event") != "connected":
-                logger.error(f"Expected 'connected' event, got: {msg.get('event')}")
-                await websocket.close(code=4400, reason="Expected connected event")
+            try:
+                first_msg = await asyncio.wait_for(
+                    websocket.receive_text(),
+                    timeout=AGENT_STREAM_HANDSHAKE_TIMEOUT_S,
+                )
+                msg = json.loads(first_msg)
+                if msg.get("event") != "connected":
+                    logger.error(f"Expected 'connected' event, got: {msg.get('event')}")
+                    await websocket.close(code=4400, reason="Expected connected event")
+                    return
+
+                start_msg = json.loads(
+                    await asyncio.wait_for(
+                        websocket.receive_text(),
+                        timeout=AGENT_STREAM_HANDSHAKE_TIMEOUT_S,
+                    )
+                )
+            except asyncio.TimeoutError:
+                logger.warning(
+                    f"Cloudonix agent-stream handshake timed out for workflow_run "
+                    f"{workflow_run_id}"
+                )
+                await websocket.close(code=4408, reason="Handshake timeout")
                 return
 
-            start_msg = json.loads(await websocket.receive_text())
             if start_msg.get("event") != "start":
                 logger.error("Expected 'start' event second")
                 await websocket.close(code=4400, reason="Expected start event")