feat: add logs in campaigns for failure or pausing (#265)

* feat: add logs in campaigns on failure

* chore: bump pipecat

* chore: update format.sh

* chore: fix github workflow

* fix: fix formatting errors

api/tests/integrations/_run_pipeline_helpers.py

@@ -29,12 +29,11 @@ from contextlib import ExitStack, contextmanager
 from typing import Any
 from unittest.mock import AsyncMock, patch
 
+from api.db.models import OrganizationModel, UserModel
+from api.enums import WorkflowRunMode
 from pipecat.frames.frames import Frame
 from pipecat.observers.base_observer import BaseObserver
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-
-from api.db.models import OrganizationModel, UserModel
-from api.enums import WorkflowRunMode
 from pipecat.tests import MockLLMService, MockTTSService
 
 USER_CONFIGURATION: dict[str, Any] = {

api/tests/integrations/test_run_pipeline.py

@@ -17,8 +17,6 @@ completion flag, and ``gathered_context`` entries.
 import asyncio
 
 import pytest
-from pipecat.tests.mock_transport import MockTransport
-from pipecat.transports.base_transport import TransportParams
 
 from api.enums import WorkflowRunMode, WorkflowRunState
 from api.services.pipecat.audio_config import create_audio_config
@@ -27,6 +25,8 @@ from api.tests.integrations._run_pipeline_helpers import (
     create_workflow_run_rows,
     patch_run_pipeline_externals,
 )
+from pipecat.tests.mock_transport import MockTransport
+from pipecat.transports.base_transport import TransportParams
 
 WORKFLOW_DEFINITION = {
     "nodes": [

api/tests/integrations/test_run_pipeline_text_greeting.py

@@ -28,10 +28,6 @@ deterministic and the synthesised audio length is short.
 import asyncio
 
 import pytest
-from pipecat.frames.frames import TranscriptionFrame
-from pipecat.tests.mock_transport import MockTransport
-from pipecat.transports.base_transport import TransportParams
-from pipecat.utils.time import time_now_iso8601
 
 from api.enums import WorkflowRunMode, WorkflowRunState
 from api.services.pipecat.audio_config import create_audio_config
@@ -40,7 +36,11 @@ from api.tests.integrations._run_pipeline_helpers import (
     create_workflow_run_rows,
     patch_run_pipeline_externals,
 )
+from pipecat.frames.frames import TranscriptionFrame
 from pipecat.tests import MockLLMService, MockTTSService
+from pipecat.tests.mock_transport import MockTransport
+from pipecat.transports.base_transport import TransportParams
+from pipecat.utils.time import time_now_iso8601
 
 GREETING_TEXT = (
     "Thanks for calling Happy Feet, this is Sarah. How can I help you today?"

api/tests/test_campaign_tasks.py

@@ -0,0 +1,87 @@
+"""
+Tests for api.tasks.campaign_tasks failure handling.
+
+Specifically: each kind of failure that pauses or fails a campaign should
+write a specific, identifiable entry into the campaign log so operators
+can tell at a glance why a campaign stopped.
+"""
+
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from api.services.campaign.errors import (
+    ConcurrentSlotAcquisitionError,
+    PhoneNumberPoolExhaustedError,
+)
+from api.tasks.campaign_tasks import process_campaign_batch
+
+
+class TestProcessCampaignBatchFailureLogs:
+    """``process_campaign_batch`` should log a *specific* event for each
+    distinct failure mode, not collapse them all into a generic
+    ``batch_failed`` entry."""
+
+    @pytest.mark.asyncio
+    async def test_phone_number_pool_exhausted_logs_specific_event(self):
+        """When PhoneNumberPoolExhaustedError propagates from process_batch,
+        the campaign log entry should use event='phone_number_pool_exhausted'
+        with a clear message — not the generic 'batch_failed' bucket."""
+        with (
+            patch("api.tasks.campaign_tasks.campaign_call_dispatcher") as mock_disp,
+            patch("api.tasks.campaign_tasks.db_client") as mock_db,
+            patch(
+                "api.tasks.campaign_tasks.get_campaign_event_publisher"
+            ) as mock_get_pub,
+        ):
+            mock_disp.process_batch = AsyncMock(
+                side_effect=PhoneNumberPoolExhaustedError(organization_id=7)
+            )
+            mock_db.update_campaign = AsyncMock()
+            mock_db.append_campaign_log = AsyncMock()
+            mock_pub = AsyncMock()
+            mock_get_pub.return_value = mock_pub
+
+            with pytest.raises(PhoneNumberPoolExhaustedError):
+                await process_campaign_batch({}, campaign_id=42)
+
+            mock_db.update_campaign.assert_called_once_with(
+                campaign_id=42, state="failed"
+            )
+
+            mock_db.append_campaign_log.assert_called_once()
+            kwargs = mock_db.append_campaign_log.call_args.kwargs
+            assert kwargs["campaign_id"] == 42
+            assert kwargs["event"] == "phone_number_pool_exhausted"
+            assert kwargs["level"] == "error"
+            assert "phone number" in kwargs["message"].lower()
+            assert kwargs["details"]["organization_id"] == 7
+
+    @pytest.mark.asyncio
+    async def test_concurrent_slot_timeout_still_logs_specific_event(self):
+        """Regression guard: the existing ConcurrentSlotAcquisitionError branch
+        should keep logging its specific reason."""
+        with (
+            patch("api.tasks.campaign_tasks.campaign_call_dispatcher") as mock_disp,
+            patch("api.tasks.campaign_tasks.db_client") as mock_db,
+            patch(
+                "api.tasks.campaign_tasks.get_campaign_event_publisher"
+            ) as mock_get_pub,
+        ):
+            mock_disp.process_batch = AsyncMock(
+                side_effect=ConcurrentSlotAcquisitionError(
+                    organization_id=7, campaign_id=42, wait_time=30.0
+                )
+            )
+            mock_db.update_campaign = AsyncMock()
+            mock_db.append_campaign_log = AsyncMock()
+            mock_pub = AsyncMock()
+            mock_get_pub.return_value = mock_pub
+
+            with pytest.raises(ConcurrentSlotAcquisitionError):
+                await process_campaign_batch({}, campaign_id=42)
+
+            mock_db.append_campaign_log.assert_called_once()
+            kwargs = mock_db.append_campaign_log.call_args.kwargs
+            assert kwargs["event"] == "batch_failed"
+            assert kwargs["details"]["reason"] == "concurrent_slot_timeout"

api/tests/test_circuit_breaker.py

@@ -198,7 +198,9 @@ class TestCircuitBreakerReset:
         result = await cb.reset(campaign_id=42)
 
         assert result is True
-        mock_redis.delete.assert_called_once_with("cb_failures:42", "cb_successes:42")
+        mock_redis.delete.assert_called_once_with(
+            "cb_failures:42", "cb_successes:42", "cb_recent_failures:42"
+        )
 
     @pytest.mark.asyncio
     async def test_reset_on_redis_error(self):
@@ -253,6 +255,7 @@ class TestRecordAndEvaluate:
         ):
             mock_db.get_campaign_by_id = AsyncMock(return_value=mock_campaign)
             mock_db.update_campaign = AsyncMock()
+            mock_db.append_campaign_log = AsyncMock()
 
             mock_publisher = AsyncMock()
             mock_get_publisher.return_value = mock_publisher
@@ -352,6 +355,206 @@ class TestRecordAndEvaluate:
             await cb.record_and_evaluate(campaign_id=42, is_failure=True)
 
 
+# =============================================================================
+# Tests for recent-failures tracking (workflow_run_id + reason)
+# =============================================================================
+
+
+class TestCircuitBreakerRecentFailures:
+    """When a call fails, the circuit breaker should remember the workflow_run_id
+    and reason in a capped Redis list, and surface those entries in the campaign
+    log entry written when the breaker trips."""
+
+    @pytest.mark.asyncio
+    async def test_failure_pushes_recent_failure_entry(self):
+        """is_failure=True with run id + reason should push to recent-failures list."""
+        from api.services.campaign.circuit_breaker import CircuitBreaker
+
+        cb = CircuitBreaker()
+
+        mock_campaign = MagicMock()
+        mock_campaign.id = 42
+        mock_campaign.state = "running"
+        mock_campaign.orchestrator_metadata = {}
+
+        with patch("api.services.campaign.circuit_breaker.db_client") as mock_db:
+            mock_db.get_campaign_by_id = AsyncMock(return_value=mock_campaign)
+            mock_db.append_campaign_log = AsyncMock()
+            cb.record_call_outcome = AsyncMock(return_value=(False, None))
+            cb._push_recent_failure = AsyncMock()
+            cb._get_recent_failures = AsyncMock(return_value=[])
+
+            await cb.record_and_evaluate(
+                campaign_id=42,
+                is_failure=True,
+                workflow_run_id=100,
+                reason="failed",
+            )
+
+            cb._push_recent_failure.assert_called_once_with(
+                campaign_id=42, workflow_run_id=100, reason="failed"
+            )
+
+    @pytest.mark.asyncio
+    async def test_success_does_not_push_recent_failure(self):
+        """is_failure=False must not push to the recent-failures list."""
+        from api.services.campaign.circuit_breaker import CircuitBreaker
+
+        cb = CircuitBreaker()
+
+        mock_campaign = MagicMock()
+        mock_campaign.id = 42
+        mock_campaign.state = "running"
+        mock_campaign.orchestrator_metadata = {}
+
+        with patch("api.services.campaign.circuit_breaker.db_client") as mock_db:
+            mock_db.get_campaign_by_id = AsyncMock(return_value=mock_campaign)
+            cb.record_call_outcome = AsyncMock(return_value=(False, None))
+            cb._push_recent_failure = AsyncMock()
+            cb._get_recent_failures = AsyncMock(return_value=[])
+
+            await cb.record_and_evaluate(
+                campaign_id=42,
+                is_failure=False,
+                workflow_run_id=100,
+                reason=None,
+            )
+
+            cb._push_recent_failure.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_trip_log_includes_recent_failures_in_details(self):
+        """When the breaker trips, the campaign log entry's details should include
+        recent_failures fetched from the Redis list."""
+        from api.services.campaign.circuit_breaker import CircuitBreaker
+
+        cb = CircuitBreaker()
+
+        mock_campaign = MagicMock()
+        mock_campaign.id = 42
+        mock_campaign.state = "running"
+        mock_campaign.orchestrator_metadata = {}
+
+        stats = {
+            "failure_rate": 0.6,
+            "failure_count": 6,
+            "success_count": 4,
+            "threshold": 0.5,
+            "window_seconds": 120,
+        }
+
+        recent = [
+            {"workflow_run_id": 100, "reason": "failed", "ts": 1700000010.0},
+            {"workflow_run_id": 99, "reason": "error", "ts": 1700000000.0},
+        ]
+
+        with (
+            patch("api.services.campaign.circuit_breaker.db_client") as mock_db,
+            patch(
+                "api.services.campaign.circuit_breaker.get_campaign_event_publisher"
+            ) as mock_get_publisher,
+        ):
+            mock_db.get_campaign_by_id = AsyncMock(return_value=mock_campaign)
+            mock_db.update_campaign = AsyncMock()
+            mock_db.append_campaign_log = AsyncMock()
+
+            mock_publisher = AsyncMock()
+            mock_get_publisher.return_value = mock_publisher
+
+            cb.record_call_outcome = AsyncMock(return_value=(True, stats))
+            cb._push_recent_failure = AsyncMock()
+            cb._get_recent_failures = AsyncMock(return_value=recent)
+
+            await cb.record_and_evaluate(
+                campaign_id=42,
+                is_failure=True,
+                workflow_run_id=100,
+                reason="failed",
+            )
+
+            mock_db.append_campaign_log.assert_called_once()
+            kwargs = mock_db.append_campaign_log.call_args.kwargs
+            assert kwargs["campaign_id"] == 42
+            assert kwargs["event"] == "circuit_breaker_tripped"
+            assert kwargs["details"]["recent_failures"] == recent
+
+    @pytest.mark.asyncio
+    async def test_push_recent_failure_uses_lpush_and_ltrim(self):
+        """_push_recent_failure should LPUSH a JSON entry and LTRIM the list
+        to keep only the most recent N (default 20)."""
+        import json
+
+        from api.services.campaign.circuit_breaker import CircuitBreaker
+
+        cb = CircuitBreaker()
+
+        mock_redis = AsyncMock()
+        mock_redis.lpush = AsyncMock(return_value=1)
+        mock_redis.ltrim = AsyncMock(return_value=True)
+        mock_redis.expire = AsyncMock(return_value=True)
+        cb.redis_client = mock_redis
+
+        await cb._push_recent_failure(
+            campaign_id=42, workflow_run_id=100, reason="failed"
+        )
+
+        # Verify the key used
+        mock_redis.lpush.assert_called_once()
+        push_args = mock_redis.lpush.call_args.args
+        assert push_args[0] == "cb_recent_failures:42"
+
+        # Verify the payload includes the run id + reason
+        entry = json.loads(push_args[1])
+        assert entry["workflow_run_id"] == 100
+        assert entry["reason"] == "failed"
+        assert "ts" in entry
+
+        # Verify the cap (LTRIM 0 19 keeps 20 entries)
+        mock_redis.ltrim.assert_called_once_with("cb_recent_failures:42", 0, 19)
+
+    @pytest.mark.asyncio
+    async def test_get_recent_failures_decodes_lrange(self):
+        """_get_recent_failures should LRANGE the list and JSON-decode entries."""
+        import json
+
+        from api.services.campaign.circuit_breaker import CircuitBreaker
+
+        cb = CircuitBreaker()
+
+        mock_redis = AsyncMock()
+        entries = [
+            json.dumps({"workflow_run_id": 100, "reason": "failed", "ts": 1.0}),
+            json.dumps({"workflow_run_id": 99, "reason": "error", "ts": 0.5}),
+        ]
+        mock_redis.lrange = AsyncMock(return_value=entries)
+        cb.redis_client = mock_redis
+
+        result = await cb._get_recent_failures(campaign_id=42)
+
+        mock_redis.lrange.assert_called_once_with("cb_recent_failures:42", 0, -1)
+        assert result == [
+            {"workflow_run_id": 100, "reason": "failed", "ts": 1.0},
+            {"workflow_run_id": 99, "reason": "error", "ts": 0.5},
+        ]
+
+    @pytest.mark.asyncio
+    async def test_reset_clears_recent_failures_key(self):
+        """reset() must also delete cb_recent_failures:{campaign_id}."""
+        from api.services.campaign.circuit_breaker import CircuitBreaker
+
+        cb = CircuitBreaker()
+
+        mock_redis = AsyncMock()
+        mock_redis.delete = AsyncMock(return_value=3)
+        cb.redis_client = mock_redis
+
+        await cb.reset(campaign_id=42)
+
+        mock_redis.delete.assert_called_once_with(
+            "cb_failures:42", "cb_successes:42", "cb_recent_failures:42"
+        )
+
+
 # =============================================================================
 # Integration tests: _process_status_update calls circuit_breaker
 # =============================================================================
@@ -405,7 +608,12 @@ class TestProcessStatusUpdateCircuitBreaker:
 
             await _process_status_update(100, status)
 
-            mock_cb.record_and_evaluate.assert_called_once_with(42, is_failure=True)
+            mock_cb.record_and_evaluate.assert_called_once_with(
+                42,
+                is_failure=True,
+                workflow_run_id=100,
+                reason="failed",
+            )
 
     @pytest.mark.asyncio
     async def test_success_status_calls_record_and_evaluate(self):

api/tests/test_custom_tools.py

@@ -12,6 +12,12 @@ from typing import Any, Dict
 from unittest.mock import AsyncMock, Mock, patch
 
 import pytest
+
+from api.services.workflow.pipecat_engine_custom_tools import get_function_schema
+from api.services.workflow.tools.custom_tool import (
+    execute_http_tool,
+    tool_to_function_schema,
+)
 from pipecat.adapters.schemas.tools_schema import ToolsSchema
 from pipecat.frames.frames import (
     FunctionCallInProgressFrame,
@@ -25,12 +31,6 @@ from pipecat.frames.frames import (
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.processors.aggregators.llm_context import LLMContext
 from pipecat.services.llm_service import FunctionCallParams
-
-from api.services.workflow.pipecat_engine_custom_tools import get_function_schema
-from api.services.workflow.tools.custom_tool import (
-    execute_http_tool,
-    tool_to_function_schema,
-)
 from pipecat.tests import MockLLMService, run_test
 
 
@@ -720,11 +720,10 @@ class TestCustomToolManagerUnit:
     @pytest.mark.asyncio
     async def test_get_tool_schemas_returns_correct_format(self):
         """Test that get_tool_schemas returns FunctionSchema objects."""
-        from pipecat.adapters.schemas.function_schema import FunctionSchema
-
         # Create a mock engine
         from api.services.workflow.pipecat_engine import PipecatEngine
         from api.services.workflow.pipecat_engine_custom_tools import CustomToolManager
+        from pipecat.adapters.schemas.function_schema import FunctionSchema
 
         mock_engine = Mock()
         mock_engine._workflow_run_id = 1

api/tests/test_custom_tools_context_integration.py

@@ -9,15 +9,15 @@ This module tests the full flow of:
 from unittest.mock import AsyncMock, patch
 
 import pytest
-from pipecat.adapters.schemas.function_schema import FunctionSchema
-from pipecat.adapters.schemas.tools_schema import ToolsSchema
-from pipecat.processors.aggregators.llm_context import LLMContext
 
 from api.services.workflow.pipecat_engine_custom_tools import (
     CustomToolManager,
     get_function_schema,
 )
 from api.tests.conftest import MockToolModel
+from pipecat.adapters.schemas.function_schema import FunctionSchema
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+from pipecat.processors.aggregators.llm_context import LLMContext
 
 
 def _update_llm_context(context, system_message, functions):

api/tests/test_pipecat_engine_context_update.py

@@ -18,6 +18,14 @@ from typing import List
 from unittest.mock import AsyncMock, patch
 
 import pytest
+
+from api.services.workflow.pipecat_engine import PipecatEngine
+from api.services.workflow.workflow import WorkflowGraph
+from api.tests.conftest import (
+    AGENT_SYSTEM_PROMPT,
+    END_CALL_SYSTEM_PROMPT,
+    START_CALL_SYSTEM_PROMPT,
+)
 from pipecat.frames.frames import LLMContextFrame
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
@@ -27,21 +35,13 @@ from pipecat.processors.aggregators.llm_response_universal import (
     LLMAssistantAggregatorParams,
     LLMContextAggregatorPair,
 )
-from pipecat.tests.mock_transport import MockTransport
-from pipecat.transports.base_transport import TransportParams
-
-from api.services.workflow.pipecat_engine import PipecatEngine
-from api.services.workflow.workflow import WorkflowGraph
-from api.tests.conftest import (
-    AGENT_SYSTEM_PROMPT,
-    END_CALL_SYSTEM_PROMPT,
-    START_CALL_SYSTEM_PROMPT,
-)
 from pipecat.tests import (
     ContextCapturingMockLLM,
     MockLLMService,
     MockTTSService,
 )
+from pipecat.tests.mock_transport import MockTransport
+from pipecat.transports.base_transport import TransportParams
 
 
 async def run_pipeline_and_capture_context(

api/tests/test_pipecat_engine_end_call.py

@@ -23,23 +23,6 @@ from typing import Any, Dict, List
 from unittest.mock import AsyncMock, patch
 
 import pytest
-from pipecat.frames.frames import Frame, LLMContextFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_context import LLMContext
-from pipecat.processors.aggregators.llm_response_universal import (
-    LLMAssistantAggregatorParams,
-    LLMContextAggregatorPair,
-    LLMUserAggregatorParams,
-)
-from pipecat.tests.mock_transport import MockTransport
-from pipecat.transports.base_transport import TransportParams
-from pipecat.turns.user_mute import (
-    CallbackUserMuteStrategy,
-    MuteUntilFirstBotCompleteUserMuteStrategy,
-)
-from pipecat.utils.enums import EndTaskReason
 
 from api.enums import ToolCategory
 from api.services.workflow.dto import (
@@ -59,7 +42,24 @@ from api.services.workflow.pipecat_engine_variable_extractor import (
 )
 from api.services.workflow.workflow import WorkflowGraph
 from api.tests.conftest import END_CALL_SYSTEM_PROMPT, START_CALL_SYSTEM_PROMPT
+from pipecat.frames.frames import Frame, LLMContextFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.llm_context import LLMContext
+from pipecat.processors.aggregators.llm_response_universal import (
+    LLMAssistantAggregatorParams,
+    LLMContextAggregatorPair,
+    LLMUserAggregatorParams,
+)
 from pipecat.tests import MockLLMService, MockTTSService
+from pipecat.tests.mock_transport import MockTransport
+from pipecat.transports.base_transport import TransportParams
+from pipecat.turns.user_mute import (
+    CallbackUserMuteStrategy,
+    MuteUntilFirstBotCompleteUserMuteStrategy,
+)
+from pipecat.utils.enums import EndTaskReason
 
 
 class EndCallTestHelper:

api/tests/test_pipecat_engine_node_switch_with_user_speech.py

@@ -15,6 +15,9 @@ import asyncio
 from unittest.mock import AsyncMock, patch
 
 import pytest
+
+from api.services.workflow.pipecat_engine import PipecatEngine
+from api.services.workflow.workflow import WorkflowGraph
 from pipecat.frames.frames import (
     Frame,
     FunctionCallResultFrame,
@@ -33,6 +36,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
     LLMUserAggregatorParams,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.tests import MockLLMService, MockTTSService
 from pipecat.tests.mock_transport import MockTransport
 from pipecat.transports.base_transport import TransportParams
 from pipecat.turns.user_mute import (
@@ -48,10 +52,6 @@ from pipecat.turns.user_stop import (
 from pipecat.turns.user_turn_strategies import UserTurnStrategies
 from pipecat.utils.time import time_now_iso8601
 
-from api.services.workflow.pipecat_engine import PipecatEngine
-from api.services.workflow.workflow import WorkflowGraph
-from pipecat.tests import MockLLMService, MockTTSService
-
 
 class UserSpeechInjector(FrameProcessor):
     """Processor that injects user speaking frames on FunctionCallResultFrame.

api/tests/test_pipecat_engine_tool_calls.py

@@ -9,6 +9,10 @@ from typing import Any, Dict, List
 from unittest.mock import AsyncMock, patch
 
 import pytest
+
+from api.services.workflow.pipecat_engine import PipecatEngine
+from api.services.workflow.workflow import WorkflowGraph
+from api.tests.conftest import END_CALL_SYSTEM_PROMPT
 from pipecat.frames.frames import LLMContextFrame
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
@@ -18,14 +22,10 @@ from pipecat.processors.aggregators.llm_response_universal import (
     LLMAssistantAggregatorParams,
     LLMContextAggregatorPair,
 )
+from pipecat.tests import MockLLMService, MockTTSService
 from pipecat.tests.mock_transport import MockTransport
 from pipecat.transports.base_transport import TransportParams
 
-from api.services.workflow.pipecat_engine import PipecatEngine
-from api.services.workflow.workflow import WorkflowGraph
-from api.tests.conftest import END_CALL_SYSTEM_PROMPT
-from pipecat.tests import MockLLMService, MockTTSService
-
 
 async def run_pipeline_with_tool_calls(
     workflow: WorkflowGraph,

api/tests/test_pipecat_engine_transition_mute.py

@@ -13,6 +13,12 @@ import asyncio
 from unittest.mock import AsyncMock, patch
 
 import pytest
+
+from api.services.workflow.pipecat_engine import PipecatEngine
+from api.services.workflow.pipecat_engine_variable_extractor import (
+    VariableExtractionManager,
+)
+from api.services.workflow.workflow import WorkflowGraph
 from pipecat.frames.frames import LLMContextFrame
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
@@ -23,6 +29,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
     LLMContextAggregatorPair,
     LLMUserAggregatorParams,
 )
+from pipecat.tests import MockLLMService, MockTTSService
 from pipecat.tests.mock_transport import MockTransport
 from pipecat.transports.base_transport import TransportParams
 from pipecat.turns.user_mute import (
@@ -31,13 +38,6 @@ from pipecat.turns.user_mute import (
     MuteUntilFirstBotCompleteUserMuteStrategy,
 )
 
-from api.services.workflow.pipecat_engine import PipecatEngine
-from api.services.workflow.pipecat_engine_variable_extractor import (
-    VariableExtractionManager,
-)
-from api.services.workflow.workflow import WorkflowGraph
-from pipecat.tests import MockLLMService, MockTTSService
-
 
 async def _build_engine_and_pipeline(
     workflow: WorkflowGraph,

api/tests/test_pipecat_engine_variable_extraction.py

@@ -16,6 +16,12 @@ from typing import Any, Dict, List
 from unittest.mock import AsyncMock, patch
 
 import pytest
+
+from api.services.workflow.pipecat_engine import PipecatEngine
+from api.services.workflow.pipecat_engine_variable_extractor import (
+    VariableExtractionManager,
+)
+from api.services.workflow.workflow import WorkflowGraph
 from pipecat.frames.frames import LLMContextFrame
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
@@ -25,16 +31,10 @@ from pipecat.processors.aggregators.llm_response_universal import (
     LLMAssistantAggregatorParams,
     LLMContextAggregatorPair,
 )
+from pipecat.tests import MockLLMService, MockTTSService
 from pipecat.tests.mock_transport import MockTransport
 from pipecat.transports.base_transport import TransportParams
 
-from api.services.workflow.pipecat_engine import PipecatEngine
-from api.services.workflow.pipecat_engine_variable_extractor import (
-    VariableExtractionManager,
-)
-from api.services.workflow.workflow import WorkflowGraph
-from pipecat.tests import MockLLMService, MockTTSService
-
 
 class TestVariableExtractionDuringTransitions:
     """Test that variable extraction is triggered for the correct node during transitions."""

api/tests/test_pipeline_cancellation.py

@@ -2,6 +2,7 @@ import asyncio
 
 import pytest
 from loguru import logger
+
 from pipecat.frames.frames import (
     EndTaskFrame,
     Frame,

api/tests/test_recording_router_processor.py

@@ -12,14 +12,6 @@ and inspect what arrives downstream.
 from typing import Optional
 
 import pytest
-from pipecat.frames.frames import (
-    LLMFullResponseEndFrame,
-    LLMTextFrame,
-    TTSAudioRawFrame,
-    TTSStartedFrame,
-    TTSStoppedFrame,
-    TTSTextFrame,
-)
 
 from api.services.pipecat.recording_audio_cache import RecordingAudio
 from api.services.pipecat.recording_router_processor import (
@@ -29,6 +21,14 @@ from api.services.workflow.pipecat_engine_context_composer import (
     RECORDING_MARKER,
     TTS_MARKER,
 )
+from pipecat.frames.frames import (
+    LLMFullResponseEndFrame,
+    LLMTextFrame,
+    TTSAudioRawFrame,
+    TTSStartedFrame,
+    TTSStoppedFrame,
+    TTSTextFrame,
+)
 from pipecat.tests import run_test
 
 # ---------------------------------------------------------------------------

api/tests/test_text_and_audio_playback.py

@@ -11,6 +11,21 @@ from typing import Any, Dict, List
 from unittest.mock import AsyncMock, Mock, patch
 
 import pytest
+
+from api.services.pipecat.recording_audio_cache import RecordingAudio
+from api.services.workflow.dto import (
+    EdgeDataDTO,
+    EndCallNodeData,
+    EndCallRFNode,
+    Position,
+    ReactFlowDTO,
+    RFEdgeDTO,
+    StartCallNodeData,
+    StartCallRFNode,
+)
+from api.services.workflow.pipecat_engine import PipecatEngine
+from api.services.workflow.pipecat_engine_custom_tools import CustomToolManager
+from api.services.workflow.workflow import WorkflowGraph
 from pipecat.frames.frames import (
     Frame,
     LLMContextFrame,
@@ -27,25 +42,10 @@ from pipecat.processors.aggregators.llm_response_universal import (
     LLMAssistantAggregatorParams,
     LLMContextAggregatorPair,
 )
+from pipecat.tests import MockLLMService, MockTTSService
 from pipecat.tests.mock_transport import MockTransport
 from pipecat.transports.base_transport import TransportParams
 
-from api.services.pipecat.recording_audio_cache import RecordingAudio
-from api.services.workflow.dto import (
-    EdgeDataDTO,
-    EndCallNodeData,
-    EndCallRFNode,
-    Position,
-    ReactFlowDTO,
-    RFEdgeDTO,
-    StartCallNodeData,
-    StartCallRFNode,
-)
-from api.services.workflow.pipecat_engine import PipecatEngine
-from api.services.workflow.pipecat_engine_custom_tools import CustomToolManager
-from api.services.workflow.workflow import WorkflowGraph
-from pipecat.tests import MockLLMService, MockTTSService
-
 # ─── Constants ──────────────────────────────────────────────────
 
 START_PROMPT = "Start Call System Prompt"

api/tests/test_tts_endframe_with_audio_write_failure.py

@@ -32,6 +32,12 @@ import asyncio
 from unittest.mock import AsyncMock, patch
 
 import pytest
+
+from api.services.workflow.pipecat_engine import PipecatEngine
+from api.services.workflow.pipecat_engine_variable_extractor import (
+    VariableExtractionManager,
+)
+from api.services.workflow.workflow import WorkflowGraph
 from pipecat.frames.frames import LLMContextFrame
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
@@ -42,6 +48,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
     LLMContextAggregatorPair,
     LLMUserAggregatorParams,
 )
+from pipecat.tests import MockLLMService, MockTTSService
 from pipecat.tests.mock_transport import MockTransport
 from pipecat.transports.base_transport import TransportParams
 from pipecat.turns.user_mute import (
@@ -50,13 +57,6 @@ from pipecat.turns.user_mute import (
 )
 from pipecat.utils.enums import EndTaskReason
 
-from api.services.workflow.pipecat_engine import PipecatEngine
-from api.services.workflow.pipecat_engine_variable_extractor import (
-    VariableExtractionManager,
-)
-from api.services.workflow.workflow import WorkflowGraph
-from pipecat.tests import MockLLMService, MockTTSService
-
 
 async def create_test_pipeline_with_failing_transport(
     workflow: WorkflowGraph,

api/tests/test_unregistered_function_call.py

@@ -1,6 +1,7 @@
 """Tests for LLM behavior when calling an unregistered function."""
 
 import pytest
+
 from pipecat.frames.frames import (
     FunctionCallInProgressFrame,
     FunctionCallResultFrame,
@@ -12,7 +13,6 @@ from pipecat.frames.frames import (
 )
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.processors.aggregators.llm_context import LLMContext
-
 from pipecat.tests import MockLLMService, run_test
 
 

api/tests/test_user_idle_handler.py

@@ -13,6 +13,9 @@ import asyncio
 from unittest.mock import AsyncMock, patch
 
 import pytest
+
+from api.services.workflow.pipecat_engine import PipecatEngine
+from api.services.workflow.workflow import WorkflowGraph
 from pipecat.frames.frames import (
     BotStoppedSpeakingFrame,
     Frame,
@@ -32,6 +35,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
     LLMUserAggregatorParams,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.tests import MockLLMService, MockTTSService
 from pipecat.tests.mock_transport import MockTransport
 from pipecat.transports.base_transport import TransportParams
 from pipecat.turns.user_mute import (
@@ -43,10 +47,6 @@ from pipecat.turns.user_stop import ExternalUserTurnStopStrategy
 from pipecat.turns.user_turn_strategies import UserTurnStrategies
 from pipecat.utils.time import time_now_iso8601
 
-from api.services.workflow.pipecat_engine import PipecatEngine
-from api.services.workflow.workflow import WorkflowGraph
-from pipecat.tests import MockLLMService, MockTTSService
-
 
 class UserSpeechInjector(FrameProcessor):
     """Processor that injects user speaking frames after the bot finishes speaking.

api/tests/test_user_muting_during_bot_speech.py

@@ -15,6 +15,12 @@ from typing import List
 from unittest.mock import AsyncMock, patch
 
 import pytest
+
+from api.services.workflow.pipecat_engine import PipecatEngine
+from api.services.workflow.pipecat_engine_variable_extractor import (
+    VariableExtractionManager,
+)
+from api.services.workflow.workflow import WorkflowGraph
 from pipecat.frames.frames import (
     BotStartedSpeakingFrame,
     BotStoppedSpeakingFrame,
@@ -35,6 +41,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
     LLMUserAggregatorParams,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.tests import MockLLMService, MockTTSService
 from pipecat.tests.mock_transport import MockTransport
 from pipecat.transports.base_transport import TransportParams
 from pipecat.turns.user_mute import (
@@ -44,13 +51,6 @@ from pipecat.turns.user_mute import (
 from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies
 from pipecat.utils.time import time_now_iso8601
 
-from api.services.workflow.pipecat_engine import PipecatEngine
-from api.services.workflow.pipecat_engine_variable_extractor import (
-    VariableExtractionManager,
-)
-from api.services.workflow.workflow import WorkflowGraph
-from pipecat.tests import MockLLMService, MockTTSService
-
 
 class BotSpeakingObserverProcessor(FrameProcessor):
     """Observer that records mute status when bot speaking events flow upstream.

api/tests/test_voicemail_detector.py

@@ -8,6 +8,7 @@ incoming speech as CONVERSATION or VOICEMAIL and how the main LLM responds.
 import asyncio
 
 import pytest
+
 from pipecat.extensions.voicemail.voicemail_detector import VoicemailDetector
 from pipecat.frames.frames import (
     EndTaskFrame,
@@ -26,6 +27,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
     LLMUserAggregatorParams,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.tests import MockLLMService
 from pipecat.turns.user_start import (
     TranscriptionUserTurnStartStrategy,
     VADUserTurnStartStrategy,
@@ -36,8 +38,6 @@ from pipecat.turns.user_stop import (
 from pipecat.turns.user_turn_strategies import UserTurnStrategies
 from pipecat.utils.time import time_now_iso8601
 
-from pipecat.tests import MockLLMService
-
 
 class FrameInjector(FrameProcessor):
     """Simple processor that can inject frames into the pipeline."""