feat: add logs in campaigns for failure or pausing (#265)

* feat: add logs in campaigns on failure

* chore: bump pipecat

* chore: update format.sh

* chore: fix github workflow

* fix: fix formatting errors

api/tasks/campaign_tasks.py

@@ -8,7 +8,10 @@ from api.services.campaign.campaign_call_dispatcher import campaign_call_dispatc
 from api.services.campaign.campaign_event_publisher import (
     get_campaign_event_publisher,
 )
-from api.services.campaign.errors import ConcurrentSlotAcquisitionError
+from api.services.campaign.errors import (
+    ConcurrentSlotAcquisitionError,
+    PhoneNumberPoolExhaustedError,
+)
 from api.services.campaign.source_sync_factory import get_sync_service
 
 
@@ -80,6 +83,13 @@ async def sync_campaign_source(ctx: Dict, campaign_id: int) -> None:
             source_sync_status="failed",
             source_sync_error=str(e),
         )
+        await db_client.append_campaign_log(
+            campaign_id=campaign_id,
+            level="error",
+            event="source_sync_failed",
+            message=f"Source sync failed: {e}",
+            details={"error": str(e)},
+        )
         raise
 
 
@@ -137,6 +147,39 @@ async def process_campaign_batch(
 
         # Update campaign state to failed
         await db_client.update_campaign(campaign_id=campaign_id, state="failed")
+        await db_client.append_campaign_log(
+            campaign_id=campaign_id,
+            level="error",
+            event="batch_failed",
+            message=f"Concurrent slot acquisition timeout: {e}",
+            details={"error": str(e), "reason": "concurrent_slot_timeout"},
+        )
+        raise
+
+    except PhoneNumberPoolExhaustedError as e:
+        logger.warning(f"Phone number pool exhausted for campaign {campaign_id}: {e}")
+
+        publisher = await get_campaign_event_publisher()
+        await publisher.publish_batch_failed(
+            campaign_id=campaign_id,
+            error=f"Phone number pool exhausted: {e}",
+            processed_count=0,
+        )
+
+        await db_client.update_campaign(campaign_id=campaign_id, state="failed")
+        await db_client.append_campaign_log(
+            campaign_id=campaign_id,
+            level="error",
+            event="phone_number_pool_exhausted",
+            message=(
+                f"Phone number pool exhausted for org {e.organization_id}: "
+                "no free from_number available to dispatch outbound calls"
+            ),
+            details={
+                "error": str(e),
+                "organization_id": e.organization_id,
+            },
+        )
         raise
 
     except Exception as e:
@@ -152,4 +195,11 @@ async def process_campaign_batch(
 
         # Update campaign state to failed
         await db_client.update_campaign(campaign_id=campaign_id, state="failed")
+        await db_client.append_campaign_log(
+            campaign_id=campaign_id,
+            level="error",
+            event="batch_failed",
+            message=f"Batch processing failed: {e}",
+            details={"error": str(e)},
+        )
         raise

api/tasks/run_integrations.py

@@ -5,8 +5,6 @@ from typing import Any, Dict, Optional
 
 import httpx
 from loguru import logger
-from pipecat.utils.enums import EndTaskReason
-from pipecat.utils.run_context import set_current_org_id, set_current_run_id
 from pydantic import ValidationError
 
 from api.constants import BACKEND_API_ENDPOINT
@@ -23,6 +21,8 @@ from api.services.workflow.dto import (
 from api.services.workflow.qa import run_per_node_qa_analysis
 from api.utils.credential_auth import build_auth_header
 from api.utils.template_renderer import render_template
+from pipecat.utils.enums import EndTaskReason
+from pipecat.utils.run_context import set_current_org_id, set_current_run_id
 
 
 def _should_skip_qa(

api/tasks/s3_upload.py

@@ -2,12 +2,12 @@ import os
 from typing import Optional
 
 from loguru import logger
-from pipecat.utils.run_context import set_current_run_id
 
 from api.db import db_client
 from api.services.pricing.workflow_run_cost import calculate_workflow_run_cost
 from api.services.storage import get_current_storage_backend, storage_fs
 from api.tasks.run_integrations import run_integrations_post_workflow_run
+from pipecat.utils.run_context import set_current_run_id
 
 
 async def upload_voicemail_audio_to_s3(