apunkt/dograh
1
0
Fork 0
mirror of https://github.com/dograh-hq/dograh.git synced 2026-07-25 12:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/main' into feat/provisional-vad-v1

Browse source 
# Conflicts:
#	pipecat
This commit is contained in:
Abhishek Kumar 2026-06-29 17:28:42 +05:30
commit 5ea80506db
38 changed files with 1401 additions and 153 deletions
Download patch file Download diff file Expand all files Collapse all files

24
api/db/telephony_configuration_client.py
View file

@ -103,6 +103,30 @@ class TelephonyConfigurationClient(BaseDBClient):
)
return int(result.scalar() or 0)
async def count_vonage_configs_missing_signature_secret(
self, organization_id: int
) -> int:
"""Count Vonage configs in this org with no signature_secret."""
async with self.async_session() as session:
result = await session.execute(
select(func.count(TelephonyConfigurationModel.id)).where(
TelephonyConfigurationModel.organization_id == organization_id,
TelephonyConfigurationModel.provider == "vonage",
(
TelephonyConfigurationModel.credentials.op("->>")(
"signature_secret"
).is_(None)
)
| (
TelephonyConfigurationModel.credentials.op("->>")(
"signature_secret"
)
== ""
),
)
)
return int(result.scalar() or 0)
async def list_all_telephony_configurations_by_provider(
self, provider: str
) -> List[TelephonyConfigurationModel]:

8
api/routes/organization.py
View file

@ -145,6 +145,7 @@ class TelephonyConfigWarningsResponse(BaseModel):
"""
telnyx_missing_webhook_public_key_count: int
vonage_missing_signature_secret_count: int
@router.get("/context", response_model=OrganizationContextResponse)
@ -200,8 +201,7 @@ async def get_telephony_providers_metadata(user: UserModel = Depends(get_user)):
async def get_telephony_config_warnings(user: UserModel = Depends(get_user)):
"""Return aggregated warning counts for the current org's telephony configs.
Today this surfaces only Telnyx configs missing ``webhook_public_key``;
additional warning types should be added as new fields on the response.
Surfaces provider configs missing webhook-verification credentials.
"""
if not user.selected_organization_id:
raise HTTPException(status_code=400, detail="No organization selected")
@ -209,8 +209,12 @@ async def get_telephony_config_warnings(user: UserModel = Depends(get_user)):
telnyx_missing = await db_client.count_telnyx_configs_missing_webhook_public_key(
user.selected_organization_id
)
vonage_missing = await db_client.count_vonage_configs_missing_signature_secret(
user.selected_organization_id
)
return TelephonyConfigWarningsResponse(
telnyx_missing_webhook_public_key_count=telnyx_missing,
vonage_missing_signature_secret_count=vonage_missing,
)

4
api/routes/telephony.py
View file

@ -684,7 +684,7 @@ async def handle_inbound_run(request: Request):
logger.error("Unable to detect provider for /inbound/run webhook")
return generic_hangup_response()
normalized_data = normalize_webhook_data(provider_class, webhook_data)
normalized_data = normalize_webhook_data(provider_class, webhook_data, headers)
logger.info(
f"/inbound/run normalized data — provider={normalized_data.provider} "
f"to={normalized_data.to_number} from={normalized_data.from_number}"
@ -871,7 +871,7 @@ async def handle_inbound_telephony(
logger.error("Unable to detect provider for webhook")
return generic_hangup_response()
normalized_data = normalize_webhook_data(provider_class, webhook_data)
normalized_data = normalize_webhook_data(provider_class, webhook_data, headers)
logger.info(f"Inbound call - Provider: {normalized_data.provider}")
logger.info(f"Normalized data: {normalized_data}")

70
api/services/pipecat/realtime/azure_realtime.py
View file

@ -11,6 +11,7 @@ from typing import Any
from loguru import logger
from api.services.pipecat.realtime.static_greeting import format_static_greeting_prompt
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
@ -49,6 +50,7 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
self._handled_initial_context: bool = False
self._bot_is_speaking: bool = False
self._deferred_function_calls: list[FunctionCallFromLLM] = []
self._pending_initial_greeting_text: str | None = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, UserMuteStartedFrame):
@ -61,7 +63,11 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
return
if isinstance(frame, TTSSpeakFrame):
if not self._handled_initial_context:
await self._handle_context(self._context)
greeting_text = frame.text.strip() if frame.text else ""
if greeting_text:
await self._handle_initial_greeting(self._context, greeting_text)
else:
await self._handle_context(self._context)
else:
logger.warning(
f"{self}: TTSSpeakFrame after initial context already handled — "
@ -118,6 +124,57 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
self._context = context
await self._process_completed_function_calls(send_new_results=True)
async def _handle_initial_greeting(self, context: LLMContext, greeting_text: str):
if context is None:
logger.warning(
f"{self}: received initial greeting trigger before context was set"
)
return
self._handled_initial_context = True
self._context = context
await self._create_initial_greeting_response(greeting_text)
async def _create_initial_greeting_response(self, greeting_text: str):
if self._disconnecting:
return
if not self._api_session_ready:
self._pending_initial_greeting_text = greeting_text
self._run_llm_when_api_session_ready = True
return
self._pending_initial_greeting_text = None
await self._ensure_conversation_setup()
await self._send_manual_response_create(
instructions=format_static_greeting_prompt(greeting_text),
tool_choice="none",
)
async def _ensure_conversation_setup(self):
if not self._llm_needs_conversation_setup:
return
adapter = self.get_llm_adapter()
llm_invocation_params = adapter.get_llm_invocation_params(self._context)
for item in llm_invocation_params["messages"]:
evt = events.ConversationItemCreateEvent(item=item)
self._messages_added_manually[evt.item.id] = True
await self.send_client_event(evt)
await self._send_session_update()
self._llm_needs_conversation_setup = False
async def _handle_evt_session_updated(self, evt):
self._api_session_ready = True
if self._pending_initial_greeting_text is not None:
greeting_text = self._pending_initial_greeting_text
self._run_llm_when_api_session_ready = False
await self._create_initial_greeting_response(greeting_text)
elif self._run_llm_when_api_session_ready:
self._run_llm_when_api_session_ready = False
await self._create_response()
async def _send_user_audio(self, frame):
if self._user_is_muted:
return
@ -171,14 +228,21 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
return "\n".join(parts) if parts else None
return None
async def _send_manual_response_create(self):
async def _send_manual_response_create(
self,
*,
instructions: str | None = None,
tool_choice: str | None = None,
):
await self.push_frame(LLMFullResponseStartFrame())
await self.start_processing_metrics()
await self.start_ttfb_metrics()
await self.send_client_event(
events.ResponseCreateEvent(
response=events.ResponseProperties(
output_modalities=self._get_enabled_modalities()
output_modalities=self._get_enabled_modalities(),
instructions=instructions,
tool_choice=tool_choice,
)
)
)

66
api/services/pipecat/realtime/gemini_live.py
View file

@ -20,11 +20,13 @@ Layers Dograh engine integration quirks onto upstream-pristine
from typing import Any
from google.genai.types import Content, Part
from loguru import logger
from api.services.pipecat.gemini_json_schema_adapter import (
DograhGeminiJSONSchemaAdapter,
)
from api.services.pipecat.realtime.static_greeting import format_static_greeting_prompt
from pipecat.frames.frames import (
BotStoppedSpeakingFrame,
Frame,
@ -63,6 +65,9 @@ class DograhGeminiLiveLLMService(GeminiLiveLLMService):
# Function calls emitted by Gemini mid-bot-turn are deferred here and
# invoked when the turn ends, so they don't race the turn's audio.
self._pending_function_calls: list[FunctionCallFromLLM] = []
# Text greeting captured from the first TTSSpeakFrame while the Gemini
# session is still connecting.
self._pending_initial_greeting_text: str | None = None
# ------------------------------------------------------------------
# Hooks from upstream GeminiLiveLLMService
@ -142,10 +147,15 @@ class DograhGeminiLiveLLMService(GeminiLiveLLMService):
if isinstance(frame, TTSSpeakFrame):
# Greeting trigger: the engine queues a TTSSpeakFrame to start the
# bot's first turn after node setup. Gemini Live renders its own
# audio, so we don't pass the frame through — we re-enter
# _handle_context to kick off the initial response.
# audio, so we don't pass the frame through. For configured static
# text greetings, ask Gemini to say the exact greeting; otherwise
# re-enter _handle_context to kick off the normal initial response.
if not self._handled_initial_context:
await self._handle_context(self._context)
greeting_text = frame.text.strip() if frame.text else ""
if greeting_text:
await self._handle_initial_greeting(self._context, greeting_text)
else:
await self._handle_context(self._context)
else:
logger.warning(
f"{self}: TTSSpeakFrame after initial context already "
@ -183,6 +193,49 @@ class DograhGeminiLiveLLMService(GeminiLiveLLMService):
self._context = context
await self._process_completed_function_calls(send_new_results=True)
async def _handle_initial_greeting(self, context: LLMContext, greeting_text: str):
"""Trigger the first Gemini turn with an exact static text greeting."""
if context is None:
logger.warning(
f"{self}: received initial greeting trigger before context was set"
)
return
self._handled_initial_context = True
self._context = context
await self._create_initial_greeting_response(greeting_text)
async def _create_initial_greeting_response(self, greeting_text: str):
"""Ask Gemini Live to speak the configured greeting exactly once."""
if self._disconnecting:
return
if not self._session:
self._pending_initial_greeting_text = greeting_text
self._run_llm_when_session_ready = True
return
self._pending_initial_greeting_text = None
prompt = format_static_greeting_prompt(greeting_text)
turn = Content(role="user", parts=[Part(text=prompt)])
logger.debug("Creating Gemini Live initial response from static greeting")
await self.start_ttfb_metrics()
try:
await self._session.send_client_content(
turns=[turn],
turn_complete=True,
)
# Gemini 3.x also needs a realtime-input nudge to begin inference.
if self._is_gemini_3:
await self._session.send_realtime_input(text=" ")
except Exception as e:
await self._handle_send_error(e)
self._ready_for_realtime_input = True
# ------------------------------------------------------------------
# Session lifecycle: drop upstream's automatic reconnect-seed and
# initial-context-seed paths. The TTSSpeakFrame trigger and the
@ -201,7 +254,12 @@ class DograhGeminiLiveLLMService(GeminiLiveLLMService):
# Context arrived before session was ready — fulfil the queued
# initial response now.
self._run_llm_when_session_ready = False
await self._create_initial_response()
if self._pending_initial_greeting_text is not None:
await self._create_initial_greeting_response(
self._pending_initial_greeting_text
)
else:
await self._create_initial_response()
await self._drain_pending_tool_results()
# Otherwise: no automatic seed. Reconnect after a session-resumption
# update relies on the server-side restored state; reconnects without

69
api/services/pipecat/realtime/grok_realtime.py
View file

@ -22,6 +22,7 @@ from typing import Any
from loguru import logger
from api.services.pipecat.realtime.static_greeting import format_static_greeting_prompt
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
@ -50,6 +51,7 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
self._handled_initial_context: bool = False
self._bot_is_speaking: bool = False
self._deferred_function_calls: list[FunctionCallFromLLM] = []
self._pending_initial_greeting_text: str | None = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, UserMuteStartedFrame):
@ -62,7 +64,11 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
return
if isinstance(frame, TTSSpeakFrame):
if not self._handled_initial_context:
await self._handle_context(self._context)
greeting_text = frame.text.strip() if frame.text else ""
if greeting_text:
await self._handle_initial_greeting(self._context, greeting_text)
else:
await self._handle_context(self._context)
else:
logger.warning(
f"{self}: TTSSpeakFrame after initial context already "
@ -120,6 +126,67 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
self._context = context
await self._process_completed_function_calls(send_new_results=True)
async def _handle_initial_greeting(self, context: LLMContext, greeting_text: str):
if context is None:
logger.warning(
f"{self}: received initial greeting trigger before context was set"
)
return
self._handled_initial_context = True
self._context = context
await self._create_initial_greeting_response(greeting_text)
async def _create_initial_greeting_response(self, greeting_text: str):
if self._disconnecting:
return
if not self._api_session_ready:
self._pending_initial_greeting_text = greeting_text
self._run_llm_when_api_session_ready = True
return
self._pending_initial_greeting_text = None
await self._ensure_conversation_setup()
item = events.ConversationItem(
type="message",
role="user",
content=[
events.ItemContent(
type="input_text",
text=format_static_greeting_prompt(greeting_text),
)
],
)
evt = events.ConversationItemCreateEvent(item=item)
self._messages_added_manually[evt.item.id] = True
await self.send_client_event(evt)
await self._send_manual_response_create()
async def _ensure_conversation_setup(self):
if not self._llm_needs_conversation_setup:
return
adapter = self.get_llm_adapter()
llm_invocation_params = adapter.get_llm_invocation_params(self._context)
for item in llm_invocation_params["messages"]:
evt = events.ConversationItemCreateEvent(item=item)
self._messages_added_manually[evt.item.id] = True
await self.send_client_event(evt)
await self._send_session_update()
self._llm_needs_conversation_setup = False
async def _handle_evt_session_updated(self, evt):
self._api_session_ready = True
if self._pending_initial_greeting_text is not None:
greeting_text = self._pending_initial_greeting_text
self._run_llm_when_api_session_ready = False
await self._create_initial_greeting_response(greeting_text)
elif self._run_llm_when_api_session_ready:
self._run_llm_when_api_session_ready = False
await self._create_response()
async def _send_user_audio(self, frame):
if self._user_is_muted:
return

77
api/services/pipecat/realtime/openai_realtime.py
View file

@ -22,6 +22,7 @@ from typing import Any
from loguru import logger
from api.services.pipecat.realtime.static_greeting import format_static_greeting_prompt
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
@ -56,6 +57,7 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
# has finished speaking, matching Dograh's Gemini Live behavior.
self._bot_is_speaking: bool = False
self._deferred_function_calls: list[FunctionCallFromLLM] = []
self._pending_initial_greeting_text: str | None = None
# ------------------------------------------------------------------
# Frame handling: mute, TTSSpeakFrame as greeting trigger
@ -73,11 +75,16 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
if isinstance(frame, TTSSpeakFrame):
# Greeting trigger: the engine queues a TTSSpeakFrame after node
# setup. OpenAI Realtime renders its own audio, so we don't pass
# the frame to TTS. Route through _handle_context so the initial
# response and later tool-result turns share the same context
# lifecycle even when Dograh has already pre-populated self._context.
# the frame to TTS. For configured static text greetings, ask the
# model to say the exact greeting; otherwise route through
# _handle_context so the initial response and later tool-result
# turns share the same context lifecycle.
if not self._handled_initial_context:
await self._handle_context(self._context)
greeting_text = frame.text.strip() if frame.text else ""
if greeting_text:
await self._handle_initial_greeting(self._context, greeting_text)
else:
await self._handle_context(self._context)
else:
logger.warning(
f"{self}: TTSSpeakFrame after initial context already "
@ -137,6 +144,57 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
self._context = context
await self._process_completed_function_calls(send_new_results=True)
async def _handle_initial_greeting(self, context: LLMContext, greeting_text: str):
if context is None:
logger.warning(
f"{self}: received initial greeting trigger before context was set"
)
return
self._handled_initial_context = True
self._context = context
await self._create_initial_greeting_response(greeting_text)
async def _create_initial_greeting_response(self, greeting_text: str):
if self._disconnecting:
return
if not self._api_session_ready:
self._pending_initial_greeting_text = greeting_text
self._run_llm_when_api_session_ready = True
return
self._pending_initial_greeting_text = None
await self._ensure_conversation_setup()
await self._send_manual_response_create(
instructions=format_static_greeting_prompt(greeting_text),
tool_choice="none",
)
async def _ensure_conversation_setup(self):
if not self._llm_needs_conversation_setup:
return
adapter = self.get_llm_adapter()
llm_invocation_params = adapter.get_llm_invocation_params(self._context)
for item in llm_invocation_params["messages"]:
evt = events.ConversationItemCreateEvent(item=item)
self._messages_added_manually[evt.item.id] = True
await self.send_client_event(evt)
await self._send_session_update()
self._llm_needs_conversation_setup = False
async def _handle_evt_session_updated(self, evt):
self._api_session_ready = True
if self._pending_initial_greeting_text is not None:
greeting_text = self._pending_initial_greeting_text
self._run_llm_when_api_session_ready = False
await self._create_initial_greeting_response(greeting_text)
elif self._run_llm_when_api_session_ready:
self._run_llm_when_api_session_ready = False
await self._create_response()
async def _send_user_audio(self, frame):
if self._user_is_muted:
return
@ -190,7 +248,12 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
return "\n".join(parts) if parts else None
return None
async def _send_manual_response_create(self):
async def _send_manual_response_create(
self,
*,
instructions: str | None = None,
tool_choice: str | None = None,
):
"""Trigger inference after manually appending conversation items."""
await self.push_frame(LLMFullResponseStartFrame())
await self.start_processing_metrics()
@ -198,7 +261,9 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
await self.send_client_event(
events.ResponseCreateEvent(
response=events.ResponseProperties(
output_modalities=self._get_enabled_modalities()
output_modalities=self._get_enabled_modalities(),
instructions=instructions,
tool_choice=tool_choice,
)
)
)

8
api/services/pipecat/realtime/static_greeting.py Normal file
View file

@ -0,0 +1,8 @@
def format_static_greeting_prompt(greeting_text: str) -> str:
return (
"The phone call has just connected. Greet the caller now: "
"say the following opening line out loud, exactly as written, "
"in a natural spoken voice, and then stop and wait for the "
"caller to respond. Do not add anything before or after it.\n\n"
f'"{greeting_text}"'
)

8
api/services/pipecat/realtime_feedback_observer.py
View file

@ -72,7 +72,7 @@ class RealtimeFeedbackObserver(BaseObserver):
- TTFB metrics (LLM generation time only)
Logs buffer persistence (only final data for post-call analysis):
- Complete user transcripts per turn (via on_user_turn_stopped)
- Complete user transcripts per turn (via on_user_turn_message_added)
- Complete assistant transcripts per turn (via on_assistant_turn_stopped)
- Function calls and TTFB metrics
@ -300,13 +300,13 @@ def register_turn_log_handlers(
):
"""Register event handlers on aggregators to persist final turn transcripts.
Hooks into on_user_turn_stopped and on_assistant_turn_stopped to store
Hooks into on_user_turn_message_added and on_assistant_turn_stopped to store
complete turn text in the logs buffer. Works for both WebRTC and telephony
calls — independent of WebSocket availability.
"""
@user_aggregator.event_handler("on_user_turn_stopped")
async def on_user_turn_stopped(aggregator, strategy, message):
@user_aggregator.event_handler("on_user_turn_message_added")
async def on_user_turn_message_added(aggregator, message):
logs_buffer.increment_turn()
try:
await logs_buffer.append(

73
api/services/pipecat/run_pipeline.py
View file

@ -113,49 +113,53 @@ def _resolve_user_turn_stop_timeout(
def _create_realtime_user_turn_config(provider: str):
"""Return user turn strategies and optional local VAD for realtime providers."""
def external_provider_turn_config():
return (
UserTurnStrategies(
start=[ExternalUserTurnStartStrategy()],
stop=[ExternalUserTurnStopStrategy(wait_for_transcript=False)],
),
None,
)
def local_vad_turn_config(*, enable_interruptions: bool):
return (
UserTurnStrategies(
start=[
VADUserTurnStartStrategy(enable_interruptions=enable_interruptions)
],
stop=[SpeechTimeoutUserTurnStopStrategy(wait_for_transcript=False)],
),
SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
)
if provider in {
ServiceProviders.GOOGLE_REALTIME.value,
ServiceProviders.GOOGLE_VERTEX_REALTIME.value,
}:
# Let Gemini Live own barge-in via its server-side VAD, but keep local
# Silero VAD for early user-turn start and speaking-state tracking.
return (
UserTurnStrategies(
start=[VADUserTurnStartStrategy(enable_interruptions=False)],
stop=[SpeechTimeoutUserTurnStopStrategy()],
),
SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
)
return local_vad_turn_config(enable_interruptions=False)
if provider == ServiceProviders.OPENAI_REALTIME.value:
# OpenAI Realtime already emits speaking-state frames and interruption
# events from the provider, so the aggregator should follow those
# external signals rather than run its own local VAD.
return (
UserTurnStrategies(
start=[ExternalUserTurnStartStrategy()],
stop=[ExternalUserTurnStopStrategy()],
),
None,
)
if provider in {
ServiceProviders.OPENAI_REALTIME.value,
ServiceProviders.AZURE_REALTIME.value,
}:
# OpenAI-compatible Realtime services already emit speaking-state frames
# and interruption events from the provider, so the aggregator should
# follow those external signals rather than run its own local VAD.
return external_provider_turn_config()
if provider == ServiceProviders.GROK_REALTIME.value:
# Grok Voice Agent emits server-side speech-start/stop and
# interruption signals, so local VAD should stay out of the way.
return (
UserTurnStrategies(
start=[ExternalUserTurnStartStrategy()],
stop=[ExternalUserTurnStopStrategy()],
),
None,
)
return external_provider_turn_config()
if provider == ServiceProviders.ULTRAVOX_REALTIME.value:
# Ultravox does not emit user-turn frames, so local VAD supplies
# lifecycle signals for Dograh observers/controllers.
return local_vad_turn_config(enable_interruptions=True)
return (
UserTurnStrategies(
start=[VADUserTurnStartStrategy()],
stop=[SpeechTimeoutUserTurnStopStrategy()],
),
SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
)
return local_vad_turn_config(enable_interruptions=True)
async def run_pipeline_telephony(
@ -773,7 +777,10 @@ async def _run_pipeline_impl(
vad_analyzer=user_vad_analyzer,
)
context_aggregator = LLMContextAggregatorPair(
context, assistant_params=assistant_params, user_params=user_params
context,
assistant_params=assistant_params,
user_params=user_params,
realtime_service_mode=is_realtime,
)
# Create usage metrics aggregator with engine's callback

8
api/services/telephony/providers/vonage/__init__.py
View file

@ -21,6 +21,7 @@ def _config_loader(value: Dict[str, Any]) -> Dict[str, Any]:
"private_key": value.get("private_key"),
"api_key": value.get("api_key"),
"api_secret": value.get("api_secret"),
"signature_secret": value.get("signature_secret"),
"from_numbers": value.get("from_numbers", []),
}
@ -49,6 +50,13 @@ _UI_METADATA = ProviderUIMetadata(
type="password",
sensitive=True,
),
ProviderUIField(
name="signature_secret",
label="Signature Secret",
type="password",
sensitive=True,
description="Vonage signature secret for signed webhook verification",
),
ProviderUIField(
name="from_numbers",
label="Phone Numbers",

7
api/services/telephony/providers/vonage/config.py
View file

@ -1,6 +1,6 @@
"""Vonage telephony configuration schemas."""
from typing import List, Literal
from typing import List, Literal, Optional
from pydantic import BaseModel, Field
@ -13,6 +13,10 @@ class VonageConfigurationRequest(BaseModel):
api_secret: str = Field(..., description="Vonage API Secret")
application_id: str = Field(..., description="Vonage Application ID")
private_key: str = Field(..., description="Private key for JWT generation")
signature_secret: Optional[str] = Field(
None,
description="Vonage signature secret used to verify signed webhooks",
)
from_numbers: List[str] = Field(
default_factory=list,
description="List of Vonage phone numbers (without + prefix)",
@ -27,4 +31,5 @@ class VonageConfigurationResponse(BaseModel):
api_key: str # Masked
api_secret: str # Masked
private_key: str # Masked
signature_secret: Optional[str] = None # Masked
from_numbers: List[str]

173
api/services/telephony/providers/vonage/provider.py
View file

@ -2,6 +2,7 @@
Vonage (Nexmo) implementation of the TelephonyProvider interface.
"""
import hashlib
import json
import random
import time
@ -44,12 +45,14 @@ class VonageProvider(TelephonyProvider):
- api_secret: Vonage API Secret
- application_id: Vonage Application ID
- private_key: Private key for JWT generation
- signature_secret: Signature secret for signed webhooks
- from_numbers: List of phone numbers to use
"""
self.api_key = config.get("api_key")
self.api_secret = config.get("api_secret")
self.application_id = config.get("application_id")
self.private_key = config.get("private_key")
self.signature_secret = config.get("signature_secret")
self.from_numbers = config.get("from_numbers", [])
# Handle both single number (string) and multiple numbers (list)
@ -186,17 +189,18 @@ class VonageProvider(TelephonyProvider):
Verify Vonage webhook signature for security.
Vonage uses JWT for webhook signatures.
"""
if not self.api_secret:
logger.error("No API secret available for webhook signature verification")
if not self.signature_secret:
logger.error(
"No signature secret available for Vonage webhook verification"
)
return False
try:
# Vonage sends JWT in Authorization header. Verify the JWT signature
decoded = jwt.decode(
jwt.decode(
signature,
self.api_secret,
self.signature_secret,
algorithms=["HS256"],
options={"verify_signature": True},
options={"verify_signature": True, "verify_aud": False},
)
return True
except jwt.InvalidTokenError:
@ -295,9 +299,13 @@ class VonageProvider(TelephonyProvider):
"ringing": TelephonyCallStatus.RINGING,
"answered": TelephonyCallStatus.ANSWERED,
"complete": TelephonyCallStatus.COMPLETED,
"completed": TelephonyCallStatus.COMPLETED,
"disconnected": TelephonyCallStatus.COMPLETED,
"failed": TelephonyCallStatus.FAILED,
"busy": TelephonyCallStatus.BUSY,
"timeout": TelephonyCallStatus.NO_ANSWER,
"unanswered": TelephonyCallStatus.NO_ANSWER,
"cancelled": TelephonyCallStatus.NO_ANSWER,
"rejected": TelephonyCallStatus.BUSY,
}
@ -349,6 +357,8 @@ class VonageProvider(TelephonyProvider):
if workflow_run.gathered_context
else None
)
if not call_uuid and workflow_run.gathered_context:
call_uuid = workflow_run.gathered_context.get("call_id")
if not call_uuid:
logger.error(
@ -400,26 +410,126 @@ class VonageProvider(TelephonyProvider):
"""
Determine if this provider can handle the incoming webhook.
"""
return False
claims = cls._decode_unverified_signed_claims(headers)
if claims.get("api_key") or claims.get("application_id"):
return True
return bool(
webhook_data.get("uuid")
and webhook_data.get("conversation_uuid")
and webhook_data.get("from")
and webhook_data.get("to")
)
@staticmethod
def parse_inbound_webhook(webhook_data: Dict[str, Any]) -> NormalizedInboundData:
def parse_inbound_webhook(
webhook_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None
) -> NormalizedInboundData:
"""
Parse Vonage-specific inbound webhook data into normalized format.
"""
claims = VonageProvider._decode_unverified_signed_claims(headers or {})
direction = webhook_data.get("direction") or "inbound"
status = webhook_data.get("status") or "started"
return NormalizedInboundData(
provider=VonageProvider.PROVIDER_NAME,
call_id=webhook_data.get("uuid", ""),
from_number=webhook_data.get("from", ""),
to_number=webhook_data.get("to", ""),
direction=webhook_data.get("direction", ""),
call_status=webhook_data.get("status", ""),
account_id=webhook_data.get("account_id"),
direction=direction,
call_status=status,
account_id=claims.get("api_key") or webhook_data.get("account_id"),
from_country=None,
to_country=None,
raw_data=webhook_data,
)
@staticmethod
def _header(headers: Dict[str, str], name: str) -> Optional[str]:
for key, value in headers.items():
if key.lower() == name.lower():
return value
return None
@classmethod
def _bearer_token(cls, headers: Dict[str, str]) -> Optional[str]:
auth_header = cls._header(headers, "authorization")
if not auth_header:
return None
parts = auth_header.split(None, 1)
if len(parts) != 2 or parts[0].lower() != "bearer":
return None
return parts[1].strip()
@classmethod
def _decode_unverified_signed_claims(
cls, headers: Dict[str, str]
) -> Dict[str, Any]:
token = cls._bearer_token(headers)
if not token:
return {}
try:
claims = jwt.decode(
token,
options={
"verify_signature": False,
"verify_aud": False,
"verify_exp": False,
},
)
except jwt.InvalidTokenError:
return {}
return claims if isinstance(claims, dict) else {}
def _verify_signed_claims(
self, headers: Dict[str, str], body: str = ""
) -> Optional[Dict[str, Any]]:
token = self._bearer_token(headers)
if not token:
logger.warning("Missing Vonage Authorization bearer token")
return None
if not self.signature_secret:
logger.error("Missing Vonage signature_secret for signed webhook")
return None
try:
claims = jwt.decode(
token,
self.signature_secret,
algorithms=["HS256"],
options={"verify_signature": True, "verify_aud": False},
)
except jwt.InvalidTokenError as exc:
logger.warning(f"Invalid Vonage signed webhook JWT: {exc}")
return None
if claims.get("iss") != "Vonage":
logger.warning("Vonage signed webhook JWT has unexpected issuer")
return None
if self.api_key and claims.get("api_key") != self.api_key:
logger.warning("Vonage signed webhook api_key does not match config")
return None
claim_application_id = claims.get("application_id")
if (
self.application_id
and claim_application_id
and claim_application_id != self.application_id
):
logger.warning("Vonage signed webhook application_id does not match config")
return None
payload_hash = claims.get("payload_hash")
if payload_hash:
actual_hash = hashlib.sha256(body.encode("utf-8")).hexdigest()
if actual_hash != payload_hash:
logger.warning("Vonage signed webhook payload hash mismatch")
return None
return claims
@staticmethod
def validate_account_id(config_data: dict, webhook_account_id: str) -> bool:
"""Validate Vonage account_id from webhook matches configuration"""
@ -437,9 +547,10 @@ class VonageProvider(TelephonyProvider):
body: str = "",
) -> bool:
"""
Vonage inbound signature verification - minimalist implementation.
Verify Vonage signed webhook JWT and optional payload hash.
"""
return True
claims = self._verify_signed_claims(headers, body)
return claims is not None
async def configure_inbound(
self, address: str, webhook_url: Optional[str]
@ -486,6 +597,15 @@ class VonageProvider(TelephonyProvider):
),
)
if not self.signature_secret:
return ProviderSyncResult(
ok=False,
message=(
"Vonage signature_secret is required because inbound calls "
"use signed webhook verification"
),
)
app_endpoint = f"{self.base_url}/v2/applications/{self.application_id}"
auth = aiohttp.BasicAuth(self.api_key, self.api_secret)
@ -510,12 +630,18 @@ class VonageProvider(TelephonyProvider):
capabilities = app_data.get("capabilities") or {}
voice = capabilities.get("voice") or {}
webhooks = voice.get("webhooks") or {}
backend_endpoint, _ = await get_backend_endpoints()
webhooks["answer_url"] = {
"address": webhook_url,
"http_method": "POST",
}
webhooks["event_url"] = {
"address": f"{backend_endpoint}/api/v1/telephony/vonage/events",
"http_method": "POST",
}
voice["webhooks"] = webhooks
voice["signed_callbacks"] = True
capabilities["voice"] = voice
update_body = {
@ -561,13 +687,24 @@ class VonageProvider(TelephonyProvider):
"""
Generate NCCO response for inbound Vonage webhook.
"""
# Minimalist NCCO response for interface compliance
ncco_response = [
{
"action": "talk",
"text": "Vonage inbound calls are not currently supported.",
},
{"action": "hangup"},
"action": "connect",
"eventUrl": [
f"{backend_endpoint}/api/v1/telephony/vonage/events/{workflow_run_id}"
],
"endpoint": [
{
"type": "websocket",
"uri": websocket_url,
"content-type": "audio/l16;rate=16000",
"headers": {
"workflow_run_id": str(workflow_run_id),
"call_uuid": normalized_data.call_id,
},
}
],
}
]
return Response(

85
api/services/telephony/providers/vonage/routes.py
View file

@ -7,16 +7,12 @@ provider registry — see ProviderSpec.router.
import json
from typing import Optional
from fastapi import APIRouter, Request
from fastapi import APIRouter, HTTPException, Request
from loguru import logger
from pipecat.utils.run_context import set_current_run_id
from api.db import db_client
from api.services.telephony.factory import get_telephony_provider_for_run
from api.services.telephony.status_processor import (
StatusCallbackRequest,
_process_status_update,
)
router = APIRouter()
@ -45,28 +41,33 @@ async def handle_ncco_webhook(
return json.loads(response_content)
@router.post("/vonage/events/{workflow_run_id}")
async def handle_vonage_events(
request: Request,
workflow_run_id: int,
):
"""Handle Vonage-specific event webhooks.
async def _read_json_body(request: Request) -> tuple[dict, str]:
body_bytes = await request.body()
try:
raw_body = body_bytes.decode("utf-8")
except UnicodeDecodeError as exc:
raise HTTPException(
status_code=400, detail="Webhook body is not valid UTF-8"
) from exc
try:
return json.loads(raw_body or "{}"), raw_body
except json.JSONDecodeError as exc:
raise HTTPException(status_code=400, detail="Webhook body is not JSON") from exc
Vonage sends all call events to a single endpoint.
Events include: started, ringing, answered, complete, failed, etc.
"""
async def _handle_vonage_event_request(request: Request, workflow_run_id: int):
set_current_run_id(workflow_run_id)
# Parse the event data
event_data = await request.json()
logger.info(f"[run {workflow_run_id}] Received Vonage event: {event_data}")
event_data, raw_body = await _read_json_body(request)
logger.info(
f"[run {workflow_run_id}] Received Vonage event "
f"uuid={event_data.get('uuid')} status={event_data.get('status')}"
)
# Get workflow run for processing
workflow_run = await db_client.get_workflow_run_by_id(workflow_run_id)
if not workflow_run:
logger.error(f"[run {workflow_run_id}] Workflow run not found")
return {"status": "error", "message": "Workflow run not found"}
# Get workflow and provider
workflow = await db_client.get_workflow_by_id(workflow_run.workflow_id)
if not workflow:
logger.error(f"[run {workflow_run_id}] Workflow not found")
@ -75,11 +76,18 @@ async def handle_vonage_events(
provider = await get_telephony_provider_for_run(
workflow_run, workflow.organization_id
)
signature_valid = await provider.verify_inbound_signature(
str(request.url), event_data, dict(request.headers), raw_body
)
if not signature_valid:
raise HTTPException(status_code=401, detail="Invalid webhook signature")
from api.services.telephony.status_processor import (
StatusCallbackRequest,
_process_status_update,
)
# Parse the event data into generic format
parsed_data = provider.parse_status_callback(event_data)
# Create StatusCallbackRequest from parsed data
status_update = StatusCallbackRequest(
call_id=parsed_data["call_id"],
status=parsed_data["status"],
@ -90,8 +98,35 @@ async def handle_vonage_events(
extra=parsed_data.get("extra", {}),
)
# Process the status update
await _process_status_update(workflow_run_id, status_update)
# Return 204 No Content as expected by Vonage
return {"status": "ok"}
@router.post("/vonage/events/{workflow_run_id}")
async def handle_vonage_events(
request: Request,
workflow_run_id: int,
):
"""Handle Vonage-specific event webhooks.
Vonage sends all call events to a single endpoint.
Events include: started, ringing, answered, complete, failed, etc.
"""
return await _handle_vonage_event_request(request, workflow_run_id)
@router.post("/vonage/events")
async def handle_vonage_events_without_run(request: Request):
"""Handle application-level events by resolving the run from call UUID."""
event_data, _ = await _read_json_body(request)
call_id = event_data.get("uuid")
if call_id:
workflow_run = await db_client.get_workflow_run_by_call_id(call_id)
if workflow_run:
return await _handle_vonage_event_request(request, workflow_run.id)
logger.info(
"Received unmatched Vonage application event "
f"uuid={event_data.get('uuid')} status={event_data.get('status')}"
)
return {"status": "ok"}

1
api/services/workflow/dto.py
View file

@ -250,7 +250,6 @@ class _ToolDocumentRefsMixin(BaseModel):
"description": (
"Text spoken via TTS at the start of the call. Supports "
"{{template_variables}}. Leave empty to skip the greeting. "
"Not supported with realtime (speech-to-speech) models."
),
"display_options": DisplayOptions(show={"greeting_type": ["text"]}),
"placeholder": "Hi {{first_name}}, this is Sarah from Acme.",

279
api/tests/telephony/vonage/test_provider.py Normal file
View file

@ -0,0 +1,279 @@
import hashlib
import json
import sys
import time
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import jwt
import pytest
from fastapi import HTTPException
from starlette.requests import Request
from api.services.telephony.providers.vonage.provider import VonageProvider
from api.services.telephony.providers.vonage.routes import handle_vonage_events
SIGNATURE_SECRET = "vonage-signature-secret"
def _body() -> str:
return json.dumps(
{
"from": "15551230001",
"to": "15551230002",
"uuid": "aaaaaaaa-bbbb-cccc-dddd-0123456789ab",
"conversation_uuid": "CON-aaaaaaaa-bbbb-cccc-dddd-0123456789ab",
"status": "answered",
"direction": "inbound",
},
separators=(",", ":"),
)
def _provider(**overrides) -> VonageProvider:
config = {
"api_key": "vonage-api-key",
"api_secret": "vonage-api-secret",
"application_id": "aaaaaaaa-bbbb-cccc-dddd-0123456789ab",
"private_key": "placeholder-private-key",
"signature_secret": SIGNATURE_SECRET,
"from_numbers": ["15551230002"],
}
config.update(overrides)
return VonageProvider(config)
def _signed_headers(
body: str,
*,
signature_secret: str = SIGNATURE_SECRET,
api_key: str = "vonage-api-key",
application_id: str = "aaaaaaaa-bbbb-cccc-dddd-0123456789ab",
) -> dict[str, str]:
token = jwt.encode(
{
"iat": int(time.time()),
"jti": "test-jti",
"iss": "Vonage",
"payload_hash": hashlib.sha256(body.encode("utf-8")).hexdigest(),
"api_key": api_key,
"application_id": application_id,
},
signature_secret,
algorithm="HS256",
)
return {"authorization": f"Bearer {token}"}
def _request(body: str, headers: dict[str, str]) -> Request:
async def receive():
return {
"type": "http.request",
"body": body.encode("utf-8"),
"more_body": False,
}
return Request(
{
"type": "http",
"method": "POST",
"path": "/api/v1/telephony/vonage/events/123",
"headers": [
(name.lower().encode("ascii"), value.encode("ascii"))
for name, value in headers.items()
],
},
receive,
)
@pytest.mark.asyncio
async def test_verify_inbound_signature_accepts_valid_vonage_signed_webhook():
body = _body()
provider = _provider()
result = await provider.verify_inbound_signature(
"https://example.test/api/v1/telephony/inbound/run",
json.loads(body),
_signed_headers(body),
body,
)
assert result is True
@pytest.mark.asyncio
async def test_verify_inbound_signature_rejects_tampered_payload():
body = _body()
provider = _provider()
result = await provider.verify_inbound_signature(
"https://example.test/api/v1/telephony/inbound/run",
json.loads(body),
_signed_headers(body),
body.replace("answered", "completed"),
)
assert result is False
@pytest.mark.asyncio
async def test_verify_inbound_signature_rejects_missing_signature_secret():
body = _body()
provider = _provider(signature_secret=None)
result = await provider.verify_inbound_signature(
"https://example.test/api/v1/telephony/inbound/run",
json.loads(body),
_signed_headers(body),
body,
)
assert result is False
@pytest.mark.asyncio
async def test_verify_inbound_signature_rejects_wrong_api_key_claim():
body = _body()
provider = _provider()
result = await provider.verify_inbound_signature(
"https://example.test/api/v1/telephony/inbound/run",
json.loads(body),
_signed_headers(body, api_key="other-api-key"),
body,
)
assert result is False
def test_parse_inbound_webhook_uses_signed_api_key_claim_for_account_id():
body = _body()
normalized = VonageProvider.parse_inbound_webhook(
json.loads(body), headers=_signed_headers(body)
)
assert normalized.provider == "vonage"
assert normalized.call_id == "aaaaaaaa-bbbb-cccc-dddd-0123456789ab"
assert normalized.account_id == "vonage-api-key"
assert normalized.direction == "inbound"
def test_can_handle_webhook_detects_signed_vonage_answer_payload():
body = _body()
assert VonageProvider.can_handle_webhook(json.loads(body), _signed_headers(body))
@pytest.mark.asyncio
async def test_start_inbound_stream_returns_websocket_ncco():
body = _body()
provider = _provider()
normalized = VonageProvider.parse_inbound_webhook(
json.loads(body), headers=_signed_headers(body)
)
response = await provider.start_inbound_stream(
websocket_url="wss://example.test/api/v1/telephony/ws/1/2/3",
workflow_run_id=123,
normalized_data=normalized,
backend_endpoint="https://example.test",
)
ncco = json.loads(response.body)
assert ncco == [
{
"action": "connect",
"eventUrl": ["https://example.test/api/v1/telephony/vonage/events/123"],
"endpoint": [
{
"type": "websocket",
"uri": "wss://example.test/api/v1/telephony/ws/1/2/3",
"content-type": "audio/l16;rate=16000",
"headers": {
"workflow_run_id": "123",
"call_uuid": "aaaaaaaa-bbbb-cccc-dddd-0123456789ab",
},
}
],
}
]
@pytest.mark.asyncio
async def test_vonage_events_route_verifies_signature_before_status_update():
body = _body()
provider = _provider()
with (
patch("api.services.telephony.providers.vonage.routes.db_client") as db_client,
patch(
"api.services.telephony.providers.vonage.routes.get_telephony_provider_for_run",
new_callable=AsyncMock,
return_value=provider,
),
):
process_status = AsyncMock()
status_processor = SimpleNamespace(
StatusCallbackRequest=SimpleNamespace,
_process_status_update=process_status,
)
db_client.get_workflow_run_by_id = AsyncMock(
return_value=SimpleNamespace(workflow_id=7)
)
db_client.get_workflow_by_id = AsyncMock(
return_value=SimpleNamespace(organization_id=11)
)
with patch.dict(
sys.modules,
{"api.services.telephony.status_processor": status_processor},
):
result = await handle_vonage_events(
_request(body, _signed_headers(body)), workflow_run_id=123
)
assert result == {"status": "ok"}
process_status.assert_awaited_once()
@pytest.mark.asyncio
async def test_vonage_events_route_rejects_invalid_signature_with_401():
body = _body()
provider = _provider()
with (
patch("api.services.telephony.providers.vonage.routes.db_client") as db_client,
patch(
"api.services.telephony.providers.vonage.routes.get_telephony_provider_for_run",
new_callable=AsyncMock,
return_value=provider,
),
):
process_status = AsyncMock()
status_processor = SimpleNamespace(
StatusCallbackRequest=SimpleNamespace,
_process_status_update=process_status,
)
db_client.get_workflow_run_by_id = AsyncMock(
return_value=SimpleNamespace(workflow_id=7)
)
db_client.get_workflow_by_id = AsyncMock(
return_value=SimpleNamespace(organization_id=11)
)
with (
patch.dict(
sys.modules,
{"api.services.telephony.status_processor": status_processor},
),
pytest.raises(HTTPException) as exc_info,
):
await handle_vonage_events(
_request(body, _signed_headers(body, signature_secret="wrong")),
workflow_run_id=123,
)
assert exc_info.value.status_code == 401
assert exc_info.value.detail == "Invalid webhook signature"
process_status.assert_not_awaited()

88
api/tests/test_azure_realtime_wrapper.py Normal file
View file

@ -0,0 +1,88 @@
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from pipecat.frames.frames import TTSSpeakFrame
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.openai.realtime import events
from api.services.pipecat.realtime.azure_realtime import (
DograhAzureRealtimeLLMService,
)
def _make_service() -> DograhAzureRealtimeLLMService:
service = DograhAzureRealtimeLLMService(
api_key="test-key",
base_url="wss://example.test/openai/realtime",
)
service._create_response = AsyncMock()
service._process_completed_function_calls = AsyncMock()
return service
@pytest.mark.asyncio
async def test_tts_greeting_sends_exact_static_greeting_prompt():
service = _make_service()
service._context = LLMContext([{"role": "user", "content": "Existing context"}])
service._api_session_ready = True
service.send_client_event = AsyncMock()
service.push_frame = AsyncMock()
service.start_processing_metrics = AsyncMock()
service.start_ttfb_metrics = AsyncMock()
await service.process_frame(
TTSSpeakFrame("Hi Sam, this is Sarah from Acme.", append_to_context=True),
FrameDirection.DOWNSTREAM,
)
sent_events = [call.args[0] for call in service.send_client_event.await_args_list]
assert isinstance(sent_events[0], events.ConversationItemCreateEvent)
assert sent_events[0].item.role == "user"
assert sent_events[0].item.content[0].text == "Existing context"
assert isinstance(sent_events[1], events.SessionUpdateEvent)
response_event = sent_events[-1]
assert isinstance(response_event, events.ResponseCreateEvent)
assert response_event.response.tool_choice == "none"
prompt = response_event.response.instructions
assert "The phone call has just connected. Greet the caller now:" in prompt
assert prompt.endswith('"Hi Sam, this is Sarah from Acme."')
assert service._llm_needs_conversation_setup is False
service._create_response.assert_not_awaited()
@pytest.mark.asyncio
async def test_tts_greeting_waits_for_session_updated_before_sending_prompt():
service = _make_service()
service._context = LLMContext([{"role": "user", "content": "Existing context"}])
await service.process_frame(
TTSSpeakFrame("Hello from Dograh.", append_to_context=True),
FrameDirection.DOWNSTREAM,
)
assert service._handled_initial_context is True
assert service._run_llm_when_api_session_ready is True
assert service._pending_initial_greeting_text == "Hello from Dograh."
service.send_client_event = AsyncMock()
service.push_frame = AsyncMock()
service.start_processing_metrics = AsyncMock()
service.start_ttfb_metrics = AsyncMock()
await service._handle_evt_session_updated(SimpleNamespace())
sent_events = [call.args[0] for call in service.send_client_event.await_args_list]
assert isinstance(sent_events[0], events.ConversationItemCreateEvent)
assert sent_events[0].item.content[0].text == "Existing context"
assert isinstance(sent_events[1], events.SessionUpdateEvent)
response_event = sent_events[-1]
assert isinstance(response_event, events.ResponseCreateEvent)
assert response_event.response.tool_choice == "none"
prompt = response_event.response.instructions
assert prompt.endswith('"Hello from Dograh."')
assert service._run_llm_when_api_session_ready is False
assert service._pending_initial_greeting_text is None
assert service._llm_needs_conversation_setup is False
service._create_response.assert_not_awaited()

57
api/tests/test_gemini_live_reconnect_tool_results.py
View file

@ -3,7 +3,7 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from pipecat.frames.frames import TranscriptionFrame
from pipecat.frames.frames import TranscriptionFrame, TTSSpeakFrame
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameDirection
@ -21,6 +21,7 @@ class _TestDograhGeminiLiveLLMService(DograhGeminiLiveLLMService):
class _FakeSession:
def __init__(self):
self.send_client_content = AsyncMock()
self.send_tool_response = AsyncMock()
self.send_realtime_input = AsyncMock()
self.close = AsyncMock()
@ -108,3 +109,57 @@ async def test_user_transcription_matches_upstream_upstream_push_behavior():
assert frame.text == "Hi there"
assert frame.finalized is False
assert direction == FrameDirection.UPSTREAM
@pytest.mark.asyncio
async def test_tts_greeting_sends_exact_static_greeting_prompt_to_gemini():
service = _make_service()
service._context = LLMContext()
service._session = _FakeSession()
await service.process_frame(
TTSSpeakFrame("Hi Sam, this is Sarah from Acme.", append_to_context=True),
FrameDirection.DOWNSTREAM,
)
service._session.send_client_content.assert_awaited_once()
kwargs = service._session.send_client_content.await_args.kwargs
assert kwargs["turn_complete"] is True
turns = kwargs["turns"]
assert len(turns) == 1
assert turns[0].role == "user"
prompt = turns[0].parts[0].text
assert "The phone call has just connected. Greet the caller now:" in prompt
assert (
'Do not add anything before or after it.\n\n"Hi Sam, this is Sarah from Acme."'
in prompt
)
assert service._handled_initial_context is True
assert service._pending_initial_greeting_text is None
assert service._ready_for_realtime_input is True
@pytest.mark.asyncio
async def test_tts_greeting_waits_for_gemini_session_before_sending_prompt():
service = _make_service()
service._context = LLMContext()
await service.process_frame(
TTSSpeakFrame("Hello from Dograh.", append_to_context=True),
FrameDirection.DOWNSTREAM,
)
assert service._handled_initial_context is True
assert service._run_llm_when_session_ready is True
assert service._pending_initial_greeting_text == "Hello from Dograh."
session = _FakeSession()
await service._handle_session_ready(session)
session.send_client_content.assert_awaited_once()
prompt = session.send_client_content.await_args.kwargs["turns"][0].parts[0].text
assert prompt.endswith('"Hello from Dograh."')
assert service._run_llm_when_session_ready is False
assert service._pending_initial_greeting_text is None

64
api/tests/test_grok_realtime_wrapper.py
View file

@ -37,17 +37,71 @@ async def test_initial_context_triggers_response_when_context_was_prepopulated()
@pytest.mark.asyncio
async def test_tts_greeting_uses_initial_context_handler():
async def test_tts_greeting_sends_exact_static_greeting_prompt():
service = _make_service()
service._context = LLMContext()
service._handle_context = AsyncMock()
service._context = LLMContext([{"role": "user", "content": "Existing context"}])
service._api_session_ready = True
service.send_client_event = AsyncMock()
service.push_frame = AsyncMock()
service.start_processing_metrics = AsyncMock()
service.start_ttfb_metrics = AsyncMock()
await service.process_frame(
TTSSpeakFrame("hello", append_to_context=True),
TTSSpeakFrame("Hi Sam, this is Sarah from Acme.", append_to_context=True),
FrameDirection.DOWNSTREAM,
)
service._handle_context.assert_awaited_once_with(service._context)
sent_events = [call.args[0] for call in service.send_client_event.await_args_list]
assert isinstance(sent_events[0], events.ConversationItemCreateEvent)
assert sent_events[0].item.role == "user"
assert sent_events[0].item.content[0].text == "Existing context"
assert isinstance(sent_events[1], events.SessionUpdateEvent)
greeting_event = sent_events[2]
assert isinstance(greeting_event, events.ConversationItemCreateEvent)
assert greeting_event.item.role == "user"
assert greeting_event.item.type == "message"
prompt = greeting_event.item.content[0].text
assert "The phone call has just connected. Greet the caller now:" in prompt
assert prompt.endswith('"Hi Sam, this is Sarah from Acme."')
assert isinstance(sent_events[-1], events.ResponseCreateEvent)
assert service._llm_needs_conversation_setup is False
service._create_response.assert_not_awaited()
@pytest.mark.asyncio
async def test_tts_greeting_waits_for_session_updated_before_sending_prompt():
service = _make_service()
service._context = LLMContext([{"role": "user", "content": "Existing context"}])
await service.process_frame(
TTSSpeakFrame("Hello from Dograh.", append_to_context=True),
FrameDirection.DOWNSTREAM,
)
assert service._handled_initial_context is True
assert service._run_llm_when_api_session_ready is True
assert service._pending_initial_greeting_text == "Hello from Dograh."
service.send_client_event = AsyncMock()
service.push_frame = AsyncMock()
service.start_processing_metrics = AsyncMock()
service.start_ttfb_metrics = AsyncMock()
await service._handle_evt_session_updated(SimpleNamespace())
sent_events = [call.args[0] for call in service.send_client_event.await_args_list]
assert isinstance(sent_events[0], events.ConversationItemCreateEvent)
assert sent_events[0].item.content[0].text == "Existing context"
assert isinstance(sent_events[1], events.SessionUpdateEvent)
greeting_event = sent_events[2]
assert isinstance(greeting_event, events.ConversationItemCreateEvent)
prompt = greeting_event.item.content[0].text
assert prompt.endswith('"Hello from Dograh."')
assert isinstance(sent_events[-1], events.ResponseCreateEvent)
assert service._run_llm_when_api_session_ready is False
assert service._pending_initial_greeting_text is None
assert service._llm_needs_conversation_setup is False
service._create_response.assert_not_awaited()
@pytest.mark.asyncio

61
api/tests/test_openai_realtime_initial_context.py
View file

@ -5,6 +5,7 @@ import pytest
from pipecat.frames.frames import TTSSpeakFrame
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.openai.realtime import events
from api.services.pipecat.realtime.openai_realtime import (
DograhOpenAIRealtimeLLMService,
@ -48,17 +49,69 @@ async def test_updated_context_uses_tool_result_path_after_initial_context():
@pytest.mark.asyncio
async def test_tts_greeting_uses_initial_context_handler():
async def test_tts_greeting_sends_exact_static_greeting_prompt():
service = _make_service()
service._context = LLMContext()
service._handle_context = AsyncMock()
service._api_session_ready = True
service.send_client_event = AsyncMock()
service.push_frame = AsyncMock()
service.start_processing_metrics = AsyncMock()
service.start_ttfb_metrics = AsyncMock()
await service.process_frame(
TTSSpeakFrame("hello", append_to_context=True),
TTSSpeakFrame("Hi Sam, this is Sarah from Acme.", append_to_context=True),
FrameDirection.DOWNSTREAM,
)
service._handle_context.assert_awaited_once_with(service._context)
sent_events = [call.args[0] for call in service.send_client_event.await_args_list]
assert not any(
isinstance(event, events.ConversationItemCreateEvent) for event in sent_events
)
assert isinstance(sent_events[0], events.SessionUpdateEvent)
response_event = sent_events[-1]
assert isinstance(response_event, events.ResponseCreateEvent)
assert response_event.response.tool_choice == "none"
prompt = response_event.response.instructions
assert "The phone call has just connected. Greet the caller now:" in prompt
assert prompt.endswith('"Hi Sam, this is Sarah from Acme."')
assert service._llm_needs_conversation_setup is False
service._create_response.assert_not_awaited()
@pytest.mark.asyncio
async def test_tts_greeting_waits_for_session_updated_before_sending_prompt():
service = _make_service()
service._context = LLMContext()
await service.process_frame(
TTSSpeakFrame("Hello from Dograh.", append_to_context=True),
FrameDirection.DOWNSTREAM,
)
assert service._handled_initial_context is True
assert service._run_llm_when_api_session_ready is True
assert service._pending_initial_greeting_text == "Hello from Dograh."
service.send_client_event = AsyncMock()
service.push_frame = AsyncMock()
service.start_processing_metrics = AsyncMock()
service.start_ttfb_metrics = AsyncMock()
await service._handle_evt_session_updated(SimpleNamespace())
sent_events = [call.args[0] for call in service.send_client_event.await_args_list]
assert not any(
isinstance(event, events.ConversationItemCreateEvent) for event in sent_events
)
assert isinstance(sent_events[0], events.SessionUpdateEvent)
response_event = sent_events[-1]
assert isinstance(response_event, events.ResponseCreateEvent)
assert response_event.response.tool_choice == "none"
prompt = response_event.response.instructions
assert prompt.endswith('"Hello from Dograh."')
assert service._run_llm_when_api_session_ready is False
assert service._pending_initial_greeting_text is None
assert service._llm_needs_conversation_setup is False
service._create_response.assert_not_awaited()

48
api/tests/test_realtime_feedback_observer.py
View file

@ -7,7 +7,23 @@ from pipecat.processors.frame_processor import FrameDirection
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import TransportParams
from api.services.pipecat.realtime_feedback_observer import RealtimeFeedbackObserver
from api.services.pipecat.in_memory_buffers import InMemoryLogsBuffer
from api.services.pipecat.realtime_feedback_observer import (
RealtimeFeedbackObserver,
register_turn_log_handlers,
)
class _FakeAggregator:
def __init__(self):
self.handlers = {}
def event_handler(self, event_name):
def decorator(handler):
self.handlers[event_name] = handler
return handler
return decorator
def _frame_pushed(frame, direction, *, source=None):
@ -98,3 +114,33 @@ async def test_observer_waits_for_tts_text_from_output_transport():
"payload": {"text": "Hello"},
}
]
@pytest.mark.asyncio
async def test_turn_log_handlers_persist_user_message_added_events():
logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
user_aggregator = _FakeAggregator()
assistant_aggregator = _FakeAggregator()
register_turn_log_handlers(logs_buffer, user_aggregator, assistant_aggregator)
assert "on_user_turn_message_added" in user_aggregator.handlers
assert "on_user_turn_stopped" not in user_aggregator.handlers
await user_aggregator.handlers["on_user_turn_message_added"](
user_aggregator,
SimpleNamespace(
content="Hi there",
timestamp="2026-01-01T00:00:00+00:00",
),
)
events = logs_buffer.get_events()
assert len(events) == 1
assert events[0]["type"] == "rtf-user-transcription"
assert events[0]["payload"] == {
"text": "Hi there",
"final": True,
"timestamp": "2026-01-01T00:00:00+00:00",
}
assert events[0]["turn"] == 1

36
api/tests/test_run_pipeline_realtime_turn_config.py
View file

@ -30,6 +30,7 @@ def test_gemini_realtime_uses_local_vad_without_local_interruptions():
assert strategies.start[0]._enable_interruptions is False
assert len(strategies.stop) == 1
assert isinstance(strategies.stop[0], SpeechTimeoutUserTurnStopStrategy)
assert strategies.stop[0].wait_for_transcript is False
def test_gemini_vertex_realtime_uses_same_turn_config_as_gemini_live():
@ -41,6 +42,9 @@ def test_gemini_vertex_realtime_uses_same_turn_config_as_gemini_live():
assert len(strategies.start) == 1
assert isinstance(strategies.start[0], VADUserTurnStartStrategy)
assert strategies.start[0]._enable_interruptions is False
assert len(strategies.stop) == 1
assert isinstance(strategies.stop[0], SpeechTimeoutUserTurnStopStrategy)
assert strategies.stop[0].wait_for_transcript is False
def test_openai_realtime_uses_provider_turn_frames_without_local_vad():
@ -54,6 +58,21 @@ def test_openai_realtime_uses_provider_turn_frames_without_local_vad():
assert strategies.start[0]._enable_interruptions is False
assert len(strategies.stop) == 1
assert isinstance(strategies.stop[0], ExternalUserTurnStopStrategy)
assert strategies.stop[0].wait_for_transcript is False
def test_azure_realtime_uses_provider_turn_frames_without_local_vad():
strategies, vad_analyzer = _create_realtime_user_turn_config(
ServiceProviders.AZURE_REALTIME.value
)
assert vad_analyzer is None
assert len(strategies.start) == 1
assert isinstance(strategies.start[0], ExternalUserTurnStartStrategy)
assert strategies.start[0]._enable_interruptions is False
assert len(strategies.stop) == 1
assert isinstance(strategies.stop[0], ExternalUserTurnStopStrategy)
assert strategies.stop[0].wait_for_transcript is False
def test_grok_realtime_uses_provider_turn_frames_without_local_vad():
@ -67,6 +86,21 @@ def test_grok_realtime_uses_provider_turn_frames_without_local_vad():
assert strategies.start[0]._enable_interruptions is False
assert len(strategies.stop) == 1
assert isinstance(strategies.stop[0], ExternalUserTurnStopStrategy)
assert strategies.stop[0].wait_for_transcript is False
def test_ultravox_realtime_uses_local_vad_with_local_interruptions():
strategies, vad_analyzer = _create_realtime_user_turn_config(
ServiceProviders.ULTRAVOX_REALTIME.value
)
assert isinstance(vad_analyzer, SileroVADAnalyzer)
assert len(strategies.start) == 1
assert isinstance(strategies.start[0], VADUserTurnStartStrategy)
assert strategies.start[0]._enable_interruptions is True
assert len(strategies.stop) == 1
assert isinstance(strategies.stop[0], SpeechTimeoutUserTurnStopStrategy)
assert strategies.stop[0].wait_for_transcript is False
def test_unknown_realtime_providers_keep_local_vad():
@ -75,8 +109,10 @@ def test_unknown_realtime_providers_keep_local_vad():
assert isinstance(vad_analyzer, SileroVADAnalyzer)
assert len(strategies.start) == 1
assert isinstance(strategies.start[0], VADUserTurnStartStrategy)
assert strategies.start[0]._enable_interruptions is True
assert len(strategies.stop) == 1
assert isinstance(strategies.stop[0], SpeechTimeoutUserTurnStopStrategy)
assert strategies.stop[0].wait_for_transcript is False
def test_external_turn_stt_uses_longer_stop_timeout():

38
api/tests/test_template_renderer.py Normal file
View file

@ -0,0 +1,38 @@
from api.utils.template_renderer import render_template
def test_initial_context_prefix_resolves_against_flat_context():
context = {
"first_name": "Abhishek",
"runtime_configuration": {
"realtime_model": "gpt-realtime-2",
},
}
assert (
render_template("Hi {{initial_context.first_name | there}}", context)
== "Hi Abhishek"
)
assert (
render_template(
"Model {{initial_context.runtime_configuration.realtime_model}}", context
)
== "Model gpt-realtime-2"
)
def test_initial_context_prefix_prefers_explicit_initial_context():
context = {
"first_name": "Flat",
"initial_context": {
"first_name": "Nested",
},
}
assert render_template("Hi {{initial_context.first_name}}", context) == "Hi Nested"
def test_initial_context_prefix_uses_fallback_when_missing_from_both_contexts():
assert (
render_template("Hi {{initial_context.first_name | there}}", {}) == "Hi there"
)

9
api/utils/telephony_helper.py
View file

@ -3,6 +3,8 @@ Telephony helper utilities.
Common functions used across telephony operations.
"""
import inspect
from fastapi import Request
from loguru import logger
from starlette.responses import HTMLResponse
@ -119,9 +121,12 @@ def _test_number_formats_with_country_code(
return False
def normalize_webhook_data(provider_class, webhook_data):
def normalize_webhook_data(provider_class, webhook_data, headers=None):
"""Normalize webhook data using the provider's parse method"""
return provider_class.parse_inbound_webhook(webhook_data)
parse_method = provider_class.parse_inbound_webhook
if headers is not None and "headers" in inspect.signature(parse_method).parameters:
return parse_method(webhook_data, headers=headers)
return parse_method(webhook_data)
def generic_hangup_response():

9
api/utils/template_renderer.py
View file

@ -12,6 +12,7 @@ from api.services.workflow.workflow_graph import TEMPLATE_VAR_PATTERN
_CURRENT_TIME_PREFIX = "current_time"
_CURRENT_WEEKDAY_PREFIX = "current_weekday"
_INITIAL_CONTEXT_PREFIX = "initial_context."
def get_nested_value(obj: Any, path: str) -> Any:
@ -184,8 +185,14 @@ def _render_string(template_str: str, context: Dict[str, Any]) -> str:
if builtin_value is not None:
return builtin_value
# Get value using nested path lookup
# Get value using nested path lookup. Prompts commonly reference
# initial_context.<key>, while some runtime callers pass the initial
# context itself as the render context.
value = get_nested_value(context, variable_path)
if value is None and variable_path.startswith(_INITIAL_CONTEXT_PREFIX):
value = get_nested_value(
context, variable_path[len(_INITIAL_CONTEXT_PREFIX) :]
)
# Apply fallback: new syntax {{var | default}} or legacy {{var | fallback:default}}
if filter_name is not None: