mirror of
https://github.com/dograh-hq/dograh.git
synced 2026-07-25 12:01:04 +02:00
chore: generate SDK and fix other realtime providers
This commit is contained in:
parent
51525b7e24
commit
d66eb8fd47
20 changed files with 535 additions and 473 deletions
|
|
@ -1,6 +1,10 @@
|
||||||
AZURE_MODELS = ["gpt-4.1-mini"]
|
AZURE_MODELS = ["gpt-4.1-mini"]
|
||||||
|
|
||||||
AZURE_REALTIME_MODELS = ["gpt-4o-realtime-preview"]
|
AZURE_REALTIME_MODELS = [
|
||||||
|
"gpt-realtime",
|
||||||
|
"gpt-realtime-1.5",
|
||||||
|
"gpt-realtime-mini",
|
||||||
|
]
|
||||||
AZURE_REALTIME_VOICES = [
|
AZURE_REALTIME_VOICES = [
|
||||||
"alloy",
|
"alloy",
|
||||||
"ash",
|
"ash",
|
||||||
|
|
@ -12,6 +16,7 @@ AZURE_REALTIME_VOICES = [
|
||||||
"verse",
|
"verse",
|
||||||
]
|
]
|
||||||
AZURE_REALTIME_API_VERSIONS = [
|
AZURE_REALTIME_API_VERSIONS = [
|
||||||
|
"v1",
|
||||||
"2025-04-01-preview",
|
"2025-04-01-preview",
|
||||||
"2024-10-01-preview",
|
"2024-10-01-preview",
|
||||||
"2024-12-17",
|
"2024-12-17",
|
||||||
|
|
|
||||||
|
|
@ -621,7 +621,7 @@ class OpenAIRealtimeLLMConfiguration(BaseLLMConfiguration):
|
||||||
|
|
||||||
|
|
||||||
GROK_REALTIME_MODELS = ["grok-voice-think-fast-1.0"]
|
GROK_REALTIME_MODELS = ["grok-voice-think-fast-1.0"]
|
||||||
GROK_REALTIME_VOICES = ["Ara", "Rex", "Sal", "Eve", "Leo"]
|
GROK_REALTIME_VOICES = ["ara", "rex", "sal", "eve", "leo"]
|
||||||
ULTRAVOX_REALTIME_MODELS = ["ultravox-v0.7", "fixie-ai/ultravox"]
|
ULTRAVOX_REALTIME_MODELS = ["ultravox-v0.7", "fixie-ai/ultravox"]
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -638,7 +638,7 @@ class GrokRealtimeLLMConfiguration(BaseLLMConfiguration):
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
voice: str = Field(
|
voice: str = Field(
|
||||||
default="Ara",
|
default="ara",
|
||||||
description="Voice the model speaks in.",
|
description="Voice the model speaks in.",
|
||||||
json_schema_extra={
|
json_schema_extra={
|
||||||
"examples": GROK_REALTIME_VOICES,
|
"examples": GROK_REALTIME_VOICES,
|
||||||
|
|
@ -756,7 +756,7 @@ class AzureRealtimeLLMConfiguration(BaseLLMConfiguration):
|
||||||
model_config = AZURE_REALTIME_PROVIDER_MODEL_CONFIG
|
model_config = AZURE_REALTIME_PROVIDER_MODEL_CONFIG
|
||||||
provider: Literal[ServiceProviders.AZURE_REALTIME] = ServiceProviders.AZURE_REALTIME
|
provider: Literal[ServiceProviders.AZURE_REALTIME] = ServiceProviders.AZURE_REALTIME
|
||||||
model: str = Field(
|
model: str = Field(
|
||||||
default="gpt-4o-realtime-preview",
|
default="gpt-realtime",
|
||||||
description="Azure OpenAI realtime deployment name.",
|
description="Azure OpenAI realtime deployment name.",
|
||||||
json_schema_extra={
|
json_schema_extra={
|
||||||
"examples": AZURE_REALTIME_MODELS,
|
"examples": AZURE_REALTIME_MODELS,
|
||||||
|
|
@ -775,8 +775,11 @@ class AzureRealtimeLLMConfiguration(BaseLLMConfiguration):
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
api_version: str = Field(
|
api_version: str = Field(
|
||||||
default="2025-04-01-preview",
|
default="v1",
|
||||||
description="Azure OpenAI API version.",
|
description=(
|
||||||
|
"Azure OpenAI Realtime protocol version. Use 'v1' for the GA API; "
|
||||||
|
"date-based versions select the deprecated preview endpoint."
|
||||||
|
),
|
||||||
json_schema_extra={
|
json_schema_extra={
|
||||||
"examples": AZURE_REALTIME_API_VERSIONS,
|
"examples": AZURE_REALTIME_API_VERSIONS,
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,9 @@
|
||||||
"""Dograh subclass of pipecat's Azure OpenAI Realtime LLM service.
|
"""Dograh subclass of pipecat's Azure OpenAI Realtime LLM service.
|
||||||
|
|
||||||
Layers Dograh engine integration quirks (mute gating, TTSSpeakFrame greeting
|
Layers Dograh engine integration quirks (mute gating, TTSSpeakFrame greeting
|
||||||
trigger, LLMMessagesAppendFrame handling, deferred tool calls) onto pipecat's
|
trigger, LLMMessagesAppendFrame handling, workflow-control deferral) onto
|
||||||
AzureRealtimeLLMService, mirroring what DograhOpenAIRealtimeLLMService does
|
pipecat's AzureRealtimeLLMService, mirroring what
|
||||||
for the standard OpenAI Realtime endpoint.
|
DograhOpenAIRealtimeLLMService does for the standard OpenAI Realtime endpoint.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|
@ -40,7 +40,7 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
|
||||||
- User-mute audio gating
|
- User-mute audio gating
|
||||||
- TTSSpeakFrame as initial-response trigger
|
- TTSSpeakFrame as initial-response trigger
|
||||||
- One-off LLMMessagesAppendFrame handling
|
- One-off LLMMessagesAppendFrame handling
|
||||||
- Deferred tool calls until bot finishes speaking
|
- Workflow-control calls deferred until bot finishes speaking
|
||||||
- finalized=True on TranscriptionFrame for consistency
|
- finalized=True on TranscriptionFrame for consistency
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
@ -49,7 +49,7 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
|
||||||
self._user_is_muted: bool = False
|
self._user_is_muted: bool = False
|
||||||
self._handled_initial_context: bool = False
|
self._handled_initial_context: bool = False
|
||||||
self._bot_is_speaking: bool = False
|
self._bot_is_speaking: bool = False
|
||||||
self._deferred_function_calls: list[FunctionCallFromLLM] = []
|
self._deferred_node_transition_function_calls: list[FunctionCallFromLLM] = []
|
||||||
self._pending_initial_greeting_text: str | None = None
|
self._pending_initial_greeting_text: str | None = None
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
|
@ -81,7 +81,7 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
|
||||||
self._bot_is_speaking = True
|
self._bot_is_speaking = True
|
||||||
elif isinstance(frame, BotStoppedSpeakingFrame):
|
elif isinstance(frame, BotStoppedSpeakingFrame):
|
||||||
self._bot_is_speaking = False
|
self._bot_is_speaking = False
|
||||||
await self._run_pending_function_calls()
|
await self._run_pending_node_transition_function_calls()
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
async def _handle_messages_append(self, frame: LLMMessagesAppendFrame):
|
async def _handle_messages_append(self, frame: LLMMessagesAppendFrame):
|
||||||
|
|
@ -247,18 +247,19 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _run_pending_function_calls(self):
|
async def _run_pending_node_transition_function_calls(self):
|
||||||
if not self._deferred_function_calls:
|
if not self._deferred_node_transition_function_calls:
|
||||||
return
|
return
|
||||||
function_calls = self._deferred_function_calls
|
function_calls = self._deferred_node_transition_function_calls
|
||||||
self._deferred_function_calls = []
|
self._deferred_node_transition_function_calls = []
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"{self}: executing {len(function_calls)} deferred function call(s) "
|
f"{self}: executing {len(function_calls)} deferred workflow-control "
|
||||||
"after bot turn ended"
|
"call(s) after bot turn ended"
|
||||||
)
|
)
|
||||||
await self.run_function_calls(function_calls)
|
await self.run_function_calls(function_calls)
|
||||||
|
|
||||||
async def _handle_evt_function_call_arguments_done(self, evt):
|
async def _handle_evt_function_call_arguments_done(self, evt):
|
||||||
|
"""Run ordinary tools immediately and defer workflow-control calls."""
|
||||||
try:
|
try:
|
||||||
args = json.loads(evt.arguments)
|
args = json.loads(evt.arguments)
|
||||||
|
|
||||||
|
|
@ -275,10 +276,14 @@ class DograhAzureRealtimeLLMService(AzureRealtimeLLMService):
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
|
||||||
if self._bot_is_speaking:
|
is_node_transition = self._function_is_node_transition(
|
||||||
self._deferred_function_calls.extend(function_calls)
|
function_call_item.name
|
||||||
|
)
|
||||||
|
if self._bot_is_speaking and is_node_transition:
|
||||||
|
self._deferred_node_transition_function_calls.extend(function_calls)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"{self}: deferring function call {function_call_item.name} "
|
f"{self}: deferring workflow-control call "
|
||||||
|
f"{function_call_item.name} "
|
||||||
"until bot stops speaking"
|
"until bot stops speaking"
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -11,8 +11,9 @@ Adds:
|
||||||
flow kicks off the bot's first response.
|
flow kicks off the bot's first response.
|
||||||
- **One-off LLMMessagesAppendFrame handling** for ephemeral realtime prompts
|
- **One-off LLMMessagesAppendFrame handling** for ephemeral realtime prompts
|
||||||
like user-idle checks, without mutating Dograh's local ``LLMContext``.
|
like user-idle checks, without mutating Dograh's local ``LLMContext``.
|
||||||
- **Function-call deferral** until the bot finishes speaking, to avoid racing
|
- **Workflow-control deferral** so node transitions, call termination, and
|
||||||
tool execution with the active audio turn.
|
transfers wait for any current bot audio to finish while ordinary tools run
|
||||||
|
immediately.
|
||||||
- **finalized=True on TranscriptionFrame** for parity with Dograh's other
|
- **finalized=True on TranscriptionFrame** for parity with Dograh's other
|
||||||
realtime providers.
|
realtime providers.
|
||||||
"""
|
"""
|
||||||
|
|
@ -50,7 +51,7 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
|
||||||
self._user_is_muted: bool = False
|
self._user_is_muted: bool = False
|
||||||
self._handled_initial_context: bool = False
|
self._handled_initial_context: bool = False
|
||||||
self._bot_is_speaking: bool = False
|
self._bot_is_speaking: bool = False
|
||||||
self._deferred_function_calls: list[FunctionCallFromLLM] = []
|
self._deferred_node_transition_function_calls: list[FunctionCallFromLLM] = []
|
||||||
self._pending_initial_greeting_text: str | None = None
|
self._pending_initial_greeting_text: str | None = None
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
|
@ -82,7 +83,7 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
|
||||||
self._bot_is_speaking = True
|
self._bot_is_speaking = True
|
||||||
elif isinstance(frame, BotStoppedSpeakingFrame):
|
elif isinstance(frame, BotStoppedSpeakingFrame):
|
||||||
self._bot_is_speaking = False
|
self._bot_is_speaking = False
|
||||||
await self._run_pending_function_calls()
|
await self._run_pending_node_transition_function_calls()
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
async def _handle_messages_append(self, frame: LLMMessagesAppendFrame):
|
async def _handle_messages_append(self, frame: LLMMessagesAppendFrame):
|
||||||
|
|
@ -251,19 +252,19 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _run_pending_function_calls(self):
|
async def _run_pending_node_transition_function_calls(self):
|
||||||
if not self._deferred_function_calls:
|
if not self._deferred_node_transition_function_calls:
|
||||||
return
|
return
|
||||||
function_calls = self._deferred_function_calls
|
function_calls = self._deferred_node_transition_function_calls
|
||||||
self._deferred_function_calls = []
|
self._deferred_node_transition_function_calls = []
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"{self}: executing {len(function_calls)} deferred function call(s) "
|
f"{self}: executing {len(function_calls)} deferred workflow-control "
|
||||||
"after bot turn ended"
|
"call(s) after bot turn ended"
|
||||||
)
|
)
|
||||||
await self.run_function_calls(function_calls)
|
await self.run_function_calls(function_calls)
|
||||||
|
|
||||||
async def _handle_evt_function_call_arguments_done(self, evt):
|
async def _handle_evt_function_call_arguments_done(self, evt):
|
||||||
"""Process or defer tool calls until the bot finishes speaking."""
|
"""Run ordinary tools immediately and defer workflow-control calls."""
|
||||||
try:
|
try:
|
||||||
args = json.loads(evt.arguments)
|
args = json.loads(evt.arguments)
|
||||||
|
|
||||||
|
|
@ -281,10 +282,11 @@ class DograhGrokRealtimeLLMService(GrokRealtimeLLMService):
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
|
||||||
if self._bot_is_speaking:
|
is_node_transition = self._function_is_node_transition(function_name)
|
||||||
self._deferred_function_calls.extend(function_calls)
|
if self._bot_is_speaking and is_node_transition:
|
||||||
|
self._deferred_node_transition_function_calls.extend(function_calls)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"{self}: deferring function call {function_name} "
|
f"{self}: deferring workflow-control call {function_name} "
|
||||||
"until bot stops speaking"
|
"until bot stops speaking"
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -3,8 +3,8 @@
|
||||||
Layers Dograh engine integration quirks onto upstream-pristine
|
Layers Dograh engine integration quirks onto upstream-pristine
|
||||||
:class:`OpenAIRealtimeLLMService`. Substantially smaller than the Gemini
|
:class:`OpenAIRealtimeLLMService`. Substantially smaller than the Gemini
|
||||||
subclass because OpenAI Realtime supports runtime ``session.update`` for
|
subclass because OpenAI Realtime supports runtime ``session.update`` for
|
||||||
both ``system_instruction`` and tools — no reconnect/defer-tool-call
|
both ``system_instruction`` and tools, so node changes do not require a
|
||||||
machinery needed.
|
reconnect.
|
||||||
|
|
||||||
Adds:
|
Adds:
|
||||||
|
|
||||||
|
|
@ -13,6 +13,9 @@ Adds:
|
||||||
flow kicks off the bot's first response.
|
flow kicks off the bot's first response.
|
||||||
- **One-off LLMMessagesAppendFrame handling** for ephemeral realtime prompts
|
- **One-off LLMMessagesAppendFrame handling** for ephemeral realtime prompts
|
||||||
like user-idle checks, without mutating Dograh's local ``LLMContext``.
|
like user-idle checks, without mutating Dograh's local ``LLMContext``.
|
||||||
|
- **Workflow-control deferral** so node transitions, call termination, and
|
||||||
|
transfers wait for any current bot audio to finish while ordinary tools run
|
||||||
|
immediately.
|
||||||
- **finalized=True on TranscriptionFrame** because every OpenAI
|
- **finalized=True on TranscriptionFrame** because every OpenAI
|
||||||
transcription via the ``completed`` event is final by construction.
|
transcription via the ``completed`` event is final by construction.
|
||||||
"""
|
"""
|
||||||
|
|
@ -53,10 +56,10 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
|
||||||
# LLMContextFrame arrives, so upstream's "first arrival means
|
# LLMContextFrame arrives, so upstream's "first arrival means
|
||||||
# self._context is None" check no longer works.
|
# self._context is None" check no longer works.
|
||||||
self._handled_initial_context: bool = False
|
self._handled_initial_context: bool = False
|
||||||
# Track bot speech locally so tool calls can be deferred until the bot
|
# Track bot speech locally so workflow-control calls can wait until the
|
||||||
# has finished speaking, matching Dograh's Gemini Live behavior.
|
# bot has finished speaking without delaying ordinary tools.
|
||||||
self._bot_is_speaking: bool = False
|
self._bot_is_speaking: bool = False
|
||||||
self._deferred_function_calls: list[FunctionCallFromLLM] = []
|
self._deferred_node_transition_function_calls: list[FunctionCallFromLLM] = []
|
||||||
self._pending_initial_greeting_text: str | None = None
|
self._pending_initial_greeting_text: str | None = None
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|
@ -100,7 +103,7 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
|
||||||
self._bot_is_speaking = True
|
self._bot_is_speaking = True
|
||||||
elif isinstance(frame, BotStoppedSpeakingFrame):
|
elif isinstance(frame, BotStoppedSpeakingFrame):
|
||||||
self._bot_is_speaking = False
|
self._bot_is_speaking = False
|
||||||
await self._run_pending_function_calls()
|
await self._run_pending_node_transition_function_calls()
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
async def _handle_messages_append(self, frame: LLMMessagesAppendFrame):
|
async def _handle_messages_append(self, frame: LLMMessagesAppendFrame):
|
||||||
|
|
@ -268,19 +271,19 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _run_pending_function_calls(self):
|
async def _run_pending_node_transition_function_calls(self):
|
||||||
if not self._deferred_function_calls:
|
if not self._deferred_node_transition_function_calls:
|
||||||
return
|
return
|
||||||
function_calls = self._deferred_function_calls
|
function_calls = self._deferred_node_transition_function_calls
|
||||||
self._deferred_function_calls = []
|
self._deferred_node_transition_function_calls = []
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"{self}: executing {len(function_calls)} deferred function call(s) "
|
f"{self}: executing {len(function_calls)} deferred workflow-control "
|
||||||
"after bot turn ended"
|
"call(s) after bot turn ended"
|
||||||
)
|
)
|
||||||
await self.run_function_calls(function_calls)
|
await self.run_function_calls(function_calls)
|
||||||
|
|
||||||
async def _handle_evt_function_call_arguments_done(self, evt):
|
async def _handle_evt_function_call_arguments_done(self, evt):
|
||||||
"""Process or defer tool calls until the bot finishes speaking."""
|
"""Run ordinary tools immediately and defer workflow-control calls."""
|
||||||
try:
|
try:
|
||||||
args = json.loads(evt.arguments)
|
args = json.loads(evt.arguments)
|
||||||
|
|
||||||
|
|
@ -297,10 +300,14 @@ class DograhOpenAIRealtimeLLMService(OpenAIRealtimeLLMService):
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
|
||||||
if self._bot_is_speaking:
|
is_node_transition = self._function_is_node_transition(
|
||||||
self._deferred_function_calls.extend(function_calls)
|
function_call_item.name
|
||||||
|
)
|
||||||
|
if self._bot_is_speaking and is_node_transition:
|
||||||
|
self._deferred_node_transition_function_calls.extend(function_calls)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"{self}: deferring function call {function_call_item.name} "
|
f"{self}: deferring workflow-control call "
|
||||||
|
f"{function_call_item.name} "
|
||||||
"until bot stops speaking"
|
"until bot stops speaking"
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,16 @@
|
||||||
"""Dograh subclass of pipecat's Ultravox realtime LLM service.
|
"""Dograh subclass of pipecat's Ultravox realtime LLM service.
|
||||||
|
|
||||||
Ultravox is audio-native and realtime, but prompt and tool configuration is
|
Ultravox is audio-native and realtime. Its native call stages allow a client
|
||||||
bound to call creation. Dograh therefore cannot lean on in-session updates or
|
tool result to atomically change the system prompt and tools while preserving
|
||||||
Gemini-style session resumption handles. This wrapper adapts Ultravox to the
|
the call's server-side conversation history. This wrapper adapts that model to
|
||||||
Dograh engine contract by:
|
the Dograh engine contract by:
|
||||||
|
|
||||||
- deferring the first call creation until the engine queues the initial node
|
- deferring the first call creation until the engine queues the initial node
|
||||||
opening via ``TTSSpeakFrame`` or ``LLMContextFrame``
|
opening via ``TTSSpeakFrame`` or ``LLMContextFrame``
|
||||||
- marking the call for recreation when ``system_instruction`` changes across
|
- returning node-transition tool results with ``responseType="new-stage"`` so
|
||||||
node transitions, then rebuilding it on the follow-up ``LLMContextFrame``
|
the existing call keeps its complete audio-native history
|
||||||
so the transition tool result is present in ``initialMessages``
|
- updating the next stage's system prompt and selected tools without a
|
||||||
- reconstructing Ultravox ``initialMessages`` from Dograh context when the
|
disconnect/reconnect cycle
|
||||||
call must be recreated after a node transition
|
|
||||||
- appending a transient resumptive user nudge to recreated ``initialMessages``
|
|
||||||
after tool-result transitions, without mutating Dograh's stored context
|
|
||||||
- handling Dograh-only frames such as user mute and idle append prompts
|
- handling Dograh-only frames such as user mute and idle append prompts
|
||||||
- tagging user transcripts with ``finalized=True`` for downstream parity
|
- tagging user transcripts with ``finalized=True`` for downstream parity
|
||||||
"""
|
"""
|
||||||
|
|
@ -34,12 +31,7 @@ from pipecat.frames.frames import (
|
||||||
UserMuteStartedFrame,
|
UserMuteStartedFrame,
|
||||||
UserMuteStoppedFrame,
|
UserMuteStoppedFrame,
|
||||||
)
|
)
|
||||||
from pipecat.processors.aggregators import async_tool_messages
|
from pipecat.processors.aggregators.llm_context import LLMContext, is_given
|
||||||
from pipecat.processors.aggregators.llm_context import (
|
|
||||||
LLMContext,
|
|
||||||
LLMSpecificMessage,
|
|
||||||
is_given,
|
|
||||||
)
|
|
||||||
from pipecat.processors.frame_processor import FrameDirection
|
from pipecat.processors.frame_processor import FrameDirection
|
||||||
from pipecat.services.llm_service import LLMService
|
from pipecat.services.llm_service import LLMService
|
||||||
from pipecat.services.settings import _NotGiven, assert_given
|
from pipecat.services.settings import _NotGiven, assert_given
|
||||||
|
|
@ -58,10 +50,6 @@ class DograhUltravoxOneShotInputParams(OneShotInputParams):
|
||||||
|
|
||||||
|
|
||||||
_ULTRAVOX_MAX_TOOL_TIMEOUT_SECS = 40.0
|
_ULTRAVOX_MAX_TOOL_TIMEOUT_SECS = 40.0
|
||||||
_RESUMPTION_USER_MESSAGE = (
|
|
||||||
"IMPORTANT: We are resuming an existing conversation. You are given previous turns ONLY for your reference. "
|
|
||||||
"Do not use that to frame your response. Follow your ORIGINAL INSTRUCTIONS ONLY."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
|
|
@ -72,12 +60,13 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
self._context: LLMContext | None = None
|
self._context: LLMContext | None = None
|
||||||
self._selected_tools = None
|
self._selected_tools = None
|
||||||
self._user_is_muted: bool = False
|
self._user_is_muted: bool = False
|
||||||
self._call_system_instruction: str | None = None
|
|
||||||
self._reconnect_required: bool = False
|
|
||||||
self._call_started: bool = False
|
self._call_started: bool = False
|
||||||
self._has_connected_once: bool = False
|
self._stage_update_required: bool = False
|
||||||
self._pending_reconnect_system_instruction: str | None = None
|
# Ultravox applies a stage update on the matching client tool result,
|
||||||
self._pending_initial_messages: list[dict[str, Any]] | None = None
|
# so retain the provider invocation ID until that result reaches us via
|
||||||
|
# the context aggregator. Unlike Gemini, this ID is part of the wire
|
||||||
|
# protocol needed to update the existing call without reconnecting.
|
||||||
|
self._pending_node_transition_tool_call_ids: set[str] = set()
|
||||||
self._pending_user_text_messages: list[str] = []
|
self._pending_user_text_messages: list[str] = []
|
||||||
|
|
||||||
async def start(self, frame):
|
async def start(self, frame):
|
||||||
|
|
@ -96,9 +85,7 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
if isinstance(frame, TTSSpeakFrame):
|
if isinstance(frame, TTSSpeakFrame):
|
||||||
if not self._socket:
|
if not self._socket:
|
||||||
await self._connect_call(
|
await self._connect_call(
|
||||||
system_instruction=self._current_system_instruction(),
|
|
||||||
greeting_text=frame.text,
|
greeting_text=frame.text,
|
||||||
initial_messages=None,
|
|
||||||
agent_speaks_first=True,
|
agent_speaks_first=True,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
|
|
@ -116,18 +103,15 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
changed = await super(UltravoxRealtimeLLMService, self)._update_settings(delta)
|
changed = await super(UltravoxRealtimeLLMService, self)._update_settings(delta)
|
||||||
if "output_medium" in changed:
|
if "output_medium" in changed:
|
||||||
await self._update_output_medium(assert_given(self._settings.output_medium))
|
await self._update_output_medium(assert_given(self._settings.output_medium))
|
||||||
if "system_instruction" in changed and self._has_connected_once:
|
if "system_instruction" in changed and self._socket:
|
||||||
# Mirror Gemini's "settings change means reconnect" intent, but
|
# The updated instruction is included in the native new-stage
|
||||||
# defer the actual new-call creation until the subsequent
|
# response when the transition tool result reaches _handle_context.
|
||||||
# LLMContextFrame arrives with the transition tool result. Ultravox
|
self._stage_update_required = True
|
||||||
# cannot accept that historical tool result over a formal
|
|
||||||
# post-connect tool-response channel the way Gemini can.
|
|
||||||
self._reconnect_required = True
|
|
||||||
handled = {"output_medium", "system_instruction"}
|
handled = {"output_medium", "system_instruction"}
|
||||||
self._warn_unhandled_updated_settings(changed.keys() - handled)
|
self._warn_unhandled_updated_settings(changed.keys() - handled)
|
||||||
return changed
|
return changed
|
||||||
|
|
||||||
async def _disconnect(self, preserve_completed_tool_calls: bool = True):
|
async def _disconnect(self):
|
||||||
self._disconnecting = True
|
self._disconnecting = True
|
||||||
await self.stop_all_metrics()
|
await self.stop_all_metrics()
|
||||||
if self._socket:
|
if self._socket:
|
||||||
|
|
@ -136,10 +120,10 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
if self._receive_task:
|
if self._receive_task:
|
||||||
await self.cancel_task(self._receive_task, timeout=1.0)
|
await self.cancel_task(self._receive_task, timeout=1.0)
|
||||||
self._receive_task = None
|
self._receive_task = None
|
||||||
if not preserve_completed_tool_calls:
|
self._completed_tool_calls = set()
|
||||||
self._completed_tool_calls = set()
|
|
||||||
self._call_started = False
|
self._call_started = False
|
||||||
self._started_placeholder_sent = set()
|
self._started_placeholder_sent = set()
|
||||||
|
self._pending_node_transition_tool_call_ids = set()
|
||||||
self._disconnecting = False
|
self._disconnecting = False
|
||||||
|
|
||||||
async def _send_user_audio(self, frame):
|
async def _send_user_audio(self, frame):
|
||||||
|
|
@ -149,39 +133,20 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
|
|
||||||
async def _handle_context(self, context: LLMContext):
|
async def _handle_context(self, context: LLMContext):
|
||||||
self._context = context
|
self._context = context
|
||||||
system_instruction = self._current_system_instruction()
|
|
||||||
|
|
||||||
if self._socket and not self._reconnect_required:
|
if not self._socket:
|
||||||
await super()._handle_context(context)
|
await self._connect_call(
|
||||||
|
greeting_text=None,
|
||||||
|
agent_speaks_first=True,
|
||||||
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
initial_messages, history_tool_call_ids = self._build_initial_messages(context)
|
current_tools = self._current_tools_schema(context)
|
||||||
if history_tool_call_ids:
|
if self._pending_node_transition_tool_call_ids and self._tools_changed(
|
||||||
self._completed_tool_calls.update(history_tool_call_ids)
|
current_tools
|
||||||
|
):
|
||||||
if self._bot_responding:
|
self._stage_update_required = True
|
||||||
self._pending_reconnect_system_instruction = system_instruction
|
await super()._handle_context(context)
|
||||||
self._pending_initial_messages = initial_messages
|
|
||||||
return
|
|
||||||
|
|
||||||
await self._reconnect_with_context(
|
|
||||||
system_instruction=system_instruction,
|
|
||||||
initial_messages=initial_messages,
|
|
||||||
)
|
|
||||||
|
|
||||||
async def _handle_response_end(self):
|
|
||||||
await super()._handle_response_end()
|
|
||||||
if self._pending_reconnect_system_instruction is None:
|
|
||||||
return
|
|
||||||
|
|
||||||
system_instruction = self._pending_reconnect_system_instruction
|
|
||||||
initial_messages = self._pending_initial_messages
|
|
||||||
self._pending_reconnect_system_instruction = None
|
|
||||||
self._pending_initial_messages = None
|
|
||||||
await self._reconnect_with_context(
|
|
||||||
system_instruction=system_instruction,
|
|
||||||
initial_messages=initial_messages,
|
|
||||||
)
|
|
||||||
|
|
||||||
async def _handle_messages_append(self, frame: LLMMessagesAppendFrame):
|
async def _handle_messages_append(self, frame: LLMMessagesAppendFrame):
|
||||||
texts = [
|
texts = [
|
||||||
|
|
@ -199,9 +164,7 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
if not self._socket:
|
if not self._socket:
|
||||||
self._pending_user_text_messages.extend(texts)
|
self._pending_user_text_messages.extend(texts)
|
||||||
await self._connect_call(
|
await self._connect_call(
|
||||||
system_instruction=self._current_system_instruction(),
|
|
||||||
greeting_text=None,
|
greeting_text=None,
|
||||||
initial_messages=None,
|
|
||||||
agent_speaks_first=False,
|
agent_speaks_first=False,
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
@ -229,17 +192,66 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
finalized=True,
|
finalized=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _requires_node_transition_context_aggregation(self) -> bool:
|
||||||
|
"""Commit any received final user transcript before changing stages.
|
||||||
|
|
||||||
|
Ultravox preserves its own audio-native history across a stage change,
|
||||||
|
but Dograh's local context still needs the final transcript before the
|
||||||
|
transition handler updates the workflow node.
|
||||||
|
"""
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def _handle_tool_invocation(
|
||||||
|
self, tool_name: str, invocation_id: str, parameters: dict[str, Any]
|
||||||
|
):
|
||||||
|
if self._function_is_node_transition(tool_name):
|
||||||
|
self._pending_node_transition_tool_call_ids.add(invocation_id)
|
||||||
|
await super()._handle_tool_invocation(tool_name, invocation_id, parameters)
|
||||||
|
|
||||||
|
async def _send_tool_result(self, tool_call_id: str, result: str):
|
||||||
|
is_node_transition = tool_call_id in self._pending_node_transition_tool_call_ids
|
||||||
|
try:
|
||||||
|
if is_node_transition and self._stage_update_required:
|
||||||
|
await self._send_node_transition_stage_result(tool_call_id, result)
|
||||||
|
else:
|
||||||
|
await super()._send_tool_result(tool_call_id, result)
|
||||||
|
finally:
|
||||||
|
if is_node_transition:
|
||||||
|
self._pending_node_transition_tool_call_ids.discard(tool_call_id)
|
||||||
|
|
||||||
|
async def _send_node_transition_stage_result(self, tool_call_id: str, result: str):
|
||||||
|
"""Apply node settings using Ultravox's native call-stage protocol."""
|
||||||
|
next_tools = self._current_tools_schema(self._context)
|
||||||
|
stage = {
|
||||||
|
"systemPrompt": self._current_system_instruction(),
|
||||||
|
"selectedTools": self._selected_tools_payload(next_tools),
|
||||||
|
# Keep the workflow handler's result as the tool-result message in
|
||||||
|
# the inherited conversation history for the next generation.
|
||||||
|
"toolResultText": result,
|
||||||
|
}
|
||||||
|
logger.debug(
|
||||||
|
f"{self}: updating Ultravox call stage for tool_call_id={tool_call_id} "
|
||||||
|
f"with {len(stage['selectedTools'])} selected tool(s)"
|
||||||
|
)
|
||||||
|
await self._send(
|
||||||
|
{
|
||||||
|
"type": "client_tool_result",
|
||||||
|
"invocationId": tool_call_id,
|
||||||
|
"result": json.dumps(stage, ensure_ascii=True, default=str),
|
||||||
|
"responseType": "new-stage",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self._selected_tools = next_tools
|
||||||
|
self._stage_update_required = False
|
||||||
|
|
||||||
async def _connect_call(
|
async def _connect_call(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
system_instruction: str | None,
|
|
||||||
greeting_text: str | None,
|
greeting_text: str | None,
|
||||||
initial_messages: list[dict[str, Any]] | None,
|
|
||||||
agent_speaks_first: bool,
|
agent_speaks_first: bool,
|
||||||
):
|
):
|
||||||
params = self._build_one_shot_params(
|
params = self._build_one_shot_params(
|
||||||
greeting_text=greeting_text,
|
greeting_text=greeting_text,
|
||||||
initial_messages=initial_messages,
|
|
||||||
agent_speaks_first=agent_speaks_first,
|
agent_speaks_first=agent_speaks_first,
|
||||||
)
|
)
|
||||||
self._params = params
|
self._params = params
|
||||||
|
|
@ -265,9 +277,7 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
logger.info(f"Joining Ultravox Realtime call via URL: {join_url}")
|
logger.info(f"Joining Ultravox Realtime call via URL: {join_url}")
|
||||||
self._socket = await websocket_client.connect(join_url)
|
self._socket = await websocket_client.connect(join_url)
|
||||||
self._receive_task = self.create_task(self._receive_messages())
|
self._receive_task = self.create_task(self._receive_messages())
|
||||||
self._call_system_instruction = system_instruction
|
|
||||||
self._call_started = False
|
self._call_started = False
|
||||||
self._has_connected_once = True
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"{self}: Ultravox call creation/join failed "
|
f"{self}: Ultravox call creation/join failed "
|
||||||
|
|
@ -365,40 +375,17 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
for pending_text in pending_texts:
|
for pending_text in pending_texts:
|
||||||
await self._send_user_text(pending_text)
|
await self._send_user_text(pending_text)
|
||||||
|
|
||||||
async def _reconnect_with_context(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
system_instruction: str | None,
|
|
||||||
initial_messages: list[dict[str, Any]] | None,
|
|
||||||
):
|
|
||||||
call_initial_messages = self._initial_messages_for_call(initial_messages)
|
|
||||||
logger.debug(
|
|
||||||
f"{self}: reconnecting Ultravox call with initialMessages="
|
|
||||||
f"{json.dumps(call_initial_messages, ensure_ascii=True, default=str)}"
|
|
||||||
)
|
|
||||||
if self._socket:
|
|
||||||
await self._disconnect(preserve_completed_tool_calls=True)
|
|
||||||
|
|
||||||
await self._connect_call(
|
|
||||||
system_instruction=system_instruction,
|
|
||||||
greeting_text=None,
|
|
||||||
initial_messages=initial_messages,
|
|
||||||
agent_speaks_first=self._should_agent_speak_first(initial_messages),
|
|
||||||
)
|
|
||||||
self._reconnect_required = False
|
|
||||||
|
|
||||||
def _build_one_shot_params(
|
def _build_one_shot_params(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
greeting_text: str | None,
|
greeting_text: str | None,
|
||||||
initial_messages: list[dict[str, Any]] | None,
|
|
||||||
agent_speaks_first: bool,
|
agent_speaks_first: bool,
|
||||||
) -> DograhUltravoxOneShotInputParams:
|
) -> DograhUltravoxOneShotInputParams:
|
||||||
current_params = self._params
|
current_params = self._params
|
||||||
extra = {
|
extra = {
|
||||||
key: value
|
key: value
|
||||||
for key, value in current_params.extra.items()
|
for key, value in current_params.extra.items()
|
||||||
if key not in {"firstSpeakerSettings", "initialMessages"}
|
if key != "firstSpeakerSettings"
|
||||||
}
|
}
|
||||||
|
|
||||||
if greeting_text is not None:
|
if greeting_text is not None:
|
||||||
|
|
@ -407,10 +394,6 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
extra["firstSpeakerSettings"] = {"agent": {}}
|
extra["firstSpeakerSettings"] = {"agent": {}}
|
||||||
else:
|
else:
|
||||||
extra["firstSpeakerSettings"] = {"user": {}}
|
extra["firstSpeakerSettings"] = {"user": {}}
|
||||||
call_initial_messages = self._initial_messages_for_call(initial_messages)
|
|
||||||
if call_initial_messages:
|
|
||||||
extra["initialMessages"] = call_initial_messages
|
|
||||||
|
|
||||||
output_medium = self._settings.output_medium
|
output_medium = self._settings.output_medium
|
||||||
if isinstance(output_medium, _NotGiven):
|
if isinstance(output_medium, _NotGiven):
|
||||||
output_medium = current_params.output_medium
|
output_medium = current_params.output_medium
|
||||||
|
|
@ -432,6 +415,14 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
return None
|
return None
|
||||||
return context.tools
|
return context.tools
|
||||||
|
|
||||||
|
def _selected_tools_payload(self, tools: Any) -> list[dict[str, Any]]:
|
||||||
|
return self._to_selected_tools(tools) if tools else []
|
||||||
|
|
||||||
|
def _tools_changed(self, tools: Any) -> bool:
|
||||||
|
return self._selected_tools_payload(tools) != self._selected_tools_payload(
|
||||||
|
self._selected_tools
|
||||||
|
)
|
||||||
|
|
||||||
def _to_selected_tools(self, tool: Any) -> list[dict[str, Any]]:
|
def _to_selected_tools(self, tool: Any) -> list[dict[str, Any]]:
|
||||||
selected_tools = super()._to_selected_tools(tool)
|
selected_tools = super()._to_selected_tools(tool)
|
||||||
for selected_tool in selected_tools:
|
for selected_tool in selected_tools:
|
||||||
|
|
@ -462,156 +453,6 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
timeout_secs = min(float(item.timeout_secs), _ULTRAVOX_MAX_TOOL_TIMEOUT_SECS)
|
timeout_secs = min(float(item.timeout_secs), _ULTRAVOX_MAX_TOOL_TIMEOUT_SECS)
|
||||||
return f"{timeout_secs:g}s"
|
return f"{timeout_secs:g}s"
|
||||||
|
|
||||||
def _initial_messages_for_call(
|
|
||||||
self, initial_messages: list[dict[str, Any]] | None
|
|
||||||
) -> list[dict[str, Any]] | None:
|
|
||||||
if not initial_messages:
|
|
||||||
return None
|
|
||||||
if not self._should_add_resumption_user_message(initial_messages):
|
|
||||||
return initial_messages
|
|
||||||
|
|
||||||
return [
|
|
||||||
*initial_messages,
|
|
||||||
{
|
|
||||||
"role": "MESSAGE_ROLE_USER",
|
|
||||||
"text": _RESUMPTION_USER_MESSAGE,
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
def _build_initial_messages(
|
|
||||||
self, context: LLMContext
|
|
||||||
) -> tuple[list[dict[str, Any]] | None, set[str]]:
|
|
||||||
initial_messages: list[dict[str, Any]] = []
|
|
||||||
tool_call_id_to_name: dict[str, str] = {}
|
|
||||||
completed_tool_call_ids: set[str] = set()
|
|
||||||
|
|
||||||
for message in context.get_messages():
|
|
||||||
if isinstance(message, LLMSpecificMessage):
|
|
||||||
continue
|
|
||||||
|
|
||||||
async_payload = async_tool_messages.parse_message(message)
|
|
||||||
if async_payload is not None:
|
|
||||||
if async_payload.kind == "intermediate":
|
|
||||||
logger.error(
|
|
||||||
f"{self}: Ultravox does not support streamed async tool results; "
|
|
||||||
f"dropping intermediate result from initialMessages for "
|
|
||||||
f"tool_call_id={async_payload.tool_call_id}."
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
if async_payload.kind == "final":
|
|
||||||
initial_message = self._build_ultravox_message(
|
|
||||||
role="MESSAGE_ROLE_TOOL_RESULT",
|
|
||||||
text=async_payload.result or "",
|
|
||||||
invocation_id=async_payload.tool_call_id,
|
|
||||||
tool_name=tool_call_id_to_name.get(async_payload.tool_call_id),
|
|
||||||
)
|
|
||||||
if initial_message is not None:
|
|
||||||
initial_messages.append(initial_message)
|
|
||||||
completed_tool_call_ids.add(async_payload.tool_call_id)
|
|
||||||
continue
|
|
||||||
|
|
||||||
role = message.get("role")
|
|
||||||
if role == "user":
|
|
||||||
initial_message = self._build_ultravox_message(
|
|
||||||
role="MESSAGE_ROLE_USER",
|
|
||||||
text=self._extract_text_content(message.get("content")),
|
|
||||||
)
|
|
||||||
if initial_message is not None:
|
|
||||||
initial_messages.append(initial_message)
|
|
||||||
elif role == "assistant":
|
|
||||||
text = self._extract_text_content(message.get("content"))
|
|
||||||
initial_message = self._build_ultravox_message(
|
|
||||||
role="MESSAGE_ROLE_AGENT",
|
|
||||||
text=text,
|
|
||||||
)
|
|
||||||
if initial_message is not None:
|
|
||||||
initial_messages.append(initial_message)
|
|
||||||
|
|
||||||
tool_calls = message.get("tool_calls")
|
|
||||||
if isinstance(tool_calls, list):
|
|
||||||
for tool_call in tool_calls:
|
|
||||||
if not isinstance(tool_call, dict):
|
|
||||||
continue
|
|
||||||
tool_id = tool_call.get("id")
|
|
||||||
function = tool_call.get("function")
|
|
||||||
tool_name = (
|
|
||||||
function.get("name") if isinstance(function, dict) else None
|
|
||||||
)
|
|
||||||
if isinstance(tool_id, str) and isinstance(tool_name, str):
|
|
||||||
tool_call_id_to_name[tool_id] = tool_name
|
|
||||||
initial_message = self._build_ultravox_message(
|
|
||||||
role="MESSAGE_ROLE_TOOL_CALL",
|
|
||||||
text="",
|
|
||||||
invocation_id=tool_id,
|
|
||||||
tool_name=tool_name,
|
|
||||||
)
|
|
||||||
if initial_message is not None:
|
|
||||||
initial_messages.append(initial_message)
|
|
||||||
elif (
|
|
||||||
role == "tool"
|
|
||||||
and message.get("content") != "IN_PROGRESS"
|
|
||||||
and message.get("content") != "CANCELLED"
|
|
||||||
):
|
|
||||||
tool_call_id = message.get("tool_call_id")
|
|
||||||
initial_message = self._build_ultravox_message(
|
|
||||||
role="MESSAGE_ROLE_TOOL_RESULT",
|
|
||||||
text=self._stringify_tool_result(message.get("content")),
|
|
||||||
invocation_id=tool_call_id
|
|
||||||
if isinstance(tool_call_id, str)
|
|
||||||
else None,
|
|
||||||
tool_name=(
|
|
||||||
tool_call_id_to_name.get(tool_call_id)
|
|
||||||
if isinstance(tool_call_id, str)
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
)
|
|
||||||
if initial_message is not None:
|
|
||||||
initial_messages.append(initial_message)
|
|
||||||
if isinstance(tool_call_id, str):
|
|
||||||
completed_tool_call_ids.add(tool_call_id)
|
|
||||||
|
|
||||||
return (initial_messages or None), completed_tool_call_ids
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _build_ultravox_message(
|
|
||||||
*,
|
|
||||||
role: str,
|
|
||||||
text: str | None,
|
|
||||||
invocation_id: str | None = None,
|
|
||||||
tool_name: str | None = None,
|
|
||||||
) -> dict[str, Any] | None:
|
|
||||||
if text is None:
|
|
||||||
return None
|
|
||||||
|
|
||||||
message: dict[str, Any] = {
|
|
||||||
"role": role,
|
|
||||||
"text": text,
|
|
||||||
}
|
|
||||||
if invocation_id is not None:
|
|
||||||
message["invocationId"] = invocation_id
|
|
||||||
if tool_name is not None:
|
|
||||||
message["toolName"] = tool_name
|
|
||||||
return message
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _should_agent_speak_first(
|
|
||||||
initial_messages: list[dict[str, Any]] | None,
|
|
||||||
) -> bool:
|
|
||||||
if not initial_messages:
|
|
||||||
return True
|
|
||||||
return initial_messages[-1].get("role") in {
|
|
||||||
"MESSAGE_ROLE_USER",
|
|
||||||
"MESSAGE_ROLE_TOOL_RESULT",
|
|
||||||
}
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _should_add_resumption_user_message(
|
|
||||||
initial_messages: list[dict[str, Any]] | None,
|
|
||||||
) -> bool:
|
|
||||||
if not initial_messages:
|
|
||||||
return False
|
|
||||||
return initial_messages[-1].get("role") == "MESSAGE_ROLE_TOOL_RESULT"
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _is_benign_websocket_close(exc: ConnectionClosed) -> bool:
|
def _is_benign_websocket_close(exc: ConnectionClosed) -> bool:
|
||||||
return any(
|
return any(
|
||||||
|
|
@ -636,18 +477,3 @@ class DograhUltravoxRealtimeLLMService(UltravoxRealtimeLLMService):
|
||||||
parts.append(text)
|
parts.append(text)
|
||||||
return "\n".join(parts) if parts else None
|
return "\n".join(parts) if parts else None
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _stringify_tool_result(content: Any) -> str:
|
|
||||||
if isinstance(content, str):
|
|
||||||
return content
|
|
||||||
if isinstance(content, list):
|
|
||||||
parts: list[str] = []
|
|
||||||
for part in content:
|
|
||||||
if isinstance(part, dict):
|
|
||||||
text = part.get("text")
|
|
||||||
if isinstance(text, str):
|
|
||||||
parts.append(text)
|
|
||||||
if parts:
|
|
||||||
return "".join(parts)
|
|
||||||
return json.dumps(content, ensure_ascii=True, default=str)
|
|
||||||
|
|
|
||||||
|
|
@ -1029,14 +1029,28 @@ def create_realtime_llm_service(user_config, audio_config: "AudioConfig"):
|
||||||
from api.services.pipecat.realtime.grok_realtime import (
|
from api.services.pipecat.realtime.grok_realtime import (
|
||||||
DograhGrokRealtimeLLMService,
|
DograhGrokRealtimeLLMService,
|
||||||
)
|
)
|
||||||
from pipecat.services.xai.realtime.events import SessionProperties
|
from pipecat.services.xai.realtime.events import (
|
||||||
|
AudioConfiguration,
|
||||||
|
AudioInput,
|
||||||
|
InputAudioTranscription,
|
||||||
|
SessionProperties,
|
||||||
|
)
|
||||||
|
|
||||||
|
grok_voice = voice or "ara"
|
||||||
|
if grok_voice.lower() in {"ara", "rex", "sal", "eve", "leo"}:
|
||||||
|
grok_voice = grok_voice.lower()
|
||||||
|
|
||||||
return DograhGrokRealtimeLLMService(
|
return DograhGrokRealtimeLLMService(
|
||||||
api_key=api_key,
|
api_key=api_key,
|
||||||
settings=DograhGrokRealtimeLLMService.Settings(
|
settings=DograhGrokRealtimeLLMService.Settings(
|
||||||
model=model,
|
model=model,
|
||||||
session_properties=SessionProperties(
|
session_properties=SessionProperties(
|
||||||
voice=voice or "Ara",
|
voice=grok_voice,
|
||||||
|
audio=AudioConfiguration(
|
||||||
|
input=AudioInput(
|
||||||
|
transcription=InputAudioTranscription(),
|
||||||
|
),
|
||||||
|
),
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
@ -1115,19 +1129,25 @@ def create_realtime_llm_service(user_config, audio_config: "AudioConfig"):
|
||||||
detail="Azure Realtime requires an endpoint.",
|
detail="Azure Realtime requires an endpoint.",
|
||||||
)
|
)
|
||||||
_validate_runtime_service_url(endpoint, "endpoint")
|
_validate_runtime_service_url(endpoint, "endpoint")
|
||||||
api_version = (
|
api_version = getattr(realtime_config, "api_version", None) or "v1"
|
||||||
getattr(realtime_config, "api_version", None) or "2025-04-01-preview"
|
|
||||||
)
|
|
||||||
# Construct the Azure Realtime WebSocket URL
|
|
||||||
# https://<resource>.openai.azure.com/openai/realtime?api-version=<ver>&deployment=<model>
|
|
||||||
parsed_endpoint = urlparse(endpoint)
|
parsed_endpoint = urlparse(endpoint)
|
||||||
|
if api_version == "v1":
|
||||||
|
# Azure's GA Realtime API uses the deployment name as `model` and
|
||||||
|
# deliberately has no date-based api-version query parameter.
|
||||||
|
path = "/openai/v1/realtime"
|
||||||
|
query = urlencode({"model": model})
|
||||||
|
else:
|
||||||
|
# Preserve explicitly configured preview deployments while users
|
||||||
|
# migrate. Microsoft deprecated this protocol on April 30, 2026.
|
||||||
|
path = "/openai/realtime"
|
||||||
|
query = urlencode({"api-version": api_version, "deployment": model})
|
||||||
wss_url = urlunparse(
|
wss_url = urlunparse(
|
||||||
(
|
(
|
||||||
"wss",
|
"wss",
|
||||||
parsed_endpoint.netloc,
|
parsed_endpoint.netloc,
|
||||||
"/openai/realtime",
|
path,
|
||||||
"",
|
"",
|
||||||
urlencode({"api-version": api_version, "deployment": model}),
|
query,
|
||||||
"",
|
"",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -86,3 +86,45 @@ async def test_tts_greeting_waits_for_session_updated_before_sending_prompt():
|
||||||
assert service._pending_initial_greeting_text is None
|
assert service._pending_initial_greeting_text is None
|
||||||
assert service._llm_needs_conversation_setup is False
|
assert service._llm_needs_conversation_setup is False
|
||||||
service._create_response.assert_not_awaited()
|
service._create_response.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_non_transition_function_call_runs_while_bot_is_speaking():
|
||||||
|
service = _make_service()
|
||||||
|
service._context = LLMContext()
|
||||||
|
service.run_function_calls = AsyncMock()
|
||||||
|
service._bot_is_speaking = True
|
||||||
|
service._pending_function_calls["call-1"] = SimpleNamespace(name="lookup_order")
|
||||||
|
|
||||||
|
await service._handle_evt_function_call_arguments_done(
|
||||||
|
SimpleNamespace(call_id="call-1", arguments='{"order_id":"123"}')
|
||||||
|
)
|
||||||
|
|
||||||
|
service.run_function_calls.assert_awaited_once()
|
||||||
|
assert service._deferred_node_transition_function_calls == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_node_transition_function_call_waits_until_bot_stops_speaking():
|
||||||
|
service = _make_service()
|
||||||
|
service._context = LLMContext()
|
||||||
|
service.run_function_calls = AsyncMock()
|
||||||
|
service._bot_is_speaking = True
|
||||||
|
service.register_function(
|
||||||
|
"customer_support",
|
||||||
|
AsyncMock(),
|
||||||
|
is_node_transition=True,
|
||||||
|
)
|
||||||
|
service._pending_function_calls["call-1"] = SimpleNamespace(name="customer_support")
|
||||||
|
|
||||||
|
await service._handle_evt_function_call_arguments_done(
|
||||||
|
SimpleNamespace(call_id="call-1", arguments='{"department":"sales"}')
|
||||||
|
)
|
||||||
|
|
||||||
|
service.run_function_calls.assert_not_awaited()
|
||||||
|
assert len(service._deferred_node_transition_function_calls) == 1
|
||||||
|
|
||||||
|
await service._run_pending_node_transition_function_calls()
|
||||||
|
|
||||||
|
service.run_function_calls.assert_awaited_once()
|
||||||
|
assert service._deferred_node_transition_function_calls == []
|
||||||
|
|
|
||||||
|
|
@ -170,6 +170,46 @@ def test_create_azure_realtime_blocks_private_endpoint_in_saas(monkeypatch):
|
||||||
assert "public IP" in exc_info.value.detail
|
assert "public IP" in exc_info.value.detail
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_azure_realtime_uses_ga_websocket_url_by_default(monkeypatch):
|
||||||
|
monkeypatch.setattr("api.utils.url_security.DEPLOYMENT_MODE", "oss")
|
||||||
|
user_config = SimpleNamespace(
|
||||||
|
realtime=SimpleNamespace(
|
||||||
|
provider=ServiceProviders.AZURE_REALTIME.value,
|
||||||
|
api_key="test-key",
|
||||||
|
endpoint="https://example.openai.azure.com",
|
||||||
|
model="my-realtime-deployment",
|
||||||
|
voice="alloy",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
service = create_realtime_llm_service(user_config, _audio_config())
|
||||||
|
|
||||||
|
assert service.base_url == (
|
||||||
|
"wss://example.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_azure_realtime_preserves_explicit_preview_websocket_url(monkeypatch):
|
||||||
|
monkeypatch.setattr("api.utils.url_security.DEPLOYMENT_MODE", "oss")
|
||||||
|
user_config = SimpleNamespace(
|
||||||
|
realtime=SimpleNamespace(
|
||||||
|
provider=ServiceProviders.AZURE_REALTIME.value,
|
||||||
|
api_key="test-key",
|
||||||
|
endpoint="https://example.openai.azure.com",
|
||||||
|
api_version="2025-04-01-preview",
|
||||||
|
model="my-preview-deployment",
|
||||||
|
voice="alloy",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
service = create_realtime_llm_service(user_config, _audio_config())
|
||||||
|
|
||||||
|
assert service.base_url == (
|
||||||
|
"wss://example.openai.azure.com/openai/realtime?"
|
||||||
|
"api-version=2025-04-01-preview&deployment=my-preview-deployment"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_azure_embedding_service_rejects_wrong_dimension():
|
def test_azure_embedding_service_rejects_wrong_dimension():
|
||||||
service = AzureOpenAIEmbeddingService(
|
service = AzureOpenAIEmbeddingService(
|
||||||
db_client=SimpleNamespace(),
|
db_client=SimpleNamespace(),
|
||||||
|
|
|
||||||
|
|
@ -134,6 +134,21 @@ def test_gemini_live_service_classes_use_dograh_gemini_adapter_class():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_vertex_live_inherits_dograh_node_transition_lifecycle():
|
||||||
|
assert (
|
||||||
|
DograhGeminiLiveVertexLLMService._requires_node_transition_context_aggregation
|
||||||
|
is DograhGeminiLiveLLMService._requires_node_transition_context_aggregation
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
DograhGeminiLiveVertexLLMService._run_or_defer_function_calls
|
||||||
|
is DograhGeminiLiveLLMService._run_or_defer_function_calls
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
DograhGeminiLiveVertexLLMService._reconnect_for_node_transition
|
||||||
|
is DograhGeminiLiveLLMService._reconnect_for_node_transition
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_gemini_live_config_accepts_json_schema_tools():
|
def test_gemini_live_config_accepts_json_schema_tools():
|
||||||
function_schema = FunctionSchema(
|
function_schema = FunctionSchema(
|
||||||
name="customer_lookup",
|
name="customer_lookup",
|
||||||
|
|
|
||||||
|
|
@ -130,7 +130,7 @@ async def test_messages_append_frame_sends_conversation_item():
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_function_call_is_deferred_until_bot_stops_speaking():
|
async def test_non_transition_function_call_runs_while_bot_is_speaking():
|
||||||
service = _make_service()
|
service = _make_service()
|
||||||
service._context = LLMContext()
|
service._context = LLMContext()
|
||||||
service.run_function_calls = AsyncMock()
|
service.run_function_calls = AsyncMock()
|
||||||
|
|
@ -145,13 +145,38 @@ async def test_function_call_is_deferred_until_bot_stops_speaking():
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
service.run_function_calls.assert_not_awaited()
|
service.run_function_calls.assert_awaited_once()
|
||||||
assert len(service._deferred_function_calls) == 1
|
assert service._deferred_node_transition_function_calls == []
|
||||||
|
|
||||||
await service._run_pending_function_calls()
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_node_transition_function_call_waits_until_bot_stops_speaking():
|
||||||
|
service = _make_service()
|
||||||
|
service._context = LLMContext()
|
||||||
|
service.run_function_calls = AsyncMock()
|
||||||
|
service._bot_is_speaking = True
|
||||||
|
service.register_function(
|
||||||
|
"customer_support",
|
||||||
|
AsyncMock(),
|
||||||
|
is_node_transition=True,
|
||||||
|
)
|
||||||
|
service._pending_function_calls["call-1"] = SimpleNamespace(name="customer_support")
|
||||||
|
|
||||||
|
await service._handle_evt_function_call_arguments_done(
|
||||||
|
SimpleNamespace(
|
||||||
|
call_id="call-1",
|
||||||
|
name="customer_support",
|
||||||
|
arguments='{"department":"sales"}',
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
service.run_function_calls.assert_not_awaited()
|
||||||
|
assert len(service._deferred_node_transition_function_calls) == 1
|
||||||
|
|
||||||
|
await service._run_pending_node_transition_function_calls()
|
||||||
|
|
||||||
service.run_function_calls.assert_awaited_once()
|
service.run_function_calls.assert_awaited_once()
|
||||||
assert service._deferred_function_calls == []
|
assert service._deferred_node_transition_function_calls == []
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
@ -190,3 +215,21 @@ def test_factory_creates_dograh_grok_realtime_service():
|
||||||
)
|
)
|
||||||
|
|
||||||
assert isinstance(service, DograhGrokRealtimeLLMService)
|
assert isinstance(service, DograhGrokRealtimeLLMService)
|
||||||
|
assert service._settings.session_properties.voice == "sal"
|
||||||
|
assert service._settings.session_properties.audio.input.transcription.model == (
|
||||||
|
"grok-transcribe"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_grok_audio_config_preserves_transcription_when_filling_sample_rates():
|
||||||
|
service = _make_service()
|
||||||
|
service._settings.session_properties.audio = events.AudioConfiguration(
|
||||||
|
input=events.AudioInput(transcription=events.InputAudioTranscription())
|
||||||
|
)
|
||||||
|
|
||||||
|
service._ensure_audio_config(input_sample_rate=16000, output_sample_rate=24000)
|
||||||
|
|
||||||
|
audio = service._settings.session_properties.audio
|
||||||
|
assert audio.input.format.rate == 16000
|
||||||
|
assert audio.input.transcription.model == "grok-transcribe"
|
||||||
|
assert audio.output.format.rate == 24000
|
||||||
|
|
|
||||||
|
|
@ -127,11 +127,11 @@ async def test_function_call_executes_immediately_when_bot_is_not_speaking():
|
||||||
)
|
)
|
||||||
|
|
||||||
service.run_function_calls.assert_awaited_once()
|
service.run_function_calls.assert_awaited_once()
|
||||||
assert service._deferred_function_calls == []
|
assert service._deferred_node_transition_function_calls == []
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_function_call_is_deferred_until_bot_stops_speaking():
|
async def test_non_transition_function_call_runs_while_bot_is_speaking():
|
||||||
service = _make_service()
|
service = _make_service()
|
||||||
service._context = LLMContext()
|
service._context = LLMContext()
|
||||||
service.run_function_calls = AsyncMock()
|
service.run_function_calls = AsyncMock()
|
||||||
|
|
@ -142,10 +142,31 @@ async def test_function_call_is_deferred_until_bot_stops_speaking():
|
||||||
SimpleNamespace(call_id="call-1", arguments='{"department":"sales"}')
|
SimpleNamespace(call_id="call-1", arguments='{"department":"sales"}')
|
||||||
)
|
)
|
||||||
|
|
||||||
service.run_function_calls.assert_not_awaited()
|
service.run_function_calls.assert_awaited_once()
|
||||||
assert len(service._deferred_function_calls) == 1
|
assert service._deferred_node_transition_function_calls == []
|
||||||
|
|
||||||
await service._run_pending_function_calls()
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_node_transition_function_call_is_deferred_until_bot_stops_speaking():
|
||||||
|
service = _make_service()
|
||||||
|
service._context = LLMContext()
|
||||||
|
service.run_function_calls = AsyncMock()
|
||||||
|
service._bot_is_speaking = True
|
||||||
|
service.register_function(
|
||||||
|
"customer_support",
|
||||||
|
AsyncMock(),
|
||||||
|
is_node_transition=True,
|
||||||
|
)
|
||||||
|
service._pending_function_calls["call-1"] = SimpleNamespace(name="customer_support")
|
||||||
|
|
||||||
|
await service._handle_evt_function_call_arguments_done(
|
||||||
|
SimpleNamespace(call_id="call-1", arguments='{"department":"sales"}')
|
||||||
|
)
|
||||||
|
|
||||||
|
service.run_function_calls.assert_not_awaited()
|
||||||
|
assert len(service._deferred_node_transition_function_calls) == 1
|
||||||
|
|
||||||
|
await service._run_pending_node_transition_function_calls()
|
||||||
|
|
||||||
service.run_function_calls.assert_awaited_once()
|
service.run_function_calls.assert_awaited_once()
|
||||||
assert service._deferred_function_calls == []
|
assert service._deferred_node_transition_function_calls == []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import json
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import AsyncMock, call
|
from unittest.mock import AsyncMock, call
|
||||||
|
|
||||||
|
|
@ -13,7 +14,6 @@ from websockets.frames import Close
|
||||||
from api.schemas.ai_model_configuration import EffectiveAIModelConfiguration
|
from api.schemas.ai_model_configuration import EffectiveAIModelConfiguration
|
||||||
from api.services.configuration.registry import UltravoxRealtimeLLMConfiguration
|
from api.services.configuration.registry import UltravoxRealtimeLLMConfiguration
|
||||||
from api.services.pipecat.realtime.ultravox_realtime import (
|
from api.services.pipecat.realtime.ultravox_realtime import (
|
||||||
_RESUMPTION_USER_MESSAGE,
|
|
||||||
DograhUltravoxOneShotInputParams,
|
DograhUltravoxOneShotInputParams,
|
||||||
DograhUltravoxRealtimeLLMService,
|
DograhUltravoxRealtimeLLMService,
|
||||||
)
|
)
|
||||||
|
|
@ -100,50 +100,35 @@ async def test_initial_context_connects_without_replay():
|
||||||
await service._handle_context(context)
|
await service._handle_context(context)
|
||||||
|
|
||||||
service._connect_call.assert_awaited_once()
|
service._connect_call.assert_awaited_once()
|
||||||
assert service._connect_call.await_args.kwargs["initial_messages"] is None
|
assert service._connect_call.await_args.kwargs["greeting_text"] is None
|
||||||
assert service._connect_call.await_args.kwargs["agent_speaks_first"] is True
|
assert service._connect_call.await_args.kwargs["agent_speaks_first"] is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_system_instruction_update_marks_reconnect_required():
|
async def test_system_instruction_update_marks_stage_update_required():
|
||||||
service = _make_service()
|
service = _make_service()
|
||||||
service._has_connected_once = True
|
service._socket = object()
|
||||||
|
|
||||||
changed = await service._update_settings(
|
changed = await service._update_settings(
|
||||||
DograhUltravoxRealtimeLLMService.Settings(system_instruction="new instruction")
|
DograhUltravoxRealtimeLLMService.Settings(system_instruction="new instruction")
|
||||||
)
|
)
|
||||||
|
|
||||||
assert "system_instruction" in changed
|
assert "system_instruction" in changed
|
||||||
assert service._reconnect_required is True
|
assert service._stage_update_required is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_system_instruction_change_reconnects_with_full_initial_messages():
|
async def test_node_transition_updates_native_stage_without_reconnecting():
|
||||||
service = _make_service()
|
service = _make_service()
|
||||||
service._socket = object()
|
service._socket = object()
|
||||||
service._has_connected_once = True
|
service._send = AsyncMock()
|
||||||
service._call_system_instruction = "old instruction"
|
service._connect_call = AsyncMock()
|
||||||
service._reconnect_required = True
|
service._pending_node_transition_tool_call_ids.add("call-transition")
|
||||||
|
service._stage_update_required = True
|
||||||
service._settings.system_instruction = "new instruction"
|
service._settings.system_instruction = "new instruction"
|
||||||
service._reconnect_with_context = AsyncMock()
|
|
||||||
|
|
||||||
context = LLMContext(
|
context = LLMContext(
|
||||||
messages=[
|
messages=[
|
||||||
{"role": "user", "content": "I want to hear the pricing."},
|
|
||||||
{
|
|
||||||
"role": "assistant",
|
|
||||||
"content": "Let me check that for you.",
|
|
||||||
"tool_calls": [
|
|
||||||
{
|
|
||||||
"id": "call-transition",
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "transition_to_next_node",
|
|
||||||
"arguments": '{"reason":"pricing requested"}',
|
|
||||||
},
|
|
||||||
}
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"role": "tool",
|
"role": "tool",
|
||||||
"tool_call_id": "call-transition",
|
"tool_call_id": "call-transition",
|
||||||
|
|
@ -155,43 +140,28 @@ async def test_system_instruction_change_reconnects_with_full_initial_messages()
|
||||||
|
|
||||||
await service._handle_context(context)
|
await service._handle_context(context)
|
||||||
|
|
||||||
service._reconnect_with_context.assert_awaited_once()
|
service._connect_call.assert_not_awaited()
|
||||||
initial_messages = service._reconnect_with_context.await_args.kwargs[
|
service._send.assert_awaited_once()
|
||||||
"initial_messages"
|
message = service._send.await_args.args[0]
|
||||||
]
|
assert message["type"] == "client_tool_result"
|
||||||
assert initial_messages == [
|
assert message["invocationId"] == "call-transition"
|
||||||
{
|
assert message["responseType"] == "new-stage"
|
||||||
"role": "MESSAGE_ROLE_USER",
|
stage = json.loads(message["result"])
|
||||||
"text": "I want to hear the pricing.",
|
assert stage["systemPrompt"] == "new instruction"
|
||||||
},
|
assert stage["toolResultText"] == '{"status":"done"}'
|
||||||
{
|
assert stage["selectedTools"][0]["temporaryTool"]["modelToolName"] == (
|
||||||
"role": "MESSAGE_ROLE_AGENT",
|
"transition_to_next_node"
|
||||||
"text": "Let me check that for you.",
|
)
|
||||||
},
|
|
||||||
{
|
|
||||||
"role": "MESSAGE_ROLE_TOOL_CALL",
|
|
||||||
"text": "",
|
|
||||||
"invocationId": "call-transition",
|
|
||||||
"toolName": "transition_to_next_node",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"role": "MESSAGE_ROLE_TOOL_RESULT",
|
|
||||||
"text": '{"status":"done"}',
|
|
||||||
"invocationId": "call-transition",
|
|
||||||
"toolName": "transition_to_next_node",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
assert "call-transition" in service._completed_tool_calls
|
assert "call-transition" in service._completed_tool_calls
|
||||||
|
assert service._pending_node_transition_tool_call_ids == set()
|
||||||
|
assert service._stage_update_required is False
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_tool_context_update_does_not_reconnect_when_system_instruction_is_unchanged():
|
async def test_ordinary_tool_result_uses_standard_tool_response():
|
||||||
service = _make_service()
|
service = _make_service()
|
||||||
service._socket = object()
|
service._socket = object()
|
||||||
service._call_system_instruction = "same instruction"
|
service._send = AsyncMock()
|
||||||
service._settings.system_instruction = "same instruction"
|
|
||||||
service._reconnect_with_context = AsyncMock()
|
|
||||||
service._send_tool_result = AsyncMock()
|
|
||||||
|
|
||||||
context = LLMContext(
|
context = LLMContext(
|
||||||
messages=[
|
messages=[
|
||||||
|
|
@ -206,13 +176,40 @@ async def test_tool_context_update_does_not_reconnect_when_system_instruction_is
|
||||||
|
|
||||||
await service._handle_context(context)
|
await service._handle_context(context)
|
||||||
|
|
||||||
service._reconnect_with_context.assert_not_awaited()
|
service._send.assert_awaited_once_with(
|
||||||
service._send_tool_result.assert_awaited_once_with(
|
{
|
||||||
"call-transition",
|
"type": "client_tool_result",
|
||||||
'{"status":"done"}',
|
"invocationId": "call-transition",
|
||||||
|
"result": '{"status":"done"}',
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_only_registered_node_transition_invocations_are_tracked():
|
||||||
|
service = _make_service()
|
||||||
|
service.run_function_calls = AsyncMock()
|
||||||
|
service.register_function(
|
||||||
|
"transition_to_next_node",
|
||||||
|
AsyncMock(),
|
||||||
|
is_node_transition=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
await service._handle_tool_invocation(
|
||||||
|
"transition_to_next_node", "call-transition", {"reason": "pricing"}
|
||||||
|
)
|
||||||
|
await service._handle_tool_invocation("lookup_price", "call-lookup", {})
|
||||||
|
|
||||||
|
assert service._pending_node_transition_tool_call_ids == {"call-transition"}
|
||||||
|
assert service.run_function_calls.await_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_ultravox_requires_transition_context_aggregation():
|
||||||
|
service = _make_service()
|
||||||
|
|
||||||
|
assert service._requires_node_transition_context_aggregation() is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_messages_append_frame_sends_user_text():
|
async def test_messages_append_frame_sends_user_text():
|
||||||
service = _make_service()
|
service = _make_service()
|
||||||
|
|
@ -287,7 +284,6 @@ def test_build_one_shot_params_uses_explicit_greeting_text():
|
||||||
|
|
||||||
params = service._build_one_shot_params(
|
params = service._build_one_shot_params(
|
||||||
greeting_text="Welcome to Dograh",
|
greeting_text="Welcome to Dograh",
|
||||||
initial_messages=None,
|
|
||||||
agent_speaks_first=True,
|
agent_speaks_first=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -296,85 +292,18 @@ def test_build_one_shot_params_uses_explicit_greeting_text():
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_build_one_shot_params_includes_initial_messages():
|
def test_build_one_shot_params_uses_current_system_instruction():
|
||||||
service = _make_service()
|
service = _make_service()
|
||||||
service._settings.system_instruction = "Base instruction"
|
service._settings.system_instruction = "Base instruction"
|
||||||
|
|
||||||
params = service._build_one_shot_params(
|
params = service._build_one_shot_params(
|
||||||
greeting_text=None,
|
greeting_text=None,
|
||||||
initial_messages=[
|
|
||||||
{"role": "MESSAGE_ROLE_USER", "text": "User asked a question."},
|
|
||||||
{"role": "MESSAGE_ROLE_TOOL_RESULT", "text": '{"status":"done"}'},
|
|
||||||
],
|
|
||||||
agent_speaks_first=True,
|
agent_speaks_first=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert params.extra["initialMessages"] == [
|
|
||||||
{"role": "MESSAGE_ROLE_USER", "text": "User asked a question."},
|
|
||||||
{"role": "MESSAGE_ROLE_TOOL_RESULT", "text": '{"status":"done"}'},
|
|
||||||
{"role": "MESSAGE_ROLE_USER", "text": _RESUMPTION_USER_MESSAGE},
|
|
||||||
]
|
|
||||||
assert params.system_prompt == "Base instruction"
|
assert params.system_prompt == "Base instruction"
|
||||||
|
|
||||||
|
|
||||||
def test_build_one_shot_params_without_tool_result_does_not_add_resumption_user_message():
|
|
||||||
service = _make_service()
|
|
||||||
service._settings.system_instruction = "Base instruction"
|
|
||||||
|
|
||||||
params = service._build_one_shot_params(
|
|
||||||
greeting_text=None,
|
|
||||||
initial_messages=[
|
|
||||||
{"role": "MESSAGE_ROLE_USER", "text": "User asked a question."},
|
|
||||||
{"role": "MESSAGE_ROLE_AGENT", "text": "Assistant replied."},
|
|
||||||
],
|
|
||||||
agent_speaks_first=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert params.system_prompt == "Base instruction"
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_agent_speak_first_when_history_ends_with_tool_result():
|
|
||||||
service = _make_service()
|
|
||||||
|
|
||||||
assert (
|
|
||||||
service._should_agent_speak_first(
|
|
||||||
[
|
|
||||||
{"role": "MESSAGE_ROLE_USER", "text": "Hello"},
|
|
||||||
{"role": "MESSAGE_ROLE_TOOL_RESULT", "text": '{"status":"done"}'},
|
|
||||||
]
|
|
||||||
)
|
|
||||||
is True
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_not_force_agent_speaks_first_when_history_ends_with_agent():
|
|
||||||
service = _make_service()
|
|
||||||
|
|
||||||
assert (
|
|
||||||
service._should_agent_speak_first(
|
|
||||||
[{"role": "MESSAGE_ROLE_AGENT", "text": "How else can I help?"}]
|
|
||||||
)
|
|
||||||
is False
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_add_resumption_user_message_only_when_history_ends_with_tool_result():
|
|
||||||
service = _make_service()
|
|
||||||
|
|
||||||
assert (
|
|
||||||
service._should_add_resumption_user_message(
|
|
||||||
[{"role": "MESSAGE_ROLE_TOOL_RESULT", "text": '{"status":"done"}'}]
|
|
||||||
)
|
|
||||||
is True
|
|
||||||
)
|
|
||||||
assert (
|
|
||||||
service._should_add_resumption_user_message(
|
|
||||||
[{"role": "MESSAGE_ROLE_AGENT", "text": "Assistant replied."}]
|
|
||||||
)
|
|
||||||
is False
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_to_selected_tools_includes_registered_timeout():
|
def test_to_selected_tools_includes_registered_timeout():
|
||||||
service = _make_service()
|
service = _make_service()
|
||||||
service.register_function(
|
service.register_function(
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
2
pipecat
2
pipecat
|
|
@ -1 +1 @@
|
||||||
Subproject commit fed8541fca8893b318b862779f1962ad9903de81
|
Subproject commit aadd1d5dd606d2871b082e6f2ca1ad1eee53785b
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# generated by datamodel-codegen:
|
# generated by datamodel-codegen:
|
||||||
# filename: dograh-openapi-XXXXXX.json.73JcjTo19T
|
# filename: dograh-openapi-XXXXXX.json.uTKtHKJw6v
|
||||||
# timestamp: 2026-07-11T10:21:09+00:00
|
# timestamp: 2026-07-15T11:25:06+00:00
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ Re-exports every typed node class so users can write
|
||||||
from dograh_sdk.typed.agent_node import AgentNode
|
from dograh_sdk.typed.agent_node import AgentNode
|
||||||
from dograh_sdk.typed.end_call import EndCall
|
from dograh_sdk.typed.end_call import EndCall
|
||||||
from dograh_sdk.typed.global_node import GlobalNode
|
from dograh_sdk.typed.global_node import GlobalNode
|
||||||
|
from dograh_sdk.typed.paygent import Paygent
|
||||||
from dograh_sdk.typed.qa import Qa
|
from dograh_sdk.typed.qa import Qa
|
||||||
from dograh_sdk.typed.start_call import StartCall
|
from dograh_sdk.typed.start_call import StartCall
|
||||||
from dograh_sdk.typed.trigger import Trigger
|
from dograh_sdk.typed.trigger import Trigger
|
||||||
|
|
@ -18,6 +19,7 @@ __all__ = [
|
||||||
"AgentNode",
|
"AgentNode",
|
||||||
"EndCall",
|
"EndCall",
|
||||||
"GlobalNode",
|
"GlobalNode",
|
||||||
|
"Paygent",
|
||||||
"Qa",
|
"Qa",
|
||||||
"StartCall",
|
"StartCall",
|
||||||
"Trigger",
|
"Trigger",
|
||||||
|
|
|
||||||
56
sdk/python/src/dograh_sdk/typed/paygent.py
Normal file
56
sdk/python/src/dograh_sdk/typed/paygent.py
Normal file
|
|
@ -0,0 +1,56 @@
|
||||||
|
"""GENERATED — do not edit by hand.
|
||||||
|
|
||||||
|
Regenerate with `python -m dograh_sdk.codegen` against the target
|
||||||
|
Dograh backend. Source of truth: the backend's model-backed node-spec
|
||||||
|
catalog served from `/api/v1/node-types`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any, ClassVar, Literal, Optional
|
||||||
|
|
||||||
|
from dograh_sdk.typed._base import TypedNode
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(kw_only=True)
|
||||||
|
class Paygent(TypedNode):
|
||||||
|
"""
|
||||||
|
Cost Tracking and Billing LLM hint: Paygent is a post-call usage-
|
||||||
|
tracking and billing integration. It does not participate in the
|
||||||
|
conversation graph and should not be connected to other nodes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
type: ClassVar[str] = 'paygent'
|
||||||
|
|
||||||
|
paygent_api_key: str
|
||||||
|
"""
|
||||||
|
API key used to authenticate requests to the Paygent REST API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
paygent_agent_id: str
|
||||||
|
"""
|
||||||
|
The agent identifier registered in your Paygent account.
|
||||||
|
"""
|
||||||
|
|
||||||
|
paygent_customer_id: str
|
||||||
|
"""
|
||||||
|
Your Paygent customer / organisation ID.
|
||||||
|
"""
|
||||||
|
|
||||||
|
name: str = 'Paygent'
|
||||||
|
"""
|
||||||
|
Short identifier for this Paygent configuration.
|
||||||
|
"""
|
||||||
|
|
||||||
|
paygent_enabled: bool = True
|
||||||
|
"""
|
||||||
|
When false, Dograh skips all Paygent tracking for this call.
|
||||||
|
"""
|
||||||
|
|
||||||
|
paygent_indicator: str = 'per-minute-call'
|
||||||
|
"""
|
||||||
|
The indicator event name sent at the end of the call (e.g. per-minute-
|
||||||
|
call).
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
@ -6,6 +6,7 @@
|
||||||
export { type AgentNode, agentNode } from "./agent-node.js";
|
export { type AgentNode, agentNode } from "./agent-node.js";
|
||||||
export { type EndCall, endCall } from "./end-call.js";
|
export { type EndCall, endCall } from "./end-call.js";
|
||||||
export { type GlobalNode, globalNode } from "./global-node.js";
|
export { type GlobalNode, globalNode } from "./global-node.js";
|
||||||
|
export { type Paygent, paygent } from "./paygent.js";
|
||||||
export { type Qa, qa } from "./qa.js";
|
export { type Qa, qa } from "./qa.js";
|
||||||
export { type StartCall, startCall } from "./start-call.js";
|
export { type StartCall, startCall } from "./start-call.js";
|
||||||
export { type Trigger, trigger } from "./trigger.js";
|
export { type Trigger, trigger } from "./trigger.js";
|
||||||
|
|
@ -16,6 +17,7 @@ import type {
|
||||||
AgentNode,
|
AgentNode,
|
||||||
EndCall,
|
EndCall,
|
||||||
GlobalNode,
|
GlobalNode,
|
||||||
|
Paygent,
|
||||||
Qa,
|
Qa,
|
||||||
StartCall,
|
StartCall,
|
||||||
Trigger,
|
Trigger,
|
||||||
|
|
@ -24,4 +26,4 @@ import type {
|
||||||
} from "./index.js";
|
} from "./index.js";
|
||||||
|
|
||||||
/** Discriminated union of every generated typed node. */
|
/** Discriminated union of every generated typed node. */
|
||||||
export type TypedNode = AgentNode | EndCall | GlobalNode | Qa | StartCall | Trigger | Tuner | Webhook;
|
export type TypedNode = AgentNode | EndCall | GlobalNode | Paygent | Qa | StartCall | Trigger | Tuner | Webhook;
|
||||||
|
|
|
||||||
44
sdk/typescript/src/typed/paygent.ts
Normal file
44
sdk/typescript/src/typed/paygent.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
// GENERATED — do not edit by hand.
|
||||||
|
//
|
||||||
|
// Regenerate with `npm run codegen` against the target Dograh backend.
|
||||||
|
// Source of truth: the backend's model-backed node-spec catalog served
|
||||||
|
// from `/api/v1/node-types`.
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cost Tracking and Billing
|
||||||
|
*
|
||||||
|
* LLM hint: Paygent is a post-call usage-tracking and billing integration. It does not participate in the conversation graph and should not be connected to other nodes.
|
||||||
|
*/
|
||||||
|
export interface Paygent {
|
||||||
|
type: "paygent";
|
||||||
|
/**
|
||||||
|
* Short identifier for this Paygent configuration.
|
||||||
|
*/
|
||||||
|
name?: string;
|
||||||
|
/**
|
||||||
|
* When false, Dograh skips all Paygent tracking for this call.
|
||||||
|
*/
|
||||||
|
paygent_enabled?: boolean;
|
||||||
|
/**
|
||||||
|
* API key used to authenticate requests to the Paygent REST API.
|
||||||
|
*/
|
||||||
|
paygent_api_key: string;
|
||||||
|
/**
|
||||||
|
* The agent identifier registered in your Paygent account.
|
||||||
|
*/
|
||||||
|
paygent_agent_id: string;
|
||||||
|
/**
|
||||||
|
* Your Paygent customer / organisation ID.
|
||||||
|
*/
|
||||||
|
paygent_customer_id: string;
|
||||||
|
/**
|
||||||
|
* The indicator event name sent at the end of the call (e.g. per-minute-call).
|
||||||
|
*/
|
||||||
|
paygent_indicator?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Factory — sets `type` for you so you don't repeat the discriminator. */
|
||||||
|
export function paygent(input: Omit<Paygent, "type">): Paygent {
|
||||||
|
return { type: "paygent", ...input };
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue