mirror of
https://github.com/dograh-hq/dograh.git
synced 2026-07-16 11:31:04 +02:00
01acf6ac30
* fix: fix transition logic for realtime providers * chore: run formatter * chore: generate SDK and fix other realtime providers * fix: fix ultravox node transitions
400 lines
18 KiB
Python
400 lines
18 KiB
Python
"""Dograh subclass of pipecat's Gemini Live LLM service.
|
|
|
|
Layers Dograh engine integration quirks onto upstream-pristine
|
|
:class:`GeminiLiveLLMService`:
|
|
|
|
- **Deferred connect.** Connection is held back until ``system_instruction``
|
|
is set via :meth:`_update_settings`, so pre-call-fetch template variables
|
|
land before the live session opens.
|
|
- **Reconnect on node transitions.** Gemini Live cannot update
|
|
``system_instruction`` mid-session, so a setting change triggers a
|
|
reconnect (deferred until the bot turn ends if currently responding).
|
|
- **Node-transition deferral.** Node-transition calls emitted mid-turn are
|
|
queued and run when the bot stops speaking, to avoid cutting off its audio.
|
|
- **User-mute audio gating.** ``UserMuteStarted/StoppedFrame`` from the
|
|
user aggregator gates whether incoming audio is forwarded to Gemini.
|
|
- **TTSSpeakFrame as greeting trigger.** The engine queues a TTSSpeakFrame
|
|
to kick off the first response after node setup; the service intercepts
|
|
it and runs the initial-context path.
|
|
"""
|
|
|
|
import asyncio
|
|
from typing import Any
|
|
|
|
from google.genai.types import Content, Part
|
|
from loguru import logger
|
|
|
|
from api.services.pipecat.gemini_json_schema_adapter import (
|
|
DograhGeminiJSONSchemaAdapter,
|
|
)
|
|
from api.services.pipecat.realtime.static_greeting import format_static_greeting_prompt
|
|
from pipecat.frames.frames import (
|
|
BotStoppedSpeakingFrame,
|
|
Frame,
|
|
TTSSpeakFrame,
|
|
UserMuteStartedFrame,
|
|
UserMuteStoppedFrame,
|
|
)
|
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
|
from pipecat.processors.frame_processor import FrameDirection
|
|
from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
|
|
from pipecat.services.llm_service import FunctionCallFromLLM
|
|
from pipecat.utils.tracing.service_decorators import traced_gemini_live
|
|
|
|
|
|
class DograhGeminiLiveLLMService(GeminiLiveLLMService):
|
|
"""Gemini Live with Dograh engine integration quirks. See module docstring."""
|
|
|
|
# Gemini input transcription is delivered independently from tool calls.
|
|
# Give late transcription messages a small window to arrive before running
|
|
# a node-transition function and tearing down the current Live connection.
|
|
_NODE_TRANSITION_TRANSCRIPTION_GRACE_SECONDS = 0.5
|
|
|
|
# Route tool schemas through Gemini's ``parameters_json_schema`` field so
|
|
# MCP/imported tools that use JSON Schema keywords (``const``, ``not``,
|
|
# nested ``anyOf``) rejected by the strict ``Schema`` model are accepted.
|
|
# Mirrors the non-realtime ``DograhGoogleLLMService`` fix;
|
|
# ``DograhGeminiLiveVertexLLMService`` inherits this via MRO.
|
|
adapter_class = DograhGeminiJSONSchemaAdapter
|
|
|
|
def __init__(self, **kwargs):
|
|
super().__init__(**kwargs)
|
|
# User-mute state, driven by broadcast UserMute{Started,Stopped}Frames.
|
|
# Audio is not forwarded to Gemini while muted.
|
|
self._user_is_muted: bool = False
|
|
# Guards initial-response triggering against double-firing across the
|
|
# initial TTSSpeakFrame and any LLMContextFrame that may arrive.
|
|
self._handled_initial_context: bool = False
|
|
# Node-transition calls emitted mid-bot-turn are deferred here so the
|
|
# transition does not tear down Gemini while it is still producing audio.
|
|
self._pending_node_transition_function_calls: list[FunctionCallFromLLM] = []
|
|
# Text greeting captured from the first TTSSpeakFrame while the Gemini
|
|
# session is still connecting.
|
|
self._pending_initial_greeting_text: str | None = None
|
|
self._transition_function_call_task: asyncio.Task | None = None
|
|
# Intentional node changes use a fresh, context-seeded connection rather
|
|
# than a potentially stale session-resumption handle. The new connection
|
|
# remains gated until the function-call result has landed in LLMContext.
|
|
self._awaiting_node_transition_context: bool = False
|
|
self._node_transition_context_received: bool = False
|
|
self._node_transition_context_seed_started: bool = False
|
|
|
|
# ------------------------------------------------------------------
|
|
# Hooks from upstream GeminiLiveLLMService
|
|
# ------------------------------------------------------------------
|
|
|
|
def _should_connect_on_start(self) -> bool:
|
|
# Hold the connection until the engine sets a system_instruction. This
|
|
# lets pre-call fetch populate template variables first.
|
|
return bool(self._settings.system_instruction)
|
|
|
|
def _requires_node_transition_context_aggregation(self) -> bool:
|
|
# A node transition replaces the current Gemini Live connection and
|
|
# seeds the new one from our local LLMContext. Wait for the upstream
|
|
# user aggregator to commit any final TranscriptionFrame before
|
|
# set_node() changes the prompt and starts that reconnect.
|
|
return True
|
|
|
|
async def cleanup(self) -> None:
|
|
"""Cancel a delayed transition before tearing down the Live session."""
|
|
if self._transition_function_call_task:
|
|
await self.cancel_task(self._transition_function_call_task)
|
|
self._transition_function_call_task = None
|
|
await super().cleanup()
|
|
|
|
async def _handle_changed_settings(self, changed: dict[str, Any]) -> set[str]:
|
|
if "system_instruction" not in changed:
|
|
return set()
|
|
|
|
# PipecatEngine updates system_instruction only from set_node(). The
|
|
# first set_node happens before a Live session exists; every later one
|
|
# is a node transition whose tool call has already been deferred until
|
|
# the current bot turn finishes.
|
|
if not self._session:
|
|
# First-time setting after deferred-connect.
|
|
await self._connect()
|
|
else:
|
|
await self._reconnect_for_node_transition()
|
|
return {"system_instruction"}
|
|
|
|
async def _run_or_defer_function_calls(
|
|
self, function_calls_llm: list[FunctionCallFromLLM]
|
|
):
|
|
if not self._contains_node_transition(function_calls_llm):
|
|
await super()._run_or_defer_function_calls(function_calls_llm)
|
|
return
|
|
|
|
# Keep a provider tool-call batch together. Splitting a mixed batch here
|
|
# would discard Pipecat's shared function-call group and could trigger an
|
|
# LLM run before every result from the original batch has arrived.
|
|
if self._bot_is_responding:
|
|
# Latest batch wins; Gemini emits tool calls as one batch per
|
|
# tool_call message, so this overwrite is intentional.
|
|
self._pending_node_transition_function_calls = function_calls_llm
|
|
logger.debug(
|
|
f"{self}: deferring {len(function_calls_llm)} node-transition "
|
|
"function call(s) "
|
|
"until bot turn ends"
|
|
)
|
|
return
|
|
|
|
self._schedule_node_transition_function_calls(function_calls_llm)
|
|
|
|
def _contains_node_transition(
|
|
self, function_calls_llm: list[FunctionCallFromLLM]
|
|
) -> bool:
|
|
return any(self._is_node_transition(fc) for fc in function_calls_llm)
|
|
|
|
def _is_node_transition(self, function_call: FunctionCallFromLLM) -> bool:
|
|
return self._function_is_node_transition(function_call.function_name)
|
|
|
|
def _schedule_node_transition_function_calls(
|
|
self, function_calls_llm: list[FunctionCallFromLLM]
|
|
) -> None:
|
|
"""Run transition calls after late input transcription has settled."""
|
|
if (
|
|
self._transition_function_call_task
|
|
and not self._transition_function_call_task.done()
|
|
):
|
|
logger.warning(
|
|
f"{self}: node-transition function call already pending; "
|
|
"ignoring duplicate batch"
|
|
)
|
|
return
|
|
|
|
async def _run_after_transcription_grace() -> None:
|
|
try:
|
|
await asyncio.sleep(self._NODE_TRANSITION_TRANSCRIPTION_GRACE_SECONDS)
|
|
await self._flush_pending_user_transcription()
|
|
await self.run_function_calls(function_calls_llm)
|
|
finally:
|
|
self._transition_function_call_task = None
|
|
|
|
self._transition_function_call_task = self.create_task(
|
|
_run_after_transcription_grace(),
|
|
name=f"{self}::node-transition-function-calls",
|
|
)
|
|
|
|
async def _flush_pending_user_transcription(self) -> None:
|
|
"""Publish any punctuationless user transcript before a node handoff."""
|
|
if self._transcription_timeout_task:
|
|
if not self._transcription_timeout_task.done():
|
|
await self.cancel_task(self._transcription_timeout_task)
|
|
self._transcription_timeout_task = None
|
|
|
|
if not self._user_transcription_buffer:
|
|
return
|
|
|
|
text = self._user_transcription_buffer
|
|
self._user_transcription_buffer = ""
|
|
logger.debug(
|
|
f"{self}: flushing pending user transcription before node transition"
|
|
)
|
|
await self._push_user_transcription(text, result=None)
|
|
|
|
# ------------------------------------------------------------------
|
|
# State-transition side effects
|
|
# ------------------------------------------------------------------
|
|
|
|
async def _set_bot_is_responding(self, responding: bool):
|
|
was_responding = self._bot_is_responding
|
|
await super()._set_bot_is_responding(responding)
|
|
if was_responding and not responding:
|
|
await self._run_pending_node_transition_function_calls()
|
|
|
|
async def _run_pending_node_transition_function_calls(self):
|
|
"""Run any node-transition calls deferred during the bot's last turn."""
|
|
if not self._pending_node_transition_function_calls:
|
|
return
|
|
fcs = self._pending_node_transition_function_calls
|
|
self._pending_node_transition_function_calls = []
|
|
logger.debug(
|
|
f"{self}: executing {len(fcs)} deferred node-transition call(s) "
|
|
"after bot turn ended"
|
|
)
|
|
self._schedule_node_transition_function_calls(fcs)
|
|
|
|
async def _reconnect_for_node_transition(self) -> None:
|
|
"""Start a fresh connection and wait to seed the completed context.
|
|
|
|
Gemini can report ``resumable=False`` while generating or executing a
|
|
function call. A workflow transition happens at exactly that boundary,
|
|
so using the last (older) resumption handle can omit the triggering user
|
|
turn. Use the local LLMContext as the source of truth for this intentional
|
|
handoff instead.
|
|
"""
|
|
self._awaiting_node_transition_context = True
|
|
self._node_transition_context_received = False
|
|
self._node_transition_context_seed_started = False
|
|
self._session_resumption_handle = None
|
|
await self._disconnect()
|
|
await self._connect(session_resumption_handle=None)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Frame handling: mute, TTSSpeakFrame, BotStoppedSpeakingFrame flush
|
|
# ------------------------------------------------------------------
|
|
|
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
|
if isinstance(frame, UserMuteStartedFrame):
|
|
self._user_is_muted = True
|
|
await self.push_frame(frame, direction)
|
|
return
|
|
if isinstance(frame, UserMuteStoppedFrame):
|
|
self._user_is_muted = False
|
|
await self.push_frame(frame, direction)
|
|
return
|
|
if isinstance(frame, TTSSpeakFrame):
|
|
# Greeting trigger: the engine queues a TTSSpeakFrame to start the
|
|
# bot's first turn after node setup. Gemini Live renders its own
|
|
# audio, so we don't pass the frame through. For configured static
|
|
# text greetings, ask Gemini to say the exact greeting; otherwise
|
|
# re-enter _handle_context to kick off the normal initial response.
|
|
if not self._handled_initial_context:
|
|
greeting_text = frame.text.strip() if frame.text else ""
|
|
if greeting_text:
|
|
await self._handle_initial_greeting(self._context, greeting_text)
|
|
else:
|
|
await self._handle_context(self._context)
|
|
else:
|
|
logger.warning(
|
|
f"{self}: TTSSpeakFrame after initial context already "
|
|
"handled — Gemini Live owns audio generation, ignoring"
|
|
)
|
|
return
|
|
if isinstance(frame, BotStoppedSpeakingFrame):
|
|
# Belt-and-suspenders: the main drain happens in
|
|
# _set_bot_is_responding(False), but if Gemini delays turn_complete
|
|
# past the audible end of the turn, flushing here ensures a pending
|
|
# node transition fires promptly.
|
|
await self._run_pending_node_transition_function_calls()
|
|
# Fall through to super for the actual push.
|
|
await super().process_frame(frame, direction)
|
|
|
|
async def _send_user_audio(self, frame):
|
|
if self._user_is_muted:
|
|
return
|
|
await super()._send_user_audio(frame)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Context lifecycle: Dograh pre-populates self._context via the engine,
|
|
# so upstream's "first arrival === self._context is None" check doesn't
|
|
# work. We gate on _handled_initial_context instead and skip the
|
|
# init-instruction reconciliation (Dograh updates system_instruction at
|
|
# runtime via _update_settings, not via init).
|
|
# ------------------------------------------------------------------
|
|
|
|
async def _handle_context(self, context: LLMContext):
|
|
if self._awaiting_node_transition_context:
|
|
self._context = context
|
|
self._node_transition_context_received = True
|
|
await self._maybe_seed_node_transition_context()
|
|
return
|
|
if not self._handled_initial_context:
|
|
self._handled_initial_context = True
|
|
self._context = context
|
|
await self._create_initial_response()
|
|
else:
|
|
self._context = context
|
|
await self._process_completed_function_calls(send_new_results=True)
|
|
|
|
async def _handle_initial_greeting(self, context: LLMContext, greeting_text: str):
|
|
"""Trigger the first Gemini turn with an exact static text greeting."""
|
|
if context is None:
|
|
logger.warning(
|
|
f"{self}: received initial greeting trigger before context was set"
|
|
)
|
|
return
|
|
|
|
self._handled_initial_context = True
|
|
self._context = context
|
|
await self._create_initial_greeting_response(greeting_text)
|
|
|
|
async def _create_initial_greeting_response(self, greeting_text: str):
|
|
"""Ask Gemini Live to speak the configured greeting exactly once."""
|
|
if self._disconnecting:
|
|
return
|
|
|
|
if not self._session:
|
|
self._pending_initial_greeting_text = greeting_text
|
|
self._run_llm_when_session_ready = True
|
|
return
|
|
|
|
self._pending_initial_greeting_text = None
|
|
prompt = format_static_greeting_prompt(greeting_text)
|
|
turn = Content(role="user", parts=[Part(text=prompt)])
|
|
|
|
logger.debug("Creating Gemini Live initial response from static greeting")
|
|
|
|
await self.start_ttfb_metrics()
|
|
|
|
try:
|
|
await self._session.send_client_content(
|
|
turns=[turn],
|
|
turn_complete=True,
|
|
)
|
|
# Gemini 3.x also needs a realtime-input nudge to begin inference.
|
|
if self._is_gemini_3:
|
|
await self._session.send_realtime_input(text=" ")
|
|
except Exception as e:
|
|
await self._handle_send_error(e)
|
|
|
|
self._ready_for_realtime_input = True
|
|
|
|
# ------------------------------------------------------------------
|
|
# Session lifecycle: drop upstream's automatic reconnect-seed and
|
|
# initial-context-seed paths. The TTSSpeakFrame trigger and the
|
|
# function-call-result LLMContextFrame are the only paths that should
|
|
# kick off bot turns in the Dograh flow.
|
|
# ------------------------------------------------------------------
|
|
|
|
@traced_gemini_live(operation="llm_setup")
|
|
async def _handle_session_ready(self, session):
|
|
logger.debug(
|
|
f"In _handle_session_ready self._run_llm_when_session_ready: {self._run_llm_when_session_ready}"
|
|
)
|
|
self._session = session
|
|
if self._awaiting_node_transition_context:
|
|
# Do not accept realtime input until the function-call result frame
|
|
# has updated the shared context and that complete history is seeded.
|
|
self._ready_for_realtime_input = False
|
|
await self._maybe_seed_node_transition_context()
|
|
return
|
|
self._ready_for_realtime_input = True
|
|
if self._run_llm_when_session_ready:
|
|
# Context arrived before session was ready — fulfil the queued
|
|
# initial response now.
|
|
self._run_llm_when_session_ready = False
|
|
if self._pending_initial_greeting_text is not None:
|
|
await self._create_initial_greeting_response(
|
|
self._pending_initial_greeting_text
|
|
)
|
|
else:
|
|
await self._create_initial_response()
|
|
await self._drain_pending_tool_results()
|
|
# Otherwise: no automatic seed. Reconnect after a session-resumption
|
|
# update relies on the server-side restored state; reconnects without
|
|
# a handle (e.g. node transitions before any handle was issued) are
|
|
# followed by a function-call-result LLMContextFrame which feeds the
|
|
# updated-context branch in _handle_context.
|
|
|
|
async def _maybe_seed_node_transition_context(self) -> None:
|
|
if (
|
|
not self._awaiting_node_transition_context
|
|
or not self._node_transition_context_received
|
|
or not self._session
|
|
or self._node_transition_context_seed_started
|
|
):
|
|
return
|
|
|
|
self._node_transition_context_seed_started = True
|
|
try:
|
|
# The complete tool result is already present in the history being
|
|
# seeded, so mark it delivered locally instead of sending a provider
|
|
# tool response for a call that the fresh session never issued.
|
|
await self._process_completed_function_calls(send_new_results=False)
|
|
await self._create_initial_response()
|
|
self._awaiting_node_transition_context = False
|
|
self._node_transition_context_received = False
|
|
await self._drain_pending_tool_results()
|
|
finally:
|
|
self._node_transition_context_seed_started = False
|