feat(openai-realtime): allow pinning input transcription language (#557)

The OpenAI realtime factory already reads `language` from the realtime
config, but builds `InputAudioTranscription()` without it, so the language
is silently dropped and the model auto-detects on every utterance.

On 8kHz telephony audio this misfires badly: in our production tests a
Brazilian Portuguese speaker was transcribed as English, French and Chinese
within a single call, which then corrupted downstream extraction.

Every other STT branch in `create_stt_service` already honours
`language` (Deepgram, Google, Cartesia, Dograh, Sarvam), and
`GoogleRealtimeLLMConfiguration` already exposes a `language` field for
Gemini Live. This brings the OpenAI realtime provider in line with both.

- expose `language` on `OpenAIRealtimeLLMConfiguration` (optional,
  defaults to None -> unchanged auto-detect behaviour)
- pass it through to `InputAudioTranscription`, which already accepts it

Verified against pipecat: the session now carries
`{"transcription": {"model": "gpt-realtime-whisper", "language": "pt"}}`.

Co-authored-by: Liberty Card <tecnologialibertycard@gmail.com>
This commit is contained in:
Joao Victor Cardoso dos Santos 2026-07-18 07:38:35 -03:00 committed by GitHub
parent f4c5954a39
commit ade0ee9104
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 35 additions and 1 deletions

View file

@ -584,6 +584,20 @@ class SarvamLLMConfiguration(BaseLLMConfiguration):
OPENAI_REALTIME_MODELS = ["gpt-realtime-2"]
# ISO 639-1 codes accepted by the Realtime API's input_audio_transcription.
# Not exhaustive — the field allows custom input.
OPENAI_REALTIME_LANGUAGES = [
"en",
"es",
"pt",
"fr",
"de",
"it",
"hi",
"ja",
"ko",
"zh",
]
OPENAI_REALTIME_VOICES = [
"alloy",
"ash",
@ -618,6 +632,17 @@ class OpenAIRealtimeLLMConfiguration(BaseLLMConfiguration):
"allow_custom_input": True,
},
)
language: str | None = Field(
default=None,
description=(
"ISO 639-1 language code for input audio transcription (e.g. 'pt', 'es'). "
"Improves transcription accuracy and latency. Leave unset to auto-detect."
),
json_schema_extra={
"examples": OPENAI_REALTIME_LANGUAGES,
"allow_custom_input": True,
},
)
GROK_REALTIME_MODELS = ["grok-voice-think-fast-1.0"]

View file

@ -1008,6 +1008,13 @@ def create_realtime_llm_service(user_config, audio_config: "AudioConfig"):
SessionProperties,
)
# Pin the transcription language when configured. Without it the model
# auto-detects per utterance, which misfires on short/noisy telephony
# audio (e.g. Portuguese transcribed as English or Chinese).
transcription_kwargs = {}
if language:
transcription_kwargs["language"] = language
return DograhOpenAIRealtimeLLMService(
api_key=api_key,
settings=DograhOpenAIRealtimeLLMService.Settings(
@ -1015,7 +1022,9 @@ def create_realtime_llm_service(user_config, audio_config: "AudioConfig"):
session_properties=SessionProperties(
audio=AudioConfiguration(
input=AudioInput(
transcription=InputAudioTranscription(),
transcription=InputAudioTranscription(
**transcription_kwargs
),
),
output=AudioOutput(
voice=voice or "alloy",