feat: add ElevenLabs realtime STT provider support (#512) (#522)

* feat: add ElevenLabs realtime STT provider support (#512)

Wire ElevenLabs scribe_v2_realtime into the STT registry and pipeline factory so BYOK transcribers can use the same provider already supported for TTS.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: address ElevenLabs STT review feedback for language, commits, and host

Pass custom language codes through instead of defaulting to English, use ElevenLabs VAD commit strategy because Dograh VAD runs downstream of STT, and document hostname-only realtime base_url handling.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: preserve ElevenLabs STT endpoint port in realtime host parsing

Use urlparse netloc instead of hostname so validated BYOK/proxy base URLs keep non-default ports when Pipecat builds the websocket endpoint.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: preserve ElevenLabs STT proxy path prefix and remove duplicate tests

Include URL path segments in realtime host normalization for BYOK proxies and delete shadowed pytest definitions.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: allow custom ElevenLabs model input

* fix: normalize ElevenLabs websocket URLs

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Abhishek Kumar <abhishek@a6k.me>

api/services/configuration/options/__init__.py

@@ -22,6 +22,7 @@ from .deepgram import (
     DEEPGRAM_LANGUAGES,
     DEEPGRAM_STT_MODELS,
 )
+from .elevenlabs import ELEVENLABS_STT_LANGUAGES, ELEVENLABS_STT_MODELS
 from .gladia import GLADIA_STT_LANGUAGES, GLADIA_STT_MODELS
 from .google import (
     GOOGLE_MODELS,
@@ -69,6 +70,8 @@ __all__ = [
     "CARTESIA_INK_WHISPER_STT_LANGUAGES",
     "CARTESIA_STT_LANGUAGES",
     "CARTESIA_STT_MODELS",
+    "ELEVENLABS_STT_LANGUAGES",
+    "ELEVENLABS_STT_MODELS",
     "DEEPGRAM_FLUX_MODELS",
     "DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGES",
     "DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGE_OPTIONS",

api/services/configuration/options/elevenlabs.py

@@ -0,0 +1,95 @@
+ELEVENLABS_STT_MODELS = ("scribe_v2_realtime",)
+
+ELEVENLABS_STT_LANGUAGES = (
+    "auto",
+    "af",
+    "am",
+    "ar",
+    "as",
+    "az",
+    "be",
+    "bg",
+    "bn",
+    "bs",
+    "ca",
+    "ceb",
+    "cs",
+    "cy",
+    "da",
+    "de",
+    "el",
+    "en",
+    "es",
+    "et",
+    "fa",
+    "fi",
+    "fil",
+    "fr",
+    "ga",
+    "gl",
+    "gu",
+    "ha",
+    "he",
+    "hi",
+    "hr",
+    "hu",
+    "hy",
+    "id",
+    "ig",
+    "is",
+    "it",
+    "ja",
+    "jv",
+    "ka",
+    "kk",
+    "km",
+    "kn",
+    "ko",
+    "ku",
+    "ky",
+    "lo",
+    "lt",
+    "lv",
+    "mi",
+    "mk",
+    "ml",
+    "mn",
+    "mr",
+    "ms",
+    "mt",
+    "my",
+    "ne",
+    "nl",
+    "no",
+    "ny",
+    "oc",
+    "or",
+    "pa",
+    "pl",
+    "ps",
+    "pt",
+    "ro",
+    "ru",
+    "sd",
+    "sk",
+    "sl",
+    "sn",
+    "so",
+    "sr",
+    "sv",
+    "sw",
+    "ta",
+    "te",
+    "tg",
+    "th",
+    "tr",
+    "uk",
+    "ur",
+    "uz",
+    "vi",
+    "wo",
+    "xh",
+    "yue",
+    "zh",
+    "zu",
+)

api/services/configuration/registry.py

@@ -22,6 +22,8 @@ from api.services.configuration.options import (
     DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGES,
     DEEPGRAM_LANGUAGES,
     DEEPGRAM_STT_MODELS,
+    ELEVENLABS_STT_LANGUAGES,
+    ELEVENLABS_STT_MODELS,
     GLADIA_STT_LANGUAGES,
     GLADIA_STT_MODELS,
     GOOGLE_MODELS,
@@ -862,7 +864,10 @@ class ElevenlabsTTSConfiguration(BaseServiceConfiguration):
     model: str = Field(
         default="eleven_flash_v2_5",
         description="ElevenLabs TTS model.",
-        json_schema_extra={"examples": ELEVENLABS_TTS_MODELS},
+        json_schema_extra={
+            "examples": ELEVENLABS_TTS_MODELS,
+            "allow_custom_input": True,
+        },
     )
     base_url: str = Field(
         default="https://api.elevenlabs.io",
@@ -1672,6 +1677,39 @@ SMALLEST_STT_LANGUAGES = [
 ]
 
 
+@register_stt
+class ElevenlabsSTTConfiguration(BaseSTTConfiguration):
+    model_config = ELEVENLABS_PROVIDER_MODEL_CONFIG
+    provider: Literal[ServiceProviders.ELEVENLABS] = ServiceProviders.ELEVENLABS
+    model: str = Field(
+        default="scribe_v2_realtime",
+        description="ElevenLabs realtime STT model.",
+        json_schema_extra={
+            "examples": ELEVENLABS_STT_MODELS,
+            "allow_custom_input": True,
+        },
+    )
+    language: str = Field(
+        default="en",
+        description=(
+            "ISO 639-1 language code for transcription. "
+            "Use 'auto' to let ElevenLabs detect the language."
+        ),
+        json_schema_extra={
+            "examples": ELEVENLABS_STT_LANGUAGES,
+            "allow_custom_input": True,
+        },
+    )
+    base_url: str = Field(
+        default="https://api.elevenlabs.io",
+        description=(
+            "ElevenLabs API base URL. Override to use a Data Residency endpoint "
+            "(e.g. https://api.eu.residency.elevenlabs.io) for GDPR / HIPAA / "
+            "regional compliance."
+        ),
+    )
+
+
 @register_stt
 class SmallestAISTTConfiguration(BaseSTTConfiguration):
     model_config = SMALLEST_PROVIDER_MODEL_CONFIG
@@ -1706,6 +1744,7 @@ STTConfig = Annotated[
         GladiaSTTConfiguration,
         AzureSpeechSTTConfiguration,
         SmallestAISTTConfiguration,
+        ElevenlabsSTTConfiguration,
     ],
     Field(discriminator="provider"),
 ]

api/services/pipecat/service_factory.py

@@ -38,6 +38,11 @@ from pipecat.services.dograh.flux.stt import DograhFluxSTTService
 from pipecat.services.dograh.llm import DograhLLMService
 from pipecat.services.dograh.stt import DograhSTTService, DograhSTTSettings
 from pipecat.services.dograh.tts import DograhTTSService, DograhTTSSettings
+from pipecat.services.elevenlabs.stt import (
+    CommitStrategy,
+    ElevenLabsRealtimeSTTService,
+    ElevenLabsRealtimeSTTSettings,
+)
 from pipecat.services.elevenlabs.tts import ElevenLabsTTSService, ElevenLabsTTSSettings
 from pipecat.services.gladia.stt import GladiaSTTService, GladiaSTTSettings
 from pipecat.services.google.llm import GoogleLLMService, GoogleLLMSettings
@@ -108,6 +113,52 @@ def dograh_stt_uses_flux_language(language: str | None) -> bool:
     return language in DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGE_OPTIONS
 
 
+def _resolve_elevenlabs_stt_language(
+    language_code: str | None,
+) -> Language | str | None:
+    if not language_code or language_code == "auto":
+        return None
+    try:
+        return Language(language_code)
+    except ValueError:
+        return language_code
+
+
+def _elevenlabs_websocket_url(base_url: str) -> str:
+    """Normalize an ElevenLabs API base URL for WebSocket clients."""
+    base_url = base_url.strip()
+    parsed = urlparse(base_url)
+    if not parsed.netloc:
+        return base_url.rstrip("/")
+
+    websocket_scheme = {
+        "http": "ws",
+        "https": "wss",
+    }.get(parsed.scheme, parsed.scheme)
+    return urlunparse(
+        parsed._replace(
+            scheme=websocket_scheme,
+            path=parsed.path.rstrip("/"),
+        )
+    )
+
+
+def _elevenlabs_realtime_stt_host(base_url: str) -> str:
+    """Return the host/path prefix Pipecat's ElevenLabs realtime STT expects.
+
+    Pipecat's realtime STT service builds
+    ``wss://{host}/v1/speech-to-text/realtime`` internally, so remove the scheme
+    from the same normalized WebSocket URL used by ElevenLabs TTS. Preserve
+    netloc (including optional ports) and any path prefix used by BYOK proxies.
+    """
+    websocket_url = _elevenlabs_websocket_url(base_url)
+    parsed = urlparse(websocket_url)
+    if parsed.netloc:
+        path = parsed.path
+        return f"{parsed.netloc}{path}" if path else parsed.netloc
+    return websocket_url
+
+
 def stt_uses_external_turns(user_config) -> bool:
     if user_config.stt.provider == ServiceProviders.DEEPGRAM.value:
         return user_config.stt.model in DEEPGRAM_FLUX_MODELS
@@ -415,6 +466,24 @@ def create_stt_service(
             ),
             sample_rate=audio_config.transport_in_sample_rate,
         )
+    elif user_config.stt.provider == ServiceProviders.ELEVENLABS.value:
+        language_code = getattr(user_config.stt, "language", None)
+        pipecat_language = _resolve_elevenlabs_stt_language(language_code)
+
+        _validate_runtime_service_url(user_config.stt.base_url, "base_url")
+        elevenlabs_host = _elevenlabs_realtime_stt_host(user_config.stt.base_url)
+
+        return ElevenLabsRealtimeSTTService(
+            api_key=user_config.stt.api_key,
+            base_url=elevenlabs_host,
+            commit_strategy=CommitStrategy.VAD,
+            settings=ElevenLabsRealtimeSTTSettings(
+                model=user_config.stt.model,
+                language=pipecat_language,
+            ),
+            should_interrupt=False,
+            sample_rate=audio_config.transport_in_sample_rate,
+        )
     else:
         raise HTTPException(
             status_code=400, detail=f"Invalid STT provider {user_config.stt.provider}"
@@ -488,13 +557,11 @@ def create_tts_service(
             voice_id = user_config.tts.voice.split(" - ")[1]
         except IndexError:
             voice_id = user_config.tts.voice
-        # ElevenLabs TTS uses WebSocket. Users configure base_url with an HTTP
-        # scheme (matching ElevenLabs documentation, e.g.
-        # https://api.eu.residency.elevenlabs.io); rewrite it to the WS scheme.
+        # ElevenLabs TTS consumes the full normalized WebSocket URL. Realtime
+        # STT uses the same normalization before adapting it to Pipecat's
+        # scheme-less base_url contract.
         _validate_runtime_service_url(user_config.tts.base_url, "base_url")
-        elevenlabs_url = user_config.tts.base_url.replace("https://", "wss://").replace(
-            "http://", "ws://"
-        )
+        elevenlabs_url = _elevenlabs_websocket_url(user_config.tts.base_url)
         return ElevenLabsTTSService(
             reconnect_on_error=False,
             api_key=user_config.tts.api_key,

api/tests/test_elevenlabs_stt_service_factory.py

@@ -0,0 +1,198 @@
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+from pipecat.services.elevenlabs.stt import CommitStrategy
+from pipecat.transcriptions.language import Language
+
+from api.services.configuration.options import (
+    ELEVENLABS_STT_LANGUAGES,
+    ELEVENLABS_STT_MODELS,
+)
+from api.services.configuration.registry import (
+    ElevenlabsSTTConfiguration,
+    ServiceProviders,
+)
+from api.services.pipecat.audio_config import AudioConfig
+from api.services.pipecat.service_factory import (
+    create_stt_service,
+    create_tts_service,
+    stt_uses_external_turns,
+)
+
+
+def _audio_config() -> AudioConfig:
+    return AudioConfig(
+        transport_in_sample_rate=16000,
+        transport_out_sample_rate=16000,
+    )
+
+
+def _elevenlabs_config(
+    language: str = "en",
+    base_url: str = "https://api.elevenlabs.io",
+) -> SimpleNamespace:
+    return SimpleNamespace(
+        stt=SimpleNamespace(
+            provider=ServiceProviders.ELEVENLABS.value,
+            api_key="test-key",
+            model="scribe_v2_realtime",
+            language=language,
+            base_url=base_url,
+        )
+    )
+
+
+def _elevenlabs_tts_config(base_url: str) -> SimpleNamespace:
+    return SimpleNamespace(
+        tts=SimpleNamespace(
+            provider=ServiceProviders.ELEVENLABS.value,
+            api_key="test-key",
+            model="eleven_flash_v2_5",
+            voice="test-voice",
+            speed=1.0,
+            base_url=base_url,
+        )
+    )
+
+
+def test_elevenlabs_stt_configuration_exposes_defaults_and_languages():
+    config = ElevenlabsSTTConfiguration(api_key="test-key")
+    language_schema = ElevenlabsSTTConfiguration.model_json_schema()["properties"][
+        "language"
+    ]
+
+    assert config.provider == ServiceProviders.ELEVENLABS
+    assert config.model == "scribe_v2_realtime"
+    assert config.language == "en"
+    assert config.base_url == "https://api.elevenlabs.io"
+    assert ELEVENLABS_STT_MODELS == ("scribe_v2_realtime",)
+    assert "auto" in ELEVENLABS_STT_LANGUAGES
+    assert "es" in ELEVENLABS_STT_LANGUAGES
+    assert language_schema["examples"] == list(ELEVENLABS_STT_LANGUAGES)
+
+
+def test_elevenlabs_stt_uses_realtime_service_with_language_mapping():
+    user_config = _elevenlabs_config(language="es")
+
+    assert not stt_uses_external_turns(user_config)
+
+    with patch(
+        "api.services.pipecat.service_factory.ElevenLabsRealtimeSTTService"
+    ) as stt_service:
+        create_stt_service(user_config, _audio_config())
+
+    stt_service.assert_called_once()
+    kwargs = stt_service.call_args.kwargs
+    assert kwargs["api_key"] == "test-key"
+    assert kwargs["base_url"] == "api.elevenlabs.io"
+    assert kwargs["commit_strategy"] == CommitStrategy.VAD
+    assert kwargs["sample_rate"] == 16000
+    assert kwargs["should_interrupt"] is False
+    assert kwargs["settings"].model == "scribe_v2_realtime"
+    assert kwargs["settings"].language == Language.ES
+
+
+def test_elevenlabs_stt_auto_language_passes_none():
+    user_config = _elevenlabs_config(language="auto")
+
+    with patch(
+        "api.services.pipecat.service_factory.ElevenLabsRealtimeSTTService"
+    ) as stt_service:
+        create_stt_service(user_config, _audio_config())
+
+    kwargs = stt_service.call_args.kwargs
+    assert kwargs["settings"].language is None
+
+
+def test_elevenlabs_stt_extracts_hostname_from_residency_base_url():
+    user_config = _elevenlabs_config(base_url="https://api.eu.residency.elevenlabs.io")
+
+    with patch(
+        "api.services.pipecat.service_factory.ElevenLabsRealtimeSTTService"
+    ) as stt_service:
+        create_stt_service(user_config, _audio_config())
+
+    kwargs = stt_service.call_args.kwargs
+    assert kwargs["base_url"] == "api.eu.residency.elevenlabs.io"
+
+
+def test_elevenlabs_stt_custom_language_passes_through():
+    user_config = _elevenlabs_config(language="custom-lang")
+
+    with patch(
+        "api.services.pipecat.service_factory.ElevenLabsRealtimeSTTService"
+    ) as stt_service:
+        create_stt_service(user_config, _audio_config())
+
+    kwargs = stt_service.call_args.kwargs
+    assert kwargs["settings"].language == "custom-lang"
+
+
+def test_elevenlabs_stt_bare_hostname_base_url_is_preserved():
+    user_config = _elevenlabs_config(base_url="api.elevenlabs.io")
+
+    with patch(
+        "api.services.pipecat.service_factory.ElevenLabsRealtimeSTTService"
+    ) as stt_service:
+        create_stt_service(user_config, _audio_config())
+
+    kwargs = stt_service.call_args.kwargs
+    assert kwargs["base_url"] == "api.elevenlabs.io"
+
+
+def test_elevenlabs_stt_preserves_non_default_port_in_base_url():
+    user_config = _elevenlabs_config(base_url="https://localhost:8443")
+
+    with patch(
+        "api.services.pipecat.service_factory.ElevenLabsRealtimeSTTService"
+    ) as stt_service:
+        create_stt_service(user_config, _audio_config())
+
+    kwargs = stt_service.call_args.kwargs
+    assert kwargs["base_url"] == "localhost:8443"
+
+
+def test_elevenlabs_stt_preserves_proxy_path_prefix_in_base_url():
+    user_config = _elevenlabs_config(base_url="https://proxy.example.com/elevenlabs")
+
+    with patch(
+        "api.services.pipecat.service_factory.ElevenLabsRealtimeSTTService"
+    ) as stt_service:
+        create_stt_service(user_config, _audio_config())
+
+    kwargs = stt_service.call_args.kwargs
+    assert kwargs["base_url"] == "proxy.example.com/elevenlabs"
+
+
+@pytest.mark.parametrize(
+    ("base_url", "expected_url"),
+    [
+        (
+            "https://api.eu.residency.elevenlabs.io/elevenlabs/",
+            "wss://api.eu.residency.elevenlabs.io/elevenlabs",
+        ),
+        ("http://localhost:8000/", "ws://localhost:8000"),
+    ],
+)
+def test_elevenlabs_tts_uses_normalized_websocket_url(base_url, expected_url):
+    user_config = _elevenlabs_tts_config(base_url)
+
+    with patch(
+        "api.services.pipecat.service_factory.ElevenLabsTTSService"
+    ) as tts_service:
+        create_tts_service(user_config, _audio_config())
+
+    assert tts_service.call_args.kwargs["url"] == expected_url
+
+
+def test_elevenlabs_stt_listed_custom_language_maps_to_pipecat_enum():
+    user_config = _elevenlabs_config(language="yue")
+
+    with patch(
+        "api.services.pipecat.service_factory.ElevenLabsRealtimeSTTService"
+    ) as stt_service:
+        create_stt_service(user_config, _audio_config())
+
+    kwargs = stt_service.call_args.kwargs
+    assert kwargs["settings"].language == Language.YUE

docs/configurations/transcriber.mdx

@@ -3,7 +3,7 @@ title: "Transcriber"
 description: "Voice Agents use STT (Speech to Text), to transcribe what the user speaks. This transcribed speech as text goes into an LLM to generate the response that gets played out to the user."
 ---
 
-Dograh platform supports Deepgram, OpenAI, Google, Azure Speech, AssemblyAI, Speechmatics, Cartesia, Gladia, Sarvam, Smallest AI, Hugging Face, and Dograh transcribers. You can take a look at the providers documentation of which language to select for your language requirements. 
+Dograh platform supports Deepgram, OpenAI, Google, Azure Speech, AssemblyAI, Speechmatics, Cartesia, Gladia, Sarvam, Smallest AI, Hugging Face, ElevenLabs, and Dograh transcribers. You can take a look at the providers documentation of which language to select for your language requirements. 
 
 <Warning>
 Transcriber providers receive the data needed to transcribe speech, such as call audio, language settings, keyterms, and request metadata. Review the provider's data processing, retention, model training, and regional hosting policies before using sensitive data.

ui/src/client/types.gen.ts

@@ -593,7 +593,9 @@ export type ByokPipelineAiModelConfiguration = {
         provider: 'azure_speech';
     } & AzureSpeechSttConfiguration) | ({
         provider: 'smallest';
-    } & SmallestAisttConfiguration);
+    } & SmallestAisttConfiguration) | ({
+        provider: 'elevenlabs';
+    } & ElevenlabsSttConfiguration);
     /**
      * Embeddings
      */
@@ -2224,6 +2226,38 @@ export type DuplicateTemplateRequest = {
     workflow_name: string;
 };
 
+/**
+ * ElevenLabs
+ */
+export type ElevenlabsSttConfiguration = {
+    /**
+     * Provider
+     */
+    provider?: 'elevenlabs';
+    /**
+     * Api Key
+     */
+    api_key: string | Array<string>;
+    /**
+     * Model
+     *
+     * ElevenLabs realtime STT model.
+     */
+    model?: string;
+    /**
+     * Language
+     *
+     * ISO 639-1 language code for transcription. Use 'auto' to let ElevenLabs detect the language.
+     */
+    language?: string;
+    /**
+     * Base Url
+     *
+     * ElevenLabs API base URL. Override to use a Data Residency endpoint (e.g. https://api.eu.residency.elevenlabs.io) for GDPR / HIPAA / regional compliance.
+     */
+    base_url?: string;
+};
+
 /**
  * ElevenLabs
  */