feat: add Azure AI multi-provider support (TTS, STT, Embeddings, Realtime) (#381)

* feat: add Azure AI multi-provider support (TTS, STT, Embeddings, Realtime)

Enables Azure AI services across all model layers so users with Azure
credits can consolidate billing on a single provider.

- Voice (TTS): AzureSpeechTTSConfiguration via azure_speech provider
- Transcriber (STT): AzureSpeechSTTConfiguration via azure_speech provider
- Embedding: AzureOpenAIEmbeddingsConfiguration via azure provider
- Realtime: AzureRealtimeLLMConfiguration via azure_realtime provider

New files:
- api/services/pipecat/realtime/azure_realtime.py
- api/services/gen_ai/embedding/azure_openai_service.py
- api/tests/test_azure_speech_service_factory.py

The UI picks up all four providers automatically from the schema —
no frontend changes required.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* fix: add validation for URL and params

---------

Co-authored-by: Vishal Dhateria <vishal@finela.ai>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: Abhishek Kumar <abhishek@a6k.me>
This commit is contained in:
Vishal Dhateria 2026-06-02 12:50:00 +05:30 committed by GitHub
parent 8a4a2e25db
commit 7ba95c0fbe
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 1082 additions and 29 deletions

View file

@ -13,7 +13,8 @@ from loguru import logger
from opentelemetry import trace
from api.db import db_client
from api.services.gen_ai import OpenAIEmbeddingService
from api.services.configuration.registry import ServiceProviders
from api.services.gen_ai import AzureOpenAIEmbeddingService, OpenAIEmbeddingService
from api.services.pipecat.tracing_config import ensure_tracing
@ -25,6 +26,9 @@ async def retrieve_from_knowledge_base(
embeddings_api_key: Optional[str] = None,
embeddings_model: Optional[str] = None,
embeddings_base_url: Optional[str] = None,
embeddings_provider: Optional[str] = None,
embeddings_endpoint: Optional[str] = None,
embeddings_api_version: Optional[str] = None,
tracing_context=None,
) -> Dict[str, Any]:
"""Retrieve relevant information from the knowledge base using vector similarity search.
@ -68,6 +72,9 @@ async def retrieve_from_knowledge_base(
embeddings_api_key,
embeddings_model,
embeddings_base_url,
embeddings_provider,
embeddings_endpoint,
embeddings_api_version,
)
# Create span with parent context
@ -105,6 +112,9 @@ async def retrieve_from_knowledge_base(
embeddings_api_key,
embeddings_model,
embeddings_base_url,
embeddings_provider,
embeddings_endpoint,
embeddings_api_version,
)
# Add result metadata to span
@ -179,6 +189,9 @@ async def retrieve_from_knowledge_base(
embeddings_api_key,
embeddings_model,
embeddings_base_url,
embeddings_provider,
embeddings_endpoint,
embeddings_api_version,
)
else:
# Tracing is disabled - perform retrieval without tracing
@ -189,6 +202,10 @@ async def retrieve_from_knowledge_base(
limit,
embeddings_api_key,
embeddings_model,
embeddings_base_url,
embeddings_provider,
embeddings_endpoint,
embeddings_api_version,
)
@ -200,6 +217,9 @@ async def _perform_retrieval(
embeddings_api_key: Optional[str] = None,
embeddings_model: Optional[str] = None,
embeddings_base_url: Optional[str] = None,
embeddings_provider: Optional[str] = None,
embeddings_endpoint: Optional[str] = None,
embeddings_api_version: Optional[str] = None,
) -> Dict[str, Any]:
"""Internal function to perform the actual retrieval operation.
@ -240,12 +260,24 @@ async def _perform_retrieval(
"Model Configurations > Embedding."
)
embedding_service = OpenAIEmbeddingService(
db_client=db_client,
api_key=embeddings_api_key,
model_id=embeddings_model or "text-embedding-3-small",
base_url=embeddings_base_url,
)
if (
embeddings_provider == ServiceProviders.AZURE.value
and embeddings_endpoint
):
embedding_service = AzureOpenAIEmbeddingService(
db_client=db_client,
api_key=embeddings_api_key,
endpoint=embeddings_endpoint,
model_id=embeddings_model or "text-embedding-3-small",
api_version=embeddings_api_version or "2024-02-15-preview",
)
else:
embedding_service = OpenAIEmbeddingService(
db_client=db_client,
api_key=embeddings_api_key,
model_id=embeddings_model or "text-embedding-3-small",
base_url=embeddings_base_url,
)
results = await embedding_service.search_similar_chunks(
query=query,