From d817d5005636b2992724296b88126b99153a4836 Mon Sep 17 00:00:00 2001 From: Haoqian Date: Wed, 24 Jun 2026 21:20:44 +0800 Subject: [PATCH] fix: support Gemini JSON schema tools (#463) * fix: support Gemini JSON schema tools * fix: harden Dograh Gemini adapter wiring * fix: route Gemini Live tool schemas through parameters_json_schema Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Abhishek Kumar Co-authored-by: Claude Opus 4.8 (1M context) --- .../pipecat/gemini_json_schema_adapter.py | 39 +++++ api/services/pipecat/realtime/gemini_live.py | 11 ++ api/services/pipecat/service_factory.py | 15 +- api/tests/test_gemini_json_schema_adapter.py | 160 ++++++++++++++++++ .../test_google_vertex_llm_service_factory.py | 4 +- 5 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 api/services/pipecat/gemini_json_schema_adapter.py create mode 100644 api/tests/test_gemini_json_schema_adapter.py diff --git a/api/services/pipecat/gemini_json_schema_adapter.py b/api/services/pipecat/gemini_json_schema_adapter.py new file mode 100644 index 00000000..c5422c80 --- /dev/null +++ b/api/services/pipecat/gemini_json_schema_adapter.py @@ -0,0 +1,39 @@ +"""Dograh-specific Gemini adapter customizations.""" + +from typing import Any + +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema +from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter + + +class DograhGeminiJSONSchemaAdapter(GeminiLLMAdapter): + """Use Gemini's full JSON Schema tool parameter field. + + Pipecat's default Gemini adapter maps ``FunctionSchema.parameters`` into + ``FunctionDeclaration.parameters``, which is backed by Google GenAI's + stricter OpenAPI-style ``Schema`` model. MCP and imported tools may contain + valid JSON Schema keywords such as ``const`` and ``not`` that are rejected + by that model. ``parameters_json_schema`` is the Google GenAI field intended + for full JSON Schema payloads. + """ + + def to_provider_tools_format( + self, tools_schema: ToolsSchema + ) -> list[dict[str, Any]]: + functions_schema = tools_schema.standard_tools + if functions_schema: + formatted_functions = [] + for func in functions_schema: + func_dict = func.to_default_dict() + parameters = func_dict.pop("parameters") + func_dict["parameters_json_schema"] = parameters + formatted_functions.append(func_dict) + formatted_standard_tools = [{"function_declarations": formatted_functions}] + else: + formatted_standard_tools = [] + + custom_gemini_tools = [] + if tools_schema.custom_tools: + custom_gemini_tools = tools_schema.custom_tools.get(AdapterType.GEMINI, []) + + return formatted_standard_tools + custom_gemini_tools diff --git a/api/services/pipecat/realtime/gemini_live.py b/api/services/pipecat/realtime/gemini_live.py index aba4880b..8203f9e6 100644 --- a/api/services/pipecat/realtime/gemini_live.py +++ b/api/services/pipecat/realtime/gemini_live.py @@ -35,10 +35,21 @@ from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService from pipecat.services.llm_service import FunctionCallFromLLM from pipecat.utils.tracing.service_decorators import traced_gemini_live +from api.services.pipecat.gemini_json_schema_adapter import ( + DograhGeminiJSONSchemaAdapter, +) + class DograhGeminiLiveLLMService(GeminiLiveLLMService): """Gemini Live with Dograh engine integration quirks. See module docstring.""" + # Route tool schemas through Gemini's ``parameters_json_schema`` field so + # MCP/imported tools that use JSON Schema keywords (``const``, ``not``, + # nested ``anyOf``) rejected by the strict ``Schema`` model are accepted. + # Mirrors the non-realtime ``DograhGoogleLLMService`` fix; + # ``DograhGeminiLiveVertexLLMService`` inherits this via MRO. + adapter_class = DograhGeminiJSONSchemaAdapter + def __init__(self, **kwargs): super().__init__(**kwargs) # User-mute state, driven by broadcast UserMute{Started,Stopped}Frames. diff --git a/api/services/pipecat/service_factory.py b/api/services/pipecat/service_factory.py index 91a9aac9..1dc27318 100644 --- a/api/services/pipecat/service_factory.py +++ b/api/services/pipecat/service_factory.py @@ -11,6 +11,9 @@ from api.services.configuration.options import ( DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGE_OPTIONS, ) from api.services.configuration.registry import ServiceProviders +from api.services.pipecat.gemini_json_schema_adapter import ( + DograhGeminiJSONSchemaAdapter, +) from api.services.pipecat.minimax_tts import MiniMaxOwnedSessionTTSService from api.utils.url_security import validate_user_configured_service_url from pipecat.services.assemblyai.stt import AssemblyAISTTService, AssemblyAISTTSettings @@ -111,6 +114,14 @@ def stt_uses_flux_turns(user_config) -> bool: return False +class DograhGoogleLLMService(GoogleLLMService): + adapter_class = DograhGeminiJSONSchemaAdapter + + +class DograhGoogleVertexLLMService(GoogleVertexLLMService): + adapter_class = DograhGeminiJSONSchemaAdapter + + def _validate_runtime_service_url(url: str, field_name: str) -> None: try: validate_user_configured_service_url( @@ -790,12 +801,12 @@ def create_llm_service_from_provider( ) elif provider == ServiceProviders.GOOGLE.value: model = _migrate_deprecated_google_model(model) - return GoogleLLMService( + return DograhGoogleLLMService( api_key=api_key, settings=GoogleLLMSettings(model=model, temperature=0.1), ) elif provider == ServiceProviders.GOOGLE_VERTEX.value: - return GoogleVertexLLMService( + return DograhGoogleVertexLLMService( credentials=credentials, project_id=project_id, location=location or "us-east4", diff --git a/api/tests/test_gemini_json_schema_adapter.py b/api/tests/test_gemini_json_schema_adapter.py new file mode 100644 index 00000000..4a9eb353 --- /dev/null +++ b/api/tests/test_gemini_json_schema_adapter.py @@ -0,0 +1,160 @@ +from unittest.mock import patch + +from google.genai.types import GenerateContentConfig, LiveConnectConfig +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema + +from api.services.configuration.registry import ServiceProviders +from api.services.pipecat.gemini_json_schema_adapter import ( + DograhGeminiJSONSchemaAdapter, +) +from api.services.pipecat.realtime.gemini_live import DograhGeminiLiveLLMService +from api.services.pipecat.realtime.gemini_live_vertex import ( + DograhGeminiLiveVertexLLMService, +) +from api.services.pipecat.service_factory import ( + DograhGoogleLLMService, + DograhGoogleVertexLLMService, + create_llm_service_from_provider, +) + + +def test_gemini_tools_use_json_schema_parameters_for_external_schemas(): + function_schema = FunctionSchema( + name="customer_lookup", + description="Look up a customer by email.", + properties={ + "customerEmail": { + "description": "Customer email address", + "anyOf": [ + {"anyOf": [{"not": {}}]}, + {"const": ""}, + ], + }, + "metadata": { + "type": "object", + "additionalProperties": {"type": "string"}, + }, + }, + required=["customerEmail"], + ) + + tools = DograhGeminiJSONSchemaAdapter().to_provider_tools_format( + ToolsSchema(standard_tools=[function_schema]) + ) + + declaration = tools[0]["function_declarations"][0] + assert "parameters" not in declaration + assert ( + declaration["parameters_json_schema"]["properties"]["customerEmail"]["anyOf"][ + 0 + ]["anyOf"][0]["not"] + == {} + ) + assert ( + declaration["parameters_json_schema"]["properties"]["customerEmail"]["anyOf"][ + 1 + ]["const"] + == "" + ) + assert declaration["parameters_json_schema"]["properties"]["metadata"][ + "additionalProperties" + ] == {"type": "string"} + + GenerateContentConfig(tools=tools) + + +def test_gemini_tools_use_json_schema_parameters_for_no_argument_tools(): + function_schema = FunctionSchema( + name="refresh_context", + description="Refresh the current context.", + properties={}, + required=[], + ) + + tools = DograhGeminiJSONSchemaAdapter().to_provider_tools_format( + ToolsSchema(standard_tools=[function_schema]) + ) + + declaration = tools[0]["function_declarations"][0] + assert "parameters" not in declaration + assert declaration["parameters_json_schema"] == { + "type": "object", + "properties": {}, + "required": [], + } + + GenerateContentConfig(tools=tools) + + +def test_google_service_classes_use_dograh_gemini_adapter_class(): + assert DograhGoogleLLMService.adapter_class is DograhGeminiJSONSchemaAdapter + assert DograhGoogleVertexLLMService.adapter_class is DograhGeminiJSONSchemaAdapter + + +def test_google_llm_service_factory_uses_dograh_service_class(): + with patch( + "api.services.pipecat.service_factory.DograhGoogleLLMService", + ) as mock_service: + result = create_llm_service_from_provider( + provider=ServiceProviders.GOOGLE.value, + model="gemini-2.5-flash", + api_key="test-api-key", + ) + + assert result is mock_service.return_value + assert mock_service.call_args.kwargs["api_key"] == "test-api-key" + assert mock_service.call_args.kwargs["settings"].model == "gemini-2.5-flash" + + +def test_google_vertex_llm_service_factory_uses_dograh_service_class(): + with patch( + "api.services.pipecat.service_factory.DograhGoogleVertexLLMService", + ) as mock_service: + result = create_llm_service_from_provider( + provider=ServiceProviders.GOOGLE_VERTEX.value, + model="gemini-2.5-pro", + api_key=None, + project_id="demo-project", + location="us-central1", + credentials='{"type":"service_account"}', + ) + + assert result is mock_service.return_value + assert mock_service.call_args.kwargs["project_id"] == "demo-project" + assert mock_service.call_args.kwargs["location"] == "us-central1" + assert mock_service.call_args.kwargs["settings"].model == "gemini-2.5-pro" + + +def test_gemini_live_service_classes_use_dograh_gemini_adapter_class(): + assert DograhGeminiLiveLLMService.adapter_class is DograhGeminiJSONSchemaAdapter + # Vertex Live inherits adapter_class from DograhGeminiLiveLLMService via MRO. + assert ( + DograhGeminiLiveVertexLLMService.adapter_class is DograhGeminiJSONSchemaAdapter + ) + + +def test_gemini_live_config_accepts_json_schema_tools(): + function_schema = FunctionSchema( + name="customer_lookup", + description="Look up a customer by email.", + properties={ + "customerEmail": { + "description": "Customer email address", + "anyOf": [{"not": {}}, {"const": ""}], + }, + }, + required=["customerEmail"], + ) + + tools = DograhGeminiJSONSchemaAdapter().to_provider_tools_format( + ToolsSchema(standard_tools=[function_schema]) + ) + + declaration = tools[0]["function_declarations"][0] + assert "parameters" not in declaration + assert "parameters_json_schema" in declaration + + # Gemini Live validates tools through LiveConnectConfig rather than + # GenerateContentConfig; it must also accept the raw JSON Schema payload. + LiveConnectConfig(tools=tools) diff --git a/api/tests/test_google_vertex_llm_service_factory.py b/api/tests/test_google_vertex_llm_service_factory.py index 966d6573..cc02e464 100644 --- a/api/tests/test_google_vertex_llm_service_factory.py +++ b/api/tests/test_google_vertex_llm_service_factory.py @@ -34,7 +34,7 @@ class TestGoogleVertexLLMConfiguration: class TestGoogleVertexLLMServiceFactory: def test_create_llm_service_from_provider_uses_vertex_service(self): with patch( - "api.services.pipecat.service_factory.GoogleVertexLLMService" + "api.services.pipecat.service_factory.DograhGoogleVertexLLMService" ) as mock_service: create_llm_service_from_provider( provider=ServiceProviders.GOOGLE_VERTEX.value, @@ -65,7 +65,7 @@ class TestGoogleVertexLLMServiceFactory: ) with patch( - "api.services.pipecat.service_factory.GoogleVertexLLMService" + "api.services.pipecat.service_factory.DograhGoogleVertexLLMService" ) as mock_service: create_llm_service(user_config)