fix: support Gemini JSON schema tools (#463)

* fix: support Gemini JSON schema tools

* fix: harden Dograh Gemini adapter wiring

* fix: route Gemini Live tool schemas through parameters_json_schema

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Abhishek Kumar <abhishek@a6k.me>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

api/services/pipecat/gemini_json_schema_adapter.py

@@ -0,0 +1,39 @@
+"""Dograh-specific Gemini adapter customizations."""
+
+from typing import Any
+
+from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
+from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter
+
+
+class DograhGeminiJSONSchemaAdapter(GeminiLLMAdapter):
+    """Use Gemini's full JSON Schema tool parameter field.
+
+    Pipecat's default Gemini adapter maps ``FunctionSchema.parameters`` into
+    ``FunctionDeclaration.parameters``, which is backed by Google GenAI's
+    stricter OpenAPI-style ``Schema`` model. MCP and imported tools may contain
+    valid JSON Schema keywords such as ``const`` and ``not`` that are rejected
+    by that model. ``parameters_json_schema`` is the Google GenAI field intended
+    for full JSON Schema payloads.
+    """
+
+    def to_provider_tools_format(
+        self, tools_schema: ToolsSchema
+    ) -> list[dict[str, Any]]:
+        functions_schema = tools_schema.standard_tools
+        if functions_schema:
+            formatted_functions = []
+            for func in functions_schema:
+                func_dict = func.to_default_dict()
+                parameters = func_dict.pop("parameters")
+                func_dict["parameters_json_schema"] = parameters
+                formatted_functions.append(func_dict)
+            formatted_standard_tools = [{"function_declarations": formatted_functions}]
+        else:
+            formatted_standard_tools = []
+
+        custom_gemini_tools = []
+        if tools_schema.custom_tools:
+            custom_gemini_tools = tools_schema.custom_tools.get(AdapterType.GEMINI, [])
+
+        return formatted_standard_tools + custom_gemini_tools

api/services/pipecat/realtime/gemini_live.py

@@ -35,10 +35,21 @@ from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService
 from pipecat.services.llm_service import FunctionCallFromLLM
 from pipecat.utils.tracing.service_decorators import traced_gemini_live
 
+from api.services.pipecat.gemini_json_schema_adapter import (
+    DograhGeminiJSONSchemaAdapter,
+)
+
 
 class DograhGeminiLiveLLMService(GeminiLiveLLMService):
     """Gemini Live with Dograh engine integration quirks. See module docstring."""
 
+    # Route tool schemas through Gemini's ``parameters_json_schema`` field so
+    # MCP/imported tools that use JSON Schema keywords (``const``, ``not``,
+    # nested ``anyOf``) rejected by the strict ``Schema`` model are accepted.
+    # Mirrors the non-realtime ``DograhGoogleLLMService`` fix;
+    # ``DograhGeminiLiveVertexLLMService`` inherits this via MRO.
+    adapter_class = DograhGeminiJSONSchemaAdapter
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         # User-mute state, driven by broadcast UserMute{Started,Stopped}Frames.

api/services/pipecat/service_factory.py

@@ -11,6 +11,9 @@ from api.services.configuration.options import (
     DEEPGRAM_FLUX_MULTILINGUAL_LANGUAGE_OPTIONS,
 )
 from api.services.configuration.registry import ServiceProviders
+from api.services.pipecat.gemini_json_schema_adapter import (
+    DograhGeminiJSONSchemaAdapter,
+)
 from api.services.pipecat.minimax_tts import MiniMaxOwnedSessionTTSService
 from api.utils.url_security import validate_user_configured_service_url
 from pipecat.services.assemblyai.stt import AssemblyAISTTService, AssemblyAISTTSettings
@@ -111,6 +114,14 @@ def stt_uses_flux_turns(user_config) -> bool:
     return False
 
 
+class DograhGoogleLLMService(GoogleLLMService):
+    adapter_class = DograhGeminiJSONSchemaAdapter
+
+
+class DograhGoogleVertexLLMService(GoogleVertexLLMService):
+    adapter_class = DograhGeminiJSONSchemaAdapter
+
+
 def _validate_runtime_service_url(url: str, field_name: str) -> None:
     try:
         validate_user_configured_service_url(
@@ -790,12 +801,12 @@ def create_llm_service_from_provider(
         )
     elif provider == ServiceProviders.GOOGLE.value:
         model = _migrate_deprecated_google_model(model)
-        return GoogleLLMService(
+        return DograhGoogleLLMService(
             api_key=api_key,
             settings=GoogleLLMSettings(model=model, temperature=0.1),
         )
     elif provider == ServiceProviders.GOOGLE_VERTEX.value:
-        return GoogleVertexLLMService(
+        return DograhGoogleVertexLLMService(
             credentials=credentials,
             project_id=project_id,
             location=location or "us-east4",

api/tests/test_gemini_json_schema_adapter.py

@@ -0,0 +1,160 @@
+from unittest.mock import patch
+
+from google.genai.types import GenerateContentConfig, LiveConnectConfig
+from pipecat.adapters.schemas.function_schema import FunctionSchema
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+
+from api.services.configuration.registry import ServiceProviders
+from api.services.pipecat.gemini_json_schema_adapter import (
+    DograhGeminiJSONSchemaAdapter,
+)
+from api.services.pipecat.realtime.gemini_live import DograhGeminiLiveLLMService
+from api.services.pipecat.realtime.gemini_live_vertex import (
+    DograhGeminiLiveVertexLLMService,
+)
+from api.services.pipecat.service_factory import (
+    DograhGoogleLLMService,
+    DograhGoogleVertexLLMService,
+    create_llm_service_from_provider,
+)
+
+
+def test_gemini_tools_use_json_schema_parameters_for_external_schemas():
+    function_schema = FunctionSchema(
+        name="customer_lookup",
+        description="Look up a customer by email.",
+        properties={
+            "customerEmail": {
+                "description": "Customer email address",
+                "anyOf": [
+                    {"anyOf": [{"not": {}}]},
+                    {"const": ""},
+                ],
+            },
+            "metadata": {
+                "type": "object",
+                "additionalProperties": {"type": "string"},
+            },
+        },
+        required=["customerEmail"],
+    )
+
+    tools = DograhGeminiJSONSchemaAdapter().to_provider_tools_format(
+        ToolsSchema(standard_tools=[function_schema])
+    )
+
+    declaration = tools[0]["function_declarations"][0]
+    assert "parameters" not in declaration
+    assert (
+        declaration["parameters_json_schema"]["properties"]["customerEmail"]["anyOf"][
+            0
+        ]["anyOf"][0]["not"]
+        == {}
+    )
+    assert (
+        declaration["parameters_json_schema"]["properties"]["customerEmail"]["anyOf"][
+            1
+        ]["const"]
+        == ""
+    )
+    assert declaration["parameters_json_schema"]["properties"]["metadata"][
+        "additionalProperties"
+    ] == {"type": "string"}
+
+    GenerateContentConfig(tools=tools)
+
+
+def test_gemini_tools_use_json_schema_parameters_for_no_argument_tools():
+    function_schema = FunctionSchema(
+        name="refresh_context",
+        description="Refresh the current context.",
+        properties={},
+        required=[],
+    )
+
+    tools = DograhGeminiJSONSchemaAdapter().to_provider_tools_format(
+        ToolsSchema(standard_tools=[function_schema])
+    )
+
+    declaration = tools[0]["function_declarations"][0]
+    assert "parameters" not in declaration
+    assert declaration["parameters_json_schema"] == {
+        "type": "object",
+        "properties": {},
+        "required": [],
+    }
+
+    GenerateContentConfig(tools=tools)
+
+
+def test_google_service_classes_use_dograh_gemini_adapter_class():
+    assert DograhGoogleLLMService.adapter_class is DograhGeminiJSONSchemaAdapter
+    assert DograhGoogleVertexLLMService.adapter_class is DograhGeminiJSONSchemaAdapter
+
+
+def test_google_llm_service_factory_uses_dograh_service_class():
+    with patch(
+        "api.services.pipecat.service_factory.DograhGoogleLLMService",
+    ) as mock_service:
+        result = create_llm_service_from_provider(
+            provider=ServiceProviders.GOOGLE.value,
+            model="gemini-2.5-flash",
+            api_key="test-api-key",
+        )
+
+    assert result is mock_service.return_value
+    assert mock_service.call_args.kwargs["api_key"] == "test-api-key"
+    assert mock_service.call_args.kwargs["settings"].model == "gemini-2.5-flash"
+
+
+def test_google_vertex_llm_service_factory_uses_dograh_service_class():
+    with patch(
+        "api.services.pipecat.service_factory.DograhGoogleVertexLLMService",
+    ) as mock_service:
+        result = create_llm_service_from_provider(
+            provider=ServiceProviders.GOOGLE_VERTEX.value,
+            model="gemini-2.5-pro",
+            api_key=None,
+            project_id="demo-project",
+            location="us-central1",
+            credentials='{"type":"service_account"}',
+        )
+
+    assert result is mock_service.return_value
+    assert mock_service.call_args.kwargs["project_id"] == "demo-project"
+    assert mock_service.call_args.kwargs["location"] == "us-central1"
+    assert mock_service.call_args.kwargs["settings"].model == "gemini-2.5-pro"
+
+
+def test_gemini_live_service_classes_use_dograh_gemini_adapter_class():
+    assert DograhGeminiLiveLLMService.adapter_class is DograhGeminiJSONSchemaAdapter
+    # Vertex Live inherits adapter_class from DograhGeminiLiveLLMService via MRO.
+    assert (
+        DograhGeminiLiveVertexLLMService.adapter_class is DograhGeminiJSONSchemaAdapter
+    )
+
+
+def test_gemini_live_config_accepts_json_schema_tools():
+    function_schema = FunctionSchema(
+        name="customer_lookup",
+        description="Look up a customer by email.",
+        properties={
+            "customerEmail": {
+                "description": "Customer email address",
+                "anyOf": [{"not": {}}, {"const": ""}],
+            },
+        },
+        required=["customerEmail"],
+    )
+
+    tools = DograhGeminiJSONSchemaAdapter().to_provider_tools_format(
+        ToolsSchema(standard_tools=[function_schema])
+    )
+
+    declaration = tools[0]["function_declarations"][0]
+    assert "parameters" not in declaration
+    assert "parameters_json_schema" in declaration
+
+    # Gemini Live validates tools through LiveConnectConfig rather than
+    # GenerateContentConfig; it must also accept the raw JSON Schema payload.
+    LiveConnectConfig(tools=tools)

api/tests/test_google_vertex_llm_service_factory.py

@@ -34,7 +34,7 @@ class TestGoogleVertexLLMConfiguration:
 class TestGoogleVertexLLMServiceFactory:
     def test_create_llm_service_from_provider_uses_vertex_service(self):
         with patch(
-            "api.services.pipecat.service_factory.GoogleVertexLLMService"
+            "api.services.pipecat.service_factory.DograhGoogleVertexLLMService"
         ) as mock_service:
             create_llm_service_from_provider(
                 provider=ServiceProviders.GOOGLE_VERTEX.value,
@@ -65,7 +65,7 @@ class TestGoogleVertexLLMServiceFactory:
         )
 
         with patch(
-            "api.services.pipecat.service_factory.GoogleVertexLLMService"
+            "api.services.pipecat.service_factory.DograhGoogleVertexLLMService"
         ) as mock_service:
             create_llm_service(user_config)