mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-14 15:52:11 +02:00
Thread existing JSON schemas from prompt definitions through the text-completion service to LLM backends' native structured output APIs. When a prompt has response-type "json" and a strict-mode compatible schema, the LLM constrains token selection at the logit level to guarantee schema-valid output. Wire-level changes: - Add response_format and schema fields to TextCompletionRequest - Update translator to encode/decode new fields - Pass new fields through LlmService, TextCompletionClient, and PromptManager Runtime schema compatibility checker: - New is_strict_mode_compatible() utility validates schemas against LLM provider constraints (additionalProperties, required fields, no unsupported constraints, no open-ended objects) - Per-prompt eligibility decision: compliant schemas use structured output, non-compliant schemas fall back to free-text + post-hoc validation LLM backend implementations: - OpenAI: response_format with json_schema, variant-aware top-level array rejection (openai variant blocks, llama/vllm variants allow) - New vllm variant for the OpenAI backend - vLLM (dedicated): response_format in raw HTTP body - Ollama: format=<schema> parameter - Claude: tool-use trick (forced tool call with schema as input_schema) - Mistral: native json_schema response_format - Llamafile, LM Studio: OpenAI SDK response_format - Azure OpenAI: AzureOpenAI SDK response_format - Azure serverless: response_format in raw HTTP body - TGI: response_format in raw HTTP body - VertexAI Gemini: response_mime_type + response_schema - VertexAI Claude: tool-use trick - Google AI Studio: response_mime_type + response_schema - Bedrock, Cohere: signature-only (no structured output yet) Post-hoc jsonschema.validate() retained as defence-in-depth. Tech spec added: docs/tech-specs/structured-output.md Update tests
268 lines
8.2 KiB
Python
268 lines
8.2 KiB
Python
"""
|
|
Unit tests for schema_compatibility.is_strict_mode_compatible
|
|
"""
|
|
|
|
import pytest
|
|
from trustgraph.base.schema_compatibility import is_strict_mode_compatible
|
|
|
|
|
|
class TestIsStrictModeCompatible:
|
|
|
|
def test_none_schema(self):
|
|
assert is_strict_mode_compatible(None) is False
|
|
|
|
def test_empty_dict(self):
|
|
assert is_strict_mode_compatible({}) is True
|
|
|
|
def test_simple_string(self):
|
|
assert is_strict_mode_compatible({"type": "string"}) is True
|
|
|
|
def test_compliant_object(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"name": {"type": "string"},
|
|
"age": {"type": "integer"},
|
|
},
|
|
"required": ["name", "age"],
|
|
"additionalProperties": False,
|
|
}
|
|
assert is_strict_mode_compatible(schema) is True
|
|
|
|
def test_missing_additional_properties(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {"name": {"type": "string"}},
|
|
"required": ["name"],
|
|
}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_additional_properties_true(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {"name": {"type": "string"}},
|
|
"required": ["name"],
|
|
"additionalProperties": True,
|
|
}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_property_not_in_required(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"name": {"type": "string"},
|
|
"nickname": {"type": "string"},
|
|
},
|
|
"required": ["name"],
|
|
"additionalProperties": False,
|
|
}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_open_ended_object_no_properties(self):
|
|
schema = {
|
|
"type": "object",
|
|
}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_implicit_object_with_properties_key(self):
|
|
schema = {
|
|
"properties": {
|
|
"x": {"type": "number"},
|
|
},
|
|
"required": ["x"],
|
|
"additionalProperties": False,
|
|
}
|
|
assert is_strict_mode_compatible(schema) is True
|
|
|
|
def test_nested_object_compliant(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"address": {
|
|
"type": "object",
|
|
"properties": {
|
|
"street": {"type": "string"},
|
|
},
|
|
"required": ["street"],
|
|
"additionalProperties": False,
|
|
},
|
|
},
|
|
"required": ["address"],
|
|
"additionalProperties": False,
|
|
}
|
|
assert is_strict_mode_compatible(schema) is True
|
|
|
|
def test_nested_object_non_compliant(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"metadata": {
|
|
"type": "object",
|
|
},
|
|
},
|
|
"required": ["metadata"],
|
|
"additionalProperties": False,
|
|
}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_array_with_compliant_items(self):
|
|
schema = {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"id": {"type": "integer"},
|
|
},
|
|
"required": ["id"],
|
|
"additionalProperties": False,
|
|
},
|
|
}
|
|
assert is_strict_mode_compatible(schema) is True
|
|
|
|
def test_array_with_non_compliant_items(self):
|
|
schema = {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {"id": {"type": "integer"}},
|
|
},
|
|
}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_array_with_simple_items(self):
|
|
schema = {
|
|
"type": "array",
|
|
"items": {"type": "string"},
|
|
}
|
|
assert is_strict_mode_compatible(schema) is True
|
|
|
|
def test_oneof_all_compliant(self):
|
|
schema = {
|
|
"oneOf": [
|
|
{"type": "string"},
|
|
{"type": "integer"},
|
|
]
|
|
}
|
|
assert is_strict_mode_compatible(schema) is True
|
|
|
|
def test_oneof_with_non_compliant_branch(self):
|
|
schema = {
|
|
"oneOf": [
|
|
{"type": "string"},
|
|
{"type": "object"},
|
|
]
|
|
}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_anyof(self):
|
|
schema = {
|
|
"anyOf": [
|
|
{"type": "string"},
|
|
{"type": "number"},
|
|
]
|
|
}
|
|
assert is_strict_mode_compatible(schema) is True
|
|
|
|
def test_allof(self):
|
|
schema = {
|
|
"allOf": [
|
|
{
|
|
"type": "object",
|
|
"properties": {"a": {"type": "string"}},
|
|
"required": ["a"],
|
|
"additionalProperties": False,
|
|
},
|
|
]
|
|
}
|
|
assert is_strict_mode_compatible(schema) is True
|
|
|
|
def test_unsupported_minimum(self):
|
|
schema = {"type": "integer", "minimum": 0}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_unsupported_maximum(self):
|
|
schema = {"type": "integer", "maximum": 100}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_unsupported_pattern(self):
|
|
schema = {"type": "string", "pattern": "^[a-z]+$"}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_unsupported_min_length(self):
|
|
schema = {"type": "string", "minLength": 1}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_unsupported_max_length(self):
|
|
schema = {"type": "string", "maxLength": 255}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_unsupported_min_items(self):
|
|
schema = {"type": "array", "items": {"type": "string"}, "minItems": 1}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_unsupported_max_items(self):
|
|
schema = {"type": "array", "items": {"type": "string"}, "maxItems": 10}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_unsupported_exclusive_minimum(self):
|
|
schema = {"type": "number", "exclusiveMinimum": 0}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_unsupported_min_max_properties(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {"a": {"type": "string"}},
|
|
"required": ["a"],
|
|
"additionalProperties": False,
|
|
"minProperties": 1,
|
|
}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_unsupported_constraint_inside_nested_property(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"score": {"type": "integer", "minimum": 0, "maximum": 100},
|
|
},
|
|
"required": ["score"],
|
|
"additionalProperties": False,
|
|
}
|
|
assert is_strict_mode_compatible(schema) is False
|
|
|
|
def test_nullable_property(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"name": {"type": ["string", "null"]},
|
|
},
|
|
"required": ["name"],
|
|
"additionalProperties": False,
|
|
}
|
|
assert is_strict_mode_compatible(schema) is True
|
|
|
|
def test_realistic_compliant_schema(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"action": {"type": "string"},
|
|
"services": {
|
|
"type": "array",
|
|
"items": {"type": "string"},
|
|
},
|
|
},
|
|
"required": ["action", "services"],
|
|
"additionalProperties": False,
|
|
}
|
|
assert is_strict_mode_compatible(schema) is True
|
|
|
|
def test_realistic_non_compliant_optional_field(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"action": {"type": "string"},
|
|
"reason": {"type": "string"},
|
|
},
|
|
"required": ["action"],
|
|
"additionalProperties": False,
|
|
}
|
|
assert is_strict_mode_compatible(schema) is False
|