Updated test suite for explainability & provenance (#696)

* Provenance tests

* Embeddings tests

* Test librarian

* Test triples stream

* Test concurrency

* Entity centric graph writes

* Agent tool service tests

* Structured data tests

* RDF tests

* Addition LLM tests

* Reliability tests
This commit is contained in:
cybermaggedon 2026-03-13 14:27:42 +00:00 committed by GitHub
parent e6623fc915
commit 29b4300808
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
36 changed files with 8799 additions and 0 deletions

View file

@ -0,0 +1,182 @@
"""
Tests for Azure OpenAI streaming: model/temperature override during streaming,
RateLimitError TooManyRequests conversion, chunk iteration, and final token
count emission.
"""
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from unittest import IsolatedAsyncioTestCase
from trustgraph.model.text_completion.azure_openai.llm import Processor
from trustgraph.base import LlmChunk
from trustgraph.exceptions import TooManyRequests
def _make_processor(mock_azure_openai_class, model="gpt-4"):
"""Create a Processor with mocked base classes."""
with patch('trustgraph.base.async_processor.AsyncProcessor.__init__',
return_value=None), \
patch('trustgraph.base.llm_service.LlmService.__init__',
return_value=None):
proc = Processor(
endpoint="https://test.openai.azure.com/",
token="test-token",
api_version="2024-12-01-preview",
model=model,
temperature=0.0,
max_output=4192,
concurrency=1,
taskgroup=AsyncMock(),
id="test-processor",
)
return proc
def _make_stream_chunk(content=None, usage=None):
"""Create a mock streaming chunk."""
chunk = MagicMock()
if content:
chunk.choices = [MagicMock()]
chunk.choices[0].delta.content = content
else:
chunk.choices = []
chunk.usage = usage
return chunk
class TestAzureOpenAIStreaming(IsolatedAsyncioTestCase):
@patch('trustgraph.model.text_completion.azure_openai.llm.AzureOpenAI')
async def test_streaming_yields_chunks(self, mock_azure_class):
mock_client = MagicMock()
mock_azure_class.return_value = mock_client
proc = _make_processor(mock_azure_class)
usage = MagicMock()
usage.prompt_tokens = 10
usage.completion_tokens = 5
stream_data = [
_make_stream_chunk(content="Hello"),
_make_stream_chunk(content=" world"),
_make_stream_chunk(usage=usage),
]
mock_client.chat.completions.create.return_value = iter(stream_data)
results = []
async for chunk in proc.generate_content_stream("sys", "user"):
results.append(chunk)
assert len(results) == 3 # 2 content + 1 final
assert results[0].text == "Hello"
assert results[0].is_final is False
assert results[1].text == " world"
assert results[2].is_final is True
assert results[2].in_token == 10
assert results[2].out_token == 5
@patch('trustgraph.model.text_completion.azure_openai.llm.AzureOpenAI')
async def test_streaming_model_override(self, mock_azure_class):
mock_client = MagicMock()
mock_azure_class.return_value = mock_client
proc = _make_processor(mock_azure_class, model="gpt-4")
usage = MagicMock()
usage.prompt_tokens = 5
usage.completion_tokens = 2
stream_data = [
_make_stream_chunk(content="ok"),
_make_stream_chunk(usage=usage),
]
mock_client.chat.completions.create.return_value = iter(stream_data)
results = []
async for chunk in proc.generate_content_stream(
"sys", "user", model="gpt-4o"
):
results.append(chunk)
# All chunks carry overridden model
for r in results:
assert r.model == "gpt-4o"
# Verify API call used overridden model
call_kwargs = mock_client.chat.completions.create.call_args[1]
assert call_kwargs["model"] == "gpt-4o"
@patch('trustgraph.model.text_completion.azure_openai.llm.AzureOpenAI')
async def test_streaming_temperature_override(self, mock_azure_class):
mock_client = MagicMock()
mock_azure_class.return_value = mock_client
proc = _make_processor(mock_azure_class)
usage = MagicMock()
usage.prompt_tokens = 5
usage.completion_tokens = 2
stream_data = [_make_stream_chunk(usage=usage)]
mock_client.chat.completions.create.return_value = iter(stream_data)
async for _ in proc.generate_content_stream(
"sys", "user", temperature=0.7
):
pass
call_kwargs = mock_client.chat.completions.create.call_args[1]
assert call_kwargs["temperature"] == 0.7
@patch('trustgraph.model.text_completion.azure_openai.llm.AzureOpenAI')
async def test_streaming_rate_limit_raises_too_many_requests(self, mock_azure_class):
from openai import RateLimitError
mock_client = MagicMock()
mock_azure_class.return_value = mock_client
proc = _make_processor(mock_azure_class)
mock_client.chat.completions.create.side_effect = RateLimitError(
"Rate limit exceeded", response=MagicMock(), body=None
)
with pytest.raises(TooManyRequests):
async for _ in proc.generate_content_stream("sys", "user"):
pass
@patch('trustgraph.model.text_completion.azure_openai.llm.AzureOpenAI')
async def test_streaming_generic_exception_propagates(self, mock_azure_class):
mock_client = MagicMock()
mock_azure_class.return_value = mock_client
proc = _make_processor(mock_azure_class)
mock_client.chat.completions.create.side_effect = Exception("API down")
with pytest.raises(Exception, match="API down"):
async for _ in proc.generate_content_stream("sys", "user"):
pass
@patch('trustgraph.model.text_completion.azure_openai.llm.AzureOpenAI')
async def test_streaming_passes_stream_options(self, mock_azure_class):
mock_client = MagicMock()
mock_azure_class.return_value = mock_client
proc = _make_processor(mock_azure_class)
usage = MagicMock()
usage.prompt_tokens = 0
usage.completion_tokens = 0
stream_data = [_make_stream_chunk(usage=usage)]
mock_client.chat.completions.create.return_value = iter(stream_data)
async for _ in proc.generate_content_stream("sys", "user"):
pass
call_kwargs = mock_client.chat.completions.create.call_args[1]
assert call_kwargs["stream"] is True
assert call_kwargs["stream_options"] == {"include_usage": True}
@patch('trustgraph.model.text_completion.azure_openai.llm.AzureOpenAI')
async def test_supports_streaming(self, mock_azure_class):
mock_client = MagicMock()
mock_azure_class.return_value = mock_client
proc = _make_processor(mock_azure_class)
assert proc.supports_streaming() is True

View file

@ -0,0 +1,199 @@
"""
Tests for Azure serverless endpoint streaming: model override during streaming,
HTTP 429 during streaming, SSE chunk parsing, and final token count emission.
"""
import json
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from unittest import IsolatedAsyncioTestCase
from trustgraph.model.text_completion.azure.llm import Processor
from trustgraph.base import LlmChunk
from trustgraph.exceptions import TooManyRequests
def _make_processor(mock_requests, model="AzureAI", temperature=0.0):
"""Create a Processor with mocked base classes."""
with patch('trustgraph.base.async_processor.AsyncProcessor.__init__',
return_value=None), \
patch('trustgraph.base.llm_service.LlmService.__init__',
return_value=None):
proc = Processor(
endpoint="https://test.azure.com/v1/chat/completions",
token="test-token",
temperature=temperature,
max_output=4192,
model=model,
concurrency=1,
taskgroup=AsyncMock(),
id="test-processor",
)
return proc
def _sse_lines(*data_items):
"""Build SSE byte lines from data items. '[DONE]' is appended."""
lines = []
for item in data_items:
if isinstance(item, dict):
lines.append(f"data: {json.dumps(item)}".encode())
else:
lines.append(f"data: {item}".encode())
lines.append(b"data: [DONE]")
return lines
class TestAzureServerlessStreaming(IsolatedAsyncioTestCase):
@patch('trustgraph.model.text_completion.azure.llm.requests')
async def test_streaming_yields_chunks(self, mock_requests):
proc = _make_processor(mock_requests)
chunks = [
{"choices": [{"delta": {"content": "Hello"}}]},
{"choices": [{"delta": {"content": " world"}}]},
{"usage": {"prompt_tokens": 10, "completion_tokens": 5}},
]
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.iter_lines.return_value = _sse_lines(*chunks)
mock_requests.post.return_value = mock_response
results = []
async for chunk in proc.generate_content_stream("sys", "user"):
results.append(chunk)
# Content chunks + final chunk
assert len(results) == 3
assert results[0].text == "Hello"
assert results[0].is_final is False
assert results[1].text == " world"
assert results[1].is_final is False
assert results[2].is_final is True
assert results[2].in_token == 10
assert results[2].out_token == 5
@patch('trustgraph.model.text_completion.azure.llm.requests')
async def test_streaming_model_override(self, mock_requests):
proc = _make_processor(mock_requests, model="default-model")
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.iter_lines.return_value = _sse_lines(
{"choices": [{"delta": {"content": "ok"}}]},
{"usage": {"prompt_tokens": 5, "completion_tokens": 2}},
)
mock_requests.post.return_value = mock_response
results = []
async for chunk in proc.generate_content_stream(
"sys", "user", model="override-model"
):
results.append(chunk)
# All chunks should carry the overridden model name
for r in results:
assert r.model == "override-model"
# Verify the request body used the overridden model
call_args = mock_requests.post.call_args
body = json.loads(call_args[1]["data"])
assert body["model"] == "override-model"
@patch('trustgraph.model.text_completion.azure.llm.requests')
async def test_streaming_temperature_override(self, mock_requests):
proc = _make_processor(mock_requests, temperature=0.0)
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.iter_lines.return_value = _sse_lines(
{"choices": [{"delta": {"content": "ok"}}]},
{"usage": {"prompt_tokens": 5, "completion_tokens": 2}},
)
mock_requests.post.return_value = mock_response
results = []
async for chunk in proc.generate_content_stream(
"sys", "user", temperature=0.9
):
results.append(chunk)
call_args = mock_requests.post.call_args
body = json.loads(call_args[1]["data"])
assert body["temperature"] == 0.9
@patch('trustgraph.model.text_completion.azure.llm.requests')
async def test_streaming_429_raises_too_many_requests(self, mock_requests):
proc = _make_processor(mock_requests)
mock_response = MagicMock()
mock_response.status_code = 429
mock_requests.post.return_value = mock_response
with pytest.raises(TooManyRequests):
async for _ in proc.generate_content_stream("sys", "user"):
pass
@patch('trustgraph.model.text_completion.azure.llm.requests')
async def test_streaming_http_error_raises_runtime(self, mock_requests):
proc = _make_processor(mock_requests)
mock_response = MagicMock()
mock_response.status_code = 503
mock_response.text = "Service Unavailable"
mock_requests.post.return_value = mock_response
with pytest.raises(RuntimeError, match="HTTP 503"):
async for _ in proc.generate_content_stream("sys", "user"):
pass
@patch('trustgraph.model.text_completion.azure.llm.requests')
async def test_streaming_includes_stream_options(self, mock_requests):
"""Verify stream=True and stream_options in request body."""
proc = _make_processor(mock_requests)
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.iter_lines.return_value = _sse_lines(
{"usage": {"prompt_tokens": 0, "completion_tokens": 0}},
)
mock_requests.post.return_value = mock_response
async for _ in proc.generate_content_stream("sys", "user"):
pass
call_args = mock_requests.post.call_args
body = json.loads(call_args[1]["data"])
assert body["stream"] is True
assert body["stream_options"]["include_usage"] is True
@patch('trustgraph.model.text_completion.azure.llm.requests')
async def test_streaming_malformed_json_skipped(self, mock_requests):
"""Malformed JSON chunks should be skipped, not crash the stream."""
proc = _make_processor(mock_requests)
mock_response = MagicMock()
mock_response.status_code = 200
lines = [
b"data: {not valid json}",
f'data: {json.dumps({"choices": [{"delta": {"content": "ok"}}]})}'.encode(),
f'data: {json.dumps({"usage": {"prompt_tokens": 1, "completion_tokens": 1}})}'.encode(),
b"data: [DONE]",
]
mock_response.iter_lines.return_value = lines
mock_requests.post.return_value = mock_response
results = []
async for chunk in proc.generate_content_stream("sys", "user"):
results.append(chunk)
# Should get the valid content chunk + final chunk
assert any(r.text == "ok" for r in results)
assert results[-1].is_final is True
@patch('trustgraph.model.text_completion.azure.llm.requests')
async def test_streaming_supports_streaming_flag(self, mock_requests):
proc = _make_processor(mock_requests)
assert proc.supports_streaming() is True

View file

@ -0,0 +1,140 @@
"""
Cross-provider rate limit contract tests: verify that every LLM provider
that handles rate limits converts its provider-specific exception to
TooManyRequests consistently.
Also tests the client-side error translation in the base client.
"""
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from unittest import IsolatedAsyncioTestCase
from trustgraph.exceptions import TooManyRequests
class TestAzureServerless429(IsolatedAsyncioTestCase):
"""Azure serverless endpoint: HTTP 429 → TooManyRequests"""
@patch('trustgraph.model.text_completion.azure.llm.requests')
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__', return_value=None)
@patch('trustgraph.base.llm_service.LlmService.__init__', return_value=None)
async def test_http_429_raises_too_many_requests(self, _llm, _async, mock_requests):
from trustgraph.model.text_completion.azure.llm import Processor
proc = Processor(
endpoint="https://test.azure.com/v1/chat",
token="t", concurrency=1, taskgroup=AsyncMock(), id="t",
)
mock_response = MagicMock()
mock_response.status_code = 429
mock_requests.post.return_value = mock_response
with pytest.raises(TooManyRequests):
await proc.generate_content("sys", "prompt")
class TestAzureOpenAIRateLimit(IsolatedAsyncioTestCase):
"""Azure OpenAI: openai.RateLimitError → TooManyRequests"""
@patch('trustgraph.model.text_completion.azure_openai.llm.AzureOpenAI')
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__', return_value=None)
@patch('trustgraph.base.llm_service.LlmService.__init__', return_value=None)
async def test_rate_limit_error_raises_too_many_requests(self, _llm, _async, mock_cls):
from openai import RateLimitError
from trustgraph.model.text_completion.azure_openai.llm import Processor
mock_client = MagicMock()
mock_cls.return_value = mock_client
proc = Processor(
endpoint="https://test.openai.azure.com/", token="t",
model="gpt-4", concurrency=1, taskgroup=AsyncMock(), id="t",
)
mock_client.chat.completions.create.side_effect = RateLimitError(
"rate limited", response=MagicMock(), body=None
)
with pytest.raises(TooManyRequests):
await proc.generate_content("sys", "prompt")
class TestOpenAIRateLimit(IsolatedAsyncioTestCase):
"""OpenAI: openai.RateLimitError → TooManyRequests"""
@patch('trustgraph.model.text_completion.openai.llm.OpenAI')
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__', return_value=None)
@patch('trustgraph.base.llm_service.LlmService.__init__', return_value=None)
async def test_rate_limit_error_raises_too_many_requests(self, _llm, _async, mock_cls):
from openai import RateLimitError
from trustgraph.model.text_completion.openai.llm import Processor
mock_client = MagicMock()
mock_cls.return_value = mock_client
proc = Processor(
api_key="k", concurrency=1, taskgroup=AsyncMock(), id="t",
)
mock_client.chat.completions.create.side_effect = RateLimitError(
"rate limited", response=MagicMock(), body=None
)
with pytest.raises(TooManyRequests):
await proc.generate_content("sys", "prompt")
class TestClaudeRateLimit(IsolatedAsyncioTestCase):
"""Claude/Anthropic: anthropic.RateLimitError → TooManyRequests"""
@patch('trustgraph.model.text_completion.claude.llm.anthropic')
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__', return_value=None)
@patch('trustgraph.base.llm_service.LlmService.__init__', return_value=None)
async def test_rate_limit_error_raises_too_many_requests(self, _llm, _async, mock_anthropic):
from trustgraph.model.text_completion.claude.llm import Processor
mock_client = MagicMock()
mock_anthropic.Anthropic.return_value = mock_client
proc = Processor(
api_key="k", concurrency=1, taskgroup=AsyncMock(), id="t",
)
mock_anthropic.RateLimitError = type("RateLimitError", (Exception,), {})
mock_client.messages.create.side_effect = mock_anthropic.RateLimitError(
"rate limited"
)
with pytest.raises(TooManyRequests):
await proc.generate_content("sys", "prompt")
class TestCohereRateLimit(IsolatedAsyncioTestCase):
"""Cohere: cohere.TooManyRequestsError → TooManyRequests"""
@patch('trustgraph.model.text_completion.cohere.llm.cohere')
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__', return_value=None)
@patch('trustgraph.base.llm_service.LlmService.__init__', return_value=None)
async def test_rate_limit_error_raises_too_many_requests(self, _llm, _async, mock_cohere):
from trustgraph.model.text_completion.cohere.llm import Processor
mock_client = MagicMock()
mock_cohere.Client.return_value = mock_client
proc = Processor(
api_key="k", concurrency=1, taskgroup=AsyncMock(), id="t",
)
mock_cohere.TooManyRequestsError = type(
"TooManyRequestsError", (Exception,), {}
)
mock_client.chat.side_effect = mock_cohere.TooManyRequestsError(
"rate limited"
)
with pytest.raises(TooManyRequests):
await proc.generate_content("sys", "prompt")
class TestClientSideRateLimitTranslation:
"""Client base class: error type 'too-many-requests' → TooManyRequests"""
def test_error_type_mapping(self):
"""The wire format error type string is 'too-many-requests'."""
from trustgraph.schema import Error
err = Error(type="too-many-requests", message="slow down")
assert err.type == "too-many-requests"