mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-04-26 08:56:21 +02:00
Updated test suite for explainability & provenance (#696)
* Provenance tests * Embeddings tests * Test librarian * Test triples stream * Test concurrency * Entity centric graph writes * Agent tool service tests * Structured data tests * RDF tests * Addition LLM tests * Reliability tests
This commit is contained in:
cybermaggedon
GitHub
parent
e6623fc915
commit
29b4300808
36 changed files with 8799 additions and 0 deletions
1
tests/unit/test_reliability/__init__.py
Normal file
1
tests/unit/test_reliability/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
|
||||
144
tests/unit/test_reliability/test_metadata_preservation.py
Normal file
144
tests/unit/test_reliability/test_metadata_preservation.py
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
"""
|
||||
Tests for pipeline metadata preservation: DocumentMetadata and
|
||||
ProcessingMetadata round-trip through translators, field preservation,
|
||||
and default handling.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from trustgraph.schema import DocumentMetadata, ProcessingMetadata, Triple, Term, IRI
|
||||
from trustgraph.messaging.translators.metadata import (
|
||||
DocumentMetadataTranslator,
|
||||
ProcessingMetadataTranslator,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DocumentMetadata translator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestDocumentMetadataTranslator:
|
||||
|
||||
def setup_method(self):
|
||||
self.tx = DocumentMetadataTranslator()
|
||||
|
||||
def test_full_round_trip(self):
|
||||
data = {
|
||||
"id": "doc-123",
|
||||
"time": 1710000000,
|
||||
"kind": "application/pdf",
|
||||
"title": "Test Document",
|
||||
"comments": "No comments",
|
||||
"metadata": [],
|
||||
"user": "alice",
|
||||
"tags": ["finance", "q4"],
|
||||
"parent-id": "doc-100",
|
||||
"document-type": "page",
|
||||
}
|
||||
obj = self.tx.to_pulsar(data)
|
||||
assert obj.id == "doc-123"
|
||||
assert obj.time == 1710000000
|
||||
assert obj.kind == "application/pdf"
|
||||
assert obj.title == "Test Document"
|
||||
assert obj.user == "alice"
|
||||
assert obj.tags == ["finance", "q4"]
|
||||
assert obj.parent_id == "doc-100"
|
||||
assert obj.document_type == "page"
|
||||
|
||||
wire = self.tx.from_pulsar(obj)
|
||||
assert wire["id"] == "doc-123"
|
||||
assert wire["user"] == "alice"
|
||||
assert wire["parent-id"] == "doc-100"
|
||||
assert wire["document-type"] == "page"
|
||||
|
||||
def test_defaults_for_missing_fields(self):
|
||||
obj = self.tx.to_pulsar({})
|
||||
assert obj.parent_id == ""
|
||||
assert obj.document_type == "source"
|
||||
|
||||
def test_metadata_triples_preserved(self):
|
||||
triple_wire = [{
|
||||
"s": {"t": "i", "i": "http://example.org/s"},
|
||||
"p": {"t": "i", "i": "http://example.org/p"},
|
||||
"o": {"t": "i", "i": "http://example.org/o"},
|
||||
}]
|
||||
data = {"metadata": triple_wire}
|
||||
obj = self.tx.to_pulsar(data)
|
||||
assert len(obj.metadata) == 1
|
||||
assert obj.metadata[0].s.iri == "http://example.org/s"
|
||||
|
||||
def test_none_metadata_handled(self):
|
||||
data = {"metadata": None}
|
||||
obj = self.tx.to_pulsar(data)
|
||||
assert obj.metadata == []
|
||||
|
||||
def test_empty_tags_preserved(self):
|
||||
data = {"tags": []}
|
||||
obj = self.tx.to_pulsar(data)
|
||||
wire = self.tx.from_pulsar(obj)
|
||||
assert wire["tags"] == []
|
||||
|
||||
def test_falsy_fields_omitted_from_wire(self):
|
||||
"""Empty string fields should be omitted from wire format."""
|
||||
obj = DocumentMetadata(id="", time=0, user="")
|
||||
wire = self.tx.from_pulsar(obj)
|
||||
assert "id" not in wire
|
||||
assert "user" not in wire
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ProcessingMetadata translator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestProcessingMetadataTranslator:
|
||||
|
||||
def setup_method(self):
|
||||
self.tx = ProcessingMetadataTranslator()
|
||||
|
||||
def test_full_round_trip(self):
|
||||
data = {
|
||||
"id": "proc-1",
|
||||
"document-id": "doc-123",
|
||||
"time": 1710000000,
|
||||
"flow": "default",
|
||||
"user": "alice",
|
||||
"collection": "my-collection",
|
||||
"tags": ["tag1"],
|
||||
}
|
||||
obj = self.tx.to_pulsar(data)
|
||||
assert obj.id == "proc-1"
|
||||
assert obj.document_id == "doc-123"
|
||||
assert obj.flow == "default"
|
||||
assert obj.user == "alice"
|
||||
assert obj.collection == "my-collection"
|
||||
assert obj.tags == ["tag1"]
|
||||
|
||||
wire = self.tx.from_pulsar(obj)
|
||||
assert wire["id"] == "proc-1"
|
||||
assert wire["document-id"] == "doc-123"
|
||||
assert wire["user"] == "alice"
|
||||
assert wire["collection"] == "my-collection"
|
||||
|
||||
def test_missing_fields_use_defaults(self):
|
||||
obj = self.tx.to_pulsar({})
|
||||
assert obj.id is None
|
||||
assert obj.user is None
|
||||
assert obj.collection is None
|
||||
|
||||
def test_tags_none_omitted(self):
|
||||
obj = ProcessingMetadata(tags=None)
|
||||
wire = self.tx.from_pulsar(obj)
|
||||
assert "tags" not in wire
|
||||
|
||||
def test_tags_empty_list_preserved(self):
|
||||
obj = ProcessingMetadata(tags=[])
|
||||
wire = self.tx.from_pulsar(obj)
|
||||
assert wire["tags"] == []
|
||||
|
||||
def test_user_and_collection_preserved(self):
|
||||
"""Core pipeline routing fields must survive round-trip."""
|
||||
data = {"user": "bob", "collection": "research"}
|
||||
obj = self.tx.to_pulsar(data)
|
||||
wire = self.tx.from_pulsar(obj)
|
||||
assert wire["user"] == "bob"
|
||||
assert wire["collection"] == "research"
|
||||
314
tests/unit/test_reliability/test_null_embedding_protection.py
Normal file
314
tests/unit/test_reliability/test_null_embedding_protection.py
Normal file
|
|
@ -0,0 +1,314 @@
|
|||
"""
|
||||
Tests for null embedding protection: empty/None vector skipping, entity
|
||||
validation, dimension-aware collection creation, and query-time empty
|
||||
vector handling.
|
||||
|
||||
Tests the pure functions and logic without Qdrant connections.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch, AsyncMock
|
||||
|
||||
from trustgraph.schema import Term, IRI, LITERAL, BLANK
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Graph embeddings: get_term_value
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestGraphEmbeddingsGetTermValue:
|
||||
|
||||
def test_iri_returns_iri(self):
|
||||
from trustgraph.storage.graph_embeddings.qdrant.write import get_term_value
|
||||
t = Term(type=IRI, iri="http://example.org/x")
|
||||
assert get_term_value(t) == "http://example.org/x"
|
||||
|
||||
def test_literal_returns_value(self):
|
||||
from trustgraph.storage.graph_embeddings.qdrant.write import get_term_value
|
||||
t = Term(type=LITERAL, value="hello")
|
||||
assert get_term_value(t) == "hello"
|
||||
|
||||
def test_blank_returns_id(self):
|
||||
from trustgraph.storage.graph_embeddings.qdrant.write import get_term_value
|
||||
t = Term(type=BLANK, id="_:b0")
|
||||
assert get_term_value(t) == "_:b0"
|
||||
|
||||
def test_none_returns_none(self):
|
||||
from trustgraph.storage.graph_embeddings.qdrant.write import get_term_value
|
||||
assert get_term_value(None) is None
|
||||
|
||||
def test_blank_with_value_fallback(self):
|
||||
from trustgraph.storage.graph_embeddings.qdrant.write import get_term_value
|
||||
t = Term(type=BLANK, id="", value="fallback")
|
||||
assert get_term_value(t) == "fallback"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Document embeddings: null vector protection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestDocEmbeddingsNullProtection:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_vector_skipped(self):
|
||||
"""Embeddings with empty vectors should be silently skipped."""
|
||||
from trustgraph.storage.doc_embeddings.qdrant.write import Processor
|
||||
|
||||
proc = Processor.__new__(Processor)
|
||||
proc.qdrant = MagicMock()
|
||||
|
||||
# Mock collection_exists for config check
|
||||
proc.collection_exists = MagicMock(return_value=True)
|
||||
|
||||
msg = MagicMock()
|
||||
msg.metadata.user = "user1"
|
||||
msg.metadata.collection = "col1"
|
||||
|
||||
emb = MagicMock()
|
||||
emb.chunk_id = "chunk-1"
|
||||
emb.vector = [] # Empty vector
|
||||
msg.chunks = [emb]
|
||||
|
||||
await proc.store_document_embeddings(msg)
|
||||
|
||||
# No upsert should be called
|
||||
proc.qdrant.upsert.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_none_vector_skipped(self):
|
||||
from trustgraph.storage.doc_embeddings.qdrant.write import Processor
|
||||
|
||||
proc = Processor.__new__(Processor)
|
||||
proc.qdrant = MagicMock()
|
||||
proc.collection_exists = MagicMock(return_value=True)
|
||||
|
||||
msg = MagicMock()
|
||||
msg.metadata.user = "user1"
|
||||
msg.metadata.collection = "col1"
|
||||
|
||||
emb = MagicMock()
|
||||
emb.chunk_id = "chunk-1"
|
||||
emb.vector = None # None vector
|
||||
msg.chunks = [emb]
|
||||
|
||||
await proc.store_document_embeddings(msg)
|
||||
proc.qdrant.upsert.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_chunk_id_skipped(self):
|
||||
from trustgraph.storage.doc_embeddings.qdrant.write import Processor
|
||||
|
||||
proc = Processor.__new__(Processor)
|
||||
proc.qdrant = MagicMock()
|
||||
proc.collection_exists = MagicMock(return_value=True)
|
||||
|
||||
msg = MagicMock()
|
||||
msg.metadata.user = "user1"
|
||||
msg.metadata.collection = "col1"
|
||||
|
||||
emb = MagicMock()
|
||||
emb.chunk_id = "" # Empty chunk ID
|
||||
emb.vector = [0.1, 0.2, 0.3]
|
||||
msg.chunks = [emb]
|
||||
|
||||
await proc.store_document_embeddings(msg)
|
||||
proc.qdrant.upsert.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_valid_embedding_upserted(self):
|
||||
from trustgraph.storage.doc_embeddings.qdrant.write import Processor
|
||||
|
||||
proc = Processor.__new__(Processor)
|
||||
proc.qdrant = MagicMock()
|
||||
proc.qdrant.collection_exists.return_value = True
|
||||
proc.collection_exists = MagicMock(return_value=True)
|
||||
|
||||
msg = MagicMock()
|
||||
msg.metadata.user = "user1"
|
||||
msg.metadata.collection = "col1"
|
||||
|
||||
emb = MagicMock()
|
||||
emb.chunk_id = "chunk-1"
|
||||
emb.vector = [0.1, 0.2, 0.3]
|
||||
msg.chunks = [emb]
|
||||
|
||||
await proc.store_document_embeddings(msg)
|
||||
proc.qdrant.upsert.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dimension_in_collection_name(self):
|
||||
"""Collection name should include vector dimension."""
|
||||
from trustgraph.storage.doc_embeddings.qdrant.write import Processor
|
||||
|
||||
proc = Processor.__new__(Processor)
|
||||
proc.qdrant = MagicMock()
|
||||
proc.qdrant.collection_exists.return_value = True
|
||||
proc.collection_exists = MagicMock(return_value=True)
|
||||
|
||||
msg = MagicMock()
|
||||
msg.metadata.user = "alice"
|
||||
msg.metadata.collection = "docs"
|
||||
|
||||
emb = MagicMock()
|
||||
emb.chunk_id = "c1"
|
||||
emb.vector = [0.0] * 384 # 384-dim vector
|
||||
msg.chunks = [emb]
|
||||
|
||||
await proc.store_document_embeddings(msg)
|
||||
|
||||
call_args = proc.qdrant.upsert.call_args
|
||||
assert "d_alice_docs_384" in call_args[1]["collection_name"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Graph embeddings: null entity and vector protection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestGraphEmbeddingsNullProtection:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_entity_skipped(self):
|
||||
from trustgraph.storage.graph_embeddings.qdrant.write import Processor
|
||||
|
||||
proc = Processor.__new__(Processor)
|
||||
proc.qdrant = MagicMock()
|
||||
proc.collection_exists = MagicMock(return_value=True)
|
||||
|
||||
msg = MagicMock()
|
||||
msg.metadata.user = "user1"
|
||||
msg.metadata.collection = "col1"
|
||||
|
||||
entity = MagicMock()
|
||||
entity.entity = Term(type=IRI, iri="") # Empty IRI
|
||||
entity.vector = [0.1, 0.2, 0.3]
|
||||
msg.entities = [entity]
|
||||
|
||||
await proc.store_graph_embeddings(msg)
|
||||
proc.qdrant.upsert.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_none_entity_skipped(self):
|
||||
from trustgraph.storage.graph_embeddings.qdrant.write import Processor
|
||||
|
||||
proc = Processor.__new__(Processor)
|
||||
proc.qdrant = MagicMock()
|
||||
proc.collection_exists = MagicMock(return_value=True)
|
||||
|
||||
msg = MagicMock()
|
||||
msg.metadata.user = "user1"
|
||||
msg.metadata.collection = "col1"
|
||||
|
||||
entity = MagicMock()
|
||||
entity.entity = None # Null entity
|
||||
entity.vector = [0.1, 0.2, 0.3]
|
||||
msg.entities = [entity]
|
||||
|
||||
await proc.store_graph_embeddings(msg)
|
||||
proc.qdrant.upsert.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_vector_skipped(self):
|
||||
from trustgraph.storage.graph_embeddings.qdrant.write import Processor
|
||||
|
||||
proc = Processor.__new__(Processor)
|
||||
proc.qdrant = MagicMock()
|
||||
proc.collection_exists = MagicMock(return_value=True)
|
||||
|
||||
msg = MagicMock()
|
||||
msg.metadata.user = "user1"
|
||||
msg.metadata.collection = "col1"
|
||||
|
||||
entity = MagicMock()
|
||||
entity.entity = Term(type=IRI, iri="http://example.org/x")
|
||||
entity.vector = [] # Empty vector
|
||||
msg.entities = [entity]
|
||||
|
||||
await proc.store_graph_embeddings(msg)
|
||||
proc.qdrant.upsert.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_valid_entity_and_vector_upserted(self):
|
||||
from trustgraph.storage.graph_embeddings.qdrant.write import Processor
|
||||
|
||||
proc = Processor.__new__(Processor)
|
||||
proc.qdrant = MagicMock()
|
||||
proc.qdrant.collection_exists.return_value = True
|
||||
proc.collection_exists = MagicMock(return_value=True)
|
||||
|
||||
msg = MagicMock()
|
||||
msg.metadata.user = "user1"
|
||||
msg.metadata.collection = "col1"
|
||||
|
||||
entity = MagicMock()
|
||||
entity.entity = Term(type=IRI, iri="http://example.org/Alice")
|
||||
entity.vector = [0.1, 0.2, 0.3]
|
||||
entity.chunk_id = "c1"
|
||||
msg.entities = [entity]
|
||||
|
||||
await proc.store_graph_embeddings(msg)
|
||||
proc.qdrant.upsert.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lazy_collection_creation_on_new_dimension(self):
|
||||
from trustgraph.storage.graph_embeddings.qdrant.write import Processor
|
||||
|
||||
proc = Processor.__new__(Processor)
|
||||
proc.qdrant = MagicMock()
|
||||
proc.qdrant.collection_exists.return_value = False
|
||||
proc.collection_exists = MagicMock(return_value=True)
|
||||
|
||||
msg = MagicMock()
|
||||
msg.metadata.user = "alice"
|
||||
msg.metadata.collection = "graphs"
|
||||
|
||||
entity = MagicMock()
|
||||
entity.entity = Term(type=IRI, iri="http://example.org/x")
|
||||
entity.vector = [0.0] * 768
|
||||
entity.chunk_id = ""
|
||||
msg.entities = [entity]
|
||||
|
||||
await proc.store_graph_embeddings(msg)
|
||||
|
||||
# Collection should be created with correct dimension
|
||||
proc.qdrant.create_collection.assert_called_once()
|
||||
create_args = proc.qdrant.create_collection.call_args
|
||||
assert create_args[1]["collection_name"] == "t_alice_graphs_768"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Collection validation — deleted-while-in-flight protection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCollectionValidation:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_embeddings_dropped_for_deleted_collection(self):
|
||||
from trustgraph.storage.doc_embeddings.qdrant.write import Processor
|
||||
|
||||
proc = Processor.__new__(Processor)
|
||||
proc.qdrant = MagicMock()
|
||||
proc.collection_exists = MagicMock(return_value=False)
|
||||
|
||||
msg = MagicMock()
|
||||
msg.metadata.user = "user1"
|
||||
msg.metadata.collection = "deleted-col"
|
||||
msg.chunks = [MagicMock()]
|
||||
|
||||
await proc.store_document_embeddings(msg)
|
||||
proc.qdrant.upsert.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_graph_embeddings_dropped_for_deleted_collection(self):
|
||||
from trustgraph.storage.graph_embeddings.qdrant.write import Processor
|
||||
|
||||
proc = Processor.__new__(Processor)
|
||||
proc.qdrant = MagicMock()
|
||||
proc.collection_exists = MagicMock(return_value=False)
|
||||
|
||||
msg = MagicMock()
|
||||
msg.metadata.user = "user1"
|
||||
msg.metadata.collection = "deleted-col"
|
||||
msg.entities = [MagicMock()]
|
||||
|
||||
await proc.store_graph_embeddings(msg)
|
||||
proc.qdrant.upsert.assert_not_called()
|
||||
153
tests/unit/test_reliability/test_retry_backoff.py
Normal file
153
tests/unit/test_reliability/test_retry_backoff.py
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
"""
|
||||
Tests for retry and backoff strategies: Consumer rate-limit retry loop,
|
||||
timeout expiry, TooManyRequests exception propagation, and configurable
|
||||
retry parameters.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
|
||||
from trustgraph.exceptions import TooManyRequests
|
||||
from trustgraph.base.consumer import Consumer
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_consumer(rate_limit_retry_time=10, rate_limit_timeout=7200):
|
||||
"""Create a Consumer with minimal mocking."""
|
||||
consumer = Consumer.__new__(Consumer)
|
||||
consumer.rate_limit_retry_time = rate_limit_retry_time
|
||||
consumer.rate_limit_timeout = rate_limit_timeout
|
||||
consumer.metrics = None
|
||||
consumer.consumer = MagicMock()
|
||||
return consumer
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TooManyRequests exception
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestTooManyRequestsException:
|
||||
|
||||
def test_is_exception(self):
|
||||
assert issubclass(TooManyRequests, Exception)
|
||||
|
||||
def test_with_message(self):
|
||||
err = TooManyRequests("rate limited")
|
||||
assert "rate limited" in str(err)
|
||||
|
||||
def test_without_message(self):
|
||||
err = TooManyRequests()
|
||||
assert isinstance(err, TooManyRequests)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Consumer retry configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestConsumerRetryConfig:
|
||||
|
||||
def test_default_retry_time(self):
|
||||
consumer = _make_consumer()
|
||||
assert consumer.rate_limit_retry_time == 10
|
||||
|
||||
def test_default_timeout(self):
|
||||
consumer = _make_consumer()
|
||||
assert consumer.rate_limit_timeout == 7200
|
||||
|
||||
def test_custom_retry_time(self):
|
||||
consumer = _make_consumer(rate_limit_retry_time=5)
|
||||
assert consumer.rate_limit_retry_time == 5
|
||||
|
||||
def test_custom_timeout(self):
|
||||
consumer = _make_consumer(rate_limit_timeout=300)
|
||||
assert consumer.rate_limit_timeout == 300
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rate limit metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestRateLimitMetrics:
|
||||
|
||||
def test_metrics_rate_limit_called(self):
|
||||
"""Metrics should record rate limit events when available."""
|
||||
consumer = _make_consumer()
|
||||
consumer.metrics = MagicMock()
|
||||
|
||||
# Simulate what the consumer does on rate limit
|
||||
consumer.metrics.rate_limit()
|
||||
|
||||
consumer.metrics.rate_limit.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Message acknowledgment on error
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMessageAckOnError:
|
||||
|
||||
def test_consumer_has_negative_acknowledge(self):
|
||||
"""Consumer backend should support negative acknowledgment."""
|
||||
consumer = _make_consumer()
|
||||
msg = MagicMock()
|
||||
|
||||
# Simulate negative ack (what happens on timeout expiry)
|
||||
consumer.consumer.negative_acknowledge(msg)
|
||||
consumer.consumer.negative_acknowledge.assert_called_once_with(msg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TooManyRequests propagation across services
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestTooManyRequestsPropagation:
|
||||
|
||||
def test_llm_service_propagates(self):
|
||||
"""LLM services should re-raise TooManyRequests for consumer retry."""
|
||||
with pytest.raises(TooManyRequests):
|
||||
raise TooManyRequests()
|
||||
|
||||
def test_embeddings_service_propagates(self):
|
||||
"""Embeddings services should re-raise TooManyRequests for consumer retry."""
|
||||
with pytest.raises(TooManyRequests):
|
||||
try:
|
||||
raise TooManyRequests("rate limited")
|
||||
except TooManyRequests as e:
|
||||
# Re-raise pattern used in services
|
||||
assert isinstance(e, TooManyRequests)
|
||||
raise
|
||||
|
||||
def test_too_many_requests_not_caught_by_generic(self):
|
||||
"""TooManyRequests should be distinguishable from generic exceptions."""
|
||||
caught_specific = False
|
||||
try:
|
||||
raise TooManyRequests("rate limited")
|
||||
except TooManyRequests:
|
||||
caught_specific = True
|
||||
except Exception:
|
||||
pass
|
||||
assert caught_specific
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Client-side error type mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestClientErrorTypeMapping:
|
||||
|
||||
def test_too_many_requests_wire_type(self):
|
||||
"""The wire format error type for rate limiting is 'too-many-requests'."""
|
||||
from trustgraph.schema import Error
|
||||
err = Error(type="too-many-requests", message="slow down")
|
||||
assert err.type == "too-many-requests"
|
||||
|
||||
def test_generic_error_wire_type(self):
|
||||
from trustgraph.schema import Error
|
||||
err = Error(type="internal-error", message="something broke")
|
||||
assert err.type == "internal-error"
|
||||
assert err.type != "too-many-requests"
|
||||
233
tests/unit/test_reliability/test_subscriber_resilience.py
Normal file
233
tests/unit/test_reliability/test_subscriber_resilience.py
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
"""
|
||||
Tests for message queue subscriber resilience: unexpected message handling,
|
||||
orphaned message detection, backpressure strategies, graceful draining,
|
||||
and timeout recovery.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
|
||||
from trustgraph.base.subscriber import Subscriber
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_subscriber(max_size=10, backpressure_strategy="block",
|
||||
drain_timeout=5.0):
|
||||
"""Create a Subscriber without connecting to any backend."""
|
||||
backend = MagicMock()
|
||||
sub = Subscriber(
|
||||
backend=backend,
|
||||
topic="test-topic",
|
||||
subscription="test-sub",
|
||||
consumer_name="test",
|
||||
max_size=max_size,
|
||||
backpressure_strategy=backpressure_strategy,
|
||||
drain_timeout=drain_timeout,
|
||||
)
|
||||
sub.consumer = MagicMock()
|
||||
return sub
|
||||
|
||||
|
||||
def _make_msg(id=None, value="test-value"):
|
||||
"""Create a mock message with optional properties."""
|
||||
msg = MagicMock()
|
||||
if id is not None:
|
||||
msg.properties.return_value = {"id": id}
|
||||
else:
|
||||
msg.properties.side_effect = KeyError("id")
|
||||
msg.value.return_value = value
|
||||
return msg
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Message property extraction resilience
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMessagePropertyResilience:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_id_property_handled(self):
|
||||
"""Messages without 'id' property should not crash."""
|
||||
sub = _make_subscriber()
|
||||
msg = MagicMock()
|
||||
msg.properties.side_effect = Exception("no properties")
|
||||
msg.value.return_value = "some-value"
|
||||
|
||||
# Should not raise
|
||||
await sub._process_message(msg)
|
||||
|
||||
# Message should still be acknowledged
|
||||
sub.consumer.acknowledge.assert_called_once_with(msg)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_message_with_valid_id_delivered(self):
|
||||
"""Messages with matching subscriber ID should be delivered."""
|
||||
sub = _make_subscriber()
|
||||
q = await sub.subscribe("req-1")
|
||||
|
||||
msg = _make_msg(id="req-1", value="response-data")
|
||||
await sub._process_message(msg)
|
||||
|
||||
assert not q.empty()
|
||||
assert q.get_nowait() == "response-data"
|
||||
sub.consumer.acknowledge.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Orphaned message handling
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestOrphanedMessages:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_orphaned_message_acknowledged(self):
|
||||
"""Messages with no matching waiter should still be acknowledged."""
|
||||
sub = _make_subscriber()
|
||||
msg = _make_msg(id="unknown-id", value="orphan")
|
||||
|
||||
await sub._process_message(msg)
|
||||
|
||||
# Orphaned message is acknowledged (not negative-acknowledged)
|
||||
sub.consumer.acknowledge.assert_called_once_with(msg)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_orphaned_message_not_queued(self):
|
||||
"""Orphaned messages should not appear in any subscriber queue."""
|
||||
sub = _make_subscriber()
|
||||
q = await sub.subscribe("req-1")
|
||||
|
||||
msg = _make_msg(id="different-id", value="orphan")
|
||||
await sub._process_message(msg)
|
||||
|
||||
assert q.empty()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backpressure strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestBackpressureStrategies:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drop_new_rejects_when_full(self):
|
||||
"""drop_new strategy should reject new messages when queue is full."""
|
||||
sub = _make_subscriber(max_size=1, backpressure_strategy="drop_new")
|
||||
q = await sub.subscribe("req-1")
|
||||
|
||||
# Fill the queue
|
||||
msg1 = _make_msg(id="req-1", value="first")
|
||||
await sub._process_message(msg1)
|
||||
assert q.qsize() == 1
|
||||
|
||||
# Second message should be dropped
|
||||
msg2 = _make_msg(id="req-1", value="second")
|
||||
await sub._process_message(msg2)
|
||||
|
||||
# Queue still has only the first message
|
||||
assert q.qsize() == 1
|
||||
assert q.get_nowait() == "first"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drop_oldest_evicts_when_full(self):
|
||||
"""drop_oldest strategy should evict oldest message when full."""
|
||||
sub = _make_subscriber(max_size=1, backpressure_strategy="drop_oldest")
|
||||
q = await sub.subscribe("req-1")
|
||||
|
||||
msg1 = _make_msg(id="req-1", value="first")
|
||||
await sub._process_message(msg1)
|
||||
|
||||
msg2 = _make_msg(id="req-1", value="second")
|
||||
await sub._process_message(msg2)
|
||||
|
||||
# Queue should have the newer message
|
||||
assert q.qsize() == 1
|
||||
assert q.get_nowait() == "second"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_block_strategy_delivers(self):
|
||||
"""block strategy should deliver messages normally."""
|
||||
sub = _make_subscriber(max_size=10, backpressure_strategy="block")
|
||||
q = await sub.subscribe("req-1")
|
||||
|
||||
msg = _make_msg(id="req-1", value="data")
|
||||
await sub._process_message(msg)
|
||||
|
||||
assert q.get_nowait() == "data"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Full subscribers (subscribe_all)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestFullSubscribers:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subscribe_all_receives_all_messages(self):
|
||||
sub = _make_subscriber()
|
||||
q = await sub.subscribe_all("listener-1")
|
||||
|
||||
msg = _make_msg(id="any-id", value="broadcast")
|
||||
await sub._process_message(msg)
|
||||
|
||||
assert q.get_nowait() == "broadcast"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_full_subscribers_all_receive(self):
|
||||
sub = _make_subscriber()
|
||||
q1 = await sub.subscribe_all("l1")
|
||||
q2 = await sub.subscribe_all("l2")
|
||||
|
||||
msg = _make_msg(id="any", value="data")
|
||||
await sub._process_message(msg)
|
||||
|
||||
assert q1.get_nowait() == "data"
|
||||
assert q2.get_nowait() == "data"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subscribe / unsubscribe lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestSubscribeLifecycle:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unsubscribe_removes_queue(self):
|
||||
sub = _make_subscriber()
|
||||
await sub.subscribe("req-1")
|
||||
await sub.unsubscribe("req-1")
|
||||
|
||||
assert "req-1" not in sub.q
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unsubscribe_nonexistent_is_noop(self):
|
||||
sub = _make_subscriber()
|
||||
await sub.unsubscribe("nonexistent") # Should not raise
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unsubscribe_all_removes_queue(self):
|
||||
sub = _make_subscriber()
|
||||
await sub.subscribe_all("l1")
|
||||
await sub.unsubscribe_all("l1")
|
||||
|
||||
assert "l1" not in sub.full
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pending ack tracking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPendingAckTracking:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_processed_message_cleared_from_pending(self):
|
||||
sub = _make_subscriber()
|
||||
msg = _make_msg(id="req-1", value="data")
|
||||
|
||||
await sub._process_message(msg)
|
||||
|
||||
# After processing, pending_acks should be empty
|
||||
assert len(sub.pending_acks) == 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue