apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-25 00:16:23 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Pub/sub abstraction: decouple from Pulsar (#751)

Browse source 
Remove Pulsar-specific concepts from application code so that
the pub/sub backend is swappable via configuration.

Rename translators:
- to_pulsar/from_pulsar → decode/encode across all translator
  classes, dispatch handlers, and tests (55+ files)
- from_response_with_completion → encode_with_completion
- Remove pulsar.schema.Record from translator base class

Queue naming (CLASS:TOPICSPACE:TOPIC):
- Replace topic() helper with queue() using new format:
  flow:tg:name, request:tg:name, response:tg:name, state:tg:name
- Queue class implies persistence/TTL (no QoS in names)
- Update Pulsar backend map_topic() to parse new format
- Librarian queues use flow class (persistent, for chunking)
- Config push uses state class (persistent, last-value)
- Remove 15 dead topic imports from schema files
- Update init_trustgraph.py namespace: config → state

Confine Pulsar to pulsar_backend.py:
- Delete legacy PulsarClient class from pubsub.py
- Move add_args to add_pubsub_args() with standalone flag
  for CLI tools (defaults to localhost)
- PulsarBackendConsumer.receive() catches _pulsar.Timeout,
  raises standard TimeoutError
- Remove Pulsar imports from: async_processor, flow_processor,
  log_level, all 11 client files, 4 storage writers, gateway
  service, gateway config receiver
- Remove log_level/LoggerLevel from client API
- Rewrite tg-monitor-prompts to use backend abstraction
- Update tg-dump-queues to use add_pubsub_args

Also: pubsub-abstraction.md tech spec covering problem statement,
design goals, as-is requirements, candidate broker assessment,
approach, and implementation order.
This commit is contained in:
cybermaggedon 2026-04-01 20:16:53 +01:00 • committed by GitHub
parent dbf8daa74a
commit 4fb0b4d8e8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
106 changed files with 1269 additions and 788 deletions
Download patch file Download diff file Expand all files Collapse all files

4
tests/unit/test_clients/test_sync_document_embeddings_client.py
View file

@ -21,17 +21,15 @@ class TestSyncDocumentEmbeddingsClient:
# Act
client = DocumentEmbeddingsClient(
log_level=1,
subscriber="test-subscriber",
input_queue="test-input",
output_queue="test-output",
pulsar_host="pulsar://test:6650",
pulsar_api_key="test-key"
)
# Assert
mock_base_init.assert_called_once_with(
log_level=1,
subscriber="test-subscriber",
input_queue="test-input",
output_queue="test-output",

8
tests/unit/test_gateway/test_dispatch_config.py
View file

@ -49,7 +49,7 @@ class TestConfigRequestor:
mock_translator_registry.get_response_translator.return_value = Mock()
# Setup translator response
mock_request_translator.to_pulsar.return_value = "translated_request"
mock_request_translator.decode.return_value = "translated_request"
# Patch ServiceRequestor async methods with regular mocks (not AsyncMock)
with patch.object(ServiceRequestor, 'start', return_value=None), \
@ -64,7 +64,7 @@ class TestConfigRequestor:
result = requestor.to_request({"test": "body"})
# Verify translator was called correctly
mock_request_translator.to_pulsar.assert_called_once_with({"test": "body"})
mock_request_translator.decode.assert_called_once_with({"test": "body"})
assert result == "translated_request"
@patch('trustgraph.gateway.dispatch.config.TranslatorRegistry')
@ -76,7 +76,7 @@ class TestConfigRequestor:
mock_translator_registry.get_response_translator.return_value = mock_response_translator
# Setup translator response
mock_response_translator.from_response_with_completion.return_value = "translated_response"
mock_response_translator.encode_with_completion.return_value = "translated_response"
requestor = ConfigRequestor(
backend=Mock(),
@ -89,5 +89,5 @@ class TestConfigRequestor:
result = requestor.from_response(mock_message)
# Verify translator was called correctly
mock_response_translator.from_response_with_completion.assert_called_once_with(mock_message)
mock_response_translator.encode_with_completion.assert_called_once_with(mock_message)
assert result == "translated_response"

56
tests/unit/test_gateway/test_streaming_translators.py
View file

@ -25,7 +25,7 @@ from trustgraph.schema import (
class TestGraphRagResponseTranslator:
"""Test GraphRagResponseTranslator streaming behavior"""
def test_from_pulsar_with_empty_response(self):
def test_encode_with_empty_response(self):
"""Test that empty response strings are preserved"""
# Arrange
translator = GraphRagResponseTranslator()
@ -36,14 +36,14 @@ class TestGraphRagResponseTranslator:
)
# Act
result = translator.from_pulsar(response)
result = translator.encode(response)
# Assert - Empty string should be included in result
assert "response" in result
assert result["response"] == ""
assert result["end_of_stream"] is True
def test_from_pulsar_with_non_empty_response(self):
def test_encode_with_non_empty_response(self):
"""Test that non-empty responses work correctly"""
# Arrange
translator = GraphRagResponseTranslator()
@ -54,13 +54,13 @@ class TestGraphRagResponseTranslator:
)
# Act
result = translator.from_pulsar(response)
result = translator.encode(response)
# Assert
assert result["response"] == "Some text"
assert result["end_of_stream"] is False
def test_from_pulsar_with_none_response(self):
def test_encode_with_none_response(self):
"""Test that None response is handled correctly"""
# Arrange
translator = GraphRagResponseTranslator()
@ -71,14 +71,14 @@ class TestGraphRagResponseTranslator:
)
# Act
result = translator.from_pulsar(response)
result = translator.encode(response)
# Assert - None should not be included
assert "response" not in result
assert result["end_of_stream"] is True
def test_from_response_with_completion_returns_correct_flag(self):
"""Test that from_response_with_completion returns correct is_final flag"""
def test_encode_with_completion_returns_correct_flag(self):
"""Test that encode_with_completion returns correct is_final flag"""
# Arrange
translator = GraphRagResponseTranslator()
@ -90,7 +90,7 @@ class TestGraphRagResponseTranslator:
)
# Act
result, is_final = translator.from_response_with_completion(response_chunk)
result, is_final = translator.encode_with_completion(response_chunk)
# Assert
assert is_final is False
@ -105,7 +105,7 @@ class TestGraphRagResponseTranslator:
)
# Act
result, is_final = translator.from_response_with_completion(final_response)
result, is_final = translator.encode_with_completion(final_response)
# Assert - is_final is based on end_of_session, not end_of_stream
assert is_final is True
@ -116,7 +116,7 @@ class TestGraphRagResponseTranslator:
class TestDocumentRagResponseTranslator:
"""Test DocumentRagResponseTranslator streaming behavior"""
def test_from_pulsar_with_empty_response(self):
def test_encode_with_empty_response(self):
"""Test that empty response strings are preserved"""
# Arrange
translator = DocumentRagResponseTranslator()
@ -127,14 +127,14 @@ class TestDocumentRagResponseTranslator:
)
# Act
result = translator.from_pulsar(response)
result = translator.encode(response)
# Assert
assert "response" in result
assert result["response"] == ""
assert result["end_of_stream"] is True
def test_from_pulsar_with_non_empty_response(self):
def test_encode_with_non_empty_response(self):
"""Test that non-empty responses work correctly"""
# Arrange
translator = DocumentRagResponseTranslator()
@ -145,7 +145,7 @@ class TestDocumentRagResponseTranslator:
)
# Act
result = translator.from_pulsar(response)
result = translator.encode(response)
# Assert
assert result["response"] == "Document content"
@ -155,7 +155,7 @@ class TestDocumentRagResponseTranslator:
class TestPromptResponseTranslator:
"""Test PromptResponseTranslator streaming behavior"""
def test_from_pulsar_with_empty_text(self):
def test_encode_with_empty_text(self):
"""Test that empty text strings are preserved"""
# Arrange
translator = PromptResponseTranslator()
@ -167,14 +167,14 @@ class TestPromptResponseTranslator:
)
# Act
result = translator.from_pulsar(response)
result = translator.encode(response)
# Assert
assert "text" in result
assert result["text"] == ""
assert result["end_of_stream"] is True
def test_from_pulsar_with_non_empty_text(self):
def test_encode_with_non_empty_text(self):
"""Test that non-empty text works correctly"""
# Arrange
translator = PromptResponseTranslator()
@ -186,13 +186,13 @@ class TestPromptResponseTranslator:
)
# Act
result = translator.from_pulsar(response)
result = translator.encode(response)
# Assert
assert result["text"] == "Some prompt response"
assert result["end_of_stream"] is False
def test_from_pulsar_with_none_text(self):
def test_encode_with_none_text(self):
"""Test that None text is handled correctly"""
# Arrange
translator = PromptResponseTranslator()
@ -204,14 +204,14 @@ class TestPromptResponseTranslator:
)
# Act
result = translator.from_pulsar(response)
result = translator.encode(response)
# Assert
assert "text" not in result
assert "object" in result
assert result["end_of_stream"] is True
def test_from_pulsar_includes_end_of_stream(self):
def test_encode_includes_end_of_stream(self):
"""Test that end_of_stream flag is always included"""
# Arrange
translator = PromptResponseTranslator()
@ -225,7 +225,7 @@ class TestPromptResponseTranslator:
)
# Act
result = translator.from_pulsar(response)
result = translator.encode(response)
# Assert
assert "end_of_stream" in result
@ -235,7 +235,7 @@ class TestPromptResponseTranslator:
class TestTextCompletionResponseTranslator:
"""Test TextCompletionResponseTranslator streaming behavior"""
def test_from_pulsar_always_includes_response(self):
def test_encode_always_includes_response(self):
"""Test that response field is always included, even if empty"""
# Arrange
translator = TextCompletionResponseTranslator()
@ -249,13 +249,13 @@ class TestTextCompletionResponseTranslator:
)
# Act
result = translator.from_pulsar(response)
result = translator.encode(response)
# Assert - Response should always be present
assert "response" in result
assert result["response"] == ""
def test_from_response_with_completion_with_empty_final(self):
def test_encode_with_completion_with_empty_final(self):
"""Test that empty final response is handled correctly"""
# Arrange
translator = TextCompletionResponseTranslator()
@ -269,7 +269,7 @@ class TestTextCompletionResponseTranslator:
)
# Act
result, is_final = translator.from_response_with_completion(response)
result, is_final = translator.encode_with_completion(response)
# Assert
assert is_final is True
@ -297,7 +297,7 @@ class TestStreamingProtocolCompliance:
response = response_class(**kwargs)
# Act
result = translator.from_pulsar(response)
result = translator.encode(response)
# Assert
assert field_name in result, f"{translator_class.__name__} should include '{field_name}' field even when empty"
@ -320,7 +320,7 @@ class TestStreamingProtocolCompliance:
response = response_class(**kwargs)
# Act
result = translator.from_pulsar(response)
result = translator.encode(response)
# Assert
assert "end_of_stream" in result, f"{translator_class.__name__} should include 'end_of_stream' flag"

12
tests/unit/test_gateway/test_text_document_translator.py
View file

@ -8,11 +8,11 @@ from trustgraph.messaging.translators.document_loading import TextDocumentTransl
class TestTextDocumentTranslator:
def test_to_pulsar_decodes_base64_text(self):
def test_decode_decodes_base64_text(self):
translator = TextDocumentTranslator()
payload = "Cancer survival: 2.74× higher hazard ratio"
msg = translator.to_pulsar(
msg = translator.decode(
{
"id": "doc-1",
"user": "alice",
@ -27,11 +27,11 @@ class TestTextDocumentTranslator:
assert msg.metadata.collection == "research"
assert msg.text == payload.encode("utf-8")
def test_to_pulsar_accepts_raw_utf8_text(self):
def test_decode_accepts_raw_utf8_text(self):
translator = TextDocumentTranslator()
payload = "Cancer survival: 2.74× higher hazard ratio"
msg = translator.to_pulsar(
msg = translator.decode(
{
"charset": "utf-8",
"text": payload,
@ -40,11 +40,11 @@ class TestTextDocumentTranslator:
assert msg.text == payload.encode("utf-8")
def test_to_pulsar_falls_back_to_raw_non_base64_ascii(self):
def test_decode_falls_back_to_raw_non_base64_ascii(self):
translator = TextDocumentTranslator()
payload = "plain-text payload"
msg = translator.to_pulsar(
msg = translator.decode(
{
"charset": "utf-8",
"text": payload,

133
tests/unit/test_pubsub/test_queue_naming.py Normal file
View file

@ -0,0 +1,133 @@
"""
Tests for queue naming and topic mapping.
"""
import pytest
import argparse
from trustgraph.schema.core.topic import queue
from trustgraph.base.pubsub import get_pubsub, add_pubsub_args
from trustgraph.base.pulsar_backend import PulsarBackend
class TestQueueFunction:
def test_flow_default(self):
assert queue('text-completion-request') == 'flow:tg:text-completion-request'
def test_request_class(self):
assert queue('config', cls='request') == 'request:tg:config'
def test_response_class(self):
assert queue('config', cls='response') == 'response:tg:config'
def test_state_class(self):
assert queue('config', cls='state') == 'state:tg:config'
def test_custom_topicspace(self):
assert queue('config', cls='request', topicspace='prod') == 'request:prod:config'
def test_default_class_is_flow(self):
result = queue('something')
assert result.startswith('flow:')
class TestPulsarMapTopic:
@pytest.fixture
def backend(self):
"""Create a PulsarBackend without connecting."""
b = object.__new__(PulsarBackend)
return b
def test_flow_maps_to_persistent(self, backend):
assert backend.map_topic('flow:tg:text-completion-request') == \
'persistent://tg/flow/text-completion-request'
def test_state_maps_to_persistent(self, backend):
assert backend.map_topic('state:tg:config') == \
'persistent://tg/state/config'
def test_request_maps_to_non_persistent(self, backend):
assert backend.map_topic('request:tg:config') == \
'non-persistent://tg/request/config'
def test_response_maps_to_non_persistent(self, backend):
assert backend.map_topic('response:tg:librarian') == \
'non-persistent://tg/response/librarian'
def test_passthrough_pulsar_uri(self, backend):
uri = 'persistent://tg/flow/something'
assert backend.map_topic(uri) == uri
def test_invalid_format_raises(self, backend):
with pytest.raises(ValueError, match="Invalid queue format"):
backend.map_topic('bad-format')
def test_invalid_class_raises(self, backend):
with pytest.raises(ValueError, match="Invalid queue class"):
backend.map_topic('unknown:tg:topic')
def test_custom_topicspace(self, backend):
assert backend.map_topic('flow:prod:my-queue') == \
'persistent://prod/flow/my-queue'
class TestGetPubsubDispatch:
def test_unknown_backend_raises(self):
with pytest.raises(ValueError, match="Unknown pub/sub backend"):
get_pubsub(pubsub_backend='redis')
class TestAddPubsubArgs:
def test_standalone_defaults_to_localhost(self):
parser = argparse.ArgumentParser()
add_pubsub_args(parser, standalone=True)
args = parser.parse_args([])
assert args.pulsar_host == 'pulsar://localhost:6650'
assert args.pulsar_listener == 'localhost'
def test_non_standalone_defaults_to_container(self):
parser = argparse.ArgumentParser()
add_pubsub_args(parser, standalone=False)
args = parser.parse_args([])
assert 'pulsar:6650' in args.pulsar_host
assert args.pulsar_listener is None
def test_cli_override_respected(self):
parser = argparse.ArgumentParser()
add_pubsub_args(parser, standalone=True)
args = parser.parse_args(['--pulsar-host', 'pulsar://custom:6650'])
assert args.pulsar_host == 'pulsar://custom:6650'
def test_pubsub_backend_default(self):
parser = argparse.ArgumentParser()
add_pubsub_args(parser)
args = parser.parse_args([])
assert args.pubsub_backend == 'pulsar'
class TestQueueDefinitions:
"""Verify the actual queue constants produce correct names."""
def test_config_request(self):
from trustgraph.schema.services.config import config_request_queue
assert config_request_queue == 'request:tg:config'
def test_config_response(self):
from trustgraph.schema.services.config import config_response_queue
assert config_response_queue == 'response:tg:config'
def test_config_push(self):
from trustgraph.schema.services.config import config_push_queue
assert config_push_queue == 'state:tg:config'
def test_librarian_request_is_persistent(self):
from trustgraph.schema.services.library import librarian_request_queue
assert librarian_request_queue.startswith('flow:')
def test_knowledge_request(self):
from trustgraph.schema.knowledge.knowledge import knowledge_request_queue
assert knowledge_request_queue == 'request:tg:knowledge'

110
tests/unit/test_rdf/test_rdf_wire_format.py
View file

@ -28,21 +28,21 @@ def triple_tx():
class TestTermTranslatorIri:
def test_iri_to_pulsar(self, term_tx):
def test_iri_decode(self, term_tx):
data = {"t": "i", "i": "http://example.org/Alice"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.type == IRI
assert term.iri == "http://example.org/Alice"
def test_iri_from_pulsar(self, term_tx):
def test_iri_encode(self, term_tx):
term = Term(type=IRI, iri="http://example.org/Bob")
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert wire == {"t": "i", "i": "http://example.org/Bob"}
def test_iri_round_trip(self, term_tx):
original = Term(type=IRI, iri="http://example.org/round")
wire = term_tx.from_pulsar(original)
restored = term_tx.to_pulsar(wire)
wire = term_tx.encode(original)
restored = term_tx.decode(wire)
assert restored == original
@ -52,21 +52,21 @@ class TestTermTranslatorIri:
class TestTermTranslatorBlank:
def test_blank_to_pulsar(self, term_tx):
def test_blank_decode(self, term_tx):
data = {"t": "b", "d": "_:b42"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.type == BLANK
assert term.id == "_:b42"
def test_blank_from_pulsar(self, term_tx):
def test_blank_encode(self, term_tx):
term = Term(type=BLANK, id="_:node1")
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert wire == {"t": "b", "d": "_:node1"}
def test_blank_round_trip(self, term_tx):
original = Term(type=BLANK, id="_:x")
wire = term_tx.from_pulsar(original)
restored = term_tx.to_pulsar(wire)
wire = term_tx.encode(original)
restored = term_tx.decode(wire)
assert restored == original
@ -76,29 +76,29 @@ class TestTermTranslatorBlank:
class TestTermTranslatorTypedLiteral:
def test_plain_literal_to_pulsar(self, term_tx):
def test_plain_literal_decode(self, term_tx):
data = {"t": "l", "v": "hello"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.type == LITERAL
assert term.value == "hello"
assert term.datatype == ""
assert term.language == ""
def test_xsd_integer_to_pulsar(self, term_tx):
def test_xsd_integer_decode(self, term_tx):
data = {
"t": "l", "v": "42",
"dt": "http://www.w3.org/2001/XMLSchema#integer",
}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.value == "42"
assert term.datatype.endswith("#integer")
def test_typed_literal_from_pulsar(self, term_tx):
def test_typed_literal_encode(self, term_tx):
term = Term(
type=LITERAL, value="3.14",
datatype="http://www.w3.org/2001/XMLSchema#double",
)
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert wire["t"] == "l"
assert wire["v"] == "3.14"
assert wire["dt"] == "http://www.w3.org/2001/XMLSchema#double"
@ -109,13 +109,13 @@ class TestTermTranslatorTypedLiteral:
type=LITERAL, value="true",
datatype="http://www.w3.org/2001/XMLSchema#boolean",
)
wire = term_tx.from_pulsar(original)
restored = term_tx.to_pulsar(wire)
wire = term_tx.encode(original)
restored = term_tx.decode(wire)
assert restored == original
def test_plain_literal_omits_dt_and_ln(self, term_tx):
term = Term(type=LITERAL, value="x")
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert "dt" not in wire
assert "ln" not in wire
@ -126,22 +126,22 @@ class TestTermTranslatorTypedLiteral:
class TestTermTranslatorLangLiteral:
def test_language_tag_to_pulsar(self, term_tx):
def test_language_tag_decode(self, term_tx):
data = {"t": "l", "v": "bonjour", "ln": "fr"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.value == "bonjour"
assert term.language == "fr"
def test_language_tag_from_pulsar(self, term_tx):
def test_language_tag_encode(self, term_tx):
term = Term(type=LITERAL, value="colour", language="en-GB")
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert wire["ln"] == "en-GB"
assert "dt" not in wire # No datatype
def test_language_tag_round_trip(self, term_tx):
original = Term(type=LITERAL, value="hola", language="es")
wire = term_tx.from_pulsar(original)
restored = term_tx.to_pulsar(wire)
wire = term_tx.encode(original)
restored = term_tx.decode(wire)
assert restored == original
@ -151,7 +151,7 @@ class TestTermTranslatorLangLiteral:
class TestTermTranslatorQuotedTriple:
def test_quoted_triple_to_pulsar(self, term_tx):
def test_quoted_triple_decode(self, term_tx):
data = {
"t": "t",
"tr": {
@ -160,20 +160,20 @@ class TestTermTranslatorQuotedTriple:
"o": {"t": "i", "i": "http://example.org/Bob"},
},
}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.type == TRIPLE
assert term.triple is not None
assert term.triple.s.iri == "http://example.org/Alice"
assert term.triple.o.iri == "http://example.org/Bob"
def test_quoted_triple_from_pulsar(self, term_tx):
def test_quoted_triple_encode(self, term_tx):
inner = Triple(
s=Term(type=IRI, iri="http://example.org/s"),
p=Term(type=IRI, iri="http://example.org/p"),
o=Term(type=LITERAL, value="val"),
)
term = Term(type=TRIPLE, triple=inner)
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert wire["t"] == "t"
assert "tr" in wire
assert wire["tr"]["s"]["i"] == "http://example.org/s"
@ -186,18 +186,18 @@ class TestTermTranslatorQuotedTriple:
o=Term(type=LITERAL, value="C", language="en"),
)
original = Term(type=TRIPLE, triple=inner)
wire = term_tx.from_pulsar(original)
restored = term_tx.to_pulsar(wire)
wire = term_tx.encode(original)
restored = term_tx.decode(wire)
assert restored.type == TRIPLE
assert restored.triple.s == original.triple.s
assert restored.triple.o == original.triple.o
def test_quoted_triple_none_triple(self, term_tx):
term = Term(type=TRIPLE, triple=None)
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert wire == {"t": "t"}
# And back
restored = term_tx.to_pulsar(wire)
restored = term_tx.decode(wire)
assert restored.type == TRIPLE
assert restored.triple is None
@ -210,7 +210,7 @@ class TestTermTranslatorQuotedTriple:
"o": {"t": "l", "v": "A feeling of expectation"},
},
}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.triple.o.type == LITERAL
assert term.triple.o.value == "A feeling of expectation"
@ -223,22 +223,22 @@ class TestTermTranslatorEdgeCases:
def test_unknown_type(self, term_tx):
data = {"t": "z"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.type == "z"
def test_empty_type(self, term_tx):
data = {}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.type == ""
def test_missing_iri_field(self, term_tx):
data = {"t": "i"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.iri == ""
def test_missing_literal_fields(self, term_tx):
data = {"t": "l"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.value == ""
assert term.datatype == ""
assert term.language == ""
@ -250,24 +250,24 @@ class TestTermTranslatorEdgeCases:
class TestTripleTranslator:
def test_triple_to_pulsar(self, triple_tx):
def test_triple_decode(self, triple_tx):
data = {
"s": {"t": "i", "i": "http://example.org/s"},
"p": {"t": "i", "i": "http://example.org/p"},
"o": {"t": "l", "v": "object"},
}
triple = triple_tx.to_pulsar(data)
triple = triple_tx.decode(data)
assert triple.s.iri == "http://example.org/s"
assert triple.o.value == "object"
assert triple.g is None
def test_triple_from_pulsar(self, triple_tx):
def test_triple_encode(self, triple_tx):
triple = Triple(
s=Term(type=IRI, iri="http://example.org/A"),
p=Term(type=IRI, iri="http://example.org/B"),
o=Term(type=LITERAL, value="C"),
)
wire = triple_tx.from_pulsar(triple)
wire = triple_tx.encode(triple)
assert wire["s"]["t"] == "i"
assert wire["o"]["v"] == "C"
assert "g" not in wire
@ -279,17 +279,17 @@ class TestTripleTranslator:
"o": {"t": "l", "v": "val"},
"g": "urn:graph:source",
}
quad = triple_tx.to_pulsar(data)
quad = triple_tx.decode(data)
assert quad.g == "urn:graph:source"
def test_quad_from_pulsar_includes_graph(self, triple_tx):
def test_quad_encode_includes_graph(self, triple_tx):
quad = Triple(
s=Term(type=IRI, iri="http://example.org/s"),
p=Term(type=IRI, iri="http://example.org/p"),
o=Term(type=LITERAL, value="v"),
g="urn:graph:retrieval",
)
wire = triple_tx.from_pulsar(quad)
wire = triple_tx.encode(quad)
assert wire["g"] == "urn:graph:retrieval"
def test_quad_round_trip(self, triple_tx):
@ -299,8 +299,8 @@ class TestTripleTranslator:
o=Term(type=LITERAL, value="v"),
g="urn:graph:source",
)
wire = triple_tx.from_pulsar(original)
restored = triple_tx.to_pulsar(wire)
wire = triple_tx.encode(original)
restored = triple_tx.decode(wire)
assert restored == original
def test_none_graph_omitted_from_wire(self, triple_tx):
@ -310,12 +310,12 @@ class TestTripleTranslator:
o=Term(type=LITERAL, value="v"),
g=None,
)
wire = triple_tx.from_pulsar(triple)
wire = triple_tx.encode(triple)
assert "g" not in wire
def test_missing_terms_handled(self, triple_tx):
data = {}
triple = triple_tx.to_pulsar(data)
triple = triple_tx.decode(data)
assert triple.s is None
assert triple.p is None
assert triple.o is None
@ -342,16 +342,16 @@ class TestSubgraphTranslator:
g="urn:graph:source",
),
]
wire_list = tx.from_pulsar(triples)
wire_list = tx.encode(triples)
assert len(wire_list) == 2
assert wire_list[1]["g"] == "urn:graph:source"
restored = tx.to_pulsar(wire_list)
restored = tx.decode(wire_list)
assert len(restored) == 2
assert restored[0] == triples[0]
assert restored[1] == triples[1]
def test_empty_subgraph(self):
tx = SubgraphTranslator()
assert tx.to_pulsar([]) == []
assert tx.from_pulsar([]) == []
assert tx.decode([]) == []
assert tx.encode([]) == []

30
tests/unit/test_reliability/test_metadata_preservation.py
View file

@ -35,7 +35,7 @@ class TestDocumentMetadataTranslator:
"parent-id": "doc-100",
"document-type": "page",
}
obj = self.tx.to_pulsar(data)
obj = self.tx.decode(data)
assert obj.id == "doc-123"
assert obj.time == 1710000000
assert obj.kind == "application/pdf"
@ -45,14 +45,14 @@ class TestDocumentMetadataTranslator:
assert obj.parent_id == "doc-100"
assert obj.document_type == "page"
wire = self.tx.from_pulsar(obj)
wire = self.tx.encode(obj)
assert wire["id"] == "doc-123"
assert wire["user"] == "alice"
assert wire["parent-id"] == "doc-100"
assert wire["document-type"] == "page"
def test_defaults_for_missing_fields(self):
obj = self.tx.to_pulsar({})
obj = self.tx.decode({})
assert obj.parent_id == ""
assert obj.document_type == "source"
@ -63,25 +63,25 @@ class TestDocumentMetadataTranslator:
"o": {"t": "i", "i": "http://example.org/o"},
}]
data = {"metadata": triple_wire}
obj = self.tx.to_pulsar(data)
obj = self.tx.decode(data)
assert len(obj.metadata) == 1
assert obj.metadata[0].s.iri == "http://example.org/s"
def test_none_metadata_handled(self):
data = {"metadata": None}
obj = self.tx.to_pulsar(data)
obj = self.tx.decode(data)
assert obj.metadata == []
def test_empty_tags_preserved(self):
data = {"tags": []}
obj = self.tx.to_pulsar(data)
wire = self.tx.from_pulsar(obj)
obj = self.tx.decode(data)
wire = self.tx.encode(obj)
assert wire["tags"] == []
def test_falsy_fields_omitted_from_wire(self):
"""Empty string fields should be omitted from wire format."""
obj = DocumentMetadata(id="", time=0, user="")
wire = self.tx.from_pulsar(obj)
wire = self.tx.encode(obj)
assert "id" not in wire
assert "user" not in wire
@ -105,7 +105,7 @@ class TestProcessingMetadataTranslator:
"collection": "my-collection",
"tags": ["tag1"],
}
obj = self.tx.to_pulsar(data)
obj = self.tx.decode(data)
assert obj.id == "proc-1"
assert obj.document_id == "doc-123"
assert obj.flow == "default"
@ -113,32 +113,32 @@ class TestProcessingMetadataTranslator:
assert obj.collection == "my-collection"
assert obj.tags == ["tag1"]
wire = self.tx.from_pulsar(obj)
wire = self.tx.encode(obj)
assert wire["id"] == "proc-1"
assert wire["document-id"] == "doc-123"
assert wire["user"] == "alice"
assert wire["collection"] == "my-collection"
def test_missing_fields_use_defaults(self):
obj = self.tx.to_pulsar({})
obj = self.tx.decode({})
assert obj.id is None
assert obj.user is None
assert obj.collection is None
def test_tags_none_omitted(self):
obj = ProcessingMetadata(tags=None)
wire = self.tx.from_pulsar(obj)
wire = self.tx.encode(obj)
assert "tags" not in wire
def test_tags_empty_list_preserved(self):
obj = ProcessingMetadata(tags=[])
wire = self.tx.from_pulsar(obj)
wire = self.tx.encode(obj)
assert wire["tags"] == []
def test_user_and_collection_preserved(self):
"""Core pipeline routing fields must survive round-trip."""
data = {"user": "bob", "collection": "research"}
obj = self.tx.to_pulsar(data)
wire = self.tx.from_pulsar(obj)
obj = self.tx.decode(data)
wire = self.tx.encode(obj)
assert wire["user"] == "bob"
assert wire["collection"] == "research"

16
tests/unit/test_retrieval/test_structured_diag/test_message_translation.py
View file

@ -28,7 +28,7 @@ class TestRequestTranslation:
}
# Translate to Pulsar
pulsar_msg = translator.to_pulsar(api_data)
pulsar_msg = translator.decode(api_data)
assert pulsar_msg.operation == "schema-selection"
assert pulsar_msg.sample == "test data sample"
@ -46,7 +46,7 @@ class TestRequestTranslation:
"options": {"delimiter": ","}
}
pulsar_msg = translator.to_pulsar(api_data)
pulsar_msg = translator.decode(api_data)
assert pulsar_msg.operation == "generate-descriptor"
assert pulsar_msg.sample == "csv data"
@ -70,7 +70,7 @@ class TestResponseTranslation:
)
# Translate to API format
api_data = translator.from_pulsar(pulsar_response)
api_data = translator.encode(pulsar_response)
assert api_data["operation"] == "schema-selection"
assert api_data["schema-matches"] == ["products", "inventory", "catalog"]
@ -86,7 +86,7 @@ class TestResponseTranslation:
error=None
)
api_data = translator.from_pulsar(pulsar_response)
api_data = translator.encode(pulsar_response)
assert api_data["operation"] == "schema-selection"
assert api_data["schema-matches"] == []
@ -103,7 +103,7 @@ class TestResponseTranslation:
error=None
)
api_data = translator.from_pulsar(pulsar_response)
api_data = translator.encode(pulsar_response)
assert api_data["operation"] == "detect-type"
assert api_data["detected-type"] == "xml"
@ -123,7 +123,7 @@ class TestResponseTranslation:
)
)
api_data = translator.from_pulsar(pulsar_response)
api_data = translator.encode(pulsar_response)
assert api_data["operation"] == "schema-selection"
# Error objects are typically handled separately by the gateway
@ -146,7 +146,7 @@ class TestResponseTranslation:
error=None
)
api_data = translator.from_pulsar(pulsar_response)
api_data = translator.encode(pulsar_response)
assert api_data["operation"] == "diagnose"
assert api_data["detected-type"] == "csv"
@ -165,7 +165,7 @@ class TestResponseTranslation:
error=None
)
api_data, is_final = translator.from_response_with_completion(pulsar_response)
api_data, is_final = translator.encode_with_completion(pulsar_response)
assert is_final is True # Structured-diag responses are always final
assert api_data["operation"] == "schema-selection"