apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-25 08:26:21 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Pub/sub abstraction: decouple from Pulsar (#751)

Browse source 
Remove Pulsar-specific concepts from application code so that
the pub/sub backend is swappable via configuration.

Rename translators:
- to_pulsar/from_pulsar → decode/encode across all translator
  classes, dispatch handlers, and tests (55+ files)
- from_response_with_completion → encode_with_completion
- Remove pulsar.schema.Record from translator base class

Queue naming (CLASS:TOPICSPACE:TOPIC):
- Replace topic() helper with queue() using new format:
  flow:tg:name, request:tg:name, response:tg:name, state:tg:name
- Queue class implies persistence/TTL (no QoS in names)
- Update Pulsar backend map_topic() to parse new format
- Librarian queues use flow class (persistent, for chunking)
- Config push uses state class (persistent, last-value)
- Remove 15 dead topic imports from schema files
- Update init_trustgraph.py namespace: config → state

Confine Pulsar to pulsar_backend.py:
- Delete legacy PulsarClient class from pubsub.py
- Move add_args to add_pubsub_args() with standalone flag
  for CLI tools (defaults to localhost)
- PulsarBackendConsumer.receive() catches _pulsar.Timeout,
  raises standard TimeoutError
- Remove Pulsar imports from: async_processor, flow_processor,
  log_level, all 11 client files, 4 storage writers, gateway
  service, gateway config receiver
- Remove log_level/LoggerLevel from client API
- Rewrite tg-monitor-prompts to use backend abstraction
- Update tg-dump-queues to use add_pubsub_args

Also: pubsub-abstraction.md tech spec covering problem statement,
design goals, as-is requirements, candidate broker assessment,
approach, and implementation order.
This commit is contained in:
cybermaggedon 2026-04-01 20:16:53 +01:00 • committed by GitHub
parent dbf8daa74a
commit 4fb0b4d8e8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
106 changed files with 1269 additions and 788 deletions
Download patch file Download diff file Expand all files Collapse all files

110
tests/unit/test_rdf/test_rdf_wire_format.py
View file

@ -28,21 +28,21 @@ def triple_tx():
class TestTermTranslatorIri:
def test_iri_to_pulsar(self, term_tx):
def test_iri_decode(self, term_tx):
data = {"t": "i", "i": "http://example.org/Alice"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.type == IRI
assert term.iri == "http://example.org/Alice"
def test_iri_from_pulsar(self, term_tx):
def test_iri_encode(self, term_tx):
term = Term(type=IRI, iri="http://example.org/Bob")
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert wire == {"t": "i", "i": "http://example.org/Bob"}
def test_iri_round_trip(self, term_tx):
original = Term(type=IRI, iri="http://example.org/round")
wire = term_tx.from_pulsar(original)
restored = term_tx.to_pulsar(wire)
wire = term_tx.encode(original)
restored = term_tx.decode(wire)
assert restored == original
@ -52,21 +52,21 @@ class TestTermTranslatorIri:
class TestTermTranslatorBlank:
def test_blank_to_pulsar(self, term_tx):
def test_blank_decode(self, term_tx):
data = {"t": "b", "d": "_:b42"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.type == BLANK
assert term.id == "_:b42"
def test_blank_from_pulsar(self, term_tx):
def test_blank_encode(self, term_tx):
term = Term(type=BLANK, id="_:node1")
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert wire == {"t": "b", "d": "_:node1"}
def test_blank_round_trip(self, term_tx):
original = Term(type=BLANK, id="_:x")
wire = term_tx.from_pulsar(original)
restored = term_tx.to_pulsar(wire)
wire = term_tx.encode(original)
restored = term_tx.decode(wire)
assert restored == original
@ -76,29 +76,29 @@ class TestTermTranslatorBlank:
class TestTermTranslatorTypedLiteral:
def test_plain_literal_to_pulsar(self, term_tx):
def test_plain_literal_decode(self, term_tx):
data = {"t": "l", "v": "hello"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.type == LITERAL
assert term.value == "hello"
assert term.datatype == ""
assert term.language == ""
def test_xsd_integer_to_pulsar(self, term_tx):
def test_xsd_integer_decode(self, term_tx):
data = {
"t": "l", "v": "42",
"dt": "http://www.w3.org/2001/XMLSchema#integer",
}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.value == "42"
assert term.datatype.endswith("#integer")
def test_typed_literal_from_pulsar(self, term_tx):
def test_typed_literal_encode(self, term_tx):
term = Term(
type=LITERAL, value="3.14",
datatype="http://www.w3.org/2001/XMLSchema#double",
)
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert wire["t"] == "l"
assert wire["v"] == "3.14"
assert wire["dt"] == "http://www.w3.org/2001/XMLSchema#double"
@ -109,13 +109,13 @@ class TestTermTranslatorTypedLiteral:
type=LITERAL, value="true",
datatype="http://www.w3.org/2001/XMLSchema#boolean",
)
wire = term_tx.from_pulsar(original)
restored = term_tx.to_pulsar(wire)
wire = term_tx.encode(original)
restored = term_tx.decode(wire)
assert restored == original
def test_plain_literal_omits_dt_and_ln(self, term_tx):
term = Term(type=LITERAL, value="x")
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert "dt" not in wire
assert "ln" not in wire
@ -126,22 +126,22 @@ class TestTermTranslatorTypedLiteral:
class TestTermTranslatorLangLiteral:
def test_language_tag_to_pulsar(self, term_tx):
def test_language_tag_decode(self, term_tx):
data = {"t": "l", "v": "bonjour", "ln": "fr"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.value == "bonjour"
assert term.language == "fr"
def test_language_tag_from_pulsar(self, term_tx):
def test_language_tag_encode(self, term_tx):
term = Term(type=LITERAL, value="colour", language="en-GB")
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert wire["ln"] == "en-GB"
assert "dt" not in wire # No datatype
def test_language_tag_round_trip(self, term_tx):
original = Term(type=LITERAL, value="hola", language="es")
wire = term_tx.from_pulsar(original)
restored = term_tx.to_pulsar(wire)
wire = term_tx.encode(original)
restored = term_tx.decode(wire)
assert restored == original
@ -151,7 +151,7 @@ class TestTermTranslatorLangLiteral:
class TestTermTranslatorQuotedTriple:
def test_quoted_triple_to_pulsar(self, term_tx):
def test_quoted_triple_decode(self, term_tx):
data = {
"t": "t",
"tr": {
@ -160,20 +160,20 @@ class TestTermTranslatorQuotedTriple:
"o": {"t": "i", "i": "http://example.org/Bob"},
},
}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.type == TRIPLE
assert term.triple is not None
assert term.triple.s.iri == "http://example.org/Alice"
assert term.triple.o.iri == "http://example.org/Bob"
def test_quoted_triple_from_pulsar(self, term_tx):
def test_quoted_triple_encode(self, term_tx):
inner = Triple(
s=Term(type=IRI, iri="http://example.org/s"),
p=Term(type=IRI, iri="http://example.org/p"),
o=Term(type=LITERAL, value="val"),
)
term = Term(type=TRIPLE, triple=inner)
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert wire["t"] == "t"
assert "tr" in wire
assert wire["tr"]["s"]["i"] == "http://example.org/s"
@ -186,18 +186,18 @@ class TestTermTranslatorQuotedTriple:
o=Term(type=LITERAL, value="C", language="en"),
)
original = Term(type=TRIPLE, triple=inner)
wire = term_tx.from_pulsar(original)
restored = term_tx.to_pulsar(wire)
wire = term_tx.encode(original)
restored = term_tx.decode(wire)
assert restored.type == TRIPLE
assert restored.triple.s == original.triple.s
assert restored.triple.o == original.triple.o
def test_quoted_triple_none_triple(self, term_tx):
term = Term(type=TRIPLE, triple=None)
wire = term_tx.from_pulsar(term)
wire = term_tx.encode(term)
assert wire == {"t": "t"}
# And back
restored = term_tx.to_pulsar(wire)
restored = term_tx.decode(wire)
assert restored.type == TRIPLE
assert restored.triple is None
@ -210,7 +210,7 @@ class TestTermTranslatorQuotedTriple:
"o": {"t": "l", "v": "A feeling of expectation"},
},
}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.triple.o.type == LITERAL
assert term.triple.o.value == "A feeling of expectation"
@ -223,22 +223,22 @@ class TestTermTranslatorEdgeCases:
def test_unknown_type(self, term_tx):
data = {"t": "z"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.type == "z"
def test_empty_type(self, term_tx):
data = {}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.type == ""
def test_missing_iri_field(self, term_tx):
data = {"t": "i"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.iri == ""
def test_missing_literal_fields(self, term_tx):
data = {"t": "l"}
term = term_tx.to_pulsar(data)
term = term_tx.decode(data)
assert term.value == ""
assert term.datatype == ""
assert term.language == ""
@ -250,24 +250,24 @@ class TestTermTranslatorEdgeCases:
class TestTripleTranslator:
def test_triple_to_pulsar(self, triple_tx):
def test_triple_decode(self, triple_tx):
data = {
"s": {"t": "i", "i": "http://example.org/s"},
"p": {"t": "i", "i": "http://example.org/p"},
"o": {"t": "l", "v": "object"},
}
triple = triple_tx.to_pulsar(data)
triple = triple_tx.decode(data)
assert triple.s.iri == "http://example.org/s"
assert triple.o.value == "object"
assert triple.g is None
def test_triple_from_pulsar(self, triple_tx):
def test_triple_encode(self, triple_tx):
triple = Triple(
s=Term(type=IRI, iri="http://example.org/A"),
p=Term(type=IRI, iri="http://example.org/B"),
o=Term(type=LITERAL, value="C"),
)
wire = triple_tx.from_pulsar(triple)
wire = triple_tx.encode(triple)
assert wire["s"]["t"] == "i"
assert wire["o"]["v"] == "C"
assert "g" not in wire
@ -279,17 +279,17 @@ class TestTripleTranslator:
"o": {"t": "l", "v": "val"},
"g": "urn:graph:source",
}
quad = triple_tx.to_pulsar(data)
quad = triple_tx.decode(data)
assert quad.g == "urn:graph:source"
def test_quad_from_pulsar_includes_graph(self, triple_tx):
def test_quad_encode_includes_graph(self, triple_tx):
quad = Triple(
s=Term(type=IRI, iri="http://example.org/s"),
p=Term(type=IRI, iri="http://example.org/p"),
o=Term(type=LITERAL, value="v"),
g="urn:graph:retrieval",
)
wire = triple_tx.from_pulsar(quad)
wire = triple_tx.encode(quad)
assert wire["g"] == "urn:graph:retrieval"
def test_quad_round_trip(self, triple_tx):
@ -299,8 +299,8 @@ class TestTripleTranslator:
o=Term(type=LITERAL, value="v"),
g="urn:graph:source",
)
wire = triple_tx.from_pulsar(original)
restored = triple_tx.to_pulsar(wire)
wire = triple_tx.encode(original)
restored = triple_tx.decode(wire)
assert restored == original
def test_none_graph_omitted_from_wire(self, triple_tx):
@ -310,12 +310,12 @@ class TestTripleTranslator:
o=Term(type=LITERAL, value="v"),
g=None,
)
wire = triple_tx.from_pulsar(triple)
wire = triple_tx.encode(triple)
assert "g" not in wire
def test_missing_terms_handled(self, triple_tx):
data = {}
triple = triple_tx.to_pulsar(data)
triple = triple_tx.decode(data)
assert triple.s is None
assert triple.p is None
assert triple.o is None
@ -342,16 +342,16 @@ class TestSubgraphTranslator:
g="urn:graph:source",
),
]
wire_list = tx.from_pulsar(triples)
wire_list = tx.encode(triples)
assert len(wire_list) == 2
assert wire_list[1]["g"] == "urn:graph:source"
restored = tx.to_pulsar(wire_list)
restored = tx.decode(wire_list)
assert len(restored) == 2
assert restored[0] == triples[0]
assert restored[1] == triples[1]
def test_empty_subgraph(self):
tx = SubgraphTranslator()
assert tx.to_pulsar([]) == []
assert tx.from_pulsar([]) == []
assert tx.decode([]) == []
assert tx.encode([]) == []