Pub/sub abstraction: decouple from Pulsar (#751)

Remove Pulsar-specific concepts from application code so that
the pub/sub backend is swappable via configuration.

Rename translators:
- to_pulsar/from_pulsar → decode/encode across all translator
  classes, dispatch handlers, and tests (55+ files)
- from_response_with_completion → encode_with_completion
- Remove pulsar.schema.Record from translator base class

Queue naming (CLASS:TOPICSPACE:TOPIC):
- Replace topic() helper with queue() using new format:
  flow:tg:name, request:tg:name, response:tg:name, state:tg:name
- Queue class implies persistence/TTL (no QoS in names)
- Update Pulsar backend map_topic() to parse new format
- Librarian queues use flow class (persistent, for chunking)
- Config push uses state class (persistent, last-value)
- Remove 15 dead topic imports from schema files
- Update init_trustgraph.py namespace: config → state

Confine Pulsar to pulsar_backend.py:
- Delete legacy PulsarClient class from pubsub.py
- Move add_args to add_pubsub_args() with standalone flag
  for CLI tools (defaults to localhost)
- PulsarBackendConsumer.receive() catches _pulsar.Timeout,
  raises standard TimeoutError
- Remove Pulsar imports from: async_processor, flow_processor,
  log_level, all 11 client files, 4 storage writers, gateway
  service, gateway config receiver
- Remove log_level/LoggerLevel from client API
- Rewrite tg-monitor-prompts to use backend abstraction
- Update tg-dump-queues to use add_pubsub_args

Also: pubsub-abstraction.md tech spec covering problem statement,
design goals, as-is requirements, candidate broker assessment,
approach, and implementation order.

trustgraph-flow/trustgraph/extract/kg/rows/processor.py

@@ -145,7 +145,7 @@ class Processor(FlowProcessor):
         
         try:
             # Convert Pulsar RowSchema to JSON-serializable dict
-            schema_dict = row_schema_translator.from_pulsar(schema)
+            schema_dict = row_schema_translator.encode(schema)
             
             # Use prompt client to extract rows based on schema
             objects = await flow("prompt-request").extract_objects(

trustgraph-flow/trustgraph/gateway/config/receiver.py

@@ -23,7 +23,6 @@ import uuid
 logger = logging.getLogger(__name__)
 import json
 
-import pulsar
 from prometheus_client import start_http_server
 
 from ... schema import ConfigPush, config_push_queue

trustgraph-flow/trustgraph/gateway/dispatch/agent.py

@@ -25,8 +25,8 @@ class AgentRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("agent")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/collection_management.py

@@ -29,8 +29,8 @@ class CollectionManagementRequestor(ServiceRequestor):
 
     def to_request(self, body):
         print("REQUEST", body, flush=True)
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
         print("RESPONSE", message, flush=True)
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/config.py

@@ -30,8 +30,8 @@ class ConfigRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("config")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/document_embeddings_import.py

@@ -44,7 +44,7 @@ class DocumentEmbeddingsImport:
     async def receive(self, msg):
 
         data = msg.json()
-        elt = self.translator.to_pulsar(data)
+        elt = self.translator.decode(data)
         await self.publisher.send(None, elt)
 
     async def run(self):

trustgraph-flow/trustgraph/gateway/dispatch/document_embeddings_query.py

@@ -25,7 +25,7 @@ class DocumentEmbeddingsQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("document-embeddings-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/document_load.py

@@ -23,5 +23,5 @@ class DocumentLoad(ServiceSender):
 
     def to_request(self, body):
         logger.info("Document received")
-        return self.translator.to_pulsar(body)
+        return self.translator.decode(body)
 

trustgraph-flow/trustgraph/gateway/dispatch/document_rag.py

@@ -25,8 +25,8 @@ class DocumentRagRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("document-rag")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/embeddings.py

@@ -25,8 +25,8 @@ class EmbeddingsRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("embeddings")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/flow.py

@@ -30,8 +30,8 @@ class FlowRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("flow")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/graph_embeddings_query.py

@@ -25,8 +25,8 @@ class GraphEmbeddingsQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("graph-embeddings-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/graph_rag.py

@@ -25,8 +25,8 @@ class GraphRagRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("graph-rag")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/knowledge.py

@@ -33,8 +33,8 @@ class KnowledgeRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("knowledge")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/librarian.py

@@ -40,8 +40,8 @@ class LibrarianRequestor(ServiceRequestor):
             body = body.copy()
             body["content"] = content
         
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/mcp_tool.py

@@ -25,8 +25,8 @@ class McpToolRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("tool")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/nlp_query.py

@@ -24,7 +24,7 @@ class NLPQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("nlp-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/prompt.py

@@ -27,8 +27,8 @@ class PromptRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("prompt")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/row_embeddings_query.py

@@ -25,7 +25,7 @@ class RowEmbeddingsQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("row-embeddings-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/rows_query.py

@@ -24,7 +24,7 @@ class RowsQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("rows-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/serialize.py

@@ -11,22 +11,22 @@ _triple_translator = TripleTranslator()
 
 def to_value(x):
     """Convert dict to Term. Delegates to TermTranslator."""
-    return _term_translator.to_pulsar(x)
+    return _term_translator.decode(x)
 
 
 def to_subgraph(x):
     """Convert list of dicts to list of Triples. Delegates to TripleTranslator."""
-    return [_triple_translator.to_pulsar(t) for t in x]
+    return [_triple_translator.decode(t) for t in x]
 
 
 def serialize_value(v):
     """Convert Term to dict. Delegates to TermTranslator."""
-    return _term_translator.from_pulsar(v)
+    return _term_translator.encode(v)
 
 
 def serialize_triple(t):
     """Convert Triple to dict. Delegates to TripleTranslator."""
-    return _triple_translator.from_pulsar(t)
+    return _triple_translator.encode(t)
 
 
 def serialize_subgraph(sg):

trustgraph-flow/trustgraph/gateway/dispatch/structured_diag.py

@@ -24,7 +24,7 @@ class StructuredDiagRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("structured-diag")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/structured_query.py

@@ -24,7 +24,7 @@ class StructuredQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("structured-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/text_completion.py

@@ -25,8 +25,8 @@ class TextCompletionRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("text-completion")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/text_load.py

@@ -23,5 +23,5 @@ class TextLoad(ServiceSender):
 
     def to_request(self, body):
         logger.info("Text document received")
-        return self.translator.to_pulsar(body)
+        return self.translator.decode(body)
 

trustgraph-flow/trustgraph/gateway/dispatch/triples_query.py

@@ -25,8 +25,8 @@ class TriplesQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("triples-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/service.py

@@ -18,7 +18,6 @@ from . dispatch.manager import DispatcherManager
 
 from . endpoint.manager import EndpointManager
 
-import pulsar
 from prometheus_client import start_http_server
 
 # Import default queue names

trustgraph-flow/trustgraph/storage/triples/cassandra/write.py

@@ -3,7 +3,6 @@
 Graph writer.  Input is graph edge.  Writes edges to Cassandra graph.
 """
 
-import pulsar
 import base64
 import os
 import argparse

trustgraph-flow/trustgraph/storage/triples/falkordb/write.py

@@ -3,7 +3,6 @@
 Graph writer.  Input is graph edge.  Writes edges to FalkorDB graph.
 """
 
-import pulsar
 import base64
 import os
 import argparse

trustgraph-flow/trustgraph/storage/triples/memgraph/write.py

@@ -3,7 +3,6 @@
 Graph writer.  Input is graph edge.  Writes edges to Memgraph.
 """
 
-import pulsar
 import base64
 import os
 import argparse

trustgraph-flow/trustgraph/storage/triples/neo4j/write.py

@@ -3,7 +3,6 @@
 Graph writer.  Input is graph edge.  Writes edges to Neo4j graph.
 """
 
-import pulsar
 import base64
 import os
 import argparse