Merge branch 'master' into docs

trustgraph-base/pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
     "prometheus-client",
     "requests",
     "python-logging-loki",
+    "pika",
 ]
 classifiers = [
     "Programming Language :: Python :: 3",

trustgraph-base/trustgraph/api/__init__.py

@@ -81,7 +81,12 @@ from .explainability import (
     Synthesis,
     Reflection,
     Analysis,
+    Observation,
     Conclusion,
+    Decomposition,
+    Finding,
+    Plan,
+    StepResult,
     EdgeSelection,
     wire_triples_to_tuples,
     extract_term_value,
@@ -160,6 +165,7 @@ __all__ = [
     "Focus",
     "Synthesis",
     "Analysis",
+    "Observation",
     "Conclusion",
     "EdgeSelection",
     "wire_triples_to_tuples",

trustgraph-base/trustgraph/api/explainability.py

@@ -40,15 +40,25 @@ TG_ANSWER_TYPE = TG + "Answer"
 TG_REFLECTION_TYPE = TG + "Reflection"
 TG_THOUGHT_TYPE = TG + "Thought"
 TG_OBSERVATION_TYPE = TG + "Observation"
+TG_TOOL_USE = TG + "ToolUse"
 TG_GRAPH_RAG_QUESTION = TG + "GraphRagQuestion"
 TG_DOC_RAG_QUESTION = TG + "DocRagQuestion"
 TG_AGENT_QUESTION = TG + "AgentQuestion"
 
+# Orchestrator entity types
+TG_DECOMPOSITION = TG + "Decomposition"
+TG_FINDING = TG + "Finding"
+TG_PLAN_TYPE = TG + "Plan"
+TG_STEP_RESULT = TG + "StepResult"
+
+# Orchestrator predicates
+TG_SUBAGENT_GOAL = TG + "subagentGoal"
+TG_PLAN_STEP = TG + "planStep"
+
 # PROV-O predicates
 PROV = "http://www.w3.org/ns/prov#"
 PROV_STARTED_AT_TIME = PROV + "startedAtTime"
 PROV_WAS_DERIVED_FROM = PROV + "wasDerivedFrom"
-PROV_WAS_GENERATED_BY = PROV + "wasGeneratedBy"
 
 RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
 RDFS_LABEL = "http://www.w3.org/2000/01/rdf-schema#label"
@@ -82,8 +92,18 @@ class ExplainEntity:
             return Exploration.from_triples(uri, triples)
         elif TG_FOCUS in types:
             return Focus.from_triples(uri, triples)
+        elif TG_DECOMPOSITION in types:
+            return Decomposition.from_triples(uri, triples)
+        elif TG_FINDING in types:
+            return Finding.from_triples(uri, triples)
+        elif TG_PLAN_TYPE in types:
+            return Plan.from_triples(uri, triples)
+        elif TG_STEP_RESULT in types:
+            return StepResult.from_triples(uri, triples)
         elif TG_SYNTHESIS in types:
             return Synthesis.from_triples(uri, triples)
+        elif TG_OBSERVATION_TYPE in types and TG_REFLECTION_TYPE not in types:
+            return Observation.from_triples(uri, triples)
         elif TG_REFLECTION_TYPE in types:
             return Reflection.from_triples(uri, triples)
         elif TG_ANALYSIS in types:
@@ -261,18 +281,16 @@ class Reflection(ExplainEntity):
 
 @dataclass
 class Analysis(ExplainEntity):
-    """Analysis entity - one think/act/observe cycle (Agent only)."""
+    """Analysis+ToolUse entity - decision + tool call (Agent only)."""
     action: str = ""
     arguments: str = ""  # JSON string
     thought: str = ""
-    observation: str = ""
 
     @classmethod
     def from_triples(cls, uri: str, triples: List[Tuple[str, str, Any]]) -> "Analysis":
         action = ""
         arguments = ""
         thought = ""
-        observation = ""
 
         for s, p, o in triples:
             if p == TG_ACTION:
@@ -281,8 +299,6 @@ class Analysis(ExplainEntity):
                 arguments = o
             elif p == TG_THOUGHT:
                 thought = o
-            elif p == TG_OBSERVATION:
-                observation = o
 
         return cls(
             uri=uri,
@@ -290,7 +306,26 @@ class Analysis(ExplainEntity):
             action=action,
             arguments=arguments,
             thought=thought,
-            observation=observation
+        )
+
+
+@dataclass
+class Observation(ExplainEntity):
+    """Observation entity - standalone tool result (Agent only)."""
+    document: str = ""
+
+    @classmethod
+    def from_triples(cls, uri: str, triples: List[Tuple[str, str, Any]]) -> "Observation":
+        document = ""
+
+        for s, p, o in triples:
+            if p == TG_DOCUMENT:
+                document = o
+
+        return cls(
+            uri=uri,
+            entity_type="observation",
+            document=document,
         )
 
 
@@ -314,6 +349,70 @@ class Conclusion(ExplainEntity):
         )
 
 
+@dataclass
+class Decomposition(ExplainEntity):
+    """Decomposition entity - supervisor broke question into sub-goals."""
+    goals: List[str] = field(default_factory=list)
+
+    @classmethod
+    def from_triples(cls, uri: str, triples: List[Tuple[str, str, Any]]) -> "Decomposition":
+        goals = []
+        for s, p, o in triples:
+            if p == TG_SUBAGENT_GOAL:
+                goals.append(o)
+        return cls(uri=uri, entity_type="decomposition", goals=goals)
+
+
+@dataclass
+class Finding(ExplainEntity):
+    """Finding entity - a subagent's result."""
+    goal: str = ""
+    document: str = ""
+
+    @classmethod
+    def from_triples(cls, uri: str, triples: List[Tuple[str, str, Any]]) -> "Finding":
+        goal = ""
+        document = ""
+        for s, p, o in triples:
+            if p == TG_SUBAGENT_GOAL:
+                goal = o
+            elif p == TG_DOCUMENT:
+                document = o
+        return cls(uri=uri, entity_type="finding", goal=goal, document=document)
+
+
+@dataclass
+class Plan(ExplainEntity):
+    """Plan entity - a structured plan of steps."""
+    steps: List[str] = field(default_factory=list)
+
+    @classmethod
+    def from_triples(cls, uri: str, triples: List[Tuple[str, str, Any]]) -> "Plan":
+        steps = []
+        for s, p, o in triples:
+            if p == TG_PLAN_STEP:
+                steps.append(o)
+        return cls(uri=uri, entity_type="plan", steps=steps)
+
+
+@dataclass
+class StepResult(ExplainEntity):
+    """StepResult entity - a plan step's result."""
+    step: str = ""
+    document: str = ""
+
+    @classmethod
+    def from_triples(cls, uri: str, triples: List[Tuple[str, str, Any]]) -> "StepResult":
+        step = ""
+        document = ""
+        for s, p, o in triples:
+            if p == TG_PLAN_STEP:
+                step = o
+            elif p == TG_DOCUMENT:
+                document = o
+        return cls(uri=uri, entity_type="step-result", step=step, document=document)
+
+
 def parse_edge_selection_triples(triples: List[Tuple[str, str, Any]]) -> EdgeSelection:
     """Parse triples for an edge selection entity."""
     uri = triples[0][0] if triples else ""
@@ -675,9 +774,9 @@ class ExplainabilityClient:
             return trace
         trace["question"] = question
 
-        # Find grounding: ?grounding prov:wasGeneratedBy question_uri
+        # Find grounding: ?grounding prov:wasDerivedFrom question_uri
         grounding_triples = self.flow.triples_query(
-            p=PROV_WAS_GENERATED_BY,
+            p=PROV_WAS_DERIVED_FROM,
             o=question_uri,
             g=graph,
             user=user,
@@ -812,9 +911,9 @@ class ExplainabilityClient:
             return trace
         trace["question"] = question
 
-        # Find grounding: ?grounding prov:wasGeneratedBy question_uri
+        # Find grounding: ?grounding prov:wasDerivedFrom question_uri
         grounding_triples = self.flow.triples_query(
-            p=PROV_WAS_GENERATED_BY,
+            p=PROV_WAS_DERIVED_FROM,
             o=question_uri,
             g=graph,
             user=user,
@@ -895,7 +994,10 @@ class ExplainabilityClient:
         """
         Fetch the complete Agent trace starting from a session URI.
 
-        Follows the provenance chain: Question -> Analysis(s) -> Conclusion
+        Follows the provenance chain for all patterns:
+        - ReAct: Question -> Analysis(s) -> Conclusion
+        - Supervisor: Question -> Decomposition -> Finding(s) -> Synthesis
+        - Plan-then-Execute: Question -> Plan -> StepResult(s) -> Synthesis
 
         Args:
             session_uri: The agent session/question URI
@@ -906,15 +1008,14 @@ class ExplainabilityClient:
             max_content: Maximum content length for conclusion
 
         Returns:
-            Dict with question, iterations (Analysis list), conclusion entities
+            Dict with question, steps (mixed entity list), conclusion/synthesis
         """
         if graph is None:
             graph = "urn:graph:retrieval"
 
         trace = {
             "question": None,
-            "iterations": [],
-            "conclusion": None,
+            "steps": [],
         }
 
         # Fetch question/session
@@ -923,65 +1024,89 @@ class ExplainabilityClient:
             return trace
         trace["question"] = question
 
-        # Follow the chain: wasGeneratedBy for first hop, wasDerivedFrom after
-        current_uri = session_uri
-        is_first = True
-        max_iterations = 50  # Safety limit
-
-        for _ in range(max_iterations):
-            # First hop uses wasGeneratedBy (entity←activity),
-            # subsequent hops use wasDerivedFrom (entity←entity)
-            if is_first:
-                derived_triples = self.flow.triples_query(
-                    p=PROV_WAS_GENERATED_BY,
-                    o=current_uri,
-                    g=graph,
-                    user=user,
-                    collection=collection,
-                    limit=10
-                )
-                # Fall back to wasDerivedFrom for backwards compatibility
-                if not derived_triples:
-                    derived_triples = self.flow.triples_query(
-                        p=PROV_WAS_DERIVED_FROM,
-                        o=current_uri,
-                        g=graph,
-                        user=user,
-                        collection=collection,
-                        limit=10
-                    )
-                is_first = False
-            else:
-                derived_triples = self.flow.triples_query(
-                    p=PROV_WAS_DERIVED_FROM,
-                    o=current_uri,
-                    g=graph,
-                    user=user,
-                    collection=collection,
-                    limit=10
-                )
-
-            if not derived_triples:
-                break
-
-            derived_uri = extract_term_value(derived_triples[0].get("s", {}))
-            if not derived_uri:
-                break
-
-            entity = self.fetch_entity(derived_uri, graph, user, collection)
-
-            if isinstance(entity, Analysis):
-                trace["iterations"].append(entity)
-                current_uri = derived_uri
-            elif isinstance(entity, Conclusion):
-                trace["conclusion"] = entity
-                break
-            else:
-                # Unknown entity type, stop
-                break
+        # Follow the provenance chain from the question
+        self._follow_provenance_chain(
+            session_uri, trace, graph, user, collection,
+            max_depth=50,
+        )
 
         return trace
 
+    def _follow_provenance_chain(
+        self, current_uri, trace, graph, user, collection,
+        max_depth=50,
+    ):
+        """Recursively follow the provenance chain, handling branches."""
+        if max_depth <= 0:
+            return
+
+        # Find entities derived from current_uri
+        derived_triples = self.flow.triples_query(
+            p=PROV_WAS_DERIVED_FROM,
+            o=current_uri,
+            g=graph, user=user, collection=collection,
+            limit=20
+        )
+
+        if not derived_triples:
+            return
+
+        derived_uris = [
+            extract_term_value(t.get("s", {}))
+            for t in derived_triples
+        ]
+
+        for derived_uri in derived_uris:
+            if not derived_uri:
+                continue
+
+            entity = self.fetch_entity(derived_uri, graph, user, collection)
+            if entity is None:
+                continue
+
+            if isinstance(entity, (Analysis, Observation, Decomposition,
+                                   Finding, Plan, StepResult)):
+                trace["steps"].append(entity)
+
+                # Continue following from this entity
+                self._follow_provenance_chain(
+                    derived_uri, trace, graph, user, collection,
+                    max_depth=max_depth - 1,
+                )
+
+            elif isinstance(entity, Question):
+                # Sub-trace: a RAG session linked to this agent step.
+                # Fetch the full sub-trace and embed it.
+                if entity.question_type == "graph-rag":
+                    sub_trace = self.fetch_graphrag_trace(
+                        derived_uri, graph, user, collection,
+                    )
+                elif entity.question_type == "document-rag":
+                    sub_trace = self.fetch_docrag_trace(
+                        derived_uri, graph, user, collection,
+                    )
+                else:
+                    sub_trace = None
+
+                if sub_trace:
+                    trace["steps"].append({
+                        "type": "sub-trace",
+                        "question": entity,
+                        "trace": sub_trace,
+                    })
+
+                    # Continue from the sub-trace's terminal entity
+                    # (Observation may derive from Synthesis)
+                    terminal = sub_trace.get("synthesis")
+                    if terminal:
+                        self._follow_provenance_chain(
+                            terminal.uri, trace, graph, user, collection,
+                            max_depth=max_depth - 1,
+                        )
+
+            elif isinstance(entity, (Conclusion, Synthesis)):
+                trace["steps"].append(entity)
+
     def list_sessions(
         self,
         graph: Optional[str] = None,
@@ -1021,10 +1146,25 @@ class ExplainabilityClient:
                 if isinstance(entity, Question):
                     questions.append(entity)
 
-        # Sort by timestamp (newest first)
-        questions.sort(key=lambda q: q.timestamp or "", reverse=True)
+        # Filter out sub-traces: sessions that have a wasDerivedFrom link
+        # (they are child sessions linked to a parent agent iteration)
+        top_level = []
+        for q in questions:
+            parent_triples = self.flow.triples_query(
+                s=q.uri,
+                p=PROV_WAS_DERIVED_FROM,
+                g=graph,
+                user=user,
+                collection=collection,
+                limit=1
+            )
+            if not parent_triples:
+                top_level.append(q)
 
-        return questions
+        # Sort by timestamp (newest first)
+        top_level.sort(key=lambda q: q.timestamp or "", reverse=True)
+
+        return top_level
 
     def detect_session_type(
         self,
@@ -1066,23 +1206,14 @@ class ExplainabilityClient:
             limit=5
         )
 
-        generated_triples = self.flow.triples_query(
-            p=PROV_WAS_GENERATED_BY,
-            o=session_uri,
-            g=graph,
-            user=user,
-            collection=collection,
-            limit=5
-        )
-
         all_child_uris = [
             extract_term_value(t.get("s", {}))
-            for t in (derived_triples + generated_triples)
+            for t in derived_triples
         ]
 
         for child_uri in all_child_uris:
             entity = self.fetch_entity(child_uri, graph, user, collection)
-            if isinstance(entity, Analysis):
+            if isinstance(entity, (Analysis, Decomposition, Plan)):
                 return "agent"
             if isinstance(entity, Exploration):
                 return "graphrag"

trustgraph-base/trustgraph/api/flow.py

@@ -1122,6 +1122,45 @@ class FlowInstance:
             
         return result
 
+    def sparql_query(
+            self, query, user="trustgraph", collection="default",
+            limit=10000
+    ):
+        """
+        Execute a SPARQL query against the knowledge graph.
+
+        Args:
+            query: SPARQL 1.1 query string
+            user: User/keyspace identifier (default: "trustgraph")
+            collection: Collection identifier (default: "default")
+            limit: Safety limit on results (default: 10000)
+
+        Returns:
+            dict with query results. Structure depends on query type:
+            - SELECT: {"query-type": "select", "variables": [...], "bindings": [...]}
+            - ASK: {"query-type": "ask", "ask-result": bool}
+            - CONSTRUCT/DESCRIBE: {"query-type": "construct", "triples": [...]}
+
+        Raises:
+            ProtocolException: If an error occurs
+        """
+
+        input = {
+            "query": query,
+            "user": user,
+            "collection": collection,
+            "limit": limit,
+        }
+
+        response = self.request("service/sparql", input)
+
+        if "error" in response and response["error"]:
+            error_type = response["error"].get("type", "unknown")
+            error_message = response["error"].get("message", "Unknown error")
+            raise ProtocolException(f"{error_type}: {error_message}")
+
+        return response
+
     def nlp_query(self, question, max_results=100):
         """
         Convert a natural language question to a GraphQL query.

trustgraph-base/trustgraph/api/library.py

@@ -22,8 +22,9 @@ logger = logging.getLogger(__name__)
 # Lower threshold provides progress feedback and resumability on slower connections
 CHUNKED_UPLOAD_THRESHOLD = 2 * 1024 * 1024
 
-# Default chunk size (5MB - S3 multipart minimum)
-DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024
+# Default chunk size (3MB - stays under broker message size limits
+# after base64 encoding ~4MB)
+DEFAULT_CHUNK_SIZE = 3 * 1024 * 1024
 
 
 def to_value(x):

trustgraph-base/trustgraph/api/socket_client.py

@@ -366,59 +366,39 @@ class SocketClient:
         # Handle GraphRAG/DocRAG message format with message_type
         if message_type == "explain":
             if include_provenance:
-                return ProvenanceEvent(
-                    explain_id=resp.get("explain_id", ""),
-                    explain_graph=resp.get("explain_graph", "")
-                )
+                return self._build_provenance_event(resp)
             return None
 
         # Handle Agent message format with chunk_type="explain"
         if chunk_type == "explain":
             if include_provenance:
-                return ProvenanceEvent(
-                    explain_id=resp.get("explain_id", ""),
-                    explain_graph=resp.get("explain_graph", "")
-                )
+                return self._build_provenance_event(resp)
             return None
 
         if chunk_type == "thought":
             return AgentThought(
                 content=resp.get("content", ""),
-                end_of_message=resp.get("end_of_message", False)
+                end_of_message=resp.get("end_of_message", False),
+                message_id=resp.get("message_id", ""),
             )
         elif chunk_type == "observation":
             return AgentObservation(
                 content=resp.get("content", ""),
-                end_of_message=resp.get("end_of_message", False)
+                end_of_message=resp.get("end_of_message", False),
+                message_id=resp.get("message_id", ""),
             )
         elif chunk_type == "answer" or chunk_type == "final-answer":
             return AgentAnswer(
                 content=resp.get("content", ""),
                 end_of_message=resp.get("end_of_message", False),
-                end_of_dialog=resp.get("end_of_dialog", False)
+                end_of_dialog=resp.get("end_of_dialog", False),
+                message_id=resp.get("message_id", ""),
             )
         elif chunk_type == "action":
             return AgentThought(
                 content=resp.get("content", ""),
                 end_of_message=resp.get("end_of_message", False)
             )
-        # Non-streaming agent format: chunk_type is empty but has thought/observation/answer fields
-        elif resp.get("thought"):
-            return AgentThought(
-                content=resp.get("thought", ""),
-                end_of_message=resp.get("end_of_message", False)
-            )
-        elif resp.get("observation"):
-            return AgentObservation(
-                content=resp.get("observation", ""),
-                end_of_message=resp.get("end_of_message", False)
-            )
-        elif resp.get("answer"):
-            return AgentAnswer(
-                content=resp.get("answer", ""),
-                end_of_message=resp.get("end_of_message", False),
-                end_of_dialog=resp.get("end_of_dialog", False)
-            )
         else:
             content = resp.get("response", resp.get("chunk", resp.get("text", "")))
             return RAGChunk(
@@ -427,6 +407,42 @@ class SocketClient:
                 error=None
             )
 
+    def _build_provenance_event(self, resp: Dict[str, Any]) -> ProvenanceEvent:
+        """Build a ProvenanceEvent from a response dict, parsing inline triples
+        into an ExplainEntity if available."""
+        explain_id = resp.get("explain_id", "")
+        explain_graph = resp.get("explain_graph", "")
+        raw_triples = resp.get("explain_triples", [])
+
+        entity = None
+        if raw_triples:
+            try:
+                from .explainability import ExplainEntity
+                # Convert wire-format triple dicts to (s, p, o) tuples
+                parsed = []
+                for t in raw_triples:
+                    s = t.get("s", {}).get("i", "") if t.get("s") else ""
+                    p = t.get("p", {}).get("i", "") if t.get("p") else ""
+                    o_term = t.get("o", {})
+                    if o_term:
+                        if o_term.get("t") == "i":
+                            o = o_term.get("i", "")
+                        else:
+                            o = o_term.get("v", "")
+                    else:
+                        o = ""
+                    parsed.append((s, p, o))
+                entity = ExplainEntity.from_triples(explain_id, parsed)
+            except Exception:
+                pass
+
+        return ProvenanceEvent(
+            explain_id=explain_id,
+            explain_graph=explain_graph,
+            entity=entity,
+            triples=raw_triples,
+        )
+
     def close(self) -> None:
         """Close the persistent WebSocket connection."""
         if self._loop and not self._loop.is_closed():
@@ -826,6 +842,31 @@ class SocketFlowInstance:
             else:
                 yield response
 
+    def sparql_query_stream(
+        self,
+        query: str,
+        user: str = "trustgraph",
+        collection: str = "default",
+        limit: int = 10000,
+        batch_size: int = 20,
+        **kwargs: Any
+    ) -> Iterator[Dict[str, Any]]:
+        """Execute a SPARQL query with streaming batches."""
+        request = {
+            "query": query,
+            "user": user,
+            "collection": collection,
+            "limit": limit,
+            "streaming": True,
+            "batch-size": batch_size,
+        }
+        request.update(kwargs)
+
+        for response in self.client._send_request_sync(
+            "sparql", self.flow_id, request, streaming_raw=True
+        ):
+            yield response
+
     def rows_query(
         self,
         query: str,

trustgraph-base/trustgraph/api/types.py

@@ -150,8 +150,10 @@ class AgentThought(StreamingChunk):
         content: Agent's thought text
         end_of_message: True if this completes the current thought
         chunk_type: Always "thought"
+        message_id: Provenance URI of the entity being built
     """
     chunk_type: str = "thought"
+    message_id: str = ""
 
 @dataclasses.dataclass
 class AgentObservation(StreamingChunk):
@@ -165,8 +167,10 @@ class AgentObservation(StreamingChunk):
         content: Observation text describing tool results
         end_of_message: True if this completes the current observation
         chunk_type: Always "observation"
+        message_id: Provenance URI of the entity being built
     """
     chunk_type: str = "observation"
+    message_id: str = ""
 
 @dataclasses.dataclass
 class AgentAnswer(StreamingChunk):
@@ -184,6 +188,7 @@ class AgentAnswer(StreamingChunk):
     """
     chunk_type: str = "final-answer"
     end_of_dialog: bool = False
+    message_id: str = ""
 
 @dataclasses.dataclass
 class RAGChunk(StreamingChunk):
@@ -208,25 +213,47 @@ class ProvenanceEvent:
     """
     Provenance event for explainability.
 
-    Emitted during GraphRAG queries when explainable mode is enabled.
+    Emitted during retrieval queries when explainable mode is enabled.
     Each event represents a provenance node created during query processing.
 
     Attributes:
         explain_id: URI of the provenance node (e.g., urn:trustgraph:question:abc123)
         explain_graph: Named graph where provenance triples are stored (e.g., urn:graph:retrieval)
-        event_type: Type of provenance event (question, exploration, focus, synthesis)
+        event_type: Type of provenance event (question, exploration, focus, synthesis, etc.)
+        entity: Parsed ExplainEntity from inline triples (if available)
+        triples: Raw triples from the response (wire format dicts)
     """
     explain_id: str
     explain_graph: str = ""
     event_type: str = ""  # Derived from explain_id
+    entity: object = None  # ExplainEntity (parsed from triples)
+    triples: list = dataclasses.field(default_factory=list)  # Raw wire-format triple dicts
 
     def __post_init__(self):
         # Extract event type from explain_id
         if "question" in self.explain_id:
             self.event_type = "question"
+        elif "grounding" in self.explain_id:
+            self.event_type = "grounding"
         elif "exploration" in self.explain_id:
             self.event_type = "exploration"
         elif "focus" in self.explain_id:
             self.event_type = "focus"
         elif "synthesis" in self.explain_id:
             self.event_type = "synthesis"
+        elif "iteration" in self.explain_id:
+            self.event_type = "iteration"
+        elif "observation" in self.explain_id:
+            self.event_type = "observation"
+        elif "conclusion" in self.explain_id:
+            self.event_type = "conclusion"
+        elif "decomposition" in self.explain_id:
+            self.event_type = "decomposition"
+        elif "finding" in self.explain_id:
+            self.event_type = "finding"
+        elif "plan" in self.explain_id:
+            self.event_type = "plan"
+        elif "step-result" in self.explain_id:
+            self.event_type = "step-result"
+        elif "session" in self.explain_id:
+            self.event_type = "session"

trustgraph-base/trustgraph/base/__init__.py

@@ -1,5 +1,5 @@
 
-from . pubsub import PulsarClient, get_pubsub
+from . pubsub import get_pubsub, add_pubsub_args
 from . async_processor import AsyncProcessor
 from . consumer import Consumer
 from . producer import Producer
@@ -14,6 +14,7 @@ from . producer_spec import ProducerSpec
 from . subscriber_spec import SubscriberSpec
 from . request_response_spec import RequestResponseSpec
 from . llm_service import LlmService, LlmResult, LlmChunk
+from . librarian_client import LibrarianClient
 from . chunking_service import ChunkingService
 from . embeddings_service import EmbeddingsService
 from . embeddings_client import EmbeddingsClientSpec

trustgraph-base/trustgraph/base/agent_client.py

@@ -57,8 +57,7 @@ class AgentClient(RequestResponse):
         await self.request(
             AgentRequest(
                 question = question,
-                plan = plan,
-                state = state,
+                state = state or "",
                 history = history,
             ),
             recipient=recipient,

trustgraph-base/trustgraph/base/agent_service.py

@@ -90,9 +90,6 @@ class AgentService(FlowProcessor):
                         type = "agent-error",
                         message = str(e),
                     ),
-                    thought = None,
-                    observation = None,
-                    answer = None,
                     end_of_message = True,
                     end_of_dialog = True,
                 ),

trustgraph-base/trustgraph/base/async_processor.py

@@ -1,24 +1,29 @@
 
 # Base class for processors.  Implements:
-# - Pulsar client, subscribe and consume basic
+# - Pub/sub client, subscribe and consume basic
 # - the async startup logic
+# - Config notify handling with subscribe-then-fetch pattern
 # - Initialising metrics
 
 import asyncio
 import argparse
-import _pulsar
 import time
 import uuid
 import logging
 import os
 from prometheus_client import start_http_server, Info
 
-from .. schema import ConfigPush, config_push_queue
+from .. schema import ConfigPush, ConfigRequest, ConfigResponse
+from .. schema import config_push_queue, config_request_queue
+from .. schema import config_response_queue
 from .. log_level import LogLevel
-from . pubsub import PulsarClient, get_pubsub
+from . pubsub import get_pubsub, add_pubsub_args
 from . producer import Producer
 from . consumer import Consumer
-from . metrics import ProcessorMetrics, ConsumerMetrics
+from . subscriber import Subscriber
+from . request_response_spec import RequestResponse
+from . metrics import ProcessorMetrics, ConsumerMetrics, ProducerMetrics
+from . metrics import SubscriberMetrics
 from . logging import add_logging_args, setup_logging
 
 default_config_queue = config_push_queue
@@ -58,9 +63,13 @@ class AsyncProcessor:
             "config_push_queue", default_config_queue
         )
 
-        # This records registered configuration handlers
+        # This records registered configuration handlers, each entry is:
+        # { "handler": async_fn, "types": set_or_none }
         self.config_handlers = []
 
+        # Track the current config version for dedup
+        self.config_version = 0
+
         # Create a random ID for this subscription to the configuration
         # service
         config_subscriber_id = str(uuid.uuid4())
@@ -69,33 +78,104 @@ class AsyncProcessor:
             processor = self.id, flow = None, name = "config",
         )
 
-        # Subscribe to config queue
+        # Subscribe to config notify queue
         self.config_sub_task = Consumer(
 
             taskgroup = self.taskgroup,
-            backend = self.pubsub_backend,  # Changed from client to backend
+            backend = self.pubsub_backend,
             subscriber = config_subscriber_id,
             flow = None,
 
             topic = self.config_push_queue,
             schema = ConfigPush,
 
-            handler = self.on_config_change,
+            handler = self.on_config_notify,
 
             metrics = config_consumer_metrics,
 
-            # This causes new subscriptions to view the entire history of
-            # configuration
-            start_of_messages = True
+            start_of_messages = False,
         )
 
         self.running = True
 
-    # This is called to start dynamic behaviour.  An over-ride point for
-    # extra functionality
+    def _create_config_client(self):
+        """Create a short-lived config request/response client."""
+        config_rr_id = str(uuid.uuid4())
+
+        config_req_metrics = ProducerMetrics(
+            processor = self.id, flow = None, name = "config-request",
+        )
+        config_resp_metrics = SubscriberMetrics(
+            processor = self.id, flow = None, name = "config-response",
+        )
+
+        return RequestResponse(
+            backend = self.pubsub_backend,
+            subscription = f"{self.id}--config--{config_rr_id}",
+            consumer_name = self.id,
+            request_topic = config_request_queue,
+            request_schema = ConfigRequest,
+            request_metrics = config_req_metrics,
+            response_topic = config_response_queue,
+            response_schema = ConfigResponse,
+            response_metrics = config_resp_metrics,
+        )
+
+    async def fetch_config(self):
+        """Fetch full config from config service using a short-lived
+        request/response client. Returns (config, version) or raises."""
+        client = self._create_config_client()
+        try:
+            await client.start()
+            resp = await client.request(
+                ConfigRequest(operation="config"),
+                timeout=10,
+            )
+            if resp.error:
+                raise RuntimeError(f"Config error: {resp.error.message}")
+            return resp.config, resp.version
+        finally:
+            await client.stop()
+
+    # This is called to start dynamic behaviour.
+    # Implements the subscribe-then-fetch pattern to avoid race conditions.
     async def start(self):
+
+        # 1. Start the notify consumer (begins buffering incoming notifys)
         await self.config_sub_task.start()
 
+        # 2. Fetch current config via request/response
+        await self.fetch_and_apply_config()
+
+        # 3. Any buffered notifys with version > fetched version will be
+        #    processed by on_config_notify, which does the version check
+
+    async def fetch_and_apply_config(self):
+        """Fetch full config from config service and apply to all handlers.
+        Retries until successful — config service may not be ready yet."""
+
+        while self.running:
+
+            try:
+                config, version = await self.fetch_config()
+
+                logger.info(f"Fetched config version {version}")
+
+                self.config_version = version
+
+                # Apply to all handlers (startup = invoke all)
+                for entry in self.config_handlers:
+                    await entry["handler"](config, version)
+
+                return
+
+            except Exception as e:
+                logger.warning(
+                    f"Config fetch failed: {e}, retrying in 2s...",
+                    exc_info=True
+                )
+                await asyncio.sleep(2)
+
     # This is called to stop all threads.  An over-ride point for extra
     # functionality
     def stop(self):
@@ -111,20 +191,66 @@ class AsyncProcessor:
     def pulsar_host(self): return self._pulsar_host
 
     # Register a new event handler for configuration change
-    def register_config_handler(self, handler):
-        self.config_handlers.append(handler)
+    def register_config_handler(self, handler, types=None):
+        self.config_handlers.append({
+            "handler": handler,
+            "types": set(types) if types else None,
+        })
 
-    # Called when a new configuration message push occurs
-    async def on_config_change(self, message, consumer, flow):
+    # Called when a config notify message arrives
+    async def on_config_notify(self, message, consumer, flow):
 
-        # Get configuration data and version number
-        config = message.value().config
-        version = message.value().version
+        notify_version = message.value().version
+        notify_types = set(message.value().types)
 
-        # Invoke message handlers
-        logger.info(f"Config change event: version={version}")
-        for ch in self.config_handlers:
-            await ch(config, version)
+        # Skip if we already have this version or newer
+        if notify_version <= self.config_version:
+            logger.debug(
+                f"Ignoring config notify v{notify_version}, "
+                f"already at v{self.config_version}"
+            )
+            return
+
+        # Check if any handler cares about the affected types
+        if notify_types:
+            any_interested = False
+            for entry in self.config_handlers:
+                handler_types = entry["types"]
+                if handler_types is None or notify_types & handler_types:
+                    any_interested = True
+                    break
+
+            if not any_interested:
+                logger.debug(
+                    f"Ignoring config notify v{notify_version}, "
+                    f"no handlers for types {notify_types}"
+                )
+                self.config_version = notify_version
+                return
+
+        logger.info(
+            f"Config notify v{notify_version} types={list(notify_types)}, "
+            f"fetching config..."
+        )
+
+        # Fetch full config using short-lived client
+        try:
+            config, version = await self.fetch_config()
+
+            self.config_version = version
+
+            # Invoke handlers that care about the affected types
+            for entry in self.config_handlers:
+                handler_types = entry["types"]
+                if handler_types is None:
+                    await entry["handler"](config, version)
+                elif not notify_types or notify_types & handler_types:
+                    await entry["handler"](config, version)
+
+        except Exception as e:
+            logger.error(
+                f"Failed to fetch config on notify: {e}", exc_info=True
+            )
 
     # This is the 'main' body of the handler.  It is a point to override
     # if needed.  By default does nothing.  Processors are implemented
@@ -182,7 +308,7 @@ class AsyncProcessor:
             prog=ident,
             description=doc
         )
-        
+
         parser.add_argument(
             '--id',
             default=ident,
@@ -223,8 +349,8 @@ class AsyncProcessor:
                 logger.info("Keyboard interrupt.")
                 return
 
-            except _pulsar.Interrupted:
-                logger.info("Pulsar Interrupted.")
+            except KeyboardInterrupt:
+                logger.info("Interrupted.")
                 return
 
             # Exceptions from a taskgroup come in as an exception group
@@ -250,15 +376,7 @@ class AsyncProcessor:
     @staticmethod
     def add_args(parser):
 
-        # Pub/sub backend selection
-        parser.add_argument(
-            '--pubsub-backend',
-            default=os.getenv('PUBSUB_BACKEND', 'pulsar'),
-            choices=['pulsar', 'mqtt'],
-            help='Pub/sub backend (default: pulsar, env: PUBSUB_BACKEND)',
-        )
-
-        PulsarClient.add_args(parser)
+        add_pubsub_args(parser)
         add_logging_args(parser)
 
         parser.add_argument(
@@ -280,4 +398,3 @@ class AsyncProcessor:
             default=8000,
             help=f'Pulsar host (default: 8000)',
         )
-

trustgraph-base/trustgraph/base/backend.py

@@ -124,18 +124,22 @@ class PubSubBackend(Protocol):
         subscription: str,
         schema: type,
         initial_position: str = 'latest',
-        consumer_type: str = 'shared',
         **options
     ) -> BackendConsumer:
         """
         Create a consumer for a topic.
 
+        Consumer behaviour is determined by the topic's class prefix:
+          - flow: shared competing consumers, durable named queue
+          - request: shared competing consumers, non-durable named queue
+          - response: exclusive per-subscriber, anonymous auto-delete queue
+          - notify: exclusive per-subscriber, anonymous auto-delete queue
+
         Args:
-            topic: Generic topic format (qos/tenant/namespace/queue)
+            topic: Queue identifier in class:topicspace:topic format
             subscription: Subscription/consumer group name
             schema: Dataclass type for messages
             initial_position: 'earliest' or 'latest' (some backends may ignore)
-            consumer_type: 'shared', 'exclusive', 'failover' (some backends may ignore)
             **options: Backend-specific options
 
         Returns:

trustgraph-base/trustgraph/base/chunking_service.py

@@ -7,23 +7,14 @@ fetching large document content.
 import asyncio
 import base64
 import logging
-import uuid
 
 from .flow_processor import FlowProcessor
 from .parameter_spec import ParameterSpec
-from .consumer import Consumer
-from .producer import Producer
-from .metrics import ConsumerMetrics, ProducerMetrics
-
-from ..schema import LibrarianRequest, LibrarianResponse, DocumentMetadata
-from ..schema import librarian_request_queue, librarian_response_queue
+from .librarian_client import LibrarianClient
 
 # Module logger
 logger = logging.getLogger(__name__)
 
-default_librarian_request_queue = librarian_request_queue
-default_librarian_response_queue = librarian_response_queue
-
 
 class ChunkingService(FlowProcessor):
     """Base service for chunking processors with parameter specification support"""
@@ -44,155 +35,18 @@ class ChunkingService(FlowProcessor):
             ParameterSpec(name="chunk-overlap")
         )
 
-        # Librarian client for fetching document content
-        librarian_request_q = params.get(
-            "librarian_request_queue", default_librarian_request_queue
-        )
-        librarian_response_q = params.get(
-            "librarian_response_queue", default_librarian_response_queue
-        )
-
-        librarian_request_metrics = ProducerMetrics(
-            processor=id, flow=None, name="librarian-request"
-        )
-
-        self.librarian_request_producer = Producer(
+        # Librarian client
+        self.librarian = LibrarianClient(
+            id=id,
             backend=self.pubsub,
-            topic=librarian_request_q,
-            schema=LibrarianRequest,
-            metrics=librarian_request_metrics,
-        )
-
-        librarian_response_metrics = ConsumerMetrics(
-            processor=id, flow=None, name="librarian-response"
-        )
-
-        self.librarian_response_consumer = Consumer(
             taskgroup=self.taskgroup,
-            backend=self.pubsub,
-            flow=None,
-            topic=librarian_response_q,
-            subscriber=f"{id}-librarian",
-            schema=LibrarianResponse,
-            handler=self.on_librarian_response,
-            metrics=librarian_response_metrics,
         )
 
-        # Pending librarian requests: request_id -> asyncio.Future
-        self.pending_requests = {}
-
         logger.debug("ChunkingService initialized with parameter specifications")
 
     async def start(self):
         await super(ChunkingService, self).start()
-        await self.librarian_request_producer.start()
-        await self.librarian_response_consumer.start()
-
-    async def on_librarian_response(self, msg, consumer, flow):
-        """Handle responses from the librarian service."""
-        response = msg.value()
-        request_id = msg.properties().get("id")
-
-        if request_id and request_id in self.pending_requests:
-            future = self.pending_requests.pop(request_id)
-            future.set_result(response)
-
-    async def fetch_document_content(self, document_id, user, timeout=120):
-        """
-        Fetch document content from librarian via Pulsar.
-        """
-        request_id = str(uuid.uuid4())
-
-        request = LibrarianRequest(
-            operation="get-document-content",
-            document_id=document_id,
-            user=user,
-        )
-
-        # Create future for response
-        future = asyncio.get_event_loop().create_future()
-        self.pending_requests[request_id] = future
-
-        try:
-            # Send request
-            await self.librarian_request_producer.send(
-                request, properties={"id": request_id}
-            )
-
-            # Wait for response
-            response = await asyncio.wait_for(future, timeout=timeout)
-
-            if response.error:
-                raise RuntimeError(
-                    f"Librarian error: {response.error.type}: {response.error.message}"
-                )
-
-            return response.content
-
-        except asyncio.TimeoutError:
-            self.pending_requests.pop(request_id, None)
-            raise RuntimeError(f"Timeout fetching document {document_id}")
-
-    async def save_child_document(self, doc_id, parent_id, user, content,
-                                   document_type="chunk", title=None, timeout=120):
-        """
-        Save a child document (chunk) to the librarian.
-
-        Args:
-            doc_id: ID for the new child document
-            parent_id: ID of the parent document
-            user: User ID
-            content: Document content (bytes or str)
-            document_type: Type of document ("chunk", etc.)
-            title: Optional title
-            timeout: Request timeout in seconds
-
-        Returns:
-            The document ID on success
-        """
-        request_id = str(uuid.uuid4())
-
-        if isinstance(content, str):
-            content = content.encode("utf-8")
-
-        doc_metadata = DocumentMetadata(
-            id=doc_id,
-            user=user,
-            kind="text/plain",
-            title=title or doc_id,
-            parent_id=parent_id,
-            document_type=document_type,
-        )
-
-        request = LibrarianRequest(
-            operation="add-child-document",
-            document_metadata=doc_metadata,
-            content=base64.b64encode(content).decode("utf-8"),
-        )
-
-        # Create future for response
-        future = asyncio.get_event_loop().create_future()
-        self.pending_requests[request_id] = future
-
-        try:
-            # Send request
-            await self.librarian_request_producer.send(
-                request, properties={"id": request_id}
-            )
-
-            # Wait for response
-            response = await asyncio.wait_for(future, timeout=timeout)
-
-            if response.error:
-                raise RuntimeError(
-                    f"Librarian error saving chunk: {response.error.type}: {response.error.message}"
-                )
-
-            return doc_id
-
-        except asyncio.TimeoutError:
-            self.pending_requests.pop(request_id, None)
-            raise RuntimeError(f"Timeout saving chunk {doc_id}")
+        await self.librarian.start()
 
     async def get_document_text(self, doc):
         """
@@ -206,14 +60,10 @@ class ChunkingService(FlowProcessor):
         """
         if doc.document_id and not doc.text:
             logger.info(f"Fetching document {doc.document_id} from librarian...")
-            content = await self.fetch_document_content(
+            text = await self.librarian.fetch_document_text(
                 document_id=doc.document_id,
                 user=doc.metadata.user,
             )
-            # Content is base64 encoded
-            if isinstance(content, str):
-                content = content.encode('utf-8')
-            text = base64.b64decode(content).decode("utf-8")
             logger.info(f"Fetched {len(text)} characters from librarian")
             return text
         else:
@@ -224,41 +74,31 @@ class ChunkingService(FlowProcessor):
         Extract chunk parameters from flow and return effective values
 
         Args:
-            msg: The message containing the document to chunk
-            consumer: The consumer spec
-            flow: The flow context
-            default_chunk_size: Default chunk size from processor config
-            default_chunk_overlap: Default chunk overlap from processor config
+            msg: The message being processed
+            consumer: The consumer instance
+            flow: The flow object containing parameters
+            default_chunk_size: Default chunk size if not configured
+            default_chunk_overlap: Default chunk overlap if not configured
 
         Returns:
-            tuple: (chunk_size, chunk_overlap) - effective values to use
+            tuple: (chunk_size, chunk_overlap) effective values
         """
-        # Extract parameters from flow (flow-configurable parameters)
-        chunk_size = flow("chunk-size")
-        chunk_overlap = flow("chunk-overlap")
 
-        # Use provided values or fall back to defaults
-        effective_chunk_size = chunk_size if chunk_size is not None else default_chunk_size
-        effective_chunk_overlap = chunk_overlap if chunk_overlap is not None else default_chunk_overlap
+        chunk_size = default_chunk_size
+        chunk_overlap = default_chunk_overlap
 
-        logger.debug(f"Using chunk-size: {effective_chunk_size}")
-        logger.debug(f"Using chunk-overlap: {effective_chunk_overlap}")
+        try:
+            cs = flow.parameters.get("chunk-size")
+            if cs is not None:
+                chunk_size = int(cs)
+        except Exception as e:
+            logger.warning(f"Could not parse chunk-size parameter: {e}")
 
-        return effective_chunk_size, effective_chunk_overlap
+        try:
+            co = flow.parameters.get("chunk-overlap")
+            if co is not None:
+                chunk_overlap = int(co)
+        except Exception as e:
+            logger.warning(f"Could not parse chunk-overlap parameter: {e}")
 
-    @staticmethod
-    def add_args(parser):
-        """Add chunking service arguments to parser"""
-        FlowProcessor.add_args(parser)
-
-        parser.add_argument(
-            '--librarian-request-queue',
-            default=default_librarian_request_queue,
-            help=f'Librarian request queue (default: {default_librarian_request_queue})',
-        )
-
-        parser.add_argument(
-            '--librarian-response-queue',
-            default=default_librarian_response_queue,
-            help=f'Librarian response queue (default: {default_librarian_response_queue})',
-        )
+        return chunk_size, chunk_overlap

trustgraph-base/trustgraph/base/consumer.py

@@ -12,6 +12,7 @@
 import asyncio
 import time
 import logging
+from concurrent.futures import ThreadPoolExecutor
 
 from .. exceptions import TooManyRequests
 
@@ -32,11 +33,12 @@ class Consumer:
             rate_limit_retry_time = 10, rate_limit_timeout = 7200,
             reconnect_time = 5,
             concurrency = 1, # Number of concurrent requests to handle
+            **kwargs,
     ):
 
         self.taskgroup = taskgroup
         self.flow = flow
-        self.backend = backend  # Changed from 'client' to 'backend'
+        self.backend = backend
         self.topic = topic
         self.subscriber = subscriber
         self.schema = schema
@@ -93,33 +95,11 @@ class Consumer:
             if self.metrics:
                 self.metrics.state("stopped")
 
-            try:
-
-                logger.info(f"Subscribing to topic: {self.topic}")
-
-                # Determine initial position
-                if self.start_of_messages:
-                    initial_pos = 'earliest'
-                else:
-                    initial_pos = 'latest'
-
-                # Create consumer via backend
-                self.consumer = await asyncio.to_thread(
-                    self.backend.create_consumer,
-                    topic = self.topic,
-                    subscription = self.subscriber,
-                    schema = self.schema,
-                    initial_position = initial_pos,
-                    consumer_type = 'shared',
-                )
-
-            except Exception as e:
-
-                logger.error(f"Consumer subscription exception: {e}", exc_info=True)
-                await asyncio.sleep(self.reconnect_time)
-                continue
-
-            logger.info(f"Successfully subscribed to topic: {self.topic}")
+            # Determine initial position
+            if self.start_of_messages:
+                initial_pos = 'earliest'
+            else:
+                initial_pos = 'latest'
 
             if self.metrics:
                 self.metrics.state("running")
@@ -128,14 +108,37 @@ class Consumer:
 
                 logger.info(f"Starting {self.concurrency} receiver threads")
 
-                async with asyncio.TaskGroup() as tg:
-
-                    tasks = []
-
-                    for i in range(0, self.concurrency):
-                        tasks.append(
-                            tg.create_task(self.consume_from_queue())
+                # Create one backend consumer per concurrent task.
+                # Each gets its own connection and dedicated thread —
+                # required for backends like RabbitMQ where connections
+                # are not thread-safe (pika BlockingConnection must be
+                # used from a single thread).
+                consumers = []
+                executors = []
+                for i in range(self.concurrency):
+                    try:
+                        logger.info(f"Subscribing to topic: {self.topic} (worker {i})")
+                        executor = ThreadPoolExecutor(max_workers=1)
+                        loop = asyncio.get_event_loop()
+                        c = await loop.run_in_executor(
+                            executor,
+                            lambda: self.backend.create_consumer(
+                                topic = self.topic,
+                                subscription = self.subscriber,
+                                schema = self.schema,
+                                initial_position = initial_pos,
+                            ),
                         )
+                        consumers.append(c)
+                        executors.append(executor)
+                        logger.info(f"Successfully subscribed to topic: {self.topic} (worker {i})")
+                    except Exception as e:
+                        logger.error(f"Consumer subscription exception (worker {i}): {e}", exc_info=True)
+                        raise
+
+                async with asyncio.TaskGroup() as tg:
+                    for c, ex in zip(consumers, executors):
+                        tg.create_task(self.consume_from_queue(c, ex))
 
                 if self.metrics:
                     self.metrics.state("stopped")
@@ -143,24 +146,38 @@ class Consumer:
             except Exception as e:
 
                 logger.error(f"Consumer loop exception: {e}", exc_info=True)
-                self.consumer.unsubscribe()
-                self.consumer.close()
-                self.consumer = None
+                for c in consumers:
+                    try:
+                        c.unsubscribe()
+                        c.close()
+                    except Exception:
+                        pass
+                for ex in executors:
+                    ex.shutdown(wait=False)
+                consumers = []
+                executors = []
                 await asyncio.sleep(self.reconnect_time)
                 continue
 
-        if self.consumer:
-            self.consumer.unsubscribe()
-            self.consumer.close()
+            finally:
+                for c in consumers:
+                    try:
+                        c.unsubscribe()
+                        c.close()
+                    except Exception:
+                        pass
+                for ex in executors:
+                    ex.shutdown(wait=False)
 
-    async def consume_from_queue(self):
+    async def consume_from_queue(self, consumer, executor=None):
 
+        loop = asyncio.get_event_loop()
         while self.running:
 
             try:
-                msg = await asyncio.to_thread(
-                    self.consumer.receive,
-                    timeout_millis=2000
+                msg = await loop.run_in_executor(
+                    executor,
+                    lambda: consumer.receive(timeout_millis=100),
                 )
             except Exception as e:
                 # Handle timeout from any backend
@@ -168,10 +185,11 @@ class Consumer:
                     continue
                 raise e
 
-            await self.handle_one_from_queue(msg)
+            await self.handle_one_from_queue(msg, consumer, executor)
 
-    async def handle_one_from_queue(self, msg):
+    async def handle_one_from_queue(self, msg, consumer, executor=None):
 
+        loop = asyncio.get_event_loop()
         expiry = time.time() + self.rate_limit_timeout
 
         # This loop is for retry on rate-limit / resource limits
@@ -182,8 +200,11 @@ class Consumer:
                 logger.warning("Gave up waiting for rate-limit retry")
 
                 # Message failed to be processed, this causes it to
-                # be retried
-                self.consumer.negative_acknowledge(msg)
+                # be retried.  Ack on the consumer's dedicated thread
+                # (pika is not thread-safe).
+                await loop.run_in_executor(
+                    executor, lambda: consumer.negative_acknowledge(msg)
+                )
 
                 if self.metrics:
                     self.metrics.process("error")
@@ -205,8 +226,11 @@ class Consumer:
 
                 logger.debug("Message processed successfully")
 
-                # Acknowledge successful processing of the message
-                self.consumer.acknowledge(msg)
+                # Acknowledge on the consumer's dedicated thread
+                # (pika is not thread-safe)
+                await loop.run_in_executor(
+                    executor, lambda: consumer.acknowledge(msg)
+                )
 
                 if self.metrics:
                     self.metrics.process("success")
@@ -232,8 +256,10 @@ class Consumer:
                 logger.error(f"Message processing exception: {e}", exc_info=True)
 
                 # Message failed to be processed, this causes it to
-                # be retried
-                self.consumer.negative_acknowledge(msg)
+                # be retried.  Ack on the consumer's dedicated thread.
+                await loop.run_in_executor(
+                    executor, lambda: consumer.negative_acknowledge(msg)
+                )
 
                 if self.metrics:
                     self.metrics.process("error")

trustgraph-base/trustgraph/base/flow_processor.py

@@ -6,8 +6,6 @@
 import json
 import logging
 
-from pulsar.schema import JsonSchema
-
 from .. schema import Error
 from .. schema import config_request_queue, config_response_queue
 from .. schema import config_push_queue
@@ -28,7 +26,9 @@ class FlowProcessor(AsyncProcessor):
         super(FlowProcessor, self).__init__(**params)
 
         # Register configuration handler
-        self.register_config_handler(self.on_configure_flows)
+        self.register_config_handler(
+            self.on_configure_flows, types=["active-flow"]
+        )
 
         # Initialise flow information state
         self.flows = {}

trustgraph-base/trustgraph/base/graph_rag_client.py

@@ -5,6 +5,7 @@ from .. schema import GraphRagQuery, GraphRagResponse
 class GraphRagClient(RequestResponse):
     async def rag(self, query, user="trustgraph", collection="default",
                   chunk_callback=None, explain_callback=None,
+                  parent_uri="",
                   timeout=600):
         """
         Execute a graph RAG query with optional streaming callbacks.
@@ -14,7 +15,7 @@ class GraphRagClient(RequestResponse):
             user: User identifier
             collection: Collection identifier
             chunk_callback: Optional async callback(text, end_of_stream) for text chunks
-            explain_callback: Optional async callback(explain_id, explain_graph) for explain notifications
+            explain_callback: Optional async callback(explain_id, explain_graph, explain_triples) for explain notifications
             timeout: Request timeout in seconds
 
         Returns:
@@ -29,7 +30,7 @@ class GraphRagClient(RequestResponse):
             # Handle explain notifications
             if resp.message_type == 'explain':
                 if explain_callback and resp.explain_id:
-                    await explain_callback(resp.explain_id, resp.explain_graph)
+                    await explain_callback(resp.explain_id, resp.explain_graph, resp.explain_triples)
                 return False  # Continue receiving
 
             # Handle text chunks
@@ -50,6 +51,7 @@ class GraphRagClient(RequestResponse):
                 query = query,
                 user = user,
                 collection = collection,
+                parent_uri = parent_uri,
             ),
             timeout=timeout,
             recipient=recipient,

trustgraph-base/trustgraph/base/librarian_client.py

@@ -0,0 +1,245 @@
+"""
+Shared librarian client for services that need to communicate
+with the librarian via pub/sub.
+
+Provides request-response and streaming operations over the message
+broker, with proper support for large documents via stream-document.
+
+Usage:
+    self.librarian = LibrarianClient(
+        id=id, backend=self.pubsub, taskgroup=self.taskgroup, **params
+    )
+    await self.librarian.start()
+    content = await self.librarian.fetch_document_content(doc_id, user)
+"""
+
+import asyncio
+import base64
+import logging
+import uuid
+
+from .consumer import Consumer
+from .producer import Producer
+from .metrics import ConsumerMetrics, ProducerMetrics
+
+from ..schema import LibrarianRequest, LibrarianResponse, DocumentMetadata
+from ..schema import librarian_request_queue, librarian_response_queue
+
+logger = logging.getLogger(__name__)
+
+
+class LibrarianClient:
+    """Client for librarian request-response over the message broker."""
+
+    def __init__(self, id, backend, taskgroup, **params):
+
+        librarian_request_q = params.get(
+            "librarian_request_queue", librarian_request_queue,
+        )
+        librarian_response_q = params.get(
+            "librarian_response_queue", librarian_response_queue,
+        )
+
+        librarian_request_metrics = ProducerMetrics(
+            processor=id, flow=None, name="librarian-request",
+        )
+
+        self._producer = Producer(
+            backend=backend,
+            topic=librarian_request_q,
+            schema=LibrarianRequest,
+            metrics=librarian_request_metrics,
+        )
+
+        librarian_response_metrics = ConsumerMetrics(
+            processor=id, flow=None, name="librarian-response",
+        )
+
+        self._consumer = Consumer(
+            taskgroup=taskgroup,
+            backend=backend,
+            flow=None,
+            topic=librarian_response_q,
+            subscriber=f"{id}-librarian",
+            schema=LibrarianResponse,
+            handler=self._on_response,
+            metrics=librarian_response_metrics,
+        )
+
+        # Single-response requests: request_id -> asyncio.Future
+        self._pending = {}
+        # Streaming requests: request_id -> asyncio.Queue
+        self._streams = {}
+
+    async def start(self):
+        """Start the librarian producer and consumer."""
+        await self._producer.start()
+        await self._consumer.start()
+
+    async def _on_response(self, msg, consumer, flow):
+        """Route librarian responses to the right waiter."""
+        response = msg.value()
+        request_id = msg.properties().get("id")
+
+        if not request_id:
+            return
+
+        if request_id in self._pending:
+            future = self._pending.pop(request_id)
+            future.set_result(response)
+        elif request_id in self._streams:
+            await self._streams[request_id].put(response)
+
+    async def request(self, request, timeout=120):
+        """Send a request to the librarian and wait for a single response."""
+        request_id = str(uuid.uuid4())
+
+        future = asyncio.get_event_loop().create_future()
+        self._pending[request_id] = future
+
+        try:
+            await self._producer.send(
+                request, properties={"id": request_id},
+            )
+            response = await asyncio.wait_for(future, timeout=timeout)
+
+            if response.error:
+                raise RuntimeError(
+                    f"Librarian error: {response.error.type}: "
+                    f"{response.error.message}"
+                )
+
+            return response
+
+        except asyncio.TimeoutError:
+            self._pending.pop(request_id, None)
+            raise RuntimeError("Timeout waiting for librarian response")
+
+    async def stream(self, request, timeout=120):
+        """Send a request and collect streamed response chunks."""
+        request_id = str(uuid.uuid4())
+
+        q = asyncio.Queue()
+        self._streams[request_id] = q
+
+        try:
+            await self._producer.send(
+                request, properties={"id": request_id},
+            )
+
+            chunks = []
+            while True:
+                response = await asyncio.wait_for(q.get(), timeout=timeout)
+
+                if response.error:
+                    raise RuntimeError(
+                        f"Librarian error: {response.error.type}: "
+                        f"{response.error.message}"
+                    )
+
+                chunks.append(response)
+
+                if response.is_final:
+                    break
+
+            return chunks
+
+        except asyncio.TimeoutError:
+            self._streams.pop(request_id, None)
+            raise RuntimeError("Timeout waiting for librarian stream")
+        finally:
+            self._streams.pop(request_id, None)
+
+    async def fetch_document_content(self, document_id, user, timeout=120):
+        """Fetch document content using streaming.
+
+        Returns base64-encoded content. Caller is responsible for decoding.
+        """
+        req = LibrarianRequest(
+            operation="stream-document",
+            document_id=document_id,
+            user=user,
+        )
+        chunks = await self.stream(req, timeout=timeout)
+
+        # Decode each chunk's base64 to raw bytes, concatenate,
+        # re-encode for the caller.
+        raw = b""
+        for chunk in chunks:
+            if chunk.content:
+                if isinstance(chunk.content, bytes):
+                    raw += base64.b64decode(chunk.content)
+                else:
+                    raw += base64.b64decode(
+                        chunk.content.encode("utf-8")
+                    )
+
+        return base64.b64encode(raw)
+
+    async def fetch_document_text(self, document_id, user, timeout=120):
+        """Fetch document content and decode as UTF-8 text."""
+        content = await self.fetch_document_content(
+            document_id, user, timeout=timeout,
+        )
+        return base64.b64decode(content).decode("utf-8")
+
+    async def fetch_document_metadata(self, document_id, user, timeout=120):
+        """Fetch document metadata from the librarian."""
+        req = LibrarianRequest(
+            operation="get-document-metadata",
+            document_id=document_id,
+            user=user,
+        )
+        response = await self.request(req, timeout=timeout)
+        return response.document_metadata
+
+    async def save_child_document(self, doc_id, parent_id, user, content,
+                                  document_type="chunk", title=None,
+                                  kind="text/plain", timeout=120):
+        """Save a child document to the librarian."""
+        if isinstance(content, str):
+            content = content.encode("utf-8")
+
+        doc_metadata = DocumentMetadata(
+            id=doc_id,
+            user=user,
+            kind=kind,
+            title=title or doc_id,
+            parent_id=parent_id,
+            document_type=document_type,
+        )
+
+        req = LibrarianRequest(
+            operation="add-child-document",
+            document_metadata=doc_metadata,
+            content=base64.b64encode(content).decode("utf-8"),
+        )
+
+        await self.request(req, timeout=timeout)
+        return doc_id
+
+    async def save_document(self, doc_id, user, content, title=None,
+                            document_type="answer", kind="text/plain",
+                            timeout=120):
+        """Save a document to the librarian."""
+        if isinstance(content, str):
+            content = content.encode("utf-8")
+
+        doc_metadata = DocumentMetadata(
+            id=doc_id,
+            user=user,
+            kind=kind,
+            title=title or doc_id,
+            document_type=document_type,
+        )
+
+        req = LibrarianRequest(
+            operation="add-document",
+            document_id=doc_id,
+            document_metadata=doc_metadata,
+            content=base64.b64encode(content).decode("utf-8"),
+            user=user,
+        )
+
+        await self.request(req, timeout=timeout)
+        return doc_id

trustgraph-base/trustgraph/base/prompt_client.py

@@ -1,21 +1,16 @@
 
 import json
 import asyncio
-import logging
 
 from . request_response_spec import RequestResponse, RequestResponseSpec
 from .. schema import PromptRequest, PromptResponse
 
-logger = logging.getLogger(__name__)
-
 class PromptClient(RequestResponse):
 
     async def prompt(self, id, variables, timeout=600, streaming=False, chunk_callback=None):
-        logger.info(f"DEBUG prompt_client: prompt called, id={id}, streaming={streaming}, chunk_callback={chunk_callback is not None}")
 
         if not streaming:
-            logger.info("DEBUG prompt_client: Non-streaming path")
-            # Non-streaming path
+
             resp = await self.request(
                 PromptRequest(
                     id = id,
@@ -36,39 +31,30 @@ class PromptClient(RequestResponse):
             return json.loads(resp.object)
 
         else:
-            logger.info("DEBUG prompt_client: Streaming path")
-            # Streaming path - just forward chunks, don't accumulate
+
             last_text = ""
             last_object = None
 
             async def forward_chunks(resp):
                 nonlocal last_text, last_object
-                logger.info(f"DEBUG prompt_client: forward_chunks called, resp.text={resp.text[:50] if resp.text else None}, end_of_stream={getattr(resp, 'end_of_stream', False)}")
 
                 if resp.error:
-                    logger.error(f"DEBUG prompt_client: Error in response: {resp.error.message}")
                     raise RuntimeError(resp.error.message)
 
                 end_stream = getattr(resp, 'end_of_stream', False)
 
-                # Always call callback if there's text OR if it's the final message
                 if resp.text is not None:
                     last_text = resp.text
-                    # Call chunk callback if provided with both chunk and end_of_stream flag
                     if chunk_callback:
-                        logger.info(f"DEBUG prompt_client: Calling chunk_callback with end_of_stream={end_stream}")
                         if asyncio.iscoroutinefunction(chunk_callback):
                             await chunk_callback(resp.text, end_stream)
                         else:
                             chunk_callback(resp.text, end_stream)
                 elif resp.object:
-                    logger.info(f"DEBUG prompt_client: Got object response")
                     last_object = resp.object
 
-                logger.info(f"DEBUG prompt_client: Returning end_of_stream={end_stream}")
                 return end_stream
 
-            logger.info("DEBUG prompt_client: Creating PromptRequest")
             req = PromptRequest(
                 id = id,
                 terms = {
@@ -77,19 +63,16 @@ class PromptClient(RequestResponse):
                 },
                 streaming = True
             )
-            logger.info(f"DEBUG prompt_client: About to call self.request with recipient, timeout={timeout}")
+
             await self.request(
                 req,
                 recipient=forward_chunks,
                 timeout=timeout
             )
-            logger.info(f"DEBUG prompt_client: self.request returned, last_text={last_text[:50] if last_text else None}")
 
             if last_text:
-                logger.info("DEBUG prompt_client: Returning last_text")
                 return last_text
 
-            logger.info("DEBUG prompt_client: Returning parsed last_object")
             return json.loads(last_object) if last_object else None
 
     async def extract_definitions(self, text, timeout=600):

trustgraph-base/trustgraph/base/pubsub.py

@@ -1,110 +1,121 @@
 
 import os
-import pulsar
-import _pulsar
-import uuid
-from pulsar.schema import JsonSchema
 import logging
 
-from .. log_level import LogLevel
-from .pulsar_backend import PulsarBackend
-
 logger = logging.getLogger(__name__)
 
+# Default connection settings from environment
+DEFAULT_PULSAR_HOST = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
+DEFAULT_PULSAR_API_KEY = os.getenv("PULSAR_API_KEY", None)
+
+DEFAULT_RABBITMQ_HOST = os.getenv("RABBITMQ_HOST", 'rabbitmq')
+DEFAULT_RABBITMQ_PORT = int(os.getenv("RABBITMQ_PORT", '5672'))
+DEFAULT_RABBITMQ_USERNAME = os.getenv("RABBITMQ_USERNAME", 'guest')
+DEFAULT_RABBITMQ_PASSWORD = os.getenv("RABBITMQ_PASSWORD", 'guest')
+DEFAULT_RABBITMQ_VHOST = os.getenv("RABBITMQ_VHOST", '/')
+
 
 def get_pubsub(**config):
     """
     Factory function to create a pub/sub backend based on configuration.
 
     Args:
-        config: Configuration dictionary from command-line args
-                Must include 'pubsub_backend' key
+        config: Configuration dictionary from command-line args.
+                Key 'pubsub_backend' selects the backend (default: 'pulsar').
 
     Returns:
-        Backend instance (PulsarBackend, MQTTBackend, etc.)
-
-    Example:
-        backend = get_pubsub(
-            pubsub_backend='pulsar',
-            pulsar_host='pulsar://localhost:6650'
-        )
+        Backend instance implementing the PubSubBackend protocol.
     """
     backend_type = config.get('pubsub_backend', 'pulsar')
 
     if backend_type == 'pulsar':
+        from .pulsar_backend import PulsarBackend
         return PulsarBackend(
-            host=config.get('pulsar_host', PulsarClient.default_pulsar_host),
-            api_key=config.get('pulsar_api_key', PulsarClient.default_pulsar_api_key),
+            host=config.get('pulsar_host', DEFAULT_PULSAR_HOST),
+            api_key=config.get('pulsar_api_key', DEFAULT_PULSAR_API_KEY),
             listener=config.get('pulsar_listener'),
         )
-    elif backend_type == 'mqtt':
-        # TODO: Implement MQTT backend
-        raise NotImplementedError("MQTT backend not yet implemented")
+    elif backend_type == 'rabbitmq':
+        from .rabbitmq_backend import RabbitMQBackend
+        return RabbitMQBackend(
+            host=config.get('rabbitmq_host', DEFAULT_RABBITMQ_HOST),
+            port=config.get('rabbitmq_port', DEFAULT_RABBITMQ_PORT),
+            username=config.get('rabbitmq_username', DEFAULT_RABBITMQ_USERNAME),
+            password=config.get('rabbitmq_password', DEFAULT_RABBITMQ_PASSWORD),
+            vhost=config.get('rabbitmq_vhost', DEFAULT_RABBITMQ_VHOST),
+        )
     else:
         raise ValueError(f"Unknown pub/sub backend: {backend_type}")
 
 
-class PulsarClient:
+STANDALONE_PULSAR_HOST = 'pulsar://localhost:6650'
 
-    default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
-    default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
 
-    def __init__(self, **params):
+def add_pubsub_args(parser, standalone=False):
+    """Add pub/sub CLI arguments to an argument parser.
 
-        self.client = None
+    Args:
+        parser: argparse.ArgumentParser
+        standalone: If True, default host is localhost (for CLI tools
+                    that run outside containers)
+    """
+    pulsar_host = STANDALONE_PULSAR_HOST if standalone else DEFAULT_PULSAR_HOST
+    pulsar_listener = 'localhost' if standalone else None
+    rabbitmq_host = 'localhost' if standalone else DEFAULT_RABBITMQ_HOST
 
-        pulsar_host = params.get("pulsar_host", self.default_pulsar_host)
-        pulsar_listener = params.get("pulsar_listener", None)
-        pulsar_api_key = params.get(
-            "pulsar_api_key",
-            self.default_pulsar_api_key
-        )
-        # Hard-code Pulsar logging to ERROR level to minimize noise
+    parser.add_argument(
+        '--pubsub-backend',
+        default=os.getenv('PUBSUB_BACKEND', 'pulsar'),
+        help='Pub/sub backend (default: pulsar, env: PUBSUB_BACKEND)',
+    )
 
-        self.pulsar_host = pulsar_host
-        self.pulsar_api_key = pulsar_api_key
+    # Pulsar options
+    parser.add_argument(
+        '-p', '--pulsar-host',
+        default=pulsar_host,
+        help=f'Pulsar host (default: {pulsar_host})',
+    )
 
-        if pulsar_api_key:
-            auth = pulsar.AuthenticationToken(pulsar_api_key)
-            self.client = pulsar.Client(
-                pulsar_host,
-                authentication=auth,
-                logger=pulsar.ConsoleLogger(_pulsar.LoggerLevel.Error)
-            )
-        else:
-            self.client = pulsar.Client(
-                pulsar_host,
-                listener_name=pulsar_listener,
-                logger=pulsar.ConsoleLogger(_pulsar.LoggerLevel.Error)
-            )
+    parser.add_argument(
+        '--pulsar-api-key',
+        default=DEFAULT_PULSAR_API_KEY,
+        help='Pulsar API key',
+    )
 
-        self.pulsar_listener = pulsar_listener
+    parser.add_argument(
+        '--pulsar-listener',
+        default=pulsar_listener,
+        help=f'Pulsar listener (default: {pulsar_listener or "none"})',
+    )
 
-    def close(self):
-        self.client.close()
+    # RabbitMQ options
+    parser.add_argument(
+        '--rabbitmq-host',
+        default=rabbitmq_host,
+        help=f'RabbitMQ host (default: {rabbitmq_host})',
+    )
 
-    def __del__(self):
+    parser.add_argument(
+        '--rabbitmq-port',
+        type=int,
+        default=DEFAULT_RABBITMQ_PORT,
+        help=f'RabbitMQ port (default: {DEFAULT_RABBITMQ_PORT})',
+    )
 
-        if hasattr(self, "client"):
-            if self.client:
-                self.client.close()
+    parser.add_argument(
+        '--rabbitmq-username',
+        default=DEFAULT_RABBITMQ_USERNAME,
+        help='RabbitMQ username',
+    )
 
-    @staticmethod
-    def add_args(parser):
+    parser.add_argument(
+        '--rabbitmq-password',
+        default=DEFAULT_RABBITMQ_PASSWORD,
+        help='RabbitMQ password',
+    )
 
-        parser.add_argument(
-            '-p', '--pulsar-host',
-            default=__class__.default_pulsar_host,
-            help=f'Pulsar host (default: {__class__.default_pulsar_host})',
-        )
-        
-        parser.add_argument(
-            '--pulsar-api-key',
-            default=__class__.default_pulsar_api_key,
-            help=f'Pulsar API key',
-        )
-
-        parser.add_argument(
-            '--pulsar-listener',
-            help=f'Pulsar listener (default: none)',
-        )
+    parser.add_argument(
+        '--rabbitmq-vhost',
+        default=DEFAULT_RABBITMQ_VHOST,
+        help=f'RabbitMQ vhost (default: {DEFAULT_RABBITMQ_VHOST})',
+    )

trustgraph-base/trustgraph/base/pulsar_backend.py

@@ -9,122 +9,14 @@ import pulsar
 import _pulsar
 import json
 import logging
-import base64
-import types
-from dataclasses import asdict, is_dataclass
-from typing import Any, get_type_hints
+from typing import Any
 
 from .backend import PubSubBackend, BackendProducer, BackendConsumer, Message
+from .serialization import dataclass_to_dict, dict_to_dataclass
 
 logger = logging.getLogger(__name__)
 
 
-def dataclass_to_dict(obj: Any) -> dict:
-    """
-    Recursively convert a dataclass to a dictionary, handling None values and bytes.
-
-    None values are excluded from the dictionary (not serialized).
-    Bytes values are decoded as UTF-8 strings for JSON serialization (matching Pulsar behavior).
-    Handles nested dataclasses, lists, and dictionaries recursively.
-    """
-    if obj is None:
-        return None
-
-    # Handle bytes - decode to UTF-8 for JSON serialization
-    if isinstance(obj, bytes):
-        return obj.decode('utf-8')
-
-    # Handle dataclass - convert to dict then recursively process all values
-    if is_dataclass(obj):
-        result = {}
-        for key, value in asdict(obj).items():
-            result[key] = dataclass_to_dict(value) if value is not None else None
-        return result
-
-    # Handle list - recursively process all items
-    if isinstance(obj, list):
-        return [dataclass_to_dict(item) for item in obj]
-
-    # Handle dict - recursively process all values
-    if isinstance(obj, dict):
-        return {k: dataclass_to_dict(v) for k, v in obj.items()}
-
-    # Return primitive types as-is
-    return obj
-
-
-def dict_to_dataclass(data: dict, cls: type) -> Any:
-    """
-    Convert a dictionary back to a dataclass instance.
-
-    Handles nested dataclasses and missing fields.
-    Uses get_type_hints() to resolve forward references (string annotations).
-    """
-    if data is None:
-        return None
-
-    if not is_dataclass(cls):
-        return data
-
-    # Get field types from the dataclass, resolving forward references
-    # get_type_hints() evaluates string annotations like "Triple | None"
-    try:
-        field_types = get_type_hints(cls)
-    except Exception:
-        # Fallback if get_type_hints fails (shouldn't happen normally)
-        field_types = {f.name: f.type for f in cls.__dataclass_fields__.values()}
-    kwargs = {}
-
-    for key, value in data.items():
-        if key in field_types:
-            field_type = field_types[key]
-
-            # Handle modern union types (X | Y)
-            if isinstance(field_type, types.UnionType):
-                # Check if it's Optional (X | None)
-                if type(None) in field_type.__args__:
-                    # Get the non-None type
-                    actual_type = next((t for t in field_type.__args__ if t is not type(None)), None)
-                    if actual_type and is_dataclass(actual_type) and isinstance(value, dict):
-                        kwargs[key] = dict_to_dataclass(value, actual_type)
-                    else:
-                        kwargs[key] = value
-                else:
-                    kwargs[key] = value
-            # Check if this is a generic type (list, dict, etc.)
-            elif hasattr(field_type, '__origin__'):
-                # Handle list[T]
-                if field_type.__origin__ == list:
-                    item_type = field_type.__args__[0] if field_type.__args__ else None
-                    if item_type and is_dataclass(item_type) and isinstance(value, list):
-                        kwargs[key] = [
-                            dict_to_dataclass(item, item_type) if isinstance(item, dict) else item
-                            for item in value
-                        ]
-                    else:
-                        kwargs[key] = value
-                # Handle old-style Optional[T] (which is Union[T, None])
-                elif hasattr(field_type, '__args__') and type(None) in field_type.__args__:
-                    # Get the non-None type from Union
-                    actual_type = next((t for t in field_type.__args__ if t is not type(None)), None)
-                    if actual_type and is_dataclass(actual_type) and isinstance(value, dict):
-                        kwargs[key] = dict_to_dataclass(value, actual_type)
-                    else:
-                        kwargs[key] = value
-                else:
-                    kwargs[key] = value
-            # Handle direct dataclass fields
-            elif is_dataclass(field_type) and isinstance(value, dict):
-                kwargs[key] = dict_to_dataclass(value, field_type)
-            # Handle bytes fields (UTF-8 encoded strings from JSON)
-            elif field_type == bytes and isinstance(value, str):
-                kwargs[key] = value.encode('utf-8')
-            else:
-                kwargs[key] = value
-
-    return cls(**kwargs)
-
-
 class PulsarMessage:
     """Wrapper for Pulsar messages to match Message protocol."""
 
@@ -181,8 +73,11 @@ class PulsarBackendConsumer:
         self._schema_cls = schema_cls
 
     def receive(self, timeout_millis: int = 2000) -> Message:
-        """Receive a message."""
-        pulsar_msg = self._consumer.receive(timeout_millis=timeout_millis)
+        """Receive a message. Raises TimeoutError if no message available."""
+        try:
+            pulsar_msg = self._consumer.receive(timeout_millis=timeout_millis)
+        except _pulsar.Timeout:
+            raise TimeoutError("No message received within timeout")
         return PulsarMessage(pulsar_msg, self._schema_cls)
 
     def acknowledge(self, message: Message) -> None:
@@ -237,38 +132,46 @@ class PulsarBackend:
         self.client = pulsar.Client(**client_args)
         logger.info(f"Pulsar client connected to {host}")
 
-    def map_topic(self, generic_topic: str) -> str:
+    def map_topic(self, queue_id: str) -> str:
         """
-        Map generic topic format to Pulsar URI.
+        Map queue identifier to Pulsar URI.
 
-        Format: qos/tenant/namespace/queue
-        Example: q1/tg/flow/my-queue -> persistent://tg/flow/my-queue
+        Format: class:topicspace:topic
+        Example: flow:tg:text-completion-request -> persistent://tg/flow/text-completion-request
 
         Args:
-            generic_topic: Generic topic string or already-formatted Pulsar URI
+            queue_id: Queue identifier string or already-formatted Pulsar URI
 
         Returns:
             Pulsar topic URI
         """
         # If already a Pulsar URI, return as-is
-        if '://' in generic_topic:
-            return generic_topic
+        if '://' in queue_id:
+            return queue_id
 
-        parts = generic_topic.split('/', 3)
-        if len(parts) != 4:
-            raise ValueError(f"Invalid topic format: {generic_topic}, expected qos/tenant/namespace/queue")
+        parts = queue_id.split(':', 2)
+        if len(parts) != 3:
+            raise ValueError(
+                f"Invalid queue format: {queue_id}, "
+                f"expected class:topicspace:topic"
+            )
 
-        qos, tenant, namespace, queue = parts
+        cls, topicspace, topic = parts
 
-        # Map QoS to persistence
-        if qos == 'q0':
-            persistence = 'non-persistent'
-        elif qos in ['q1', 'q2']:
+        # Map class to Pulsar persistence and namespace
+        if cls == 'flow':
             persistence = 'persistent'
+        elif cls in ('request', 'response'):
+            persistence = 'non-persistent'
+        elif cls == 'notify':
+            persistence = 'non-persistent'
         else:
-            raise ValueError(f"Invalid QoS level: {qos}, expected q0, q1, or q2")
+            raise ValueError(
+                f"Invalid queue class: {cls}, "
+                f"expected flow, request, response, or notify"
+            )
 
-        return f"{persistence}://{tenant}/{namespace}/{queue}"
+        return f"{persistence}://{topicspace}/{cls}/{topic}"
 
     def create_producer(self, topic: str, schema: type, **options) -> BackendProducer:
         """
@@ -304,18 +207,20 @@ class PulsarBackend:
         subscription: str,
         schema: type,
         initial_position: str = 'latest',
-        consumer_type: str = 'shared',
         **options
     ) -> BackendConsumer:
         """
         Create a Pulsar consumer.
 
+        Consumer type is derived from the topic's class prefix:
+          - flow/request: Shared (competing consumers)
+          - response/notify: Exclusive (per-subscriber)
+
         Args:
-            topic: Generic topic format (qos/tenant/namespace/queue)
+            topic: Queue identifier in class:topicspace:topic format
             subscription: Subscription name
             schema: Dataclass type for messages
             initial_position: 'earliest' or 'latest'
-            consumer_type: 'shared', 'exclusive', or 'failover'
             **options: Backend-specific options
 
         Returns:
@@ -323,17 +228,18 @@ class PulsarBackend:
         """
         pulsar_topic = self.map_topic(topic)
 
+        # Extract class from topic for consumer type mapping
+        cls = topic.split(':', 1)[0] if ':' in topic else 'flow'
+
         # Map initial position
         if initial_position == 'earliest':
             pos = pulsar.InitialPosition.Earliest
         else:
             pos = pulsar.InitialPosition.Latest
 
-        # Map consumer type
-        if consumer_type == 'exclusive':
+        # Map consumer type from class
+        if cls in ('response', 'notify'):
             ctype = pulsar.ConsumerType.Exclusive
-        elif consumer_type == 'failover':
-            ctype = pulsar.ConsumerType.Failover
         else:
             ctype = pulsar.ConsumerType.Shared
 

trustgraph-base/trustgraph/base/rabbitmq_backend.py

@@ -0,0 +1,384 @@
+"""
+RabbitMQ backend implementation for pub/sub abstraction.
+
+Uses a single topic exchange per topicspace. The logical queue name
+becomes the routing key. Consumer behavior is determined by the
+subscription name:
+
+- Same subscription + same topic = shared queue (competing consumers)
+- Different subscriptions = separate queues (broadcast / fan-out)
+
+This mirrors Pulsar's subscription model using idiomatic RabbitMQ.
+
+Architecture:
+  Producer  -->  [tg exchange]  --routing key-->  [named queue]  --> Consumer
+                                --routing key-->  [named queue]  --> Consumer
+                                --routing key-->  [exclusive q]  --> Subscriber
+
+Uses basic_consume (push) instead of basic_get (polling) for
+efficient message delivery.
+"""
+
+import json
+import time
+import logging
+import queue
+import threading
+import pika
+from typing import Any
+
+from .backend import PubSubBackend, BackendProducer, BackendConsumer, Message
+from .serialization import dataclass_to_dict, dict_to_dataclass
+
+logger = logging.getLogger(__name__)
+
+
+class RabbitMQMessage:
+    """Wrapper for RabbitMQ messages to match Message protocol."""
+
+    def __init__(self, method, properties, body, schema_cls):
+        self._method = method
+        self._properties = properties
+        self._body = body
+        self._schema_cls = schema_cls
+        self._value = None
+
+    def value(self) -> Any:
+        """Deserialize and return the message value as a dataclass."""
+        if self._value is None:
+            data_dict = json.loads(self._body.decode('utf-8'))
+            self._value = dict_to_dataclass(data_dict, self._schema_cls)
+        return self._value
+
+    def properties(self) -> dict:
+        """Return message properties from AMQP headers."""
+        headers = self._properties.headers or {}
+        return dict(headers)
+
+
+class RabbitMQBackendProducer:
+    """Publishes messages to a topic exchange with a routing key.
+
+    Uses thread-local connections so each thread gets its own
+    connection/channel. This avoids wire corruption from concurrent
+    threads writing to the same socket (pika is not thread-safe).
+    """
+
+    def __init__(self, connection_params, exchange_name, routing_key,
+                 durable):
+        self._connection_params = connection_params
+        self._exchange_name = exchange_name
+        self._routing_key = routing_key
+        self._durable = durable
+        self._local = threading.local()
+
+    def _get_channel(self):
+        """Get or create a thread-local connection and channel."""
+        conn = getattr(self._local, 'connection', None)
+        chan = getattr(self._local, 'channel', None)
+
+        if conn is None or not conn.is_open or chan is None or not chan.is_open:
+            # Close stale connection if any
+            if conn is not None:
+                try:
+                    conn.close()
+                except Exception:
+                    pass
+
+            conn = pika.BlockingConnection(self._connection_params)
+            chan = conn.channel()
+            chan.exchange_declare(
+                exchange=self._exchange_name,
+                exchange_type='topic',
+                durable=True,
+            )
+            self._local.connection = conn
+            self._local.channel = chan
+
+        return chan
+
+    def send(self, message: Any, properties: dict = {}) -> None:
+        data_dict = dataclass_to_dict(message)
+        json_data = json.dumps(data_dict)
+
+        amqp_properties = pika.BasicProperties(
+            delivery_mode=2 if self._durable else 1,
+            content_type='application/json',
+            headers=properties if properties else None,
+        )
+
+        for attempt in range(2):
+            try:
+                channel = self._get_channel()
+                channel.basic_publish(
+                    exchange=self._exchange_name,
+                    routing_key=self._routing_key,
+                    body=json_data.encode('utf-8'),
+                    properties=amqp_properties,
+                )
+                return
+            except Exception as e:
+                logger.warning(
+                    f"RabbitMQ send failed (attempt {attempt + 1}): {e}"
+                )
+                # Force reconnect on next attempt
+                self._local.connection = None
+                self._local.channel = None
+                if attempt == 1:
+                    raise
+
+    def flush(self) -> None:
+        pass
+
+    def close(self) -> None:
+        """Close the thread-local connection if any."""
+        conn = getattr(self._local, 'connection', None)
+        if conn is not None:
+            try:
+                conn.close()
+            except Exception:
+                pass
+            self._local.connection = None
+            self._local.channel = None
+
+
+class RabbitMQBackendConsumer:
+    """Consumes from a queue bound to a topic exchange.
+
+    Uses basic_consume (push model) with messages delivered to an
+    internal thread-safe queue. process_data_events() drives both
+    message delivery and heartbeat processing.
+    """
+
+    def __init__(self, connection_params, exchange_name, routing_key,
+                 queue_name, schema_cls, durable, exclusive=False,
+                 auto_delete=False):
+        self._connection_params = connection_params
+        self._exchange_name = exchange_name
+        self._routing_key = routing_key
+        self._queue_name = queue_name
+        self._schema_cls = schema_cls
+        self._durable = durable
+        self._exclusive = exclusive
+        self._auto_delete = auto_delete
+        self._connection = None
+        self._channel = None
+        self._consumer_tag = None
+        self._incoming = queue.Queue()
+
+    def _connect(self):
+        self._connection = pika.BlockingConnection(self._connection_params)
+        self._channel = self._connection.channel()
+
+        # Declare the topic exchange
+        self._channel.exchange_declare(
+            exchange=self._exchange_name,
+            exchange_type='topic',
+            durable=True,
+        )
+
+        # Declare the queue — anonymous if exclusive
+        result = self._channel.queue_declare(
+            queue=self._queue_name,
+            durable=self._durable,
+            exclusive=self._exclusive,
+            auto_delete=self._auto_delete,
+        )
+        # Capture actual name (important for anonymous queues where name='')
+        self._queue_name = result.method.queue
+
+        self._channel.queue_bind(
+            queue=self._queue_name,
+            exchange=self._exchange_name,
+            routing_key=self._routing_key,
+        )
+
+        self._channel.basic_qos(prefetch_count=1)
+
+        # Register push-based consumer
+        self._consumer_tag = self._channel.basic_consume(
+            queue=self._queue_name,
+            on_message_callback=self._on_message,
+            auto_ack=False,
+        )
+
+    def _on_message(self, channel, method, properties, body):
+        """Callback invoked by pika when a message arrives."""
+        self._incoming.put((method, properties, body))
+
+    def _is_alive(self):
+        return (
+            self._connection is not None
+            and self._connection.is_open
+            and self._channel is not None
+            and self._channel.is_open
+        )
+
+    def receive(self, timeout_millis: int = 2000) -> Message:
+        """Receive a message. Raises TimeoutError if none available."""
+        if not self._is_alive():
+            self._connect()
+
+        timeout_seconds = timeout_millis / 1000.0
+        deadline = time.monotonic() + timeout_seconds
+
+        while time.monotonic() < deadline:
+            # Check if a message was already delivered
+            try:
+                method, properties, body = self._incoming.get_nowait()
+                return RabbitMQMessage(
+                    method, properties, body, self._schema_cls,
+                )
+            except queue.Empty:
+                pass
+
+            # Drive pika's I/O — delivers messages and processes heartbeats
+            remaining = deadline - time.monotonic()
+            if remaining > 0:
+                self._connection.process_data_events(
+                    time_limit=min(0.1, remaining),
+                )
+
+        raise TimeoutError("No message received within timeout")
+
+    def acknowledge(self, message: Message) -> None:
+        if isinstance(message, RabbitMQMessage) and message._method:
+            self._channel.basic_ack(
+                delivery_tag=message._method.delivery_tag,
+            )
+
+    def negative_acknowledge(self, message: Message) -> None:
+        if isinstance(message, RabbitMQMessage) and message._method:
+            self._channel.basic_nack(
+                delivery_tag=message._method.delivery_tag,
+                requeue=True,
+            )
+
+    def unsubscribe(self) -> None:
+        if self._consumer_tag and self._channel and self._channel.is_open:
+            try:
+                self._channel.basic_cancel(self._consumer_tag)
+            except Exception:
+                pass
+            self._consumer_tag = None
+
+    def close(self) -> None:
+        self.unsubscribe()
+        try:
+            if self._channel and self._channel.is_open:
+                self._channel.close()
+        except Exception:
+            pass
+        try:
+            if self._connection and self._connection.is_open:
+                self._connection.close()
+        except Exception:
+            pass
+        self._channel = None
+        self._connection = None
+
+
+class RabbitMQBackend:
+    """RabbitMQ pub/sub backend using a topic exchange per topicspace."""
+
+    def __init__(self, host='localhost', port=5672, username='guest',
+                 password='guest', vhost='/'):
+        self._connection_params = pika.ConnectionParameters(
+            host=host,
+            port=port,
+            virtual_host=vhost,
+            credentials=pika.PlainCredentials(username, password),
+            heartbeat=0,
+        )
+        logger.info(f"RabbitMQ backend: {host}:{port} vhost={vhost}")
+
+    def _parse_queue_id(self, queue_id: str) -> tuple[str, str, str, bool]:
+        """
+        Parse queue identifier into exchange, routing key, and durability.
+
+        Format: class:topicspace:topic
+        Returns: (exchange_name, routing_key, class, durable)
+        """
+        if ':' not in queue_id:
+            return 'tg', queue_id, 'flow', False
+
+        parts = queue_id.split(':', 2)
+        if len(parts) != 3:
+            raise ValueError(
+                f"Invalid queue format: {queue_id}, "
+                f"expected class:topicspace:topic"
+            )
+
+        cls, topicspace, topic = parts
+
+        if cls == 'flow':
+            durable = True
+        elif cls in ('request', 'response', 'notify'):
+            durable = False
+        else:
+            raise ValueError(
+                f"Invalid queue class: {cls}, "
+                f"expected flow, request, response, or notify"
+            )
+
+        # Exchange per topicspace, routing key includes class
+        exchange_name = topicspace
+        routing_key = f"{cls}.{topic}"
+
+        return exchange_name, routing_key, cls, durable
+
+    # Keep map_queue_name for backward compatibility with tests
+    def map_queue_name(self, queue_id: str) -> tuple[str, bool]:
+        exchange, routing_key, cls, durable = self._parse_queue_id(queue_id)
+        return f"{exchange}.{routing_key}", durable
+
+    def create_producer(self, topic: str, schema: type,
+                        **options) -> BackendProducer:
+        exchange, routing_key, cls, durable = self._parse_queue_id(topic)
+        logger.debug(
+            f"Creating producer: exchange={exchange}, "
+            f"routing_key={routing_key}"
+        )
+        return RabbitMQBackendProducer(
+            self._connection_params, exchange, routing_key, durable,
+        )
+
+    def create_consumer(self, topic: str, subscription: str, schema: type,
+                        initial_position: str = 'latest',
+                        **options) -> BackendConsumer:
+        """Create a consumer with a queue bound to the topic exchange.
+
+        Behaviour is determined by the topic's class prefix:
+          - flow: named durable queue, competing consumers (round-robin)
+          - request: named non-durable queue, competing consumers
+          - response: anonymous ephemeral queue, per-subscriber (auto-delete)
+          - notify: anonymous ephemeral queue, per-subscriber (auto-delete)
+        """
+        exchange, routing_key, cls, durable = self._parse_queue_id(topic)
+
+        if cls in ('response', 'notify'):
+            # Per-subscriber: anonymous queue, auto-deleted on disconnect
+            queue_name = ''
+            queue_durable = False
+            exclusive = True
+            auto_delete = True
+        else:
+            # Shared: named queue, competing consumers
+            queue_name = f"{exchange}.{routing_key}.{subscription}"
+            queue_durable = durable
+            exclusive = False
+            auto_delete = False
+
+        logger.debug(
+            f"Creating consumer: exchange={exchange}, "
+            f"routing_key={routing_key}, queue={queue_name or '(anonymous)'}, "
+            f"cls={cls}"
+        )
+
+        return RabbitMQBackendConsumer(
+            self._connection_params, exchange, routing_key,
+            queue_name, schema, queue_durable, exclusive, auto_delete,
+        )
+
+    def close(self) -> None:
+        pass

trustgraph-base/trustgraph/base/serialization.py

@@ -0,0 +1,115 @@
+"""
+JSON serialization helpers for dataclass ↔ dict conversion.
+
+Used by pub/sub backends that use JSON as their wire format.
+"""
+
+import types
+from dataclasses import asdict, is_dataclass
+from typing import Any, get_type_hints
+
+
+def dataclass_to_dict(obj: Any) -> dict:
+    """
+    Recursively convert a dataclass to a dictionary, handling None values and bytes.
+
+    None values are excluded from the dictionary (not serialized).
+    Bytes values are decoded as UTF-8 strings for JSON serialization.
+    Handles nested dataclasses, lists, and dictionaries recursively.
+    """
+    if obj is None:
+        return None
+
+    # Handle bytes - decode to UTF-8 for JSON serialization
+    if isinstance(obj, bytes):
+        return obj.decode('utf-8')
+
+    # Handle dataclass - convert to dict then recursively process all values
+    if is_dataclass(obj):
+        result = {}
+        for key, value in asdict(obj).items():
+            result[key] = dataclass_to_dict(value) if value is not None else None
+        return result
+
+    # Handle list - recursively process all items
+    if isinstance(obj, list):
+        return [dataclass_to_dict(item) for item in obj]
+
+    # Handle dict - recursively process all values
+    if isinstance(obj, dict):
+        return {k: dataclass_to_dict(v) for k, v in obj.items()}
+
+    # Return primitive types as-is
+    return obj
+
+
+def dict_to_dataclass(data: dict, cls: type) -> Any:
+    """
+    Convert a dictionary back to a dataclass instance.
+
+    Handles nested dataclasses and missing fields.
+    Uses get_type_hints() to resolve forward references (string annotations).
+    """
+    if data is None:
+        return None
+
+    if not is_dataclass(cls):
+        return data
+
+    # Get field types from the dataclass, resolving forward references
+    # get_type_hints() evaluates string annotations like "Triple | None"
+    try:
+        field_types = get_type_hints(cls)
+    except Exception:
+        # Fallback if get_type_hints fails (shouldn't happen normally)
+        field_types = {f.name: f.type for f in cls.__dataclass_fields__.values()}
+    kwargs = {}
+
+    for key, value in data.items():
+        if key in field_types:
+            field_type = field_types[key]
+
+            # Handle modern union types (X | Y)
+            if isinstance(field_type, types.UnionType):
+                # Check if it's Optional (X | None)
+                if type(None) in field_type.__args__:
+                    # Get the non-None type
+                    actual_type = next((t for t in field_type.__args__ if t is not type(None)), None)
+                    if actual_type and is_dataclass(actual_type) and isinstance(value, dict):
+                        kwargs[key] = dict_to_dataclass(value, actual_type)
+                    else:
+                        kwargs[key] = value
+                else:
+                    kwargs[key] = value
+            # Check if this is a generic type (list, dict, etc.)
+            elif hasattr(field_type, '__origin__'):
+                # Handle list[T]
+                if field_type.__origin__ == list:
+                    item_type = field_type.__args__[0] if field_type.__args__ else None
+                    if item_type and is_dataclass(item_type) and isinstance(value, list):
+                        kwargs[key] = [
+                            dict_to_dataclass(item, item_type) if isinstance(item, dict) else item
+                            for item in value
+                        ]
+                    else:
+                        kwargs[key] = value
+                # Handle old-style Optional[T] (which is Union[T, None])
+                elif hasattr(field_type, '__args__') and type(None) in field_type.__args__:
+                    # Get the non-None type from Union
+                    actual_type = next((t for t in field_type.__args__ if t is not type(None)), None)
+                    if actual_type and is_dataclass(actual_type) and isinstance(value, dict):
+                        kwargs[key] = dict_to_dataclass(value, actual_type)
+                    else:
+                        kwargs[key] = value
+                else:
+                    kwargs[key] = value
+            # Handle direct dataclass fields
+            elif is_dataclass(field_type) and isinstance(value, dict):
+                kwargs[key] = dict_to_dataclass(value, field_type)
+            # Handle bytes fields (UTF-8 encoded strings from JSON)
+            elif field_type == bytes and isinstance(value, str):
+                kwargs[key] = value.encode('utf-8')
+            else:
+                kwargs[key] = value
+
+    return cls(**kwargs)

trustgraph-base/trustgraph/base/subscriber.py

@@ -7,6 +7,7 @@ import asyncio
 import time
 import logging
 import uuid
+from concurrent.futures import ThreadPoolExecutor
 
 # Module logger
 logger = logging.getLogger(__name__)
@@ -38,6 +39,7 @@ class Subscriber:
         self.pending_acks = {}  # Track messages awaiting delivery
 
         self.consumer = None
+        self.executor = None
 
     def __del__(self):
 
@@ -45,15 +47,6 @@ class Subscriber:
 
     async def start(self):
 
-        # Create consumer via backend
-        self.consumer = await asyncio.to_thread(
-            self.backend.create_consumer,
-            topic=self.topic,
-            subscription=self.subscription,
-            schema=self.schema,
-            consumer_type='shared',
-        )
-
         self.task = asyncio.create_task(self.run())
 
     async def stop(self):
@@ -80,6 +73,20 @@ class Subscriber:
 
             try:
 
+                # Create consumer and dedicated thread if needed
+                # (first run or after failure)
+                if self.consumer is None:
+                    self.executor = ThreadPoolExecutor(max_workers=1)
+                    loop = asyncio.get_event_loop()
+                    self.consumer = await loop.run_in_executor(
+                        self.executor,
+                        lambda: self.backend.create_consumer(
+                            topic=self.topic,
+                            subscription=self.subscription,
+                            schema=self.schema,
+                        ),
+                    )
+
                 if self.metrics:
                     self.metrics.state("running")
 
@@ -128,9 +135,12 @@ class Subscriber:
                     # Process messages only if not draining
                     if not self.draining:
                         try:
-                            msg = await asyncio.to_thread(
-                                self.consumer.receive,
-                                timeout_millis=250
+                            loop = asyncio.get_event_loop()
+                            msg = await loop.run_in_executor(
+                                self.executor,
+                                lambda: self.consumer.receive(
+                                    timeout_millis=250
+                                ),
                             )
                         except Exception as e:
                             # Handle timeout from any backend
@@ -172,15 +182,18 @@ class Subscriber:
                     except Exception:
                         pass  # Already closed or error
                     self.consumer = None
-                
-         
+
+                if self.executor:
+                    self.executor.shutdown(wait=False)
+                    self.executor = None
+
             if self.metrics:
                 self.metrics.state("stopped")
 
             if not self.running and not self.draining:
                 return
-            
-            # If handler drops out, sleep a retry
+
+            # Sleep before retry
             await asyncio.sleep(1)
 
     async def subscribe(self, id):

trustgraph-base/trustgraph/clients/agent_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 
 from .. schema import AgentRequest, AgentResponse
 from .. schema import agent_request_queue
@@ -7,15 +6,11 @@ from .. schema import agent_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class AgentClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -27,7 +22,6 @@ class AgentClient(BaseClient):
         if output_queue is None: output_queue = agent_response_queue
 
         super(AgentClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

trustgraph-base/trustgraph/clients/base.py

@@ -1,10 +1,6 @@
 
-import pulsar
-import _pulsar
-import hashlib
 import uuid
 import time
-from pulsar.schema import JsonSchema
 
 from .. exceptions import *
 from ..base.pubsub import get_pubsub
@@ -12,24 +8,17 @@ from ..base.pubsub import get_pubsub
 # Default timeout for a request/response.  In seconds.
 DEFAULT_TIMEOUT=300
 
-# Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class BaseClient:
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
             input_schema=None,
             output_schema=None,
-            pulsar_host="pulsar://pulsar:6650",
-            pulsar_api_key=None,
-            listener=None,
+            **pubsub_config,
     ):
 
         if input_queue == None: raise RuntimeError("Need input_queue")
@@ -41,12 +30,7 @@ class BaseClient:
             subscriber = str(uuid.uuid4())
 
         # Create backend using factory
-        self.backend = get_pubsub(
-            pulsar_host=pulsar_host,
-            pulsar_api_key=pulsar_api_key,
-            pulsar_listener=listener,
-            pubsub_backend='pulsar'
-        )
+        self.backend = get_pubsub(**pubsub_config)
 
         self.producer = self.backend.create_producer(
             topic=input_queue,
@@ -58,7 +42,6 @@ class BaseClient:
             topic=output_queue,
             subscription=subscriber,
             schema=output_schema,
-            consumer_type='shared',
         )
 
         self.input_schema = input_schema
@@ -87,7 +70,7 @@ class BaseClient:
 
             try:
                 msg = self.consumer.receive(timeout_millis=2500)
-            except pulsar.exceptions.Timeout:
+            except TimeoutError:
                 continue
 
             mid = msg.properties()["id"]
@@ -139,4 +122,3 @@ class BaseClient:
 
         if hasattr(self, "backend"):
             self.backend.close()
-

trustgraph-base/trustgraph/clients/config_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 import json
 import dataclasses
 
@@ -9,10 +8,6 @@ from .. schema import config_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 @dataclasses.dataclass
 class Definition:
@@ -34,13 +29,11 @@ class Topic:
 class ConfigClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
-            pulsar_host="pulsar://pulsar:6650",
-            listener=None,
-            pulsar_api_key=None,
+            **pubsub_config,
     ):
 
         if input_queue == None:
@@ -50,15 +43,12 @@ class ConfigClient(BaseClient):
             output_queue = config_response_queue
 
         super(ConfigClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,
-            pulsar_host=pulsar_host,
-            pulsar_api_key=pulsar_api_key,
             input_schema=ConfigRequest,
             output_schema=ConfigResponse,
-            listener=listener,
+            **pubsub_config,
         )
 
     def get(self, keys, timeout=300):

trustgraph-base/trustgraph/clients/document_embeddings_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 
 from .. schema import DocumentEmbeddingsRequest, DocumentEmbeddingsResponse
 from .. schema import document_embeddings_request_queue
@@ -7,15 +6,11 @@ from .. schema import document_embeddings_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class DocumentEmbeddingsClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -30,7 +25,6 @@ class DocumentEmbeddingsClient(BaseClient):
             output_queue = document_embeddings_response_queue
             
         super(DocumentEmbeddingsClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

trustgraph-base/trustgraph/clients/document_rag_client.py

@@ -1,21 +1,15 @@
 
-import _pulsar
 
 from .. schema import DocumentRagQuery, DocumentRagResponse
 from .. schema import document_rag_request_queue, document_rag_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class DocumentRagClient(BaseClient):
 
     def __init__(
             self,
-            log_level=ERROR,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -30,7 +24,6 @@ class DocumentRagClient(BaseClient):
             output_queue = document_rag_response_queue
   
         super(DocumentRagClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,
@@ -50,7 +43,7 @@ class DocumentRagClient(BaseClient):
             user: User identifier
             collection: Collection identifier
             chunk_callback: Optional callback(text, end_of_stream) for text chunks
-            explain_callback: Optional callback(explain_id, explain_graph) for explain notifications
+            explain_callback: Optional callback(explain_id, explain_graph, explain_triples) for explain notifications
             timeout: Request timeout in seconds
 
         Returns:
@@ -62,7 +55,7 @@ class DocumentRagClient(BaseClient):
             # Handle explain notifications (response is None/empty, explain_id present)
             if x.explain_id and not x.response:
                 if explain_callback:
-                    explain_callback(x.explain_id, x.explain_graph)
+                    explain_callback(x.explain_id, x.explain_graph, x.explain_triples)
                 return False  # Continue receiving
 
             # Handle text chunks

trustgraph-base/trustgraph/clients/embeddings_client.py

@@ -1,20 +1,14 @@
 
-from pulsar.schema import JsonSchema
 from .. schema import EmbeddingsRequest, EmbeddingsResponse
 from . base import BaseClient
 
-import _pulsar
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class EmbeddingsClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             input_queue=None,
             output_queue=None,
             subscriber=None,
@@ -23,7 +17,6 @@ class EmbeddingsClient(BaseClient):
     ):
 
         super(EmbeddingsClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

trustgraph-base/trustgraph/clients/graph_embeddings_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 
 from .. schema import GraphEmbeddingsRequest, GraphEmbeddingsResponse
 from .. schema import graph_embeddings_request_queue
@@ -7,15 +6,11 @@ from .. schema import graph_embeddings_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class GraphEmbeddingsClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -30,7 +25,6 @@ class GraphEmbeddingsClient(BaseClient):
             output_queue = graph_embeddings_response_queue
             
         super(GraphEmbeddingsClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

trustgraph-base/trustgraph/clients/graph_rag_client.py

@@ -1,21 +1,15 @@
 
-import _pulsar
 
 from .. schema import GraphRagQuery, GraphRagResponse
 from .. schema import graph_rag_request_queue, graph_rag_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class GraphRagClient(BaseClient):
 
     def __init__(
             self,
-            log_level=ERROR,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -30,7 +24,6 @@ class GraphRagClient(BaseClient):
             output_queue = graph_rag_response_queue
   
         super(GraphRagClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,
@@ -54,7 +47,7 @@ class GraphRagClient(BaseClient):
             user: User identifier
             collection: Collection identifier
             chunk_callback: Optional callback(text, end_of_stream) for text chunks
-            explain_callback: Optional callback(explain_id, explain_graph) for explain notifications
+            explain_callback: Optional callback(explain_id, explain_graph, explain_triples) for explain notifications
             timeout: Request timeout in seconds
 
         Returns:
@@ -66,7 +59,7 @@ class GraphRagClient(BaseClient):
             # Handle explain notifications
             if x.message_type == 'explain':
                 if explain_callback and x.explain_id:
-                    explain_callback(x.explain_id, x.explain_graph)
+                    explain_callback(x.explain_id, x.explain_graph, x.explain_triples)
                 return False  # Continue receiving
 
             # Handle text chunks

trustgraph-base/trustgraph/clients/llm_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 
 from .. schema import TextCompletionRequest, TextCompletionResponse
 from .. schema import text_completion_request_queue
@@ -8,15 +7,11 @@ from . base import BaseClient
 from .. exceptions import LlmError
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class LlmClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -28,7 +23,6 @@ class LlmClient(BaseClient):
         if output_queue is None: output_queue = text_completion_response_queue
 
         super(LlmClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

trustgraph-base/trustgraph/clients/prompt_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 import json
 import dataclasses
 
@@ -9,10 +8,6 @@ from .. schema import prompt_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 @dataclasses.dataclass
 class Definition:
@@ -34,7 +29,7 @@ class Topic:
 class PromptClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -49,7 +44,6 @@ class PromptClient(BaseClient):
             output_queue = prompt_response_queue
 
         super(PromptClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

trustgraph-base/trustgraph/clients/row_embeddings_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 
 from .. schema import RowEmbeddingsRequest, RowEmbeddingsResponse
 from .. schema import row_embeddings_request_queue
@@ -7,15 +6,11 @@ from .. schema import row_embeddings_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class RowEmbeddingsClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -30,7 +25,6 @@ class RowEmbeddingsClient(BaseClient):
             output_queue = row_embeddings_response_queue
 
         super(RowEmbeddingsClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

trustgraph-base/trustgraph/clients/triples_query_client.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import _pulsar
 
 from .. schema import TriplesQueryRequest, TriplesQueryResponse, Term, IRI, LITERAL
 from .. schema import triples_request_queue
@@ -8,15 +7,11 @@ from .. schema import triples_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class TriplesQueryClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -31,7 +26,6 @@ class TriplesQueryClient(BaseClient):
             output_queue = triples_response_queue
 
         super(TriplesQueryClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

trustgraph-base/trustgraph/log_level.py

@@ -1,6 +1,6 @@
 
 from enum import Enum
-import _pulsar
+
 
 class LogLevel(Enum):
     DEBUG = 'debug'
@@ -10,11 +10,3 @@ class LogLevel(Enum):
 
     def __str__(self):
         return self.value
-
-    def to_pulsar(self):
-        if self == LogLevel.DEBUG: return _pulsar.LoggerLevel.Debug
-        if self == LogLevel.INFO: return _pulsar.LoggerLevel.Info
-        if self == LogLevel.WARN: return _pulsar.LoggerLevel.Warn
-        if self == LogLevel.ERROR: return _pulsar.LoggerLevel.Error
-        raise RuntimeError("Log level mismatch")
-

trustgraph-base/trustgraph/messaging/__init__.py

@@ -27,6 +27,7 @@ from .translators.nlp_query import QuestionToStructuredQueryRequestTranslator, Q
 from .translators.structured_query import StructuredQueryRequestTranslator, StructuredQueryResponseTranslator
 from .translators.diagnosis import StructuredDataDiagnosisRequestTranslator, StructuredDataDiagnosisResponseTranslator
 from .translators.collection import CollectionManagementRequestTranslator, CollectionManagementResponseTranslator
+from .translators.sparql_query import SparqlQueryRequestTranslator, SparqlQueryResponseTranslator
 
 # Register all service translators
 TranslatorRegistry.register_service(
@@ -149,6 +150,12 @@ TranslatorRegistry.register_service(
     CollectionManagementResponseTranslator()
 )
 
+TranslatorRegistry.register_service(
+    "sparql-query",
+    SparqlQueryRequestTranslator(),
+    SparqlQueryResponseTranslator()
+)
+
 # Register single-direction translators for document loading
 TranslatorRegistry.register_request("document", DocumentTranslator())
 TranslatorRegistry.register_request("text-document", TextDocumentTranslator())

trustgraph-base/trustgraph/messaging/translators/agent.py

@@ -1,12 +1,13 @@
 from typing import Dict, Any, Tuple
 from ...schema import AgentRequest, AgentResponse
 from .base import MessageTranslator
+from .primitives import TripleTranslator
 
 
 class AgentRequestTranslator(MessageTranslator):
     """Translator for AgentRequest schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> AgentRequest:
+    def decode(self, data: Dict[str, Any]) -> AgentRequest:
         return AgentRequest(
             question=data["question"],
             state=data.get("state", None),
@@ -16,9 +17,17 @@ class AgentRequestTranslator(MessageTranslator):
             collection=data.get("collection", "default"),
             streaming=data.get("streaming", False),
             session_id=data.get("session_id", ""),
+            conversation_id=data.get("conversation_id", ""),
+            pattern=data.get("pattern", ""),
+            task_type=data.get("task_type", ""),
+            framing=data.get("framing", ""),
+            correlation_id=data.get("correlation_id", ""),
+            parent_session_id=data.get("parent_session_id", ""),
+            subagent_goal=data.get("subagent_goal", ""),
+            expected_siblings=data.get("expected_siblings", 0),
         )
 
-    def from_pulsar(self, obj: AgentRequest) -> Dict[str, Any]:
+    def encode(self, obj: AgentRequest) -> Dict[str, Any]:
         return {
             "question": obj.question,
             "state": obj.state,
@@ -28,36 +37,38 @@ class AgentRequestTranslator(MessageTranslator):
             "collection": getattr(obj, "collection", "default"),
             "streaming": getattr(obj, "streaming", False),
             "session_id": getattr(obj, "session_id", ""),
+            "conversation_id": getattr(obj, "conversation_id", ""),
+            "pattern": getattr(obj, "pattern", ""),
+            "task_type": getattr(obj, "task_type", ""),
+            "framing": getattr(obj, "framing", ""),
+            "correlation_id": getattr(obj, "correlation_id", ""),
+            "parent_session_id": getattr(obj, "parent_session_id", ""),
+            "subagent_goal": getattr(obj, "subagent_goal", ""),
+            "expected_siblings": getattr(obj, "expected_siblings", 0),
         }
 
 
 class AgentResponseTranslator(MessageTranslator):
     """Translator for AgentResponse schema objects"""
-    
-    def to_pulsar(self, data: Dict[str, Any]) -> AgentResponse:
+
+    def __init__(self):
+        self.triple_translator = TripleTranslator()
+
+    def decode(self, data: Dict[str, Any]) -> AgentResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
-    
-    def from_pulsar(self, obj: AgentResponse) -> Dict[str, Any]:
+
+    def encode(self, obj: AgentResponse) -> Dict[str, Any]:
         result = {}
 
-        # Check if this is a streaming response (has chunk_type)
-        if hasattr(obj, 'chunk_type') and obj.chunk_type:
+        if obj.chunk_type:
             result["chunk_type"] = obj.chunk_type
-            if obj.content:
-                result["content"] = obj.content
-            result["end_of_message"] = getattr(obj, "end_of_message", False)
-            result["end_of_dialog"] = getattr(obj, "end_of_dialog", False)
-        else:
-            # Legacy format (non-streaming)
-            if obj.answer:
-                result["answer"] = obj.answer
-            if obj.thought:
-                result["thought"] = obj.thought
-            if obj.observation:
-                result["observation"] = obj.observation
-            # Include completion flags for legacy format too
-            result["end_of_message"] = getattr(obj, "end_of_message", False)
-            result["end_of_dialog"] = getattr(obj, "end_of_dialog", False)
+        if obj.content:
+            result["content"] = obj.content
+        result["end_of_message"] = getattr(obj, "end_of_message", False)
+        result["end_of_dialog"] = getattr(obj, "end_of_dialog", False)
+
+        if getattr(obj, "message_id", ""):
+            result["message_id"] = obj.message_id
 
         # Include explainability fields if present
         explain_id = getattr(obj, "explain_id", None)
@@ -68,19 +79,20 @@ class AgentResponseTranslator(MessageTranslator):
         if explain_graph is not None:
             result["explain_graph"] = explain_graph
 
+        # Include explain_triples for explain messages
+        explain_triples = getattr(obj, "explain_triples", [])
+        if explain_triples:
+            result["explain_triples"] = [
+                self.triple_translator.encode(t) for t in explain_triples
+            ]
+
         # Always include error if present
         if hasattr(obj, 'error') and obj.error and obj.error.message:
             result["error"] = {"message": obj.error.message, "code": obj.error.code}
 
         return result
 
-    def from_response_with_completion(self, obj: AgentResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: AgentResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        # For streaming responses, check end_of_dialog
-        if hasattr(obj, 'chunk_type') and obj.chunk_type:
-            is_final = getattr(obj, 'end_of_dialog', False)
-        else:
-            # For legacy responses, check if answer is present
-            is_final = (obj.answer is not None)
-
-        return self.from_pulsar(obj), is_final
+        is_final = getattr(obj, 'end_of_dialog', False)
+        return self.encode(obj), is_final

trustgraph-base/trustgraph/messaging/translators/base.py

@@ -1,43 +1,46 @@
 from abc import ABC, abstractmethod
 from typing import Dict, Any, Tuple
-from pulsar.schema import Record
 
 
 class Translator(ABC):
-    """Base class for bidirectional Pulsar ↔ dict translation"""
-    
+    """Base class for bidirectional schema ↔ dict translation.
+
+    Translates between external API dicts (JSON from HTTP/WebSocket)
+    and internal schema objects (dataclasses).
+    """
+
     @abstractmethod
-    def to_pulsar(self, data: Dict[str, Any]) -> Record:
-        """Convert dict to Pulsar schema object"""
+    def decode(self, data: Dict[str, Any]) -> Any:
+        """Convert external dict to schema object."""
         pass
-    
-    @abstractmethod  
-    def from_pulsar(self, obj: Record) -> Dict[str, Any]:
-        """Convert Pulsar schema object to dict"""
+
+    @abstractmethod
+    def encode(self, obj: Any) -> Dict[str, Any]:
+        """Convert schema object to external dict."""
         pass
 
 
 class MessageTranslator(Translator):
-    """For complete request/response message translation"""
-    
-    def from_response_with_completion(self, obj: Record) -> Tuple[Dict[str, Any], bool]:
-        """Returns (response_dict, is_final) - for streaming responses"""
-        return self.from_pulsar(obj), True
+    """For complete request/response message translation."""
+
+    def encode_with_completion(self, obj: Any) -> Tuple[Dict[str, Any], bool]:
+        """Returns (response_dict, is_final) — for streaming responses."""
+        return self.encode(obj), True
 
 
 class SendTranslator(Translator):
-    """For fire-and-forget send operations (like ServiceSender)"""
-    
-    def from_pulsar(self, obj: Record) -> Dict[str, Any]:
-        """Usually not needed for send-only operations"""
-        raise NotImplementedError("Send translators typically don't need from_pulsar")
+    """For fire-and-forget send operations."""
+
+    def encode(self, obj: Any) -> Dict[str, Any]:
+        """Usually not needed for send-only operations."""
+        raise NotImplementedError("Send translators don't need encode")
 
 
-def handle_optional_fields(obj: Record, fields: list) -> Dict[str, Any]:
-    """Helper to extract optional fields from Pulsar object"""
+def handle_optional_fields(obj: Any, fields: list) -> Dict[str, Any]:
+    """Helper to extract optional fields from a schema object."""
     result = {}
     for field in fields:
         value = getattr(obj, field, None)
         if value is not None:
             result[field] = value
-    return result
+    return result

trustgraph-base/trustgraph/messaging/translators/collection.py

@@ -6,7 +6,7 @@ from .base import MessageTranslator
 class CollectionManagementRequestTranslator(MessageTranslator):
     """Translator for CollectionManagementRequest schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> CollectionManagementRequest:
+    def decode(self, data: Dict[str, Any]) -> CollectionManagementRequest:
         return CollectionManagementRequest(
             operation=data.get("operation"),
             user=data.get("user"),
@@ -19,7 +19,7 @@ class CollectionManagementRequestTranslator(MessageTranslator):
             limit=data.get("limit")
         )
 
-    def from_pulsar(self, obj: CollectionManagementRequest) -> Dict[str, Any]:
+    def encode(self, obj: CollectionManagementRequest) -> Dict[str, Any]:
         result = {}
 
         if obj.operation is not None:
@@ -47,7 +47,7 @@ class CollectionManagementRequestTranslator(MessageTranslator):
 class CollectionManagementResponseTranslator(MessageTranslator):
     """Translator for CollectionManagementResponse schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> CollectionManagementResponse:
+    def decode(self, data: Dict[str, Any]) -> CollectionManagementResponse:
 
         # Handle error
         error = None
@@ -76,10 +76,9 @@ class CollectionManagementResponseTranslator(MessageTranslator):
             collections=collections
         )
 
-    def from_pulsar(self, obj: CollectionManagementResponse) -> Dict[str, Any]:
+    def encode(self, obj: CollectionManagementResponse) -> Dict[str, Any]:
         result = {}
 
-        print("COLLECTIONMGMT", obj, flush=True)
 
         if obj.error is not None:
             result["error"] = {
@@ -99,6 +98,4 @@ class CollectionManagementResponseTranslator(MessageTranslator):
                     "tags": list(coll.tags) if coll.tags else []
                 })
 
-        print("RESULT IS", result, flush=True)
-
         return result

trustgraph-base/trustgraph/messaging/translators/config.py

@@ -6,7 +6,7 @@ from .base import MessageTranslator
 class ConfigRequestTranslator(MessageTranslator):
     """Translator for ConfigRequest schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> ConfigRequest:
+    def decode(self, data: Dict[str, Any]) -> ConfigRequest:
         keys = None
         if "keys" in data:
             keys = [
@@ -35,7 +35,7 @@ class ConfigRequestTranslator(MessageTranslator):
             values=values
         )
     
-    def from_pulsar(self, obj: ConfigRequest) -> Dict[str, Any]:
+    def encode(self, obj: ConfigRequest) -> Dict[str, Any]:
         result = {}
         
         if obj.operation is not None:
@@ -69,10 +69,10 @@ class ConfigRequestTranslator(MessageTranslator):
 class ConfigResponseTranslator(MessageTranslator):
     """Translator for ConfigResponse schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> ConfigResponse:
+    def decode(self, data: Dict[str, Any]) -> ConfigResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
     
-    def from_pulsar(self, obj: ConfigResponse) -> Dict[str, Any]:
+    def encode(self, obj: ConfigResponse) -> Dict[str, Any]:
         result = {}
         
         if obj.version is not None:
@@ -96,6 +96,6 @@ class ConfigResponseTranslator(MessageTranslator):
         
         return result
     
-    def from_response_with_completion(self, obj: ConfigResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: ConfigResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), True
+        return self.encode(obj), True

trustgraph-base/trustgraph/messaging/translators/diagnosis.py

@@ -7,7 +7,7 @@ from .base import MessageTranslator
 class StructuredDataDiagnosisRequestTranslator(MessageTranslator):
     """Translator for StructuredDataDiagnosisRequest schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> StructuredDataDiagnosisRequest:
+    def decode(self, data: Dict[str, Any]) -> StructuredDataDiagnosisRequest:
         return StructuredDataDiagnosisRequest(
             operation=data["operation"],
             sample=data["sample"],
@@ -16,7 +16,7 @@ class StructuredDataDiagnosisRequestTranslator(MessageTranslator):
             options=data.get("options", {})
         )
 
-    def from_pulsar(self, obj: StructuredDataDiagnosisRequest) -> Dict[str, Any]:
+    def encode(self, obj: StructuredDataDiagnosisRequest) -> Dict[str, Any]:
         result = {
             "operation": obj.operation,
             "sample": obj.sample,
@@ -36,10 +36,10 @@ class StructuredDataDiagnosisRequestTranslator(MessageTranslator):
 class StructuredDataDiagnosisResponseTranslator(MessageTranslator):
     """Translator for StructuredDataDiagnosisResponse schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> StructuredDataDiagnosisResponse:
+    def decode(self, data: Dict[str, Any]) -> StructuredDataDiagnosisResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
 
-    def from_pulsar(self, obj: StructuredDataDiagnosisResponse) -> Dict[str, Any]:
+    def encode(self, obj: StructuredDataDiagnosisResponse) -> Dict[str, Any]:
         result = {
             "operation": obj.operation
         }
@@ -64,6 +64,6 @@ class StructuredDataDiagnosisResponseTranslator(MessageTranslator):
 
         return result
 
-    def from_response_with_completion(self, obj: StructuredDataDiagnosisResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: StructuredDataDiagnosisResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), True
+        return self.encode(obj), True

trustgraph-base/trustgraph/messaging/translators/document_loading.py

@@ -4,10 +4,33 @@ from ...schema import Document, TextDocument, Chunk, DocumentEmbeddings, ChunkEm
 from .base import SendTranslator
 
 
+def _decode_text_payload(payload: str | bytes, charset: str) -> str:
+    """
+    Decode text-load payloads.
+
+    Historical clients send base64-encoded text, but direct REST callers may
+    send raw UTF-8 text. Support both so Unicode text-load requests do not fail
+    at the gateway translation layer.
+    """
+    if isinstance(payload, bytes):
+        if not payload.isascii():
+            return payload.decode(charset)
+        candidate = payload.decode("ascii")
+    else:
+        if not payload.isascii():
+            return payload
+        candidate = payload
+
+    try:
+        return base64.b64decode(candidate, validate=True).decode(charset)
+    except (ValueError, UnicodeDecodeError):
+        return candidate
+
+
 class DocumentTranslator(SendTranslator):
     """Translator for Document schema objects (PDF docs etc.)"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> Document:
+    def decode(self, data: Dict[str, Any]) -> Document:
         # Handle base64 content validation
         doc = base64.b64decode(data["data"])
 
@@ -22,7 +45,7 @@ class DocumentTranslator(SendTranslator):
             data=base64.b64encode(doc).decode("utf-8")
         )
 
-    def from_pulsar(self, obj: Document) -> Dict[str, Any]:
+    def encode(self, obj: Document) -> Dict[str, Any]:
         result = {
             "data": obj.data
         }
@@ -46,11 +69,10 @@ class DocumentTranslator(SendTranslator):
 class TextDocumentTranslator(SendTranslator):
     """Translator for TextDocument schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> TextDocument:
+    def decode(self, data: Dict[str, Any]) -> TextDocument:
         charset = data.get("charset", "utf-8")
 
-        # Text is base64 encoded in input
-        text = base64.b64decode(data["text"]).decode(charset)
+        text = _decode_text_payload(data["text"], charset)
 
         from ...schema import Metadata
         return TextDocument(
@@ -63,7 +85,7 @@ class TextDocumentTranslator(SendTranslator):
             text=text.encode("utf-8")
         )
 
-    def from_pulsar(self, obj: TextDocument) -> Dict[str, Any]:
+    def encode(self, obj: TextDocument) -> Dict[str, Any]:
         result = {
             "text": obj.text.decode("utf-8") if isinstance(obj.text, bytes) else obj.text
         }
@@ -87,7 +109,7 @@ class TextDocumentTranslator(SendTranslator):
 class ChunkTranslator(SendTranslator):
     """Translator for Chunk schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> Chunk:
+    def decode(self, data: Dict[str, Any]) -> Chunk:
         from ...schema import Metadata
         return Chunk(
             metadata=Metadata(
@@ -99,7 +121,7 @@ class ChunkTranslator(SendTranslator):
             chunk=data["chunk"].encode("utf-8") if isinstance(data["chunk"], str) else data["chunk"]
         )
 
-    def from_pulsar(self, obj: Chunk) -> Dict[str, Any]:
+    def encode(self, obj: Chunk) -> Dict[str, Any]:
         result = {
             "chunk": obj.chunk.decode("utf-8") if isinstance(obj.chunk, bytes) else obj.chunk
         }
@@ -123,7 +145,7 @@ class ChunkTranslator(SendTranslator):
 class DocumentEmbeddingsTranslator(SendTranslator):
     """Translator for DocumentEmbeddings schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> DocumentEmbeddings:
+    def decode(self, data: Dict[str, Any]) -> DocumentEmbeddings:
         metadata = data.get("metadata", {})
 
         chunks = [
@@ -145,7 +167,7 @@ class DocumentEmbeddingsTranslator(SendTranslator):
             chunks=chunks
         )
 
-    def from_pulsar(self, obj: DocumentEmbeddings) -> Dict[str, Any]:
+    def encode(self, obj: DocumentEmbeddings) -> Dict[str, Any]:
         result = {
             "chunks": [
                 {
@@ -169,4 +191,4 @@ class DocumentEmbeddingsTranslator(SendTranslator):
 
             result["metadata"] = metadata_dict
 
-        return result
+        return result

trustgraph-base/trustgraph/messaging/translators/embeddings.py

@@ -6,12 +6,12 @@ from .base import MessageTranslator
 class EmbeddingsRequestTranslator(MessageTranslator):
     """Translator for EmbeddingsRequest schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> EmbeddingsRequest:
+    def decode(self, data: Dict[str, Any]) -> EmbeddingsRequest:
         return EmbeddingsRequest(
             texts=data["texts"]
         )
 
-    def from_pulsar(self, obj: EmbeddingsRequest) -> Dict[str, Any]:
+    def encode(self, obj: EmbeddingsRequest) -> Dict[str, Any]:
         return {
             "texts": obj.texts
         }
@@ -20,14 +20,14 @@ class EmbeddingsRequestTranslator(MessageTranslator):
 class EmbeddingsResponseTranslator(MessageTranslator):
     """Translator for EmbeddingsResponse schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> EmbeddingsResponse:
+    def decode(self, data: Dict[str, Any]) -> EmbeddingsResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
     
-    def from_pulsar(self, obj: EmbeddingsResponse) -> Dict[str, Any]:
+    def encode(self, obj: EmbeddingsResponse) -> Dict[str, Any]:
         return {
             "vectors": obj.vectors
         }
     
-    def from_response_with_completion(self, obj: EmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: EmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), True
+        return self.encode(obj), True

trustgraph-base/trustgraph/messaging/translators/embeddings_query.py

@@ -11,7 +11,7 @@ from .primitives import ValueTranslator
 class DocumentEmbeddingsRequestTranslator(MessageTranslator):
     """Translator for DocumentEmbeddingsRequest schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> DocumentEmbeddingsRequest:
+    def decode(self, data: Dict[str, Any]) -> DocumentEmbeddingsRequest:
         return DocumentEmbeddingsRequest(
             vector=data["vector"],
             limit=int(data.get("limit", 10)),
@@ -19,7 +19,7 @@ class DocumentEmbeddingsRequestTranslator(MessageTranslator):
             collection=data.get("collection", "default")
         )
 
-    def from_pulsar(self, obj: DocumentEmbeddingsRequest) -> Dict[str, Any]:
+    def encode(self, obj: DocumentEmbeddingsRequest) -> Dict[str, Any]:
         return {
             "vector": obj.vector,
             "limit": obj.limit,
@@ -31,10 +31,10 @@ class DocumentEmbeddingsRequestTranslator(MessageTranslator):
 class DocumentEmbeddingsResponseTranslator(MessageTranslator):
     """Translator for DocumentEmbeddingsResponse schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> DocumentEmbeddingsResponse:
+    def decode(self, data: Dict[str, Any]) -> DocumentEmbeddingsResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
 
-    def from_pulsar(self, obj: DocumentEmbeddingsResponse) -> Dict[str, Any]:
+    def encode(self, obj: DocumentEmbeddingsResponse) -> Dict[str, Any]:
         result = {}
 
         if obj.chunks is not None:
@@ -48,15 +48,15 @@ class DocumentEmbeddingsResponseTranslator(MessageTranslator):
 
         return result
 
-    def from_response_with_completion(self, obj: DocumentEmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: DocumentEmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), True
+        return self.encode(obj), True
 
 
 class GraphEmbeddingsRequestTranslator(MessageTranslator):
     """Translator for GraphEmbeddingsRequest schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> GraphEmbeddingsRequest:
+    def decode(self, data: Dict[str, Any]) -> GraphEmbeddingsRequest:
         return GraphEmbeddingsRequest(
             vector=data["vector"],
             limit=int(data.get("limit", 10)),
@@ -64,7 +64,7 @@ class GraphEmbeddingsRequestTranslator(MessageTranslator):
             collection=data.get("collection", "default")
         )
 
-    def from_pulsar(self, obj: GraphEmbeddingsRequest) -> Dict[str, Any]:
+    def encode(self, obj: GraphEmbeddingsRequest) -> Dict[str, Any]:
         return {
             "vector": obj.vector,
             "limit": obj.limit,
@@ -79,16 +79,16 @@ class GraphEmbeddingsResponseTranslator(MessageTranslator):
     def __init__(self):
         self.value_translator = ValueTranslator()
 
-    def to_pulsar(self, data: Dict[str, Any]) -> GraphEmbeddingsResponse:
+    def decode(self, data: Dict[str, Any]) -> GraphEmbeddingsResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
 
-    def from_pulsar(self, obj: GraphEmbeddingsResponse) -> Dict[str, Any]:
+    def encode(self, obj: GraphEmbeddingsResponse) -> Dict[str, Any]:
         result = {}
 
         if obj.entities is not None:
             result["entities"] = [
                 {
-                    "entity": self.value_translator.from_pulsar(match.entity),
+                    "entity": self.value_translator.encode(match.entity),
                     "score": match.score
                 }
                 for match in obj.entities
@@ -96,15 +96,15 @@ class GraphEmbeddingsResponseTranslator(MessageTranslator):
 
         return result
 
-    def from_response_with_completion(self, obj: GraphEmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: GraphEmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), True
+        return self.encode(obj), True
 
 
 class RowEmbeddingsRequestTranslator(MessageTranslator):
     """Translator for RowEmbeddingsRequest schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> RowEmbeddingsRequest:
+    def decode(self, data: Dict[str, Any]) -> RowEmbeddingsRequest:
         return RowEmbeddingsRequest(
             vector=data["vector"],
             limit=int(data.get("limit", 10)),
@@ -114,7 +114,7 @@ class RowEmbeddingsRequestTranslator(MessageTranslator):
             index_name=data.get("index_name")
         )
 
-    def from_pulsar(self, obj: RowEmbeddingsRequest) -> Dict[str, Any]:
+    def encode(self, obj: RowEmbeddingsRequest) -> Dict[str, Any]:
         result = {
             "vector": obj.vector,
             "limit": obj.limit,
@@ -130,10 +130,10 @@ class RowEmbeddingsRequestTranslator(MessageTranslator):
 class RowEmbeddingsResponseTranslator(MessageTranslator):
     """Translator for RowEmbeddingsResponse schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> RowEmbeddingsResponse:
+    def decode(self, data: Dict[str, Any]) -> RowEmbeddingsResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
 
-    def from_pulsar(self, obj: RowEmbeddingsResponse) -> Dict[str, Any]:
+    def encode(self, obj: RowEmbeddingsResponse) -> Dict[str, Any]:
         result = {}
 
         if obj.error is not None:
@@ -155,6 +155,6 @@ class RowEmbeddingsResponseTranslator(MessageTranslator):
 
         return result
 
-    def from_response_with_completion(self, obj: RowEmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: RowEmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), True
+        return self.encode(obj), True

trustgraph-base/trustgraph/messaging/translators/flow.py

@@ -6,7 +6,7 @@ from .base import MessageTranslator
 class FlowRequestTranslator(MessageTranslator):
     """Translator for FlowRequest schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> FlowRequest:
+    def decode(self, data: Dict[str, Any]) -> FlowRequest:
         return FlowRequest(
             operation=data.get("operation"),
             blueprint_name=data.get("blueprint-name"),
@@ -16,7 +16,7 @@ class FlowRequestTranslator(MessageTranslator):
             parameters=data.get("parameters")
         )
     
-    def from_pulsar(self, obj: FlowRequest) -> Dict[str, Any]:
+    def encode(self, obj: FlowRequest) -> Dict[str, Any]:
         result = {}
 
         if obj.operation is not None:
@@ -38,10 +38,10 @@ class FlowRequestTranslator(MessageTranslator):
 class FlowResponseTranslator(MessageTranslator):
     """Translator for FlowResponse schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> FlowResponse:
+    def decode(self, data: Dict[str, Any]) -> FlowResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
     
-    def from_pulsar(self, obj: FlowResponse) -> Dict[str, Any]:
+    def encode(self, obj: FlowResponse) -> Dict[str, Any]:
         result = {}
 
         if obj.blueprint_names is not None:
@@ -59,6 +59,6 @@ class FlowResponseTranslator(MessageTranslator):
 
         return result
     
-    def from_response_with_completion(self, obj: FlowResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: FlowResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), True
+        return self.encode(obj), True

trustgraph-base/trustgraph/messaging/translators/knowledge.py

@@ -14,7 +14,7 @@ class KnowledgeRequestTranslator(MessageTranslator):
         self.value_translator = ValueTranslator()
         self.subgraph_translator = SubgraphTranslator()
 
-    def to_pulsar(self, data: Dict[str, Any]) -> KnowledgeRequest:
+    def decode(self, data: Dict[str, Any]) -> KnowledgeRequest:
         triples = None
         if "triples" in data:
             triples = Triples(
@@ -24,7 +24,7 @@ class KnowledgeRequestTranslator(MessageTranslator):
                     user=data["triples"]["metadata"]["user"],
                     collection=data["triples"]["metadata"]["collection"]
                 ),
-                triples=self.subgraph_translator.to_pulsar(data["triples"]["triples"]),
+                triples=self.subgraph_translator.decode(data["triples"]["triples"]),
             )
 
         graph_embeddings = None
@@ -38,7 +38,7 @@ class KnowledgeRequestTranslator(MessageTranslator):
                 ),
                 entities=[
                     EntityEmbeddings(
-                        entity=self.value_translator.to_pulsar(ent["entity"]),
+                        entity=self.value_translator.decode(ent["entity"]),
                         vectors=ent["vectors"],
                     )
                     for ent in data["graph-embeddings"]["entities"]
@@ -55,7 +55,7 @@ class KnowledgeRequestTranslator(MessageTranslator):
             graph_embeddings=graph_embeddings,
         )
 
-    def from_pulsar(self, obj: KnowledgeRequest) -> Dict[str, Any]:
+    def encode(self, obj: KnowledgeRequest) -> Dict[str, Any]:
         result = {}
 
         if obj.operation:
@@ -77,7 +77,7 @@ class KnowledgeRequestTranslator(MessageTranslator):
                     "user": obj.triples.metadata.user,
                     "collection": obj.triples.metadata.collection,
                 },
-                "triples": self.subgraph_translator.from_pulsar(obj.triples.triples),
+                "triples": self.subgraph_translator.encode(obj.triples.triples),
             }
 
         if obj.graph_embeddings:
@@ -91,7 +91,7 @@ class KnowledgeRequestTranslator(MessageTranslator):
                 "entities": [
                     {
                         "vector": entity.vector,
-                        "entity": self.value_translator.from_pulsar(entity.entity),
+                        "entity": self.value_translator.encode(entity.entity),
                     }
                     for entity in obj.graph_embeddings.entities
                 ],
@@ -107,10 +107,10 @@ class KnowledgeResponseTranslator(MessageTranslator):
         self.value_translator = ValueTranslator()
         self.subgraph_translator = SubgraphTranslator()
 
-    def to_pulsar(self, data: Dict[str, Any]) -> KnowledgeResponse:
+    def decode(self, data: Dict[str, Any]) -> KnowledgeResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
 
-    def from_pulsar(self, obj: KnowledgeResponse) -> Dict[str, Any]:
+    def encode(self, obj: KnowledgeResponse) -> Dict[str, Any]:
         # Response to list operation
         if obj.ids is not None:
             return {"ids": obj.ids}
@@ -125,7 +125,7 @@ class KnowledgeResponseTranslator(MessageTranslator):
                         "user": obj.triples.metadata.user,
                         "collection": obj.triples.metadata.collection,
                     },
-                    "triples": self.subgraph_translator.from_pulsar(obj.triples.triples),
+                    "triples": self.subgraph_translator.encode(obj.triples.triples),
                 }
             }
 
@@ -142,7 +142,7 @@ class KnowledgeResponseTranslator(MessageTranslator):
                     "entities": [
                         {
                             "vector": entity.vector,
-                            "entity": self.value_translator.from_pulsar(entity.entity),
+                            "entity": self.value_translator.encode(entity.entity),
                         }
                         for entity in obj.graph_embeddings.entities
                     ],
@@ -156,9 +156,9 @@ class KnowledgeResponseTranslator(MessageTranslator):
         # Empty response (successful delete)
         return {}
     
-    def from_response_with_completion(self, obj: KnowledgeResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: KnowledgeResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        response = self.from_pulsar(obj)
+        response = self.encode(obj)
         
         # Check if this is a final response
         is_final = (

trustgraph-base/trustgraph/messaging/translators/library.py

@@ -11,16 +11,16 @@ class LibraryRequestTranslator(MessageTranslator):
         self.doc_metadata_translator = DocumentMetadataTranslator()
         self.proc_metadata_translator = ProcessingMetadataTranslator()
     
-    def to_pulsar(self, data: Dict[str, Any]) -> LibrarianRequest:
+    def decode(self, data: Dict[str, Any]) -> LibrarianRequest:
         # Document metadata
         doc_metadata = None
         if "document-metadata" in data:
-            doc_metadata = self.doc_metadata_translator.to_pulsar(data["document-metadata"])
+            doc_metadata = self.doc_metadata_translator.decode(data["document-metadata"])
         
         # Processing metadata
         proc_metadata = None
         if "processing-metadata" in data:
-            proc_metadata = self.proc_metadata_translator.to_pulsar(data["processing-metadata"])
+            proc_metadata = self.proc_metadata_translator.decode(data["processing-metadata"])
         
         # Criteria
         criteria = []
@@ -61,7 +61,7 @@ class LibraryRequestTranslator(MessageTranslator):
             include_children=data.get("include-children", False),
         )
     
-    def from_pulsar(self, obj: LibrarianRequest) -> Dict[str, Any]:
+    def encode(self, obj: LibrarianRequest) -> Dict[str, Any]:
         result = {}
         
         if obj.operation:
@@ -71,9 +71,9 @@ class LibraryRequestTranslator(MessageTranslator):
         if obj.processing_id:
             result["processing-id"] = obj.processing_id
         if obj.document_metadata:
-            result["document-metadata"] = self.doc_metadata_translator.from_pulsar(obj.document_metadata)
+            result["document-metadata"] = self.doc_metadata_translator.encode(obj.document_metadata)
         if obj.processing_metadata:
-            result["processing-metadata"] = self.proc_metadata_translator.from_pulsar(obj.processing_metadata)
+            result["processing-metadata"] = self.proc_metadata_translator.encode(obj.processing_metadata)
         if obj.content:
             result["content"] = obj.content.decode("utf-8") if isinstance(obj.content, bytes) else obj.content
         if obj.user:
@@ -100,10 +100,10 @@ class LibraryResponseTranslator(MessageTranslator):
         self.doc_metadata_translator = DocumentMetadataTranslator()
         self.proc_metadata_translator = ProcessingMetadataTranslator()
     
-    def to_pulsar(self, data: Dict[str, Any]) -> LibrarianResponse:
+    def decode(self, data: Dict[str, Any]) -> LibrarianResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
     
-    def from_pulsar(self, obj: LibrarianResponse) -> Dict[str, Any]:
+    def encode(self, obj: LibrarianResponse) -> Dict[str, Any]:
         result = {}
 
         if obj.error:
@@ -113,20 +113,20 @@ class LibraryResponseTranslator(MessageTranslator):
             }
 
         if obj.document_metadata:
-            result["document-metadata"] = self.doc_metadata_translator.from_pulsar(obj.document_metadata)
+            result["document-metadata"] = self.doc_metadata_translator.encode(obj.document_metadata)
 
         if obj.content:
             result["content"] = obj.content.decode("utf-8") if isinstance(obj.content, bytes) else obj.content
 
         if obj.document_metadatas is not None:
             result["document-metadatas"] = [
-                self.doc_metadata_translator.from_pulsar(dm)
+                self.doc_metadata_translator.encode(dm)
                 for dm in obj.document_metadatas
             ]
 
         if obj.processing_metadatas is not None:
             result["processing-metadatas"] = [
-                self.proc_metadata_translator.from_pulsar(pm)
+                self.proc_metadata_translator.encode(pm)
                 for pm in obj.processing_metadatas
             ]
 
@@ -172,6 +172,6 @@ class LibraryResponseTranslator(MessageTranslator):
 
         return result
     
-    def from_response_with_completion(self, obj: LibrarianResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: LibrarianResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), obj.is_final
+        return self.encode(obj), obj.is_final

trustgraph-base/trustgraph/messaging/translators/metadata.py

@@ -10,7 +10,7 @@ class DocumentMetadataTranslator(Translator):
     def __init__(self):
         self.subgraph_translator = SubgraphTranslator()
     
-    def to_pulsar(self, data: Dict[str, Any]) -> DocumentMetadata:
+    def decode(self, data: Dict[str, Any]) -> DocumentMetadata:
         metadata = data.get("metadata", [])
         return DocumentMetadata(
             id=data.get("id"),
@@ -18,14 +18,14 @@ class DocumentMetadataTranslator(Translator):
             kind=data.get("kind"),
             title=data.get("title"),
             comments=data.get("comments"),
-            metadata=self.subgraph_translator.to_pulsar(metadata) if metadata is not None else [],
+            metadata=self.subgraph_translator.decode(metadata) if metadata is not None else [],
             user=data.get("user"),
             tags=data.get("tags"),
             parent_id=data.get("parent-id", ""),
             document_type=data.get("document-type", "source"),
         )
     
-    def from_pulsar(self, obj: DocumentMetadata) -> Dict[str, Any]:
+    def encode(self, obj: DocumentMetadata) -> Dict[str, Any]:
         result = {}
 
         if obj.id:
@@ -39,7 +39,7 @@ class DocumentMetadataTranslator(Translator):
         if obj.comments:
             result["comments"] = obj.comments
         if obj.metadata is not None:
-            result["metadata"] = self.subgraph_translator.from_pulsar(obj.metadata)
+            result["metadata"] = self.subgraph_translator.encode(obj.metadata)
         if obj.user:
             result["user"] = obj.user
         if obj.tags is not None:
@@ -55,7 +55,7 @@ class DocumentMetadataTranslator(Translator):
 class ProcessingMetadataTranslator(Translator):
     """Translator for ProcessingMetadata schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> ProcessingMetadata:
+    def decode(self, data: Dict[str, Any]) -> ProcessingMetadata:
         return ProcessingMetadata(
             id=data.get("id"),
             document_id=data.get("document-id"),
@@ -66,7 +66,7 @@ class ProcessingMetadataTranslator(Translator):
             tags=data.get("tags")
         )
     
-    def from_pulsar(self, obj: ProcessingMetadata) -> Dict[str, Any]:
+    def encode(self, obj: ProcessingMetadata) -> Dict[str, Any]:
         result = {}
         
         if obj.id:

trustgraph-base/trustgraph/messaging/translators/nlp_query.py

@@ -6,13 +6,13 @@ from .base import MessageTranslator
 class QuestionToStructuredQueryRequestTranslator(MessageTranslator):
     """Translator for QuestionToStructuredQueryRequest schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> QuestionToStructuredQueryRequest:
+    def decode(self, data: Dict[str, Any]) -> QuestionToStructuredQueryRequest:
         return QuestionToStructuredQueryRequest(
             question=data.get("question", ""),
             max_results=data.get("max_results", 100)
         )
     
-    def from_pulsar(self, obj: QuestionToStructuredQueryRequest) -> Dict[str, Any]:
+    def encode(self, obj: QuestionToStructuredQueryRequest) -> Dict[str, Any]:
         return {
             "question": obj.question,
             "max_results": obj.max_results
@@ -22,10 +22,10 @@ class QuestionToStructuredQueryRequestTranslator(MessageTranslator):
 class QuestionToStructuredQueryResponseTranslator(MessageTranslator):
     """Translator for QuestionToStructuredQueryResponse schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> QuestionToStructuredQueryResponse:
+    def decode(self, data: Dict[str, Any]) -> QuestionToStructuredQueryResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
     
-    def from_pulsar(self, obj: QuestionToStructuredQueryResponse) -> Dict[str, Any]:
+    def encode(self, obj: QuestionToStructuredQueryResponse) -> Dict[str, Any]:
         result = {
             "graphql_query": obj.graphql_query,
             "variables": dict(obj.variables) if obj.variables else {},
@@ -42,6 +42,6 @@ class QuestionToStructuredQueryResponseTranslator(MessageTranslator):
         
         return result
     
-    def from_response_with_completion(self, obj: QuestionToStructuredQueryResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: QuestionToStructuredQueryResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), True
+        return self.encode(obj), True

trustgraph-base/trustgraph/messaging/translators/primitives.py

@@ -17,7 +17,7 @@ class TermTranslator(Translator):
     - "tr": triple (for TRIPLE type, nested)
     """
 
-    def to_pulsar(self, data: Dict[str, Any]) -> Term:
+    def decode(self, data: Dict[str, Any]) -> Term:
         term_type = data.get("t", "")
 
         if term_type == IRI:
@@ -38,7 +38,7 @@ class TermTranslator(Translator):
             # Nested triple - use TripleTranslator
             triple_data = data.get("tr")
             if triple_data:
-                triple = _triple_translator_to_pulsar(triple_data)
+                triple = _triple_translator_decode(triple_data)
             else:
                 triple = None
             return Term(type=TRIPLE, triple=triple)
@@ -47,7 +47,7 @@ class TermTranslator(Translator):
             # Unknown or empty type
             return Term(type=term_type)
 
-    def from_pulsar(self, obj: Term) -> Dict[str, Any]:
+    def encode(self, obj: Term) -> Dict[str, Any]:
         result: Dict[str, Any] = {"t": obj.type}
 
         if obj.type == IRI:
@@ -65,33 +65,33 @@ class TermTranslator(Translator):
 
         elif obj.type == TRIPLE:
             if obj.triple:
-                result["tr"] = _triple_translator_from_pulsar(obj.triple)
+                result["tr"] = _triple_translator_encode(obj.triple)
 
         return result
 
 
 # Module-level helper functions to avoid circular instantiation
-def _triple_translator_to_pulsar(data: Dict[str, Any]) -> Triple:
+def _triple_translator_decode(data: Dict[str, Any]) -> Triple:
     term_translator = TermTranslator()
     return Triple(
-        s=term_translator.to_pulsar(data["s"]) if data.get("s") else None,
-        p=term_translator.to_pulsar(data["p"]) if data.get("p") else None,
-        o=term_translator.to_pulsar(data["o"]) if data.get("o") else None,
+        s=term_translator.decode(data["s"]) if data.get("s") else None,
+        p=term_translator.decode(data["p"]) if data.get("p") else None,
+        o=term_translator.decode(data["o"]) if data.get("o") else None,
         g=data.get("g"),
     )
 
 
-def _triple_translator_from_pulsar(obj: Triple) -> Dict[str, Any]:
+def _triple_translator_encode(obj: Triple) -> Dict[str, Any]:
     """Convert Triple object to wire format dict."""
     term_translator = TermTranslator()
     result: Dict[str, Any] = {}
 
     if obj.s:
-        result["s"] = term_translator.from_pulsar(obj.s)
+        result["s"] = term_translator.encode(obj.s)
     if obj.p:
-        result["p"] = term_translator.from_pulsar(obj.p)
+        result["p"] = term_translator.encode(obj.p)
     if obj.o:
-        result["o"] = term_translator.from_pulsar(obj.o)
+        result["o"] = term_translator.encode(obj.o)
     if obj.g:
         result["g"] = obj.g
 
@@ -104,23 +104,23 @@ class TripleTranslator(Translator):
     def __init__(self):
         self.term_translator = TermTranslator()
 
-    def to_pulsar(self, data: Dict[str, Any]) -> Triple:
+    def decode(self, data: Dict[str, Any]) -> Triple:
         return Triple(
-            s=self.term_translator.to_pulsar(data["s"]) if data.get("s") else None,
-            p=self.term_translator.to_pulsar(data["p"]) if data.get("p") else None,
-            o=self.term_translator.to_pulsar(data["o"]) if data.get("o") else None,
+            s=self.term_translator.decode(data["s"]) if data.get("s") else None,
+            p=self.term_translator.decode(data["p"]) if data.get("p") else None,
+            o=self.term_translator.decode(data["o"]) if data.get("o") else None,
             g=data.get("g"),
         )
 
-    def from_pulsar(self, obj: Triple) -> Dict[str, Any]:
+    def encode(self, obj: Triple) -> Dict[str, Any]:
         result: Dict[str, Any] = {}
 
         if obj.s:
-            result["s"] = self.term_translator.from_pulsar(obj.s)
+            result["s"] = self.term_translator.encode(obj.s)
         if obj.p:
-            result["p"] = self.term_translator.from_pulsar(obj.p)
+            result["p"] = self.term_translator.encode(obj.p)
         if obj.o:
-            result["o"] = self.term_translator.from_pulsar(obj.o)
+            result["o"] = self.term_translator.encode(obj.o)
         if obj.g:
             result["g"] = obj.g
 
@@ -137,17 +137,17 @@ class SubgraphTranslator(Translator):
     def __init__(self):
         self.triple_translator = TripleTranslator()
     
-    def to_pulsar(self, data: List[Dict[str, Any]]) -> List[Triple]:
-        return [self.triple_translator.to_pulsar(t) for t in data]
+    def decode(self, data: List[Dict[str, Any]]) -> List[Triple]:
+        return [self.triple_translator.decode(t) for t in data]
     
-    def from_pulsar(self, obj: List[Triple]) -> List[Dict[str, Any]]:
-        return [self.triple_translator.from_pulsar(t) for t in obj]
+    def encode(self, obj: List[Triple]) -> List[Dict[str, Any]]:
+        return [self.triple_translator.encode(t) for t in obj]
 
 
 class RowSchemaTranslator(Translator):
     """Translator for RowSchema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> RowSchema:
+    def decode(self, data: Dict[str, Any]) -> RowSchema:
         """Convert dict to RowSchema Pulsar object"""
         fields = []
         for field_data in data.get("fields", []):
@@ -169,7 +169,7 @@ class RowSchemaTranslator(Translator):
             fields=fields
         )
     
-    def from_pulsar(self, obj: RowSchema) -> Dict[str, Any]:
+    def encode(self, obj: RowSchema) -> Dict[str, Any]:
         """Convert RowSchema Pulsar object to JSON-serializable dictionary"""
         result = {
             "name": obj.name,
@@ -200,7 +200,7 @@ class RowSchemaTranslator(Translator):
 class FieldTranslator(Translator):
     """Translator for Field objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> Field:
+    def decode(self, data: Dict[str, Any]) -> Field:
         """Convert dict to Field Pulsar object"""
         return Field(
             name=data.get("name", ""),
@@ -213,7 +213,7 @@ class FieldTranslator(Translator):
             enum_values=data.get("enum_values", [])
         )
     
-    def from_pulsar(self, obj: Field) -> Dict[str, Any]:
+    def encode(self, obj: Field) -> Dict[str, Any]:
         """Convert Field Pulsar object to JSON-serializable dictionary"""
         result = {
             "name": obj.name,

trustgraph-base/trustgraph/messaging/translators/prompt.py

@@ -7,7 +7,7 @@ from .base import MessageTranslator
 class PromptRequestTranslator(MessageTranslator):
     """Translator for PromptRequest schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> PromptRequest:
+    def decode(self, data: Dict[str, Any]) -> PromptRequest:
         # Handle both "terms" and "variables" input keys
         terms = data.get("terms", {})
         if "variables" in data:
@@ -23,7 +23,7 @@ class PromptRequestTranslator(MessageTranslator):
             streaming=data.get("streaming", False)
         )
     
-    def from_pulsar(self, obj: PromptRequest) -> Dict[str, Any]:
+    def encode(self, obj: PromptRequest) -> Dict[str, Any]:
         result = {}
         
         if obj.id:
@@ -37,10 +37,10 @@ class PromptRequestTranslator(MessageTranslator):
 class PromptResponseTranslator(MessageTranslator):
     """Translator for PromptResponse schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> PromptResponse:
+    def decode(self, data: Dict[str, Any]) -> PromptResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
     
-    def from_pulsar(self, obj: PromptResponse) -> Dict[str, Any]:
+    def encode(self, obj: PromptResponse) -> Dict[str, Any]:
         result = {}
 
         # Include text field if present (even if empty string)
@@ -55,8 +55,8 @@ class PromptResponseTranslator(MessageTranslator):
 
         return result
     
-    def from_response_with_completion(self, obj: PromptResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: PromptResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
         # Check end_of_stream field to determine if this is the final message
         is_final = getattr(obj, 'end_of_stream', True)
-        return self.from_pulsar(obj), is_final
+        return self.encode(obj), is_final

trustgraph-base/trustgraph/messaging/translators/retrieval.py

@@ -1,12 +1,13 @@
 from typing import Dict, Any, Tuple
 from ...schema import DocumentRagQuery, DocumentRagResponse, GraphRagQuery, GraphRagResponse
 from .base import MessageTranslator
+from .primitives import TripleTranslator
 
 
 class DocumentRagRequestTranslator(MessageTranslator):
     """Translator for DocumentRagQuery schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> DocumentRagQuery:
+    def decode(self, data: Dict[str, Any]) -> DocumentRagQuery:
         return DocumentRagQuery(
             query=data["query"],
             user=data.get("user", "trustgraph"),
@@ -15,7 +16,7 @@ class DocumentRagRequestTranslator(MessageTranslator):
             streaming=data.get("streaming", False)
         )
 
-    def from_pulsar(self, obj: DocumentRagQuery) -> Dict[str, Any]:
+    def encode(self, obj: DocumentRagQuery) -> Dict[str, Any]:
         return {
             "query": obj.query,
             "user": obj.user,
@@ -28,10 +29,13 @@ class DocumentRagRequestTranslator(MessageTranslator):
 class DocumentRagResponseTranslator(MessageTranslator):
     """Translator for DocumentRagResponse schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> DocumentRagResponse:
+    def __init__(self):
+        self.triple_translator = TripleTranslator()
+
+    def decode(self, data: Dict[str, Any]) -> DocumentRagResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
 
-    def from_pulsar(self, obj: DocumentRagResponse) -> Dict[str, Any]:
+    def encode(self, obj: DocumentRagResponse) -> Dict[str, Any]:
         result = {}
 
         # Include message_type for distinguishing chunk vs explain messages
@@ -53,6 +57,13 @@ class DocumentRagResponseTranslator(MessageTranslator):
         if explain_graph is not None:
             result["explain_graph"] = explain_graph
 
+        # Include explain_triples for explain messages
+        explain_triples = getattr(obj, "explain_triples", [])
+        if explain_triples:
+            result["explain_triples"] = [
+                self.triple_translator.encode(t) for t in explain_triples
+            ]
+
         # Include end_of_stream flag (LLM stream complete)
         result["end_of_stream"] = getattr(obj, "end_of_stream", False)
 
@@ -65,17 +76,17 @@ class DocumentRagResponseTranslator(MessageTranslator):
 
         return result
 
-    def from_response_with_completion(self, obj: DocumentRagResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: DocumentRagResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
         # Session is complete when end_of_session is True
         is_final = getattr(obj, 'end_of_session', False)
-        return self.from_pulsar(obj), is_final
+        return self.encode(obj), is_final
 
 
 class GraphRagRequestTranslator(MessageTranslator):
     """Translator for GraphRagQuery schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> GraphRagQuery:
+    def decode(self, data: Dict[str, Any]) -> GraphRagQuery:
         return GraphRagQuery(
             query=data["query"],
             user=data.get("user", "trustgraph"),
@@ -89,7 +100,7 @@ class GraphRagRequestTranslator(MessageTranslator):
             streaming=data.get("streaming", False)
         )
 
-    def from_pulsar(self, obj: GraphRagQuery) -> Dict[str, Any]:
+    def encode(self, obj: GraphRagQuery) -> Dict[str, Any]:
         return {
             "query": obj.query,
             "user": obj.user,
@@ -107,10 +118,13 @@ class GraphRagRequestTranslator(MessageTranslator):
 class GraphRagResponseTranslator(MessageTranslator):
     """Translator for GraphRagResponse schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> GraphRagResponse:
+    def __init__(self):
+        self.triple_translator = TripleTranslator()
+
+    def decode(self, data: Dict[str, Any]) -> GraphRagResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
 
-    def from_pulsar(self, obj: GraphRagResponse) -> Dict[str, Any]:
+    def encode(self, obj: GraphRagResponse) -> Dict[str, Any]:
         result = {}
 
         # Include message_type
@@ -132,6 +146,13 @@ class GraphRagResponseTranslator(MessageTranslator):
         if explain_graph is not None:
             result["explain_graph"] = explain_graph
 
+        # Include explain_triples for explain messages
+        explain_triples = getattr(obj, "explain_triples", [])
+        if explain_triples:
+            result["explain_triples"] = [
+                self.triple_translator.encode(t) for t in explain_triples
+            ]
+
         # Include end_of_stream flag (LLM stream complete)
         result["end_of_stream"] = getattr(obj, "end_of_stream", False)
 
@@ -144,8 +165,8 @@ class GraphRagResponseTranslator(MessageTranslator):
 
         return result
 
-    def from_response_with_completion(self, obj: GraphRagResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: GraphRagResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
         # Session is complete when end_of_session is True
         is_final = getattr(obj, 'end_of_session', False)
-        return self.from_pulsar(obj), is_final
+        return self.encode(obj), is_final

trustgraph-base/trustgraph/messaging/translators/rows_query.py

@@ -7,7 +7,7 @@ import json
 class RowsQueryRequestTranslator(MessageTranslator):
     """Translator for RowsQueryRequest schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> RowsQueryRequest:
+    def decode(self, data: Dict[str, Any]) -> RowsQueryRequest:
         return RowsQueryRequest(
             user=data.get("user", "trustgraph"),
             collection=data.get("collection", "default"),
@@ -16,7 +16,7 @@ class RowsQueryRequestTranslator(MessageTranslator):
             operation_name=data.get("operation_name", None)
         )
 
-    def from_pulsar(self, obj: RowsQueryRequest) -> Dict[str, Any]:
+    def encode(self, obj: RowsQueryRequest) -> Dict[str, Any]:
         result = {
             "user": obj.user,
             "collection": obj.collection,
@@ -33,10 +33,10 @@ class RowsQueryRequestTranslator(MessageTranslator):
 class RowsQueryResponseTranslator(MessageTranslator):
     """Translator for RowsQueryResponse schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> RowsQueryResponse:
+    def decode(self, data: Dict[str, Any]) -> RowsQueryResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
 
-    def from_pulsar(self, obj: RowsQueryResponse) -> Dict[str, Any]:
+    def encode(self, obj: RowsQueryResponse) -> Dict[str, Any]:
         result = {}
 
         # Handle GraphQL response data
@@ -74,6 +74,6 @@ class RowsQueryResponseTranslator(MessageTranslator):
 
         return result
 
-    def from_response_with_completion(self, obj: RowsQueryResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: RowsQueryResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), True
+        return self.encode(obj), True

trustgraph-base/trustgraph/messaging/translators/sparql_query.py

@@ -0,0 +1,115 @@
+from typing import Dict, Any, Tuple
+from ...schema import (
+    SparqlQueryRequest, SparqlQueryResponse, SparqlBinding,
+    Error, Term, Triple, IRI, LITERAL, BLANK,
+)
+from .base import MessageTranslator
+from .primitives import TermTranslator, TripleTranslator
+
+
+class SparqlQueryRequestTranslator(MessageTranslator):
+    """Translator for SparqlQueryRequest schema objects."""
+
+    def decode(self, data: Dict[str, Any]) -> SparqlQueryRequest:
+        return SparqlQueryRequest(
+            user=data.get("user", "trustgraph"),
+            collection=data.get("collection", "default"),
+            query=data.get("query", ""),
+            limit=int(data.get("limit", 10000)),
+            streaming=data.get("streaming", False),
+            batch_size=int(data.get("batch-size", 20)),
+        )
+
+    def encode(self, obj: SparqlQueryRequest) -> Dict[str, Any]:
+        return {
+            "user": obj.user,
+            "collection": obj.collection,
+            "query": obj.query,
+            "limit": obj.limit,
+            "streaming": obj.streaming,
+            "batch-size": obj.batch_size,
+        }
+
+
+class SparqlQueryResponseTranslator(MessageTranslator):
+    """Translator for SparqlQueryResponse schema objects."""
+
+    def __init__(self):
+        self.term_translator = TermTranslator()
+        self.triple_translator = TripleTranslator()
+
+    def decode(self, data: Dict[str, Any]) -> SparqlQueryResponse:
+        raise NotImplementedError(
+            "Response translation to schema not typically needed"
+        )
+
+    def _encode_term(self, v):
+        """Encode a Term, handling both Term objects and dicts from
+        pub/sub deserialization."""
+        if v is None:
+            return None
+        if isinstance(v, dict):
+            # Reconstruct Term from dict (pub/sub deserializes nested
+            # dataclasses as dicts)
+            term = Term(
+                type=v.get("type", ""),
+                iri=v.get("iri", ""),
+                id=v.get("id", ""),
+                value=v.get("value", ""),
+                datatype=v.get("datatype", ""),
+                language=v.get("language", ""),
+            )
+            return self.term_translator.encode(term)
+        return self.term_translator.encode(v)
+
+    def _encode_error(self, error):
+        """Encode an Error, handling both Error objects and dicts."""
+        if isinstance(error, dict):
+            return {
+                "type": error.get("type", ""),
+                "message": error.get("message", ""),
+            }
+        return {
+            "type": error.type,
+            "message": error.message,
+        }
+
+    def encode(self, obj: SparqlQueryResponse) -> Dict[str, Any]:
+        result = {
+            "query-type": obj.query_type,
+        }
+
+        if obj.error:
+            result["error"] = self._encode_error(obj.error)
+
+        if obj.query_type == "select":
+            result["variables"] = obj.variables
+            bindings = []
+            for binding in obj.bindings:
+                # binding may be a SparqlBinding or a dict
+                if isinstance(binding, dict):
+                    values = binding.get("values", [])
+                else:
+                    values = binding.values
+                bindings.append({
+                    "values": [
+                        self._encode_term(v) for v in values
+                    ]
+                })
+            result["bindings"] = bindings
+
+        elif obj.query_type == "ask":
+            result["ask-result"] = obj.ask_result
+
+        elif obj.query_type in ("construct", "describe"):
+            result["triples"] = [
+                self.triple_translator.encode(t)
+                for t in obj.triples
+            ]
+
+        return result
+
+    def encode_with_completion(
+        self, obj: SparqlQueryResponse
+    ) -> Tuple[Dict[str, Any], bool]:
+        return self.encode(obj), obj.is_final

trustgraph-base/trustgraph/messaging/translators/structured_query.py

@@ -7,14 +7,14 @@ import json
 class StructuredQueryRequestTranslator(MessageTranslator):
     """Translator for StructuredQueryRequest schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> StructuredQueryRequest:
+    def decode(self, data: Dict[str, Any]) -> StructuredQueryRequest:
         return StructuredQueryRequest(
             question=data.get("question", ""),
             user=data.get("user", "trustgraph"),        # Default fallback
             collection=data.get("collection", "default") # Default fallback
         )
     
-    def from_pulsar(self, obj: StructuredQueryRequest) -> Dict[str, Any]:
+    def encode(self, obj: StructuredQueryRequest) -> Dict[str, Any]:
         return {
             "question": obj.question,
             "user": obj.user,
@@ -25,10 +25,10 @@ class StructuredQueryRequestTranslator(MessageTranslator):
 class StructuredQueryResponseTranslator(MessageTranslator):
     """Translator for StructuredQueryResponse schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> StructuredQueryResponse:
+    def decode(self, data: Dict[str, Any]) -> StructuredQueryResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
     
-    def from_pulsar(self, obj: StructuredQueryResponse) -> Dict[str, Any]:
+    def encode(self, obj: StructuredQueryResponse) -> Dict[str, Any]:
         result = {}
         
         # Handle structured query response data
@@ -55,6 +55,6 @@ class StructuredQueryResponseTranslator(MessageTranslator):
         
         return result
     
-    def from_response_with_completion(self, obj: StructuredQueryResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: StructuredQueryResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), True
+        return self.encode(obj), True

trustgraph-base/trustgraph/messaging/translators/text_completion.py

@@ -6,14 +6,14 @@ from .base import MessageTranslator
 class TextCompletionRequestTranslator(MessageTranslator):
     """Translator for TextCompletionRequest schema objects"""
 
-    def to_pulsar(self, data: Dict[str, Any]) -> TextCompletionRequest:
+    def decode(self, data: Dict[str, Any]) -> TextCompletionRequest:
         return TextCompletionRequest(
             system=data["system"],
             prompt=data["prompt"],
             streaming=data.get("streaming", False)
         )
     
-    def from_pulsar(self, obj: TextCompletionRequest) -> Dict[str, Any]:
+    def encode(self, obj: TextCompletionRequest) -> Dict[str, Any]:
         return {
             "system": obj.system,
             "prompt": obj.prompt
@@ -23,10 +23,10 @@ class TextCompletionRequestTranslator(MessageTranslator):
 class TextCompletionResponseTranslator(MessageTranslator):
     """Translator for TextCompletionResponse schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> TextCompletionResponse:
+    def decode(self, data: Dict[str, Any]) -> TextCompletionResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
     
-    def from_pulsar(self, obj: TextCompletionResponse) -> Dict[str, Any]:
+    def encode(self, obj: TextCompletionResponse) -> Dict[str, Any]:
         result = {"response": obj.response}
 
         if obj.in_token:
@@ -41,8 +41,8 @@ class TextCompletionResponseTranslator(MessageTranslator):
 
         return result
     
-    def from_response_with_completion(self, obj: TextCompletionResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: TextCompletionResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
         # Check end_of_stream field to determine if this is the final message
         is_final = getattr(obj, 'end_of_stream', True)
-        return self.from_pulsar(obj), is_final
+        return self.encode(obj), is_final

trustgraph-base/trustgraph/messaging/translators/tool.py

@@ -6,7 +6,7 @@ from .base import MessageTranslator
 class ToolRequestTranslator(MessageTranslator):
     """Translator for ToolRequest schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> ToolRequest:
+    def decode(self, data: Dict[str, Any]) -> ToolRequest:
         # Handle both "name" and "parameters" input keys
         name = data.get("name", "")
         if "parameters" in data:
@@ -19,7 +19,7 @@ class ToolRequestTranslator(MessageTranslator):
             parameters = parameters,
         )
     
-    def from_pulsar(self, obj: ToolRequest) -> Dict[str, Any]:
+    def encode(self, obj: ToolRequest) -> Dict[str, Any]:
         result = {}
         
         if obj.name:
@@ -32,10 +32,10 @@ class ToolRequestTranslator(MessageTranslator):
 class ToolResponseTranslator(MessageTranslator):
     """Translator for ToolResponse schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> ToolResponse:
+    def decode(self, data: Dict[str, Any]) -> ToolResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
     
-    def from_pulsar(self, obj: ToolResponse) -> Dict[str, Any]:
+    def encode(self, obj: ToolResponse) -> Dict[str, Any]:
 
         result = {}
         
@@ -46,6 +46,6 @@ class ToolResponseTranslator(MessageTranslator):
         
         return result
     
-    def from_response_with_completion(self, obj: ToolResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: ToolResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), True
+        return self.encode(obj), True

trustgraph-base/trustgraph/messaging/translators/triples.py

@@ -10,10 +10,10 @@ class TriplesQueryRequestTranslator(MessageTranslator):
     def __init__(self):
         self.value_translator = ValueTranslator()
     
-    def to_pulsar(self, data: Dict[str, Any]) -> TriplesQueryRequest:
-        s = self.value_translator.to_pulsar(data["s"]) if "s" in data else None
-        p = self.value_translator.to_pulsar(data["p"]) if "p" in data else None
-        o = self.value_translator.to_pulsar(data["o"]) if "o" in data else None
+    def decode(self, data: Dict[str, Any]) -> TriplesQueryRequest:
+        s = self.value_translator.decode(data["s"]) if "s" in data else None
+        p = self.value_translator.decode(data["p"]) if "p" in data else None
+        o = self.value_translator.decode(data["o"]) if "o" in data else None
         g = data.get("g")  # None=default graph, "*"=all graphs
 
         return TriplesQueryRequest(
@@ -28,7 +28,7 @@ class TriplesQueryRequestTranslator(MessageTranslator):
             batch_size=int(data.get("batch-size", 20)),
         )
     
-    def from_pulsar(self, obj: TriplesQueryRequest) -> Dict[str, Any]:
+    def encode(self, obj: TriplesQueryRequest) -> Dict[str, Any]:
         result = {
             "limit": obj.limit,
             "user": obj.user,
@@ -38,11 +38,11 @@ class TriplesQueryRequestTranslator(MessageTranslator):
         }
 
         if obj.s:
-            result["s"] = self.value_translator.from_pulsar(obj.s)
+            result["s"] = self.value_translator.encode(obj.s)
         if obj.p:
-            result["p"] = self.value_translator.from_pulsar(obj.p)
+            result["p"] = self.value_translator.encode(obj.p)
         if obj.o:
-            result["o"] = self.value_translator.from_pulsar(obj.o)
+            result["o"] = self.value_translator.encode(obj.o)
         if obj.g is not None:
             result["g"] = obj.g
 
@@ -55,14 +55,14 @@ class TriplesQueryResponseTranslator(MessageTranslator):
     def __init__(self):
         self.subgraph_translator = SubgraphTranslator()
     
-    def to_pulsar(self, data: Dict[str, Any]) -> TriplesQueryResponse:
+    def decode(self, data: Dict[str, Any]) -> TriplesQueryResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
     
-    def from_pulsar(self, obj: TriplesQueryResponse) -> Dict[str, Any]:
+    def encode(self, obj: TriplesQueryResponse) -> Dict[str, Any]:
         return {
-            "response": self.subgraph_translator.from_pulsar(obj.triples)
+            "response": self.subgraph_translator.encode(obj.triples)
         }
     
-    def from_response_with_completion(self, obj: TriplesQueryResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: TriplesQueryResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
-        return self.from_pulsar(obj), obj.is_final
+        return self.encode(obj), obj.is_final

trustgraph-base/trustgraph/provenance/__init__.py

@@ -53,6 +53,12 @@ from . uris import (
     agent_thought_uri,
     agent_observation_uri,
     agent_final_uri,
+    # Orchestrator provenance URIs
+    agent_decomposition_uri,
+    agent_finding_uri,
+    agent_plan_uri,
+    agent_step_result_uri,
+    agent_synthesis_uri,
     # Document RAG provenance URIs
     docrag_question_uri,
     docrag_grounding_uri,
@@ -90,10 +96,14 @@ from . namespaces import (
     TG_ANALYSIS, TG_CONCLUSION,
     # Unifying types
     TG_ANSWER_TYPE, TG_REFLECTION_TYPE, TG_THOUGHT_TYPE, TG_OBSERVATION_TYPE,
+    TG_TOOL_USE,
     # Question subtypes (to distinguish retrieval mechanism)
     TG_GRAPH_RAG_QUESTION, TG_DOC_RAG_QUESTION, TG_AGENT_QUESTION,
     # Agent provenance predicates
     TG_THOUGHT, TG_ACTION, TG_ARGUMENTS, TG_OBSERVATION,
+    TG_SUBAGENT_GOAL, TG_PLAN_STEP,
+    # Orchestrator entity types
+    TG_DECOMPOSITION, TG_FINDING, TG_PLAN_TYPE, TG_STEP_RESULT,
     # Document reference predicate
     TG_DOCUMENT,
     # Named graphs
@@ -123,7 +133,14 @@ from . triples import (
 from . agent import (
     agent_session_triples,
     agent_iteration_triples,
+    agent_observation_triples,
     agent_final_triples,
+    # Orchestrator provenance triple builders
+    agent_decomposition_triples,
+    agent_finding_triples,
+    agent_plan_triples,
+    agent_step_result_triples,
+    agent_synthesis_triples,
 )
 
 # Vocabulary bootstrap
@@ -159,6 +176,12 @@ __all__ = [
     "agent_thought_uri",
     "agent_observation_uri",
     "agent_final_uri",
+    # Orchestrator provenance URIs
+    "agent_decomposition_uri",
+    "agent_finding_uri",
+    "agent_plan_uri",
+    "agent_step_result_uri",
+    "agent_synthesis_uri",
     # Document RAG provenance URIs
     "docrag_question_uri",
     "docrag_grounding_uri",
@@ -189,10 +212,14 @@ __all__ = [
     "TG_ANALYSIS", "TG_CONCLUSION",
     # Unifying types
     "TG_ANSWER_TYPE", "TG_REFLECTION_TYPE", "TG_THOUGHT_TYPE", "TG_OBSERVATION_TYPE",
+    "TG_TOOL_USE",
     # Question subtypes
     "TG_GRAPH_RAG_QUESTION", "TG_DOC_RAG_QUESTION", "TG_AGENT_QUESTION",
     # Agent provenance predicates
     "TG_THOUGHT", "TG_ACTION", "TG_ARGUMENTS", "TG_OBSERVATION",
+    "TG_SUBAGENT_GOAL", "TG_PLAN_STEP",
+    # Orchestrator entity types
+    "TG_DECOMPOSITION", "TG_FINDING", "TG_PLAN_TYPE", "TG_STEP_RESULT",
     # Document reference predicate
     "TG_DOCUMENT",
     # Named graphs
@@ -214,7 +241,14 @@ __all__ = [
     # Agent provenance triple builders
     "agent_session_triples",
     "agent_iteration_triples",
+    "agent_observation_triples",
     "agent_final_triples",
+    # Orchestrator provenance triple builders
+    "agent_decomposition_triples",
+    "agent_finding_triples",
+    "agent_plan_triples",
+    "agent_step_result_triples",
+    "agent_synthesis_triples",
     # Utility
     "set_graph",
     # Vocabulary

trustgraph-base/trustgraph/provenance/agent.py

@@ -1,10 +1,15 @@
 """
 Helper functions to build PROV-O triples for agent provenance.
 
-Agent provenance tracks the reasoning trace of ReAct agent sessions:
+Agent provenance tracks the reasoning trace of agent sessions:
 - Question: The root activity with query and timestamp
-- Analysis: Each think/act/observe cycle
-- Conclusion: The final answer
+- Analysis: Each think/act/observe cycle (ReAct)
+- Conclusion: The final answer (ReAct)
+- Decomposition: Supervisor broke question into sub-goals
+- Finding: A subagent's result (Supervisor)
+- Plan: Structured plan of steps (Plan-then-Execute)
+- StepResult: A plan step's result (Plan-then-Execute)
+- Synthesis: Final synthesised answer (Supervisor, Plan-then-Execute)
 """
 
 import json
@@ -15,12 +20,15 @@ from .. schema import Triple, Term, IRI, LITERAL
 
 from . namespaces import (
     RDF_TYPE, RDFS_LABEL,
-    PROV_ACTIVITY, PROV_ENTITY, PROV_WAS_DERIVED_FROM,
-    PROV_WAS_GENERATED_BY, PROV_STARTED_AT_TIME,
-    TG_QUERY, TG_THOUGHT, TG_ACTION, TG_ARGUMENTS, TG_OBSERVATION,
+    PROV_ENTITY, PROV_WAS_DERIVED_FROM,
+    PROV_STARTED_AT_TIME,
+    TG_QUERY, TG_THOUGHT, TG_ACTION, TG_ARGUMENTS,
     TG_QUESTION, TG_ANALYSIS, TG_CONCLUSION, TG_DOCUMENT,
     TG_ANSWER_TYPE, TG_REFLECTION_TYPE, TG_THOUGHT_TYPE, TG_OBSERVATION_TYPE,
+    TG_TOOL_USE,
     TG_AGENT_QUESTION,
+    TG_DECOMPOSITION, TG_FINDING, TG_PLAN_TYPE, TG_STEP_RESULT,
+    TG_SYNTHESIS, TG_SUBAGENT_GOAL, TG_PLAN_STEP,
 )
 
 
@@ -43,6 +51,7 @@ def agent_session_triples(
     session_uri: str,
     query: str,
     timestamp: Optional[str] = None,
+    parent_uri: Optional[str] = None,
 ) -> List[Triple]:
     """
     Build triples for an agent session start (Question).
@@ -50,11 +59,13 @@ def agent_session_triples(
     Creates:
     - Activity declaration with tg:Question type
     - Query text and timestamp
+    - wasDerivedFrom link to parent (for subagent sessions)
 
     Args:
         session_uri: URI of the session (from agent_session_uri)
         query: The user's query text
         timestamp: ISO timestamp (defaults to now)
+        parent_uri: URI of the parent entity (e.g. Decomposition) for subagents
 
     Returns:
         List of Triple objects
@@ -62,8 +73,8 @@ def agent_session_triples(
     if timestamp is None:
         timestamp = datetime.utcnow().isoformat() + "Z"
 
-    return [
-        _triple(session_uri, RDF_TYPE, _iri(PROV_ACTIVITY)),
+    triples = [
+        _triple(session_uri, RDF_TYPE, _iri(PROV_ENTITY)),
         _triple(session_uri, RDF_TYPE, _iri(TG_QUESTION)),
         _triple(session_uri, RDF_TYPE, _iri(TG_AGENT_QUESTION)),
         _triple(session_uri, RDFS_LABEL, _literal("Agent Question")),
@@ -71,6 +82,13 @@ def agent_session_triples(
         _triple(session_uri, TG_QUERY, _literal(query)),
     ]
 
+    if parent_uri:
+        triples.append(
+            _triple(session_uri, PROV_WAS_DERIVED_FROM, _iri(parent_uri))
+        )
+
+    return triples
+
 
 def agent_iteration_triples(
     iteration_uri: str,
@@ -80,19 +98,15 @@ def agent_iteration_triples(
     arguments: Dict[str, Any] = None,
     thought_uri: Optional[str] = None,
     thought_document_id: Optional[str] = None,
-    observation_uri: Optional[str] = None,
-    observation_document_id: Optional[str] = None,
 ) -> List[Triple]:
     """
-    Build triples for one agent iteration (Analysis - think/act/observe cycle).
+    Build triples for one agent iteration (Analysis+ToolUse).
 
     Creates:
-    - Entity declaration with tg:Analysis type
-    - wasGeneratedBy link to question (if first iteration)
-    - wasDerivedFrom link to previous iteration (if not first)
+    - Entity declaration with tg:Analysis and tg:ToolUse types
+    - wasDerivedFrom link to question (if first iteration) or previous
     - Action and arguments metadata
     - Thought sub-entity (tg:Reflection, tg:Thought) with librarian document
-    - Observation sub-entity (tg:Reflection, tg:Observation) with librarian document
 
     Args:
         iteration_uri: URI of this iteration (from agent_iteration_uri)
@@ -102,8 +116,6 @@ def agent_iteration_triples(
         arguments: Arguments passed to the tool (will be JSON-encoded)
         thought_uri: URI for the thought sub-entity
         thought_document_id: Document URI for thought in librarian
-        observation_uri: URI for the observation sub-entity
-        observation_document_id: Document URI for observation in librarian
 
     Returns:
         List of Triple objects
@@ -114,6 +126,7 @@ def agent_iteration_triples(
     triples = [
         _triple(iteration_uri, RDF_TYPE, _iri(PROV_ENTITY)),
         _triple(iteration_uri, RDF_TYPE, _iri(TG_ANALYSIS)),
+        _triple(iteration_uri, RDF_TYPE, _iri(TG_TOOL_USE)),
         _triple(iteration_uri, RDFS_LABEL, _literal(f"Analysis: {action}")),
         _triple(iteration_uri, TG_ACTION, _literal(action)),
         _triple(iteration_uri, TG_ARGUMENTS, _literal(json.dumps(arguments))),
@@ -121,7 +134,7 @@ def agent_iteration_triples(
 
     if question_uri:
         triples.append(
-            _triple(iteration_uri, PROV_WAS_GENERATED_BY, _iri(question_uri))
+            _triple(iteration_uri, PROV_WAS_DERIVED_FROM, _iri(question_uri))
         )
     elif previous_uri:
         triples.append(
@@ -135,26 +148,48 @@ def agent_iteration_triples(
             _triple(thought_uri, RDF_TYPE, _iri(TG_REFLECTION_TYPE)),
             _triple(thought_uri, RDF_TYPE, _iri(TG_THOUGHT_TYPE)),
             _triple(thought_uri, RDFS_LABEL, _literal("Thought")),
-            _triple(thought_uri, PROV_WAS_GENERATED_BY, _iri(iteration_uri)),
+            _triple(thought_uri, PROV_WAS_DERIVED_FROM, _iri(iteration_uri)),
         ])
         if thought_document_id:
             triples.append(
                 _triple(thought_uri, TG_DOCUMENT, _iri(thought_document_id))
             )
 
-    # Observation sub-entity
-    if observation_uri:
-        triples.extend([
-            _triple(iteration_uri, TG_OBSERVATION, _iri(observation_uri)),
-            _triple(observation_uri, RDF_TYPE, _iri(TG_REFLECTION_TYPE)),
-            _triple(observation_uri, RDF_TYPE, _iri(TG_OBSERVATION_TYPE)),
-            _triple(observation_uri, RDFS_LABEL, _literal("Observation")),
-            _triple(observation_uri, PROV_WAS_GENERATED_BY, _iri(iteration_uri)),
-        ])
-        if observation_document_id:
-            triples.append(
-                _triple(observation_uri, TG_DOCUMENT, _iri(observation_document_id))
-            )
+    return triples
+
+
+def agent_observation_triples(
+    observation_uri: str,
+    iteration_uri: str,
+    document_id: Optional[str] = None,
+) -> List[Triple]:
+    """
+    Build triples for an agent observation (standalone entity).
+
+    Creates:
+    - Entity declaration with prov:Entity and tg:Observation types
+    - wasDerivedFrom link to the iteration (Analysis+ToolUse)
+    - Document reference to librarian (if provided)
+
+    Args:
+        observation_uri: URI of the observation entity
+        iteration_uri: URI of the iteration this observation derives from
+        document_id: Librarian document ID for the observation content
+
+    Returns:
+        List of Triple objects
+    """
+    triples = [
+        _triple(observation_uri, RDF_TYPE, _iri(PROV_ENTITY)),
+        _triple(observation_uri, RDF_TYPE, _iri(TG_OBSERVATION_TYPE)),
+        _triple(observation_uri, RDFS_LABEL, _literal("Observation")),
+        _triple(observation_uri, PROV_WAS_DERIVED_FROM, _iri(iteration_uri)),
+    ]
+
+    if document_id:
+        triples.append(
+            _triple(observation_uri, TG_DOCUMENT, _iri(document_id))
+        )
 
     return triples
 
@@ -192,7 +227,7 @@ def agent_final_triples(
 
     if question_uri:
         triples.append(
-            _triple(final_uri, PROV_WAS_GENERATED_BY, _iri(question_uri))
+            _triple(final_uri, PROV_WAS_DERIVED_FROM, _iri(question_uri))
         )
     elif previous_uri:
         triples.append(
@@ -203,3 +238,108 @@ def agent_final_triples(
         triples.append(_triple(final_uri, TG_DOCUMENT, _iri(document_id)))
 
     return triples
+
+
+def agent_decomposition_triples(
+    uri: str,
+    session_uri: str,
+    goals: List[str],
+) -> List[Triple]:
+    """Build triples for a supervisor decomposition step."""
+    triples = [
+        _triple(uri, RDF_TYPE, _iri(PROV_ENTITY)),
+        _triple(uri, RDF_TYPE, _iri(TG_DECOMPOSITION)),
+        _triple(uri, RDFS_LABEL,
+                _literal(f"Decomposed into {len(goals)} research threads")),
+        _triple(uri, PROV_WAS_DERIVED_FROM, _iri(session_uri)),
+    ]
+    for goal in goals:
+        triples.append(_triple(uri, TG_SUBAGENT_GOAL, _literal(goal)))
+    return triples
+
+
+def agent_finding_triples(
+    uri: str,
+    decomposition_uri: str,
+    goal: str,
+    document_id: Optional[str] = None,
+) -> List[Triple]:
+    """Build triples for a subagent finding."""
+    triples = [
+        _triple(uri, RDF_TYPE, _iri(PROV_ENTITY)),
+        _triple(uri, RDF_TYPE, _iri(TG_FINDING)),
+        _triple(uri, RDF_TYPE, _iri(TG_ANSWER_TYPE)),
+        _triple(uri, RDFS_LABEL, _literal(f"Finding: {goal[:60]}")),
+        _triple(uri, PROV_WAS_DERIVED_FROM, _iri(decomposition_uri)),
+        _triple(uri, TG_SUBAGENT_GOAL, _literal(goal)),
+    ]
+    if document_id:
+        triples.append(_triple(uri, TG_DOCUMENT, _iri(document_id)))
+    return triples
+
+
+def agent_plan_triples(
+    uri: str,
+    session_uri: str,
+    steps: List[str],
+) -> List[Triple]:
+    """Build triples for a plan-then-execute plan."""
+    triples = [
+        _triple(uri, RDF_TYPE, _iri(PROV_ENTITY)),
+        _triple(uri, RDF_TYPE, _iri(TG_PLAN_TYPE)),
+        _triple(uri, RDFS_LABEL,
+                _literal(f"Plan with {len(steps)} steps")),
+        _triple(uri, PROV_WAS_DERIVED_FROM, _iri(session_uri)),
+    ]
+    for step in steps:
+        triples.append(_triple(uri, TG_PLAN_STEP, _literal(step)))
+    return triples
+
+
+def agent_step_result_triples(
+    uri: str,
+    plan_uri: str,
+    goal: str,
+    document_id: Optional[str] = None,
+) -> List[Triple]:
+    """Build triples for a plan step result."""
+    triples = [
+        _triple(uri, RDF_TYPE, _iri(PROV_ENTITY)),
+        _triple(uri, RDF_TYPE, _iri(TG_STEP_RESULT)),
+        _triple(uri, RDF_TYPE, _iri(TG_ANSWER_TYPE)),
+        _triple(uri, RDFS_LABEL, _literal(f"Step result: {goal[:60]}")),
+        _triple(uri, PROV_WAS_DERIVED_FROM, _iri(plan_uri)),
+        _triple(uri, TG_PLAN_STEP, _literal(goal)),
+    ]
+    if document_id:
+        triples.append(_triple(uri, TG_DOCUMENT, _iri(document_id)))
+    return triples
+
+
+def agent_synthesis_triples(
+    uri: str,
+    previous_uris,
+    document_id: Optional[str] = None,
+) -> List[Triple]:
+    """Build triples for a synthesis answer.
+
+    Args:
+        uri: URI of the synthesis entity
+        previous_uris: Single URI string or list of URIs to derive from
+        document_id: Librarian document ID for the answer content
+    """
+    triples = [
+        _triple(uri, RDF_TYPE, _iri(PROV_ENTITY)),
+        _triple(uri, RDF_TYPE, _iri(TG_SYNTHESIS)),
+        _triple(uri, RDF_TYPE, _iri(TG_ANSWER_TYPE)),
+        _triple(uri, RDFS_LABEL, _literal("Synthesis")),
+    ]
+
+    if isinstance(previous_uris, str):
+        previous_uris = [previous_uris]
+    for prev in previous_uris:
+        triples.append(_triple(uri, PROV_WAS_DERIVED_FROM, _iri(prev)))
+
+    if document_id:
+        triples.append(_triple(uri, TG_DOCUMENT, _iri(document_id)))
+    return triples

trustgraph-base/trustgraph/provenance/namespaces.py

@@ -94,11 +94,18 @@ TG_SYNTHESIS = TG + "Synthesis"
 TG_ANALYSIS = TG + "Analysis"
 TG_CONCLUSION = TG + "Conclusion"
 
+# Orchestrator entity types
+TG_DECOMPOSITION = TG + "Decomposition"  # Supervisor decomposed into sub-goals
+TG_FINDING = TG + "Finding"              # Subagent result
+TG_PLAN_TYPE = TG + "Plan"               # Plan-then-execute plan
+TG_STEP_RESULT = TG + "StepResult"       # Plan step result
+
 # Unifying types for answer and intermediate commentary
-TG_ANSWER_TYPE = TG + "Answer"          # Final answer (Synthesis, Conclusion)
+TG_ANSWER_TYPE = TG + "Answer"          # Final answer (Synthesis, Conclusion, Finding, StepResult)
 TG_REFLECTION_TYPE = TG + "Reflection"  # Intermediate commentary (Thought, Observation)
 TG_THOUGHT_TYPE = TG + "Thought"        # Agent reasoning
 TG_OBSERVATION_TYPE = TG + "Observation" # Agent tool result
+TG_TOOL_USE = TG + "ToolUse"            # Analysis+ToolUse mixin
 
 # Question subtypes (to distinguish retrieval mechanism)
 TG_GRAPH_RAG_QUESTION = TG + "GraphRagQuestion"
@@ -110,6 +117,8 @@ TG_THOUGHT = TG + "thought"        # Links iteration to thought sub-entity
 TG_ACTION = TG + "action"
 TG_ARGUMENTS = TG + "arguments"
 TG_OBSERVATION = TG + "observation" # Links iteration to observation sub-entity
+TG_SUBAGENT_GOAL = TG + "subagentGoal"  # Goal string on Decomposition/Finding
+TG_PLAN_STEP = TG + "planStep"          # Step goal string on Plan/StepResult
 
 # Named graph URIs for RDF datasets
 # These separate different types of data while keeping them in the same collection

trustgraph-base/trustgraph/provenance/triples.py

@@ -353,18 +353,21 @@ def question_triples(
     question_uri: str,
     query: str,
     timestamp: Optional[str] = None,
+    parent_uri: Optional[str] = None,
 ) -> List[Triple]:
     """
-    Build triples for a question activity.
+    Build triples for a question entity.
 
     Creates:
-    - Activity declaration for the question
+    - Entity declaration for the question
     - Query text and timestamp
+    - Optional wasDerivedFrom link to parent (for sub-traces)
 
     Args:
         question_uri: URI of the question (from question_uri)
         query: The user's query text
         timestamp: ISO timestamp (defaults to now)
+        parent_uri: Optional parent URI to link as wasDerivedFrom (for sub-traces)
 
     Returns:
         List of Triple objects
@@ -372,8 +375,8 @@ def question_triples(
     if timestamp is None:
         timestamp = datetime.utcnow().isoformat() + "Z"
 
-    return [
-        _triple(question_uri, RDF_TYPE, _iri(PROV_ACTIVITY)),
+    triples = [
+        _triple(question_uri, RDF_TYPE, _iri(PROV_ENTITY)),
         _triple(question_uri, RDF_TYPE, _iri(TG_QUESTION)),
         _triple(question_uri, RDF_TYPE, _iri(TG_GRAPH_RAG_QUESTION)),
         _triple(question_uri, RDFS_LABEL, _literal("GraphRAG Question")),
@@ -381,6 +384,13 @@ def question_triples(
         _triple(question_uri, TG_QUERY, _literal(query)),
     ]
 
+    if parent_uri:
+        triples.append(
+            _triple(question_uri, PROV_WAS_DERIVED_FROM, _iri(parent_uri))
+        )
+
+    return triples
+
 
 def grounding_triples(
     grounding_uri: str,
@@ -407,7 +417,7 @@ def grounding_triples(
         _triple(grounding_uri, RDF_TYPE, _iri(PROV_ENTITY)),
         _triple(grounding_uri, RDF_TYPE, _iri(TG_GROUNDING)),
         _triple(grounding_uri, RDFS_LABEL, _literal("Grounding")),
-        _triple(grounding_uri, PROV_WAS_GENERATED_BY, _iri(question_uri)),
+        _triple(grounding_uri, PROV_WAS_DERIVED_FROM, _iri(question_uri)),
     ]
 
     for concept in concepts:
@@ -455,11 +465,18 @@ def exploration_triples(
     return triples
 
 
-def _quoted_triple(s: str, p: str, o: str) -> Term:
-    """Create a quoted triple term (RDF-star) from string values."""
+def _quoted_triple(s, p, o) -> Term:
+    """Create a quoted triple term (RDF-star).
+
+    Accepts either Term objects (preserving original types) or plain
+    strings (treated as IRIs for backward compatibility).
+    """
+    s_term = s if isinstance(s, Term) else _iri(s)
+    p_term = p if isinstance(p, Term) else _iri(p)
+    o_term = o if isinstance(o, Term) else _iri(o)
     return Term(
         type=TRIPLE,
-        triple=Triple(s=_iri(s), p=_iri(p), o=_iri(o))
+        triple=Triple(s=s_term, p=p_term, o=o_term)
     )
 
 
@@ -575,18 +592,21 @@ def docrag_question_triples(
     question_uri: str,
     query: str,
     timestamp: Optional[str] = None,
+    parent_uri: Optional[str] = None,
 ) -> List[Triple]:
     """
-    Build triples for a document RAG question activity.
+    Build triples for a document RAG question entity.
 
     Creates:
-    - Activity declaration with tg:Question type
+    - Entity declaration with tg:Question type
     - Query text and timestamp
+    - Optional wasDerivedFrom link to parent (for sub-traces)
 
     Args:
         question_uri: URI of the question (from docrag_question_uri)
         query: The user's query text
         timestamp: ISO timestamp (defaults to now)
+        parent_uri: Optional parent URI to link as wasDerivedFrom (for sub-traces)
 
     Returns:
         List of Triple objects
@@ -594,8 +614,8 @@ def docrag_question_triples(
     if timestamp is None:
         timestamp = datetime.utcnow().isoformat() + "Z"
 
-    return [
-        _triple(question_uri, RDF_TYPE, _iri(PROV_ACTIVITY)),
+    triples = [
+        _triple(question_uri, RDF_TYPE, _iri(PROV_ENTITY)),
         _triple(question_uri, RDF_TYPE, _iri(TG_QUESTION)),
         _triple(question_uri, RDF_TYPE, _iri(TG_DOC_RAG_QUESTION)),
         _triple(question_uri, RDFS_LABEL, _literal("DocumentRAG Question")),
@@ -603,6 +623,13 @@ def docrag_question_triples(
         _triple(question_uri, TG_QUERY, _literal(query)),
     ]
 
+    if parent_uri:
+        triples.append(
+            _triple(question_uri, PROV_WAS_DERIVED_FROM, _iri(parent_uri))
+        )
+
+    return triples
+
 
 def docrag_exploration_triples(
     exploration_uri: str,

trustgraph-base/trustgraph/provenance/uris.py

@@ -234,6 +234,31 @@ def agent_final_uri(session_id: str) -> str:
     return f"urn:trustgraph:agent:{session_id}/final"
 
 
+def agent_decomposition_uri(session_id: str) -> str:
+    """Generate URI for a supervisor decomposition step."""
+    return f"urn:trustgraph:agent:{session_id}/decompose"
+
+
+def agent_finding_uri(session_id: str, index: int) -> str:
+    """Generate URI for a subagent finding."""
+    return f"urn:trustgraph:agent:{session_id}/finding/{index}"
+
+
+def agent_plan_uri(session_id: str) -> str:
+    """Generate URI for a plan-then-execute plan."""
+    return f"urn:trustgraph:agent:{session_id}/plan"
+
+
+def agent_step_result_uri(session_id: str, index: int) -> str:
+    """Generate URI for a plan step result."""
+    return f"urn:trustgraph:agent:{session_id}/step/{index}"
+
+
+def agent_synthesis_uri(session_id: str) -> str:
+    """Generate URI for a synthesis answer."""
+    return f"urn:trustgraph:agent:{session_id}/synthesis"
+
+
 # Document RAG provenance URIs
 # These URIs use the urn:trustgraph:docrag: namespace to distinguish
 # document RAG provenance from graph RAG provenance

trustgraph-base/trustgraph/provenance/vocabulary.py

@@ -27,6 +27,8 @@ from . namespaces import (
     TG_DOCUMENT_TYPE, TG_PAGE_TYPE, TG_CHUNK_TYPE, TG_SUBGRAPH_TYPE,
     TG_CONCEPT, TG_ENTITY, TG_GROUNDING,
     TG_ANSWER_TYPE, TG_REFLECTION_TYPE, TG_THOUGHT_TYPE, TG_OBSERVATION_TYPE,
+    TG_DECOMPOSITION, TG_FINDING, TG_PLAN_TYPE, TG_STEP_RESULT,
+    TG_SUBAGENT_GOAL, TG_PLAN_STEP,
 )
 
 
@@ -87,6 +89,10 @@ TG_CLASS_LABELS = [
     _label_triple(TG_REFLECTION_TYPE, "Reflection"),
     _label_triple(TG_THOUGHT_TYPE, "Thought"),
     _label_triple(TG_OBSERVATION_TYPE, "Observation"),
+    _label_triple(TG_DECOMPOSITION, "Decomposition"),
+    _label_triple(TG_FINDING, "Finding"),
+    _label_triple(TG_PLAN_TYPE, "Plan"),
+    _label_triple(TG_STEP_RESULT, "Step Result"),
 ]
 
 # TrustGraph predicate labels
@@ -109,6 +115,8 @@ TG_PREDICATE_LABELS = [
     _label_triple(TG_SOURCE_CHAR_LENGTH, "source character length"),
     _label_triple(TG_CONCEPT, "concept"),
     _label_triple(TG_ENTITY, "entity"),
+    _label_triple(TG_SUBAGENT_GOAL, "subagent goal"),
+    _label_triple(TG_PLAN_STEP, "plan step"),
 ]
 
 

trustgraph-base/trustgraph/schema/core/topic.py

@@ -1,23 +1,26 @@
 
-def topic(queue_name, qos='q1', tenant='tg', namespace='flow'):
+def queue(topic, cls='flow', topicspace='tg'):
     """
-    Create a generic topic identifier that can be mapped by backends.
+    Create a queue identifier in CLASS:TOPICSPACE:TOPIC format.
 
     Args:
-        queue_name: The queue/topic name
-        qos: Quality of service
-             - 'q0' = best-effort (no ack)
-             - 'q1' = at-least-once (ack required)
-             - 'q2' = exactly-once (two-phase ack)
-        tenant: Tenant identifier for multi-tenancy
-        namespace: Namespace within tenant
+        topic: The logical queue name (e.g. 'config', 'librarian')
+        cls: Queue class determining operational characteristics:
+             - 'flow' = persistent shared work queue (competing consumers)
+             - 'request' = non-persistent RPC request queue (shared)
+             - 'response' = non-persistent RPC response queue (per-subscriber)
+             - 'notify' = ephemeral broadcast (per-subscriber, auto-delete)
+        topicspace: Deployment isolation prefix (default: 'tg')
 
     Returns:
-        Generic topic string: qos/tenant/namespace/queue_name
+        Queue identifier string: cls:topicspace:topic
 
     Examples:
-        topic('my-queue')  # q1/tg/flow/my-queue
-        topic('config', qos='q2', namespace='config')  # q2/tg/config/config
+        queue('text-completion-request')
+            # flow:tg:text-completion-request
+        queue('config', cls='request')
+            # request:tg:config
+        queue('config', cls='notify')
+            # notify:tg:config
     """
-    return f"{qos}/{tenant}/{namespace}/{queue_name}"
-
+    return f"{cls}:{topicspace}:{topic}"

trustgraph-base/trustgraph/schema/knowledge/document.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass
 
 from ..core.metadata import Metadata
-from ..core.topic import topic
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/knowledge/embeddings.py

@@ -2,7 +2,6 @@ from dataclasses import dataclass, field
 
 from ..core.metadata import Metadata
 from ..core.primitives import Term, RowSchema
-from ..core.topic import topic
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/knowledge/graph.py

@@ -2,7 +2,6 @@ from dataclasses import dataclass, field
 
 from ..core.primitives import Term, Triple
 from ..core.metadata import Metadata
-from ..core.topic import topic
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/knowledge/knowledge.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, field
 from ..core.primitives import Triple, Error
-from ..core.topic import topic
+from ..core.topic import queue
 from ..core.metadata import Metadata
 from .document import Document, TextDocument
 from .graph import Triples
@@ -52,9 +52,5 @@ class KnowledgeResponse:
     triples: Triples | None = None
     graph_embeddings: GraphEmbeddings | None = None
 
-knowledge_request_queue = topic(
-    'knowledge', qos='q0', namespace='request'
-)
-knowledge_response_queue = topic(
-    'knowledge', qos='q0', namespace='response',
-)
+knowledge_request_queue = queue('knowledge', cls='request')
+knowledge_response_queue = queue('knowledge', cls='response')

trustgraph-base/trustgraph/schema/knowledge/nlp.py

@@ -1,6 +1,5 @@
 from dataclasses import dataclass
 
-from ..core.topic import topic
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/knowledge/object.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass, field
 
 from ..core.metadata import Metadata
-from ..core.topic import topic
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/knowledge/rows.py

@@ -2,7 +2,6 @@ from dataclasses import dataclass, field
 
 from ..core.metadata import Metadata
 from ..core.primitives import RowSchema
-from ..core.topic import topic
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/knowledge/structured.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass, field
 
 from ..core.metadata import Metadata
-from ..core.topic import topic
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/services/__init__.py

@@ -13,4 +13,5 @@ from .rows_query import *
 from .diagnosis import *
 from .collection import *
 from .storage import *
-from .tool_service import *
+from .tool_service import *
+from .sparql_query import *

trustgraph-base/trustgraph/schema/services/agent.py

@@ -1,13 +1,21 @@
 
 from dataclasses import dataclass, field
+from typing import Optional
 
-from ..core.topic import topic
-from ..core.primitives import Error
+from ..core.primitives import Error, Triple
 
 ############################################################################
 
 # Prompt services, abstract the prompt generation
 
+@dataclass
+class PlanStep:
+    goal: str = ""
+    tool_hint: str = ""                   # Suggested tool for this step
+    depends_on: list[int] = field(default_factory=list)  # Indices of prerequisite steps
+    status: str = "pending"               # pending, running, completed, failed
+    result: str = ""                      # Result of step execution
+
 @dataclass
 class AgentStep:
     thought: str = ""
@@ -15,6 +23,9 @@ class AgentStep:
     arguments: dict[str, str] = field(default_factory=dict)
     observation: str = ""
     user: str = ""              # User context for the step
+    step_type: str = ""         # "react", "plan", "execute", "decompose", "synthesise"
+    plan: list[PlanStep] = field(default_factory=list)      # Plan steps (for plan-then-execute)
+    subagent_results: dict[str, str] = field(default_factory=dict)  # Subagent results keyed by goal
 
 @dataclass
 class AgentRequest:
@@ -27,6 +38,16 @@ class AgentRequest:
     streaming: bool = False     # Enable streaming response delivery (default false)
     session_id: str = ""        # For provenance tracking across iterations
 
+    # Orchestration fields
+    conversation_id: str = ""   # Groups related requests into a conversation
+    pattern: str = ""           # Selected pattern: "react", "plan-then-execute", "supervisor"
+    task_type: str = ""         # Task type from config: "general", "research", etc.
+    framing: str = ""           # Domain framing text injected into prompts
+    correlation_id: str = ""    # Links fan-out subagents to parent for fan-in
+    parent_session_id: str = "" # Session ID of the supervisor that spawned this subagent
+    subagent_goal: str = ""     # Specific goal for a subagent (set by supervisor)
+    expected_siblings: int = 0  # Number of sibling subagents in this fan-out
+
 @dataclass
 class AgentResponse:
     # Streaming-first design
@@ -36,14 +57,14 @@ class AgentResponse:
     end_of_dialog: bool = False    # Entire agent dialog is complete
 
     # Explainability fields
-    explain_id: str | None = None     # Provenance URI (announced as created)
-    explain_graph: str | None = None  # Named graph where explain was stored
+    explain_id: str | None = None     # Root URI for this explain step
+    explain_graph: str | None = None  # Named graph (e.g., urn:graph:retrieval)
+    explain_triples: list[Triple] = field(default_factory=list)  # Provenance triples for this step
+
+    # Orchestration fields
+    message_id: str = ""              # Unique ID for this response message
 
-    # Legacy fields (deprecated but kept for backward compatibility)
-    answer: str = ""
     error: Error | None = None
-    thought: str = ""
-    observation: str = ""
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/services/collection.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass, field
 from datetime import datetime
 
 from ..core.primitives import Error
-from ..core.topic import topic
+from ..core.topic import queue
 
 ############################################################################
 
@@ -50,10 +50,6 @@ class CollectionManagementResponse:
 
 # Topics
 
-collection_request_queue = topic(
-    'collection', qos='q0', namespace='request'
-)
-collection_response_queue = topic(
-    'collection', qos='q0', namespace='response'
-)
+collection_request_queue = queue('collection', cls='request')
+collection_response_queue = queue('collection', cls='response')
 

trustgraph-base/trustgraph/schema/services/config.py

@@ -1,7 +1,7 @@
 
 from dataclasses import dataclass, field
 
-from ..core.topic import topic
+from ..core.topic import queue
 from ..core.primitives import Error
 
 ############################################################################
@@ -58,17 +58,11 @@ class ConfigResponse:
 @dataclass
 class ConfigPush:
     version: int = 0
-    config: dict[str, dict[str, str]] = field(default_factory=dict)
+    types: list[str] = field(default_factory=list)
 
-config_request_queue = topic(
-    'config', qos='q0', namespace='request'
-)
-config_response_queue = topic(
-    'config', qos='q0', namespace='response'
-)
-config_push_queue = topic(
-    'config', qos='q2', namespace='config'
-)
+config_request_queue = queue('config', cls='request')
+config_response_queue = queue('config', cls='response')
+config_push_queue = queue('config', cls='notify')
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/services/flow.py

@@ -1,7 +1,7 @@
 
 from dataclasses import dataclass, field
 
-from ..core.topic import topic
+from ..core.topic import queue
 from ..core.primitives import Error
 
 ############################################################################
@@ -61,12 +61,8 @@ class FlowResponse:
     # Everything
     error: Error | None = None
 
-flow_request_queue = topic(
-    'flow', qos='q0', namespace='request'
-)
-flow_response_queue = topic(
-    'flow', qos='q0', namespace='response'
-)
+flow_request_queue = queue('flow', cls='request')
+flow_response_queue = queue('flow', cls='response')
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/services/library.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, field
 from ..core.primitives import Triple, Error
-from ..core.topic import topic
+from ..core.topic import queue
 from ..core.metadata import Metadata
 # Note: Document imports will be updated after knowledge schemas are converted
 
@@ -24,10 +24,13 @@ from ..core.metadata import Metadata
 #   <- (document_metadata)
 #   <- (error)
 
-# get-document-content
+# get-document-content [DEPRECATED — use stream-document instead]
 #   -> (document_id)
 #   <- (content)
 #   <- (error)
+#   NOTE: Returns entire document in a single message. Fails for documents
+#   exceeding the broker's max message size. Use stream-document which
+#   returns content in chunks.
 
 # add-processing
 #   -> (processing_id, processing_metadata)
@@ -220,9 +223,5 @@ class LibrarianResponse:
 # FIXME: Is this right?  Using persistence on librarian so that
 # message chunking works
 
-librarian_request_queue = topic(
-    'librarian', qos='q1', namespace='request'
-)
-librarian_response_queue = topic(
-    'librarian', qos='q1', namespace='response',
-)
+librarian_request_queue = queue('librarian', cls='request')
+librarian_response_queue = queue('librarian', cls='response')

trustgraph-base/trustgraph/schema/services/llm.py

@@ -1,7 +1,6 @@
 
 from dataclasses import dataclass, field
 
-from ..core.topic import topic
 from ..core.primitives import Error
 
 ############################################################################

trustgraph-base/trustgraph/schema/services/lookup.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass
 
 from ..core.primitives import Error, Term, Triple
-from ..core.topic import topic
 from ..core.metadata import Metadata
 
 ############################################################################

trustgraph-base/trustgraph/schema/services/nlp_query.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass, field
 
 from ..core.primitives import Error
-from ..core.topic import topic
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/services/prompt.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass, field
 
 from ..core.primitives import Error
-from ..core.topic import topic
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/services/query.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass, field
 
 from ..core.primitives import Error, Term, Triple
-from ..core.topic import topic
+from ..core.topic import queue
 
 ############################################################################
 
@@ -69,12 +69,8 @@ class DocumentEmbeddingsResponse:
     error: Error | None = None
     chunks: list[ChunkMatch] = field(default_factory=list)
 
-document_embeddings_request_queue = topic(
-    "document-embeddings-request", qos='q0', tenant='trustgraph', namespace='flow'
-)
-document_embeddings_response_queue = topic(
-    "document-embeddings-response", qos='q0', tenant='trustgraph', namespace='flow'
-)
+document_embeddings_request_queue = queue('document-embeddings', cls='request')
+document_embeddings_response_queue = queue('document-embeddings', cls='response')
 
 ############################################################################
 
@@ -104,9 +100,5 @@ class RowEmbeddingsResponse:
     error: Error | None = None
     matches: list[RowIndexMatch] = field(default_factory=list)
 
-row_embeddings_request_queue = topic(
-    "row-embeddings-request", qos='q0', tenant='trustgraph', namespace='flow'
-)
-row_embeddings_response_queue = topic(
-    "row-embeddings-response", qos='q0', tenant='trustgraph', namespace='flow'
-)
+row_embeddings_request_queue = queue('row-embeddings', cls='request')
+row_embeddings_response_queue = queue('row-embeddings', cls='response')

trustgraph-base/trustgraph/schema/services/retrieval.py

@@ -1,6 +1,5 @@
-from dataclasses import dataclass
-from ..core.topic import topic
-from ..core.primitives import Error, Term
+from dataclasses import dataclass, field
+from ..core.primitives import Error, Term, Triple
 
 ############################################################################
 
@@ -18,14 +17,16 @@ class GraphRagQuery:
     edge_score_limit: int = 0
     edge_limit: int = 0
     streaming: bool = False
+    parent_uri: str = ""
 
 @dataclass
 class GraphRagResponse:
     error: Error | None = None
     response: str = ""
     end_of_stream: bool = False       # LLM response stream complete
-    explain_id: str | None = None     # Single explain URI (announced as created)
-    explain_graph: str | None = None  # Named graph where explain was stored (e.g., urn:graph:retrieval)
+    explain_id: str | None = None     # Root URI for this explain step
+    explain_graph: str | None = None  # Named graph (e.g., urn:graph:retrieval)
+    explain_triples: list[Triple] = field(default_factory=list)  # Provenance triples for this step
     message_type: str = ""            # "chunk" or "explain"
     end_of_session: bool = False      # Entire session complete
 
@@ -46,7 +47,8 @@ class DocumentRagResponse:
     error: Error | None = None
     response: str | None = ""
     end_of_stream: bool = False       # LLM response stream complete
-    explain_id: str | None = None     # Single explain URI (announced as created)
-    explain_graph: str | None = None  # Named graph where explain was stored (e.g., urn:graph:retrieval)
+    explain_id: str | None = None     # Root URI for this explain step
+    explain_graph: str | None = None  # Named graph (e.g., urn:graph:retrieval)
+    explain_triples: list[Triple] = field(default_factory=list)  # Provenance triples for this step
     message_type: str = ""            # "chunk" or "explain"
     end_of_session: bool = False      # Entire session complete

trustgraph-base/trustgraph/schema/services/rows_query.py

@@ -2,7 +2,6 @@ from dataclasses import dataclass, field
 from typing import Optional
 
 from ..core.primitives import Error
-from ..core.topic import topic
 
 ############################################################################
 

trustgraph-base/trustgraph/schema/services/sparql_query.py

@@ -0,0 +1,44 @@
+from dataclasses import dataclass, field
+
+from ..core.primitives import Error, Term, Triple
+from ..core.topic import queue
+
+############################################################################
+
+# SPARQL query
+
+@dataclass
+class SparqlBinding:
+    """A single row of SPARQL SELECT results.
+    Values are ordered to match the variables list in SparqlQueryResponse.
+    """
+    values: list[Term | None] = field(default_factory=list)
+
+@dataclass
+class SparqlQueryRequest:
+    user: str = ""
+    collection: str = ""
+    query: str = ""           # SPARQL query string
+    limit: int = 10000        # Safety limit on results
+    streaming: bool = False   # Enable streaming mode
+    batch_size: int = 20      # Bindings per batch in streaming mode
+
+@dataclass
+class SparqlQueryResponse:
+    error: Error | None = None
+    query_type: str = ""      # "select", "ask", "construct", "describe"
+
+    # For SELECT queries
+    variables: list[str] = field(default_factory=list)
+    bindings: list[SparqlBinding] = field(default_factory=list)
+
+    # For ASK queries
+    ask_result: bool = False
+
+    # For CONSTRUCT/DESCRIBE queries
+    triples: list[Triple] = field(default_factory=list)
+
+    is_final: bool = True     # False for intermediate batches in streaming
+
+sparql_query_request_queue = queue('sparql-query', cls='request')
+sparql_query_response_queue = queue('sparql-query', cls='response')

trustgraph-base/trustgraph/schema/services/structured_query.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass, field
 
 from ..core.primitives import Error
-from ..core.topic import topic
 
 ############################################################################