Merge branch 'master' into docs

trustgraph-flow/pyproject.toml

@@ -56,6 +56,7 @@ Homepage = "https://github.com/trustgraph-ai/trustgraph"
 
 [project.scripts]
 agent-manager-react = "trustgraph.agent.react:run"
+agent-orchestrator = "trustgraph.agent.orchestrator:run"
 api-gateway = "trustgraph.gateway:run"
 chunker-recursive = "trustgraph.chunking.recursive:run"
 chunker-token = "trustgraph.chunking.token:run"
@@ -100,6 +101,7 @@ pdf-ocr-mistral = "trustgraph.decoding.mistral_ocr:run"
 prompt-template = "trustgraph.prompt.template:run"
 rev-gateway = "trustgraph.rev_gateway:run"
 run-processing = "trustgraph.processing:run"
+sparql-query = "trustgraph.query.sparql:run"
 structured-query = "trustgraph.retrieval.structured_query:run"
 structured-diag = "trustgraph.retrieval.structured_diag:run"
 text-completion-azure = "trustgraph.model.text_completion.azure:run"

trustgraph-flow/trustgraph/agent/mcp_tool/service.py

@@ -24,7 +24,7 @@ class Service(ToolService):
             **params
         )
 
-        self.register_config_handler(self.on_mcp_config)
+        self.register_config_handler(self.on_mcp_config, types=["mcp"])
 
         self.mcp_services = {}
 

trustgraph-flow/trustgraph/agent/orchestrator/__init__.py

@@ -0,0 +1,2 @@
+
+from . service import *

trustgraph-flow/trustgraph/agent/orchestrator/__main__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from . service import run
+
+if __name__ == '__main__':
+    run()

trustgraph-flow/trustgraph/agent/orchestrator/aggregator.py

@@ -0,0 +1,166 @@
+"""
+Aggregator — tracks in-flight fan-out correlations and triggers
+synthesis when all subagents have completed.
+
+Subagent completions arrive as AgentRequest messages on the agent
+request queue with step_type="subagent-completion". The orchestrator
+intercepts these and feeds them to the aggregator. When all expected
+siblings for a correlation ID have reported, the aggregator builds
+a synthesis request for the supervisor pattern.
+"""
+
+import asyncio
+import json
+import logging
+import time
+import uuid
+
+from ... schema import AgentRequest, AgentStep
+
+logger = logging.getLogger(__name__)
+
+# How long to wait for stalled correlations before giving up (seconds)
+DEFAULT_TIMEOUT = 300
+
+
+class Aggregator:
+    """
+    Tracks in-flight fan-out correlations and triggers synthesis
+    when all subagents complete.
+
+    State is held in-memory; if the process restarts, in-flight
+    correlations are lost (acceptable for v1).
+    """
+
+    def __init__(self, timeout=DEFAULT_TIMEOUT):
+        self.timeout = timeout
+
+        # correlation_id -> {
+        #   "parent_session_id": str,
+        #   "expected": int,
+        #   "results": {goal: answer},
+        #   "request_template": AgentRequest or None,
+        #   "created_at": float,
+        # }
+        self.correlations = {}
+
+    def register_fanout(self, correlation_id, parent_session_id,
+                        expected_siblings, request_template=None):
+        """
+        Register a new fan-out. Called by the supervisor after emitting
+        subagent requests.
+        """
+        self.correlations[correlation_id] = {
+            "parent_session_id": parent_session_id,
+            "expected": expected_siblings,
+            "results": {},
+            "request_template": request_template,
+            "created_at": time.time(),
+        }
+        logger.debug(
+            f"Aggregator: registered fan-out {correlation_id}, "
+            f"expecting {expected_siblings} subagents"
+        )
+
+    def record_completion(self, correlation_id, subagent_goal, result):
+        """
+        Record a subagent completion.
+
+        Returns:
+            True if all siblings are now complete, False otherwise.
+            Returns None if the correlation_id is unknown.
+        """
+        if correlation_id not in self.correlations:
+            logger.warning(
+                f"Aggregator: unknown correlation_id {correlation_id}"
+            )
+            return None
+
+        entry = self.correlations[correlation_id]
+        entry["results"][subagent_goal] = result
+
+        completed = len(entry["results"])
+        expected = entry["expected"]
+
+        logger.debug(
+            f"Aggregator: {correlation_id} — "
+            f"{completed}/{expected} subagents complete"
+        )
+
+        return completed >= expected
+
+    def get_original_request(self, correlation_id):
+        """Peek at the stored request template without consuming it."""
+        entry = self.correlations.get(correlation_id)
+        if entry is None:
+            return None
+        return entry["request_template"]
+
+    def get_results(self, correlation_id):
+        """Get all results for a correlation and remove the tracking entry."""
+        entry = self.correlations.pop(correlation_id, None)
+        if entry is None:
+            return None, None, None
+        return (
+            entry["results"],
+            entry["parent_session_id"],
+            entry["request_template"],
+        )
+
+    def build_synthesis_request(self, correlation_id, original_question,
+                                user, collection):
+        """
+        Build the AgentRequest that triggers the synthesis phase.
+        """
+        results, parent_session_id, template = self.get_results(correlation_id)
+
+        if results is None:
+            raise RuntimeError(
+                f"No results for correlation_id {correlation_id}"
+            )
+
+        # Build history with decompose step + results
+        synthesis_step = AgentStep(
+            thought="All subagents completed",
+            action="aggregate",
+            arguments={},
+            observation=json.dumps(results),
+            step_type="synthesise",
+            subagent_results=results,
+        )
+
+        history = []
+        if template and template.history:
+            history = list(template.history)
+        history.append(synthesis_step)
+
+        return AgentRequest(
+            question=original_question,
+            state="",
+            group=template.group if template else [],
+            history=history,
+            user=user,
+            collection=collection,
+            streaming=template.streaming if template else False,
+            session_id=parent_session_id,
+            conversation_id=template.conversation_id if template else "",
+            pattern="supervisor",
+            task_type=template.task_type if template else "",
+            framing=template.framing if template else "",
+            correlation_id="",
+            parent_session_id="",
+            subagent_goal="",
+            expected_siblings=0,
+        )
+
+    def cleanup_stale(self):
+        """Remove correlations that have timed out."""
+        now = time.time()
+        stale = [
+            cid for cid, entry in self.correlations.items()
+            if now - entry["created_at"] > self.timeout
+        ]
+        for cid in stale:
+            logger.warning(f"Aggregator: timing out stale correlation {cid}")
+            self.correlations.pop(cid, None)
+        return stale

trustgraph-flow/trustgraph/agent/orchestrator/meta_router.py

@@ -0,0 +1,168 @@
+"""
+MetaRouter — selects the task type and execution pattern for a query.
+
+Uses the config API to look up available task types and patterns, then
+asks the LLM to classify the query and select the best pattern.
+Falls back to ("react", "general", "") if config is empty.
+"""
+
+import json
+import logging
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_PATTERN = "react"
+DEFAULT_TASK_TYPE = "general"
+DEFAULT_FRAMING = ""
+
+
+class MetaRouter:
+
+    def __init__(self, config=None):
+        """
+        Args:
+            config: The full config dict from the config service.
+                    May contain "agent-pattern" and "agent-task-type" keys.
+        """
+        self.patterns = {}
+        self.task_types = {}
+
+        if config:
+            # Load from config API
+            if "agent-pattern" in config:
+                for pid, pval in config["agent-pattern"].items():
+                    try:
+                        self.patterns[pid] = json.loads(pval)
+                    except (json.JSONDecodeError, TypeError):
+                        self.patterns[pid] = {"name": pid}
+
+            if "agent-task-type" in config:
+                for tid, tval in config["agent-task-type"].items():
+                    try:
+                        self.task_types[tid] = json.loads(tval)
+                    except (json.JSONDecodeError, TypeError):
+                        self.task_types[tid] = {"name": tid}
+
+        # If config has no patterns/task-types, default to react/general
+        if not self.patterns:
+            self.patterns = {
+                "react": {"name": "react", "description": "Interleaved reasoning and action"},
+            }
+        if not self.task_types:
+            self.task_types = {
+                "general": {"name": "general", "description": "General queries", "valid_patterns": ["react"], "framing": ""},
+            }
+
+    async def identify_task_type(self, question, context):
+        """
+        Use the LLM to classify the question into one of the known task types.
+
+        Args:
+            question: The user's query.
+            context: UserAwareContext (flow wrapper).
+
+        Returns:
+            (task_type_id, framing) tuple.
+        """
+        if len(self.task_types) <= 1:
+            tid = next(iter(self.task_types), DEFAULT_TASK_TYPE)
+            framing = self.task_types.get(tid, {}).get("framing", DEFAULT_FRAMING)
+            return tid, framing
+
+        try:
+            client = context("prompt-request")
+            response = await client.prompt(
+                id="task-type-classify",
+                variables={
+                    "question": question,
+                    "task_types": [
+                        {"name": tid, "description": tdata.get("description", tid)}
+                        for tid, tdata in self.task_types.items()
+                    ],
+                },
+            )
+            selected = response.strip().lower().replace('"', '').replace("'", "")
+
+            if selected in self.task_types:
+                framing = self.task_types[selected].get("framing", DEFAULT_FRAMING)
+                logger.info(f"MetaRouter: identified task type '{selected}'")
+                return selected, framing
+            else:
+                logger.warning(
+                    f"MetaRouter: LLM returned unknown task type '{selected}', "
+                    f"falling back to '{DEFAULT_TASK_TYPE}'"
+                )
+        except Exception as e:
+            logger.warning(f"MetaRouter: task type classification failed: {e}")
+
+        framing = self.task_types.get(DEFAULT_TASK_TYPE, {}).get(
+            "framing", DEFAULT_FRAMING
+        )
+        return DEFAULT_TASK_TYPE, framing
+
+    async def select_pattern(self, question, task_type, context):
+        """
+        Use the LLM to select the best execution pattern for this task type.
+
+        Args:
+            question: The user's query.
+            task_type: The identified task type ID.
+            context: UserAwareContext (flow wrapper).
+
+        Returns:
+            Pattern ID string.
+        """
+        task_config = self.task_types.get(task_type, {})
+        valid_patterns = task_config.get("valid_patterns", list(self.patterns.keys()))
+
+        if len(valid_patterns) <= 1:
+            return valid_patterns[0] if valid_patterns else DEFAULT_PATTERN
+
+        try:
+            client = context("prompt-request")
+            response = await client.prompt(
+                id="pattern-select",
+                variables={
+                    "question": question,
+                    "task_type": task_type,
+                    "task_type_description": task_config.get("description", task_type),
+                    "patterns": [
+                        {"name": pid, "description": self.patterns.get(pid, {}).get("description", pid)}
+                        for pid in valid_patterns
+                        if pid in self.patterns
+                    ],
+                },
+            )
+            selected = response.strip().lower().replace('"', '').replace("'", "")
+
+            if selected in valid_patterns:
+                logger.info(f"MetaRouter: selected pattern '{selected}'")
+                return selected
+            else:
+                logger.warning(
+                    f"MetaRouter: LLM returned invalid pattern '{selected}', "
+                    f"falling back to '{valid_patterns[0]}'"
+                )
+                return valid_patterns[0]
+        except Exception as e:
+            logger.warning(f"MetaRouter: pattern selection failed: {e}")
+            return valid_patterns[0] if valid_patterns else DEFAULT_PATTERN
+
+    async def route(self, question, context):
+        """
+        Full routing pipeline: identify task type, then select pattern.
+
+        Args:
+            question: The user's query.
+            context: UserAwareContext (flow wrapper).
+
+        Returns:
+            (pattern, task_type, framing) tuple.
+        """
+        task_type, framing = await self.identify_task_type(question, context)
+        pattern = await self.select_pattern(question, task_type, context)
+        logger.info(
+            f"MetaRouter: route result — "
+            f"pattern={pattern}, task_type={task_type}, framing={framing!r}"
+        )
+        return pattern, task_type, framing

trustgraph-flow/trustgraph/agent/orchestrator/pattern_base.py

@@ -0,0 +1,683 @@
+"""
+Base class for agent patterns.
+
+Provides shared infrastructure used by all patterns: tool filtering,
+provenance emission, streaming callbacks, history management, and
+librarian integration.
+"""
+
+import json
+import logging
+import uuid
+from datetime import datetime
+
+from ... schema import AgentRequest, AgentResponse, AgentStep, Error
+from ... schema import Triples, Metadata
+
+from trustgraph.provenance import (
+    agent_session_uri,
+    agent_iteration_uri,
+    agent_thought_uri,
+    agent_observation_uri,
+    agent_final_uri,
+    agent_decomposition_uri,
+    agent_finding_uri,
+    agent_plan_uri,
+    agent_step_result_uri,
+    agent_synthesis_uri,
+    agent_session_triples,
+    agent_iteration_triples,
+    agent_observation_triples,
+    agent_final_triples,
+    agent_decomposition_triples,
+    agent_finding_triples,
+    agent_plan_triples,
+    agent_step_result_triples,
+    agent_synthesis_triples,
+    set_graph,
+    GRAPH_RETRIEVAL,
+)
+
+from ..react.types import Action, Final
+from ..tool_filter import filter_tools_by_group_and_state, get_next_state
+
+logger = logging.getLogger(__name__)
+
+
+class UserAwareContext:
+    """Wraps flow interface to inject user context for tools that need it."""
+
+    def __init__(self, flow, user, respond=None, streaming=False):
+        self._flow = flow
+        self._user = user
+        self.respond = respond
+        self.streaming = streaming
+        self.current_explain_uri = None
+        self.last_sub_explain_uri = None
+
+    def __call__(self, service_name):
+        client = self._flow(service_name)
+        if service_name in (
+            "structured-query-request",
+            "row-embeddings-query-request",
+        ):
+            client._current_user = self._user
+        return client
+
+
+class PatternBase:
+    """
+    Shared infrastructure for all agent patterns.
+
+    Subclasses implement iterate() to perform one iteration of their
+    pattern-specific logic.
+    """
+
+    def __init__(self, processor):
+        self.processor = processor
+
+    def is_subagent(self, request):
+        """Check if this request is running as a subagent of a supervisor."""
+        return bool(getattr(request, 'correlation_id', ''))
+
+    async def emit_subagent_completion(self, request, next, answer_text):
+        """Signal completion back to the orchestrator via the agent request
+        queue. Instead of sending the final answer to the client, send a
+        completion message so the aggregator can collect it."""
+
+        completion_step = AgentStep(
+            thought="Subagent completed",
+            action="complete",
+            arguments={},
+            observation=answer_text,
+            step_type="subagent-completion",
+        )
+
+        completion_request = AgentRequest(
+            question=request.question,
+            state="",
+            group=getattr(request, 'group', []),
+            history=[completion_step],
+            user=request.user,
+            collection=getattr(request, 'collection', 'default'),
+            streaming=False,
+            session_id=getattr(request, 'session_id', ''),
+            conversation_id=getattr(request, 'conversation_id', ''),
+            pattern="",
+            correlation_id=request.correlation_id,
+            parent_session_id=getattr(request, 'parent_session_id', ''),
+            subagent_goal=getattr(request, 'subagent_goal', ''),
+            expected_siblings=getattr(request, 'expected_siblings', 0),
+        )
+
+        await next(completion_request)
+        logger.debug(
+            f"Subagent completion emitted for "
+            f"correlation={request.correlation_id}, "
+            f"goal={getattr(request, 'subagent_goal', '')}"
+        )
+
+    def filter_tools(self, tools, request):
+        """Apply group/state filtering to the tool set."""
+        return filter_tools_by_group_and_state(
+            tools=tools,
+            requested_groups=getattr(request, 'group', None),
+            current_state=getattr(request, 'state', None),
+        )
+
+    def make_context(self, flow, user, respond=None, streaming=False):
+        """Create a user-aware context wrapper."""
+        return UserAwareContext(flow, user, respond=respond, streaming=streaming)
+
+    def build_history(self, request):
+        """Convert AgentStep history into Action objects."""
+        if not request.history:
+            return []
+        return [
+            Action(
+                thought=h.thought,
+                name=h.action,
+                arguments=h.arguments,
+                observation=h.observation,
+            )
+            for h in request.history
+        ]
+
+    # ---- Streaming callbacks ------------------------------------------------
+
+    def make_think_callback(self, respond, streaming, message_id=""):
+        """Create the think callback for streaming/non-streaming."""
+        async def think(x, is_final=False):
+            logger.debug(f"Think: {x} (is_final={is_final})")
+            if streaming:
+                r = AgentResponse(
+                    chunk_type="thought",
+                    content=x,
+                    end_of_message=is_final,
+                    end_of_dialog=False,
+                    message_id=message_id,
+                )
+            else:
+                r = AgentResponse(
+                    chunk_type="thought",
+                    content=x,
+                    end_of_message=True,
+                    end_of_dialog=False,
+                    message_id=message_id,
+                )
+            await respond(r)
+        return think
+
+    def make_observe_callback(self, respond, streaming, message_id=""):
+        """Create the observe callback for streaming/non-streaming."""
+        async def observe(x, is_final=False):
+            logger.debug(f"Observe: {x} (is_final={is_final})")
+            if streaming:
+                r = AgentResponse(
+                    chunk_type="observation",
+                    content=x,
+                    end_of_message=is_final,
+                    end_of_dialog=False,
+                    message_id=message_id,
+                )
+            else:
+                r = AgentResponse(
+                    chunk_type="observation",
+                    content=x,
+                    end_of_message=True,
+                    end_of_dialog=False,
+                    message_id=message_id,
+                )
+            await respond(r)
+        return observe
+
+    def make_answer_callback(self, respond, streaming, message_id=""):
+        """Create the answer callback for streaming/non-streaming."""
+        async def answer(x):
+            logger.debug(f"Answer: {x}")
+            if streaming:
+                r = AgentResponse(
+                    chunk_type="answer",
+                    content=x,
+                    end_of_message=False,
+                    end_of_dialog=False,
+                    message_id=message_id,
+                )
+            else:
+                r = AgentResponse(
+                    chunk_type="answer",
+                    content=x,
+                    end_of_message=True,
+                    end_of_dialog=False,
+                    message_id=message_id,
+                )
+            await respond(r)
+        return answer
+
+    # ---- Provenance emission ------------------------------------------------
+
+    async def emit_session_triples(self, flow, session_uri, question, user,
+                                   collection, respond, streaming,
+                                   parent_uri=None):
+        """Emit provenance triples for a new session."""
+        timestamp = datetime.utcnow().isoformat() + "Z"
+        triples = set_graph(
+            agent_session_triples(
+                session_uri, question, timestamp,
+                parent_uri=parent_uri,
+            ),
+            GRAPH_RETRIEVAL,
+        )
+        await flow("explainability").send(Triples(
+            metadata=Metadata(
+                id=session_uri,
+                user=user,
+                collection=collection,
+            ),
+            triples=triples,
+        ))
+        logger.debug(f"Emitted session triples for {session_uri}")
+
+        await respond(AgentResponse(
+            chunk_type="explain",
+            content="",
+            explain_id=session_uri,
+            explain_graph=GRAPH_RETRIEVAL,
+            explain_triples=triples,
+        ))
+
+    async def emit_iteration_triples(self, flow, session_id, iteration_num,
+                                     session_uri, act, request, respond,
+                                     streaming):
+        """Emit provenance triples for an iteration (Analysis+ToolUse)."""
+        iteration_uri = agent_iteration_uri(session_id, iteration_num)
+
+        if iteration_num > 1:
+            # Chain through previous Observation (last entity in prior cycle)
+            iter_question_uri = None
+            iter_previous_uri = agent_observation_uri(session_id, iteration_num - 1)
+        else:
+            iter_question_uri = session_uri
+            iter_previous_uri = None
+
+        # Save thought to librarian
+        thought_doc_id = None
+        if act.thought:
+            thought_doc_id = (
+                f"urn:trustgraph:agent:{session_id}/i{iteration_num}/thought"
+            )
+            try:
+                await self.processor.save_answer_content(
+                    doc_id=thought_doc_id,
+                    user=request.user,
+                    content=act.thought,
+                    title=f"Agent Thought: {act.name}",
+                )
+            except Exception as e:
+                logger.warning(f"Failed to save thought to librarian: {e}")
+                thought_doc_id = None
+
+        thought_entity_uri = agent_thought_uri(session_id, iteration_num)
+
+        iter_triples = set_graph(
+            agent_iteration_triples(
+                iteration_uri,
+                question_uri=iter_question_uri,
+                previous_uri=iter_previous_uri,
+                action=act.name,
+                arguments=act.arguments,
+                thought_uri=thought_entity_uri if thought_doc_id else None,
+                thought_document_id=thought_doc_id,
+            ),
+            GRAPH_RETRIEVAL,
+        )
+        await flow("explainability").send(Triples(
+            metadata=Metadata(
+                id=iteration_uri,
+                user=request.user,
+                collection=getattr(request, 'collection', 'default'),
+            ),
+            triples=iter_triples,
+        ))
+        logger.debug(f"Emitted iteration triples for {iteration_uri}")
+
+        await respond(AgentResponse(
+            chunk_type="explain",
+            content="",
+            explain_id=iteration_uri,
+            explain_graph=GRAPH_RETRIEVAL,
+            explain_triples=iter_triples,
+        ))
+
+    async def emit_observation_triples(self, flow, session_id, iteration_num,
+                                       observation_text, request, respond,
+                                       context=None):
+        """Emit provenance triples for a standalone Observation entity."""
+        iteration_uri = agent_iteration_uri(session_id, iteration_num)
+        observation_entity_uri = agent_observation_uri(session_id, iteration_num)
+
+        # Derive from the last sub-trace entity if available (e.g. Synthesis),
+        # otherwise fall back to the iteration (Analysis+ToolUse).
+        parent_uri = iteration_uri
+        if context and getattr(context, 'last_sub_explain_uri', None):
+            parent_uri = context.last_sub_explain_uri
+
+        # Save observation to librarian
+        observation_doc_id = None
+        if observation_text:
+            observation_doc_id = (
+                f"urn:trustgraph:agent:{session_id}/i{iteration_num}/observation"
+            )
+            try:
+                await self.processor.save_answer_content(
+                    doc_id=observation_doc_id,
+                    user=request.user,
+                    content=observation_text,
+                    title=f"Agent Observation",
+                )
+            except Exception as e:
+                logger.warning(f"Failed to save observation to librarian: {e}")
+                observation_doc_id = None
+
+        obs_triples = set_graph(
+            agent_observation_triples(
+                observation_entity_uri,
+                parent_uri,
+                document_id=observation_doc_id,
+            ),
+            GRAPH_RETRIEVAL,
+        )
+        await flow("explainability").send(Triples(
+            metadata=Metadata(
+                id=observation_entity_uri,
+                user=request.user,
+                collection=getattr(request, 'collection', 'default'),
+            ),
+            triples=obs_triples,
+        ))
+        logger.debug(f"Emitted observation triples for {observation_entity_uri}")
+
+        await respond(AgentResponse(
+            chunk_type="explain",
+            content="",
+            explain_id=observation_entity_uri,
+            explain_graph=GRAPH_RETRIEVAL,
+            explain_triples=obs_triples,
+        ))
+
+    async def emit_final_triples(self, flow, session_id, iteration_num,
+                                 session_uri, answer_text, request, respond,
+                                 streaming):
+        """Emit provenance triples for the final answer and save to librarian."""
+        final_uri = agent_final_uri(session_id)
+
+        if iteration_num > 1:
+            # Chain through last Observation (last entity in prior cycle)
+            final_question_uri = None
+            final_previous_uri = agent_observation_uri(session_id, iteration_num - 1)
+        else:
+            final_question_uri = session_uri
+            final_previous_uri = None
+
+        # Save answer to librarian
+        answer_doc_id = None
+        if answer_text:
+            answer_doc_id = f"urn:trustgraph:agent:{session_id}/answer"
+            try:
+                await self.processor.save_answer_content(
+                    doc_id=answer_doc_id,
+                    user=request.user,
+                    content=answer_text,
+                    title=f"Agent Answer: {request.question[:50]}...",
+                )
+                logger.debug(f"Saved answer to librarian: {answer_doc_id}")
+            except Exception as e:
+                logger.warning(f"Failed to save answer to librarian: {e}")
+                answer_doc_id = None
+
+        final_triples = set_graph(
+            agent_final_triples(
+                final_uri,
+                question_uri=final_question_uri,
+                previous_uri=final_previous_uri,
+                document_id=answer_doc_id,
+            ),
+            GRAPH_RETRIEVAL,
+        )
+        await flow("explainability").send(Triples(
+            metadata=Metadata(
+                id=final_uri,
+                user=request.user,
+                collection=getattr(request, 'collection', 'default'),
+            ),
+            triples=final_triples,
+        ))
+        logger.debug(f"Emitted final triples for {final_uri}")
+
+        await respond(AgentResponse(
+            chunk_type="explain",
+            content="",
+            explain_id=final_uri,
+            explain_graph=GRAPH_RETRIEVAL,
+            explain_triples=final_triples,
+        ))
+
+    # ---- Orchestrator provenance helpers ------------------------------------
+
+    async def emit_decomposition_triples(
+        self, flow, session_id, session_uri, goals, user, collection,
+        respond, streaming,
+    ):
+        """Emit provenance for a supervisor decomposition step."""
+        uri = agent_decomposition_uri(session_id)
+        triples = set_graph(
+            agent_decomposition_triples(uri, session_uri, goals),
+            GRAPH_RETRIEVAL,
+        )
+        await flow("explainability").send(Triples(
+            metadata=Metadata(id=uri, user=user, collection=collection),
+            triples=triples,
+        ))
+        await respond(AgentResponse(
+            chunk_type="explain", content="",
+            explain_id=uri, explain_graph=GRAPH_RETRIEVAL,
+            explain_triples=triples,
+        ))
+
+    async def emit_finding_triples(
+        self, flow, session_id, index, goal, answer_text, user, collection,
+        respond, streaming, subagent_session_id="",
+    ):
+        """Emit provenance for a subagent finding."""
+        uri = agent_finding_uri(session_id, index)
+
+        # Derive from the subagent's conclusion if available,
+        # otherwise fall back to the decomposition.
+        if subagent_session_id:
+            parent_uri = agent_final_uri(subagent_session_id)
+        else:
+            parent_uri = agent_decomposition_uri(session_id)
+
+        doc_id = f"urn:trustgraph:agent:{session_id}/finding/{index}/doc"
+        try:
+            await self.processor.save_answer_content(
+                doc_id=doc_id, user=user,
+                content=answer_text,
+                title=f"Finding: {goal[:60]}",
+            )
+        except Exception as e:
+            logger.warning(f"Failed to save finding to librarian: {e}")
+            doc_id = None
+
+        triples = set_graph(
+            agent_finding_triples(uri, parent_uri, goal, doc_id),
+            GRAPH_RETRIEVAL,
+        )
+        await flow("explainability").send(Triples(
+            metadata=Metadata(id=uri, user=user, collection=collection),
+            triples=triples,
+        ))
+        await respond(AgentResponse(
+            chunk_type="explain", content="",
+            explain_id=uri, explain_graph=GRAPH_RETRIEVAL,
+            explain_triples=triples,
+        ))
+
+    async def emit_plan_triples(
+        self, flow, session_id, session_uri, steps, user, collection,
+        respond, streaming,
+    ):
+        """Emit provenance for a plan creation."""
+        uri = agent_plan_uri(session_id)
+        triples = set_graph(
+            agent_plan_triples(uri, session_uri, steps),
+            GRAPH_RETRIEVAL,
+        )
+        await flow("explainability").send(Triples(
+            metadata=Metadata(id=uri, user=user, collection=collection),
+            triples=triples,
+        ))
+        await respond(AgentResponse(
+            chunk_type="explain", content="",
+            explain_id=uri, explain_graph=GRAPH_RETRIEVAL,
+            explain_triples=triples,
+        ))
+
+    async def emit_step_result_triples(
+        self, flow, session_id, index, goal, answer_text, user, collection,
+        respond, streaming,
+    ):
+        """Emit provenance for a plan step result."""
+        uri = agent_step_result_uri(session_id, index)
+        plan_uri = agent_plan_uri(session_id)
+
+        doc_id = f"urn:trustgraph:agent:{session_id}/step/{index}/doc"
+        try:
+            await self.processor.save_answer_content(
+                doc_id=doc_id, user=user,
+                content=answer_text,
+                title=f"Step result: {goal[:60]}",
+            )
+        except Exception as e:
+            logger.warning(f"Failed to save step result to librarian: {e}")
+            doc_id = None
+
+        triples = set_graph(
+            agent_step_result_triples(uri, plan_uri, goal, doc_id),
+            GRAPH_RETRIEVAL,
+        )
+        await flow("explainability").send(Triples(
+            metadata=Metadata(id=uri, user=user, collection=collection),
+            triples=triples,
+        ))
+        await respond(AgentResponse(
+            chunk_type="explain", content="",
+            explain_id=uri, explain_graph=GRAPH_RETRIEVAL,
+            explain_triples=triples,
+        ))
+
+    async def emit_synthesis_triples(
+        self, flow, session_id, previous_uris, answer_text, user, collection,
+        respond, streaming,
+    ):
+        """Emit provenance for a synthesis answer."""
+        uri = agent_synthesis_uri(session_id)
+
+        doc_id = f"urn:trustgraph:agent:{session_id}/synthesis/doc"
+        try:
+            await self.processor.save_answer_content(
+                doc_id=doc_id, user=user,
+                content=answer_text,
+                title="Synthesis",
+            )
+        except Exception as e:
+            logger.warning(f"Failed to save synthesis to librarian: {e}")
+            doc_id = None
+
+        triples = set_graph(
+            agent_synthesis_triples(uri, previous_uris, doc_id),
+            GRAPH_RETRIEVAL,
+        )
+        await flow("explainability").send(Triples(
+            metadata=Metadata(id=uri, user=user, collection=collection),
+            triples=triples,
+        ))
+        await respond(AgentResponse(
+            chunk_type="explain", content="",
+            explain_id=uri, explain_graph=GRAPH_RETRIEVAL,
+            explain_triples=triples,
+        ))
+
+    # ---- Response helpers ---------------------------------------------------
+
+    async def prompt_as_answer(self, client, prompt_id, variables,
+                               respond, streaming, message_id=""):
+        """Call a prompt template, forwarding chunks as answer
+        AgentResponse messages when streaming is enabled.
+
+        Returns the full accumulated answer text (needed for provenance).
+        """
+        if streaming:
+            accumulated = []
+
+            async def on_chunk(text, end_of_stream):
+                if text:
+                    accumulated.append(text)
+                    await respond(AgentResponse(
+                        chunk_type="answer",
+                        content=text,
+                        end_of_message=False,
+                        end_of_dialog=False,
+                        message_id=message_id,
+                    ))
+
+            await client.prompt(
+                id=prompt_id,
+                variables=variables,
+                streaming=True,
+                chunk_callback=on_chunk,
+            )
+
+            return "".join(accumulated)
+        else:
+            return await client.prompt(
+                id=prompt_id,
+                variables=variables,
+            )
+
+    async def send_final_response(self, respond, streaming, answer_text,
+                                  already_streamed=False, message_id=""):
+        """Send the answer content and end-of-dialog marker.
+
+        Args:
+            already_streamed: If True, answer chunks were already sent
+                via streaming callbacks (e.g. ReactPattern). Only the
+                end-of-dialog marker is emitted.
+            message_id: Provenance URI for the answer entity.
+        """
+        if streaming and not already_streamed:
+            # Answer wasn't streamed yet — send it as a chunk first
+            if answer_text:
+                await respond(AgentResponse(
+                    chunk_type="answer",
+                    content=answer_text,
+                    end_of_message=False,
+                    end_of_dialog=False,
+                    message_id=message_id,
+                ))
+        if streaming:
+            # End-of-dialog marker
+            await respond(AgentResponse(
+                chunk_type="answer",
+                content="",
+                end_of_message=True,
+                end_of_dialog=True,
+                message_id=message_id,
+            ))
+        else:
+            await respond(AgentResponse(
+                chunk_type="answer",
+                content=answer_text,
+                end_of_message=True,
+                end_of_dialog=True,
+                message_id=message_id,
+            ))
+
+    def build_next_request(self, request, history, session_id, collection,
+                           streaming, next_state):
+        """Build the AgentRequest for the next iteration."""
+        return AgentRequest(
+            question=request.question,
+            state=next_state,
+            group=getattr(request, 'group', []),
+            history=[
+                AgentStep(
+                    thought=h.thought,
+                    action=h.name,
+                    arguments={k: str(v) for k, v in h.arguments.items()},
+                    observation=h.observation,
+                )
+                for h in history
+            ],
+            user=request.user,
+            collection=collection,
+            streaming=streaming,
+            session_id=session_id,
+            # Preserve orchestration fields
+            conversation_id=getattr(request, 'conversation_id', ''),
+            pattern=getattr(request, 'pattern', ''),
+            task_type=getattr(request, 'task_type', ''),
+            framing=getattr(request, 'framing', ''),
+            correlation_id=getattr(request, 'correlation_id', ''),
+            parent_session_id=getattr(request, 'parent_session_id', ''),
+            subagent_goal=getattr(request, 'subagent_goal', ''),
+            expected_siblings=getattr(request, 'expected_siblings', 0),
+        )
+
+    async def iterate(self, request, respond, next, flow):
+        """
+        Perform one iteration of this pattern.
+
+        Must be implemented by subclasses.
+        """
+        raise NotImplementedError

trustgraph-flow/trustgraph/agent/orchestrator/plan_pattern.py

@@ -0,0 +1,383 @@
+"""
+PlanThenExecutePattern — structured planning followed by step execution.
+
+Phase 1 (planning): LLM produces a structured plan of steps.
+Phase 2 (execution): Each step is executed via single-shot tool call.
+"""
+
+import json
+import logging
+import uuid
+
+from ... schema import AgentRequest, AgentResponse, AgentStep, PlanStep
+
+from trustgraph.provenance import (
+    agent_step_result_uri as make_step_result_uri,
+    agent_thought_uri,
+    agent_observation_uri,
+    agent_synthesis_uri,
+)
+
+from . pattern_base import PatternBase
+
+logger = logging.getLogger(__name__)
+
+
+class PlanThenExecutePattern(PatternBase):
+    """
+    Plan-then-Execute pattern.
+
+    History tracks the current phase via AgentStep.step_type:
+      - "plan" step: contains the plan in step.plan
+      - "execute" step: a normal react iteration executing a plan step
+
+    On the first call (empty history), a planning iteration is run.
+    Subsequent calls execute the next pending plan step via ReACT.
+    """
+
+    async def iterate(self, request, respond, next, flow):
+
+        streaming = getattr(request, 'streaming', False)
+        session_id = getattr(request, 'session_id', '') or str(uuid.uuid4())
+        collection = getattr(request, 'collection', 'default')
+
+        history = self.build_history(request)
+        iteration_num = len(request.history) + 1
+        session_uri = self.processor.provenance_session_uri(session_id)
+
+        # Emit session provenance on first iteration
+        if iteration_num == 1:
+            await self.emit_session_triples(
+                flow, session_uri, request.question,
+                request.user, collection, respond, streaming,
+            )
+
+        logger.info(
+            f"PlanThenExecutePattern iteration {iteration_num}: "
+            f"{request.question}"
+        )
+
+        if iteration_num >= self.processor.max_iterations:
+            raise RuntimeError("Too many agent iterations")
+
+        # Determine current phase by checking history for a plan step
+        plan = self._extract_plan(request.history)
+
+        if plan is None:
+            await self._planning_iteration(
+                request, respond, next, flow,
+                session_id, collection, streaming, session_uri,
+                iteration_num,
+            )
+        else:
+            await self._execution_iteration(
+                request, respond, next, flow,
+                session_id, collection, streaming, session_uri,
+                iteration_num, plan,
+            )
+
+    def _extract_plan(self, history):
+        """Find the most recent plan from history.
+
+        Checks execute steps first (they carry the updated plan with
+        completion statuses), then falls back to the original plan step.
+        """
+        if not history:
+            return None
+        for step in reversed(history):
+            if step.plan:
+                return list(step.plan)
+        return None
+
+    def _find_next_pending_step(self, plan):
+        """Return index of the next pending step, or None if all done."""
+        for i, step in enumerate(plan):
+            if getattr(step, 'status', 'pending') == 'pending':
+                return i
+        return None
+
+    async def _planning_iteration(self, request, respond, next, flow,
+                                  session_id, collection, streaming,
+                                  session_uri, iteration_num):
+        """Ask the LLM to produce a structured plan."""
+
+        think = self.make_think_callback(respond, streaming)
+
+        tools = self.filter_tools(self.processor.agent.tools, request)
+        framing = getattr(request, 'framing', '')
+
+        context = self.make_context(
+            flow, request.user,
+            respond=respond, streaming=streaming,
+        )
+        client = context("prompt-request")
+
+        # Use the plan-create prompt template
+        plan_steps = await client.prompt(
+            id="plan-create",
+            variables={
+                "question": request.question,
+                "framing": framing,
+                "tools": [
+                    {"name": t.name, "description": t.description}
+                    for t in tools.values()
+                ],
+            },
+        )
+
+        # Validate we got a list
+        if not isinstance(plan_steps, list) or not plan_steps:
+            logger.warning("plan-create returned invalid result, falling back to single step")
+            plan_steps = [{"goal": "Answer the question directly", "tool_hint": "", "depends_on": []}]
+
+        # Emit thought about the plan
+        thought_text = f"Created plan with {len(plan_steps)} steps"
+        await think(thought_text, is_final=True)
+
+        # Emit plan provenance
+        step_goals = [ps.get("goal", "") for ps in plan_steps]
+        await self.emit_plan_triples(
+            flow, session_id, session_uri, step_goals,
+            request.user, collection, respond, streaming,
+        )
+
+        # Build PlanStep objects
+        plan_agent_steps = [
+            PlanStep(
+                goal=ps.get("goal", ""),
+                tool_hint=ps.get("tool_hint", ""),
+                depends_on=ps.get("depends_on", []),
+                status="pending",
+                result="",
+            )
+            for ps in plan_steps
+        ]
+
+        # Create a plan step in history
+        plan_history_step = AgentStep(
+            thought=thought_text,
+            action="plan",
+            arguments={},
+            observation=json.dumps(plan_steps),
+            step_type="plan",
+            plan=plan_agent_steps,
+        )
+
+        # Build next request with plan in history
+        new_history = list(request.history) + [plan_history_step]
+        r = AgentRequest(
+            question=request.question,
+            state=request.state,
+            group=getattr(request, 'group', []),
+            history=new_history,
+            user=request.user,
+            collection=collection,
+            streaming=streaming,
+            session_id=session_id,
+            conversation_id=getattr(request, 'conversation_id', ''),
+            pattern=getattr(request, 'pattern', ''),
+            task_type=getattr(request, 'task_type', ''),
+            framing=getattr(request, 'framing', ''),
+            correlation_id=getattr(request, 'correlation_id', ''),
+            parent_session_id=getattr(request, 'parent_session_id', ''),
+            subagent_goal=getattr(request, 'subagent_goal', ''),
+            expected_siblings=getattr(request, 'expected_siblings', 0),
+        )
+        await next(r)
+
+    async def _execution_iteration(self, request, respond, next, flow,
+                                   session_id, collection, streaming,
+                                   session_uri, iteration_num, plan):
+        """Execute the next pending plan step via single-shot tool call."""
+
+        pending_idx = self._find_next_pending_step(plan)
+
+        if pending_idx is None:
+            # All steps done — synthesise final answer
+            await self._synthesise(
+                request, respond, next, flow,
+                session_id, collection, streaming,
+                session_uri, iteration_num, plan,
+            )
+            return
+
+        current_step = plan[pending_idx]
+        goal = getattr(current_step, 'goal', '') or str(current_step)
+
+        logger.info(f"Executing plan step {pending_idx}: {goal}")
+
+        thought_msg_id = agent_thought_uri(session_id, iteration_num)
+        observation_msg_id = agent_observation_uri(session_id, iteration_num)
+
+        think = self.make_think_callback(respond, streaming, message_id=thought_msg_id)
+        observe = self.make_observe_callback(respond, streaming, message_id=observation_msg_id)
+
+        # Gather results from dependencies
+        previous_results = []
+        depends_on = getattr(current_step, 'depends_on', [])
+        if depends_on:
+            for dep_idx in depends_on:
+                if 0 <= dep_idx < len(plan):
+                    dep_step = plan[dep_idx]
+                    dep_result = getattr(dep_step, 'result', '')
+                    if dep_result:
+                        previous_results.append({
+                            "index": dep_idx,
+                            "result": dep_result,
+                        })
+
+        tools = self.filter_tools(self.processor.agent.tools, request)
+        context = self.make_context(
+            flow, request.user,
+            respond=respond, streaming=streaming,
+        )
+
+        # Set current explain URI so tools can link sub-traces
+        context.current_explain_uri = make_step_result_uri(
+            session_id, pending_idx,
+        )
+
+        client = context("prompt-request")
+
+        # Single-shot: ask LLM which tool + arguments to use for this goal
+        tool_call = await client.prompt(
+            id="plan-step-execute",
+            variables={
+                "goal": goal,
+                "previous_results": previous_results,
+                "tools": [
+                    {
+                        "name": t.name,
+                        "description": t.description,
+                        "arguments": [
+                            {"name": a.name, "type": a.type, "description": a.description}
+                            for a in t.arguments
+                        ],
+                    }
+                    for t in tools.values()
+                ],
+            },
+        )
+
+        tool_name = tool_call.get("tool", "")
+        tool_arguments = tool_call.get("arguments", {})
+
+        await think(
+            f"Step {pending_idx}: {goal} → calling {tool_name}",
+            is_final=True,
+        )
+
+        # Invoke the tool directly
+        if tool_name in tools:
+            tool = tools[tool_name]
+            resp = await tool.implementation(context).invoke(**tool_arguments)
+            step_result = resp.strip() if isinstance(resp, str) else str(resp).strip()
+        else:
+            logger.warning(
+                f"Plan step {pending_idx}: LLM selected unknown tool "
+                f"'{tool_name}', available: {list(tools.keys())}"
+            )
+            step_result = f"Error: tool '{tool_name}' not found"
+
+        await observe(step_result, is_final=True)
+
+        # Update plan step status
+        plan[pending_idx] = PlanStep(
+            goal=goal,
+            tool_hint=getattr(current_step, 'tool_hint', ''),
+            depends_on=getattr(current_step, 'depends_on', []),
+            status="completed",
+            result=step_result,
+        )
+
+        # Emit step result provenance
+        await self.emit_step_result_triples(
+            flow, session_id, pending_idx, goal, step_result,
+            request.user, collection, respond, streaming,
+        )
+
+        # Build execution step for history
+        exec_step = AgentStep(
+            thought=f"Executing plan step {pending_idx}: {goal}",
+            action=tool_name,
+            arguments={k: str(v) for k, v in tool_arguments.items()},
+            observation=step_result,
+            step_type="execute",
+            plan=plan,
+        )
+
+        new_history = list(request.history) + [exec_step]
+
+        r = AgentRequest(
+            question=request.question,
+            state=request.state,
+            group=getattr(request, 'group', []),
+            history=new_history,
+            user=request.user,
+            collection=collection,
+            streaming=streaming,
+            session_id=session_id,
+            conversation_id=getattr(request, 'conversation_id', ''),
+            pattern=getattr(request, 'pattern', ''),
+            task_type=getattr(request, 'task_type', ''),
+            framing=getattr(request, 'framing', ''),
+            correlation_id=getattr(request, 'correlation_id', ''),
+            parent_session_id=getattr(request, 'parent_session_id', ''),
+            subagent_goal=getattr(request, 'subagent_goal', ''),
+            expected_siblings=getattr(request, 'expected_siblings', 0),
+        )
+        await next(r)
+
+    async def _synthesise(self, request, respond, next, flow,
+                          session_id, collection, streaming,
+                          session_uri, iteration_num, plan):
+        """Synthesise a final answer from all completed plan step results."""
+
+        think = self.make_think_callback(respond, streaming)
+        framing = getattr(request, 'framing', '')
+
+        context = self.make_context(
+            flow, request.user,
+            respond=respond, streaming=streaming,
+        )
+        client = context("prompt-request")
+
+        # Use the plan-synthesise prompt template
+        steps_data = []
+        for i, step in enumerate(plan):
+            steps_data.append({
+                "index": i,
+                "goal": getattr(step, 'goal', f'Step {i}'),
+                "result": getattr(step, 'result', ''),
+            })
+
+        await think("Synthesising final answer from plan results", is_final=True)
+
+        synthesis_msg_id = agent_synthesis_uri(session_id)
+
+        response_text = await self.prompt_as_answer(
+            client, "plan-synthesise",
+            variables={
+                "question": request.question,
+                "framing": framing,
+                "steps": steps_data,
+            },
+            respond=respond,
+            streaming=streaming,
+            message_id=synthesis_msg_id,
+        )
+
+        # Emit synthesis provenance (links back to last step result)
+        last_step_uri = make_step_result_uri(session_id, len(plan) - 1)
+        await self.emit_synthesis_triples(
+            flow, session_id, last_step_uri,
+            response_text, request.user, collection, respond, streaming,
+        )
+
+        if self.is_subagent(request):
+            await self.emit_subagent_completion(request, next, response_text)
+        else:
+            await self.send_final_response(
+                respond, streaming, response_text, already_streamed=streaming,
+                message_id=synthesis_msg_id,
+            )

trustgraph-flow/trustgraph/agent/orchestrator/react_pattern.py

@@ -0,0 +1,171 @@
+"""
+ReactPattern — extracted from the existing agent_manager.py.
+
+Implements the ReACT (Reasoning + Acting) loop: think, select a tool,
+observe the result, repeat until a final answer is produced.
+"""
+
+import json
+import logging
+import uuid
+
+from ... schema import AgentRequest, AgentResponse, AgentStep
+
+from trustgraph.provenance import (
+    agent_iteration_uri,
+    agent_thought_uri,
+    agent_observation_uri,
+    agent_final_uri,
+    agent_decomposition_uri,
+)
+
+from ..react.agent_manager import AgentManager
+from ..react.types import Action, Final
+from ..tool_filter import get_next_state
+
+from . pattern_base import PatternBase
+
+logger = logging.getLogger(__name__)
+
+
+class ReactPattern(PatternBase):
+    """
+    ReACT pattern: interleaved reasoning and action.
+
+    Each iterate() call performs one reason/act cycle. If the LLM
+    produces a Final answer the dialog completes; otherwise the action
+    result is appended to history and a next-request is emitted.
+    """
+
+    async def iterate(self, request, respond, next, flow):
+
+        streaming = getattr(request, 'streaming', False)
+        session_id = getattr(request, 'session_id', '') or str(uuid.uuid4())
+        collection = getattr(request, 'collection', 'default')
+
+        history = self.build_history(request)
+        iteration_num = len(history) + 1
+        session_uri = self.processor.provenance_session_uri(session_id)
+
+        # Emit session provenance on first iteration
+        if iteration_num == 1:
+            # Subagents link back to the parent's decomposition
+            parent_session_id = getattr(request, 'parent_session_id', '')
+            parent_uri = (
+                agent_decomposition_uri(parent_session_id)
+                if parent_session_id else None
+            )
+            await self.emit_session_triples(
+                flow, session_uri, request.question,
+                request.user, collection, respond, streaming,
+                parent_uri=parent_uri,
+            )
+
+        logger.info(f"ReactPattern iteration {iteration_num}: {request.question}")
+
+        if len(history) >= self.processor.max_iterations:
+            raise RuntimeError("Too many agent iterations")
+
+        # Compute URIs upfront for message_id
+        thought_msg_id = agent_thought_uri(session_id, iteration_num)
+        observation_msg_id = agent_observation_uri(session_id, iteration_num)
+        answer_msg_id = agent_final_uri(session_id)
+
+        # Build callbacks
+        think = self.make_think_callback(respond, streaming, message_id=thought_msg_id)
+        observe = self.make_observe_callback(respond, streaming, message_id=observation_msg_id)
+        answer_cb = self.make_answer_callback(respond, streaming, message_id=answer_msg_id)
+
+        # Filter tools
+        filtered_tools = self.filter_tools(
+            self.processor.agent.tools, request,
+        )
+
+        # Create temporary agent with filtered tools and optional framing
+        additional_context = self.processor.agent.additional_context
+        framing = getattr(request, 'framing', '')
+        if framing:
+            if additional_context:
+                additional_context = f"{additional_context}\n\n{framing}"
+            else:
+                additional_context = framing
+
+        temp_agent = AgentManager(
+            tools=filtered_tools,
+            additional_context=additional_context,
+        )
+
+        context = self.make_context(
+            flow, request.user,
+            respond=respond, streaming=streaming,
+        )
+
+        # Set current explain URI so tools can link sub-traces
+        context.current_explain_uri = agent_iteration_uri(
+            session_id, iteration_num,
+        )
+
+        # Callback: emit Analysis+ToolUse triples before tool executes
+        async def on_action(act):
+            await self.emit_iteration_triples(
+                flow, session_id, iteration_num, session_uri,
+                act, request, respond, streaming,
+            )
+
+        act = await temp_agent.react(
+            question=request.question,
+            history=history,
+            think=think,
+            observe=observe,
+            answer=answer_cb,
+            context=context,
+            streaming=streaming,
+            on_action=on_action,
+        )
+
+        logger.debug(f"Action: {act}")
+
+        if isinstance(act, Final):
+
+            if isinstance(act.final, str):
+                f = act.final
+            else:
+                f = json.dumps(act.final)
+
+            # Emit final provenance
+            await self.emit_final_triples(
+                flow, session_id, iteration_num, session_uri,
+                f, request, respond, streaming,
+            )
+
+            if self.is_subagent(request):
+                await self.emit_subagent_completion(request, next, f)
+            else:
+                await self.send_final_response(
+                    respond, streaming, f, already_streamed=streaming,
+                    message_id=answer_msg_id,
+                )
+            return
+
+        # Emit observation provenance after tool execution
+        await self.emit_observation_triples(
+            flow, session_id, iteration_num,
+            act.observation, request, respond,
+            context=context,
+        )
+
+        history.append(act)
+
+        # Handle state transitions
+        next_state = request.state
+        if act.name in filtered_tools:
+            executed_tool = filtered_tools[act.name]
+            next_state = get_next_state(executed_tool, request.state or "undefined")
+
+        r = self.build_next_request(
+            request, history, session_id, collection,
+            streaming, next_state,
+        )
+        await next(r)
+
+        logger.debug("ReactPattern iteration complete")

trustgraph-flow/trustgraph/agent/orchestrator/service.py

@@ -0,0 +1,594 @@
+"""
+Agent orchestrator service — multi-pattern drop-in replacement for
+agent-manager-react.
+
+Uses the same service identity and Pulsar queues. Adds meta-routing
+to select between ReactPattern, PlanThenExecutePattern, and
+SupervisorPattern at runtime.
+"""
+
+import asyncio
+import base64
+import json
+import functools
+import logging
+import uuid
+from datetime import datetime
+
+from ... base import AgentService, TextCompletionClientSpec, PromptClientSpec
+from ... base import GraphRagClientSpec, ToolClientSpec, StructuredQueryClientSpec
+from ... base import RowEmbeddingsQueryClientSpec, EmbeddingsClientSpec
+from ... base import ProducerSpec
+from ... base import Consumer, Producer
+from ... base import ConsumerMetrics, ProducerMetrics
+
+from ... schema import AgentRequest, AgentResponse, AgentStep, Error
+from ... schema import Triples, Metadata
+from ... schema import LibrarianRequest, LibrarianResponse, DocumentMetadata
+from ... schema import librarian_request_queue, librarian_response_queue
+
+from trustgraph.provenance import (
+    agent_session_uri,
+    GRAPH_RETRIEVAL,
+)
+
+from ..react.tools import (
+    KnowledgeQueryImpl, TextCompletionImpl, McpToolImpl, PromptImpl,
+    StructuredQueryImpl, RowEmbeddingsQueryImpl, ToolServiceImpl,
+)
+from ..react.agent_manager import AgentManager
+from ..tool_filter import validate_tool_config
+from ..react.types import Final, Action, Tool, Argument
+
+from . meta_router import MetaRouter
+from . pattern_base import PatternBase, UserAwareContext
+from . react_pattern import ReactPattern
+from . plan_pattern import PlanThenExecutePattern
+from . supervisor_pattern import SupervisorPattern
+from . aggregator import Aggregator
+
+logger = logging.getLogger(__name__)
+
+default_ident = "agent-manager"
+default_max_iterations = 10
+default_librarian_request_queue = librarian_request_queue
+default_librarian_response_queue = librarian_response_queue
+
+
+class Processor(AgentService):
+
+    def __init__(self, **params):
+
+        id = params.get("id")
+
+        self.max_iterations = int(
+            params.get("max_iterations", default_max_iterations)
+        )
+
+        self.config_key = params.get("config_type", "agent")
+
+        super(Processor, self).__init__(
+            **params | {
+                "id": id,
+                "max_iterations": self.max_iterations,
+                "config_type": self.config_key,
+            }
+        )
+
+        self.agent = AgentManager(
+            tools={},
+            additional_context="",
+        )
+
+        self.tool_service_clients = {}
+
+        # Patterns
+        self.react_pattern = ReactPattern(self)
+        self.plan_pattern = PlanThenExecutePattern(self)
+        self.supervisor_pattern = SupervisorPattern(self)
+
+        # Aggregator for supervisor fan-in
+        self.aggregator = Aggregator()
+
+        # Meta-router (initialised on first config load)
+        self.meta_router = None
+
+        self.register_config_handler(
+            self.on_tools_config, types=["tool", "tool-service"]
+        )
+
+        self.register_specification(
+            TextCompletionClientSpec(
+                request_name="text-completion-request",
+                response_name="text-completion-response",
+            )
+        )
+
+        self.register_specification(
+            GraphRagClientSpec(
+                request_name="graph-rag-request",
+                response_name="graph-rag-response",
+            )
+        )
+
+        self.register_specification(
+            PromptClientSpec(
+                request_name="prompt-request",
+                response_name="prompt-response",
+            )
+        )
+
+        self.register_specification(
+            ToolClientSpec(
+                request_name="mcp-tool-request",
+                response_name="mcp-tool-response",
+            )
+        )
+
+        self.register_specification(
+            StructuredQueryClientSpec(
+                request_name="structured-query-request",
+                response_name="structured-query-response",
+            )
+        )
+
+        self.register_specification(
+            EmbeddingsClientSpec(
+                request_name="embeddings-request",
+                response_name="embeddings-response",
+            )
+        )
+
+        self.register_specification(
+            RowEmbeddingsQueryClientSpec(
+                request_name="row-embeddings-query-request",
+                response_name="row-embeddings-query-response",
+            )
+        )
+
+        # Explainability producer
+        self.register_specification(
+            ProducerSpec(
+                name="explainability",
+                schema=Triples,
+            )
+        )
+
+        # Librarian client
+        librarian_request_q = params.get(
+            "librarian_request_queue", default_librarian_request_queue
+        )
+        librarian_response_q = params.get(
+            "librarian_response_queue", default_librarian_response_queue
+        )
+
+        librarian_request_metrics = ProducerMetrics(
+            processor=id, flow=None, name="librarian-request"
+        )
+
+        self.librarian_request_producer = Producer(
+            backend=self.pubsub,
+            topic=librarian_request_q,
+            schema=LibrarianRequest,
+            metrics=librarian_request_metrics,
+        )
+
+        librarian_response_metrics = ConsumerMetrics(
+            processor=id, flow=None, name="librarian-response"
+        )
+
+        self.librarian_response_consumer = Consumer(
+            taskgroup=self.taskgroup,
+            backend=self.pubsub,
+            flow=None,
+            topic=librarian_response_q,
+            subscriber=f"{id}-librarian",
+            schema=LibrarianResponse,
+            handler=self.on_librarian_response,
+            metrics=librarian_response_metrics,
+        )
+
+        self.pending_librarian_requests = {}
+
+    async def start(self):
+        await super(Processor, self).start()
+        await self.librarian_request_producer.start()
+        await self.librarian_response_consumer.start()
+
+    async def on_librarian_response(self, msg, consumer, flow):
+        response = msg.value()
+        request_id = msg.properties().get("id")
+
+        if request_id in self.pending_librarian_requests:
+            future = self.pending_librarian_requests.pop(request_id)
+            future.set_result(response)
+
+    async def save_answer_content(self, doc_id, user, content, title=None,
+                                  timeout=120):
+        request_id = str(uuid.uuid4())
+
+        doc_metadata = DocumentMetadata(
+            id=doc_id,
+            user=user,
+            kind="text/plain",
+            title=title or "Agent Answer",
+            document_type="answer",
+        )
+
+        request = LibrarianRequest(
+            operation="add-document",
+            document_id=doc_id,
+            document_metadata=doc_metadata,
+            content=base64.b64encode(content.encode("utf-8")).decode("utf-8"),
+            user=user,
+        )
+
+        future = asyncio.get_event_loop().create_future()
+        self.pending_librarian_requests[request_id] = future
+
+        try:
+            await self.librarian_request_producer.send(
+                request, properties={"id": request_id}
+            )
+            response = await asyncio.wait_for(future, timeout=timeout)
+
+            if response.error:
+                raise RuntimeError(
+                    f"Librarian error saving answer: "
+                    f"{response.error.type}: {response.error.message}"
+                )
+            return doc_id
+
+        except asyncio.TimeoutError:
+            self.pending_librarian_requests.pop(request_id, None)
+            raise RuntimeError(f"Timeout saving answer document {doc_id}")
+
+    def provenance_session_uri(self, session_id):
+        return agent_session_uri(session_id)
+
+    async def on_tools_config(self, config, version):
+
+        logger.info(f"Loading configuration version {version}")
+
+        try:
+            tools = {}
+
+            # Load tool-service configurations
+            tool_services = {}
+            if "tool-service" in config:
+                for service_id, service_value in config["tool-service"].items():
+                    service_data = json.loads(service_value)
+                    tool_services[service_id] = service_data
+                    logger.debug(f"Loaded tool-service config: {service_id}")
+
+            logger.info(
+                f"Loaded {len(tool_services)} tool-service configurations"
+            )
+
+            # Load tool configurations
+            if "tool" in config:
+                for tool_id, tool_value in config["tool"].items():
+                    data = json.loads(tool_value)
+                    impl_id = data.get("type")
+                    name = data.get("name")
+
+                    if impl_id == "knowledge-query":
+                        impl = functools.partial(
+                            KnowledgeQueryImpl,
+                            collection=data.get("collection"),
+                        )
+                        arguments = KnowledgeQueryImpl.get_arguments()
+                    elif impl_id == "text-completion":
+                        impl = TextCompletionImpl
+                        arguments = TextCompletionImpl.get_arguments()
+                    elif impl_id == "mcp-tool":
+                        config_args = data.get("arguments", [])
+                        arguments = [
+                            Argument(
+                                name=arg.get("name"),
+                                type=arg.get("type"),
+                                description=arg.get("description"),
+                            )
+                            for arg in config_args
+                        ]
+                        impl = functools.partial(
+                            McpToolImpl,
+                            mcp_tool_id=data.get("mcp-tool"),
+                            arguments=arguments,
+                        )
+                    elif impl_id == "prompt":
+                        config_args = data.get("arguments", [])
+                        arguments = [
+                            Argument(
+                                name=arg.get("name"),
+                                type=arg.get("type"),
+                                description=arg.get("description"),
+                            )
+                            for arg in config_args
+                        ]
+                        impl = functools.partial(
+                            PromptImpl,
+                            template_id=data.get("template"),
+                            arguments=arguments,
+                        )
+                    elif impl_id == "structured-query":
+                        impl = functools.partial(
+                            StructuredQueryImpl,
+                            collection=data.get("collection"),
+                            user=None,
+                        )
+                        arguments = StructuredQueryImpl.get_arguments()
+                    elif impl_id == "row-embeddings-query":
+                        impl = functools.partial(
+                            RowEmbeddingsQueryImpl,
+                            schema_name=data.get("schema-name"),
+                            collection=data.get("collection"),
+                            user=None,
+                            index_name=data.get("index-name"),
+                            limit=int(data.get("limit", 10)),
+                        )
+                        arguments = RowEmbeddingsQueryImpl.get_arguments()
+                    elif impl_id == "tool-service":
+                        service_ref = data.get("service")
+                        if not service_ref:
+                            raise RuntimeError(
+                                f"Tool {name} has type 'tool-service' "
+                                f"but no 'service' reference"
+                            )
+                        if service_ref not in tool_services:
+                            raise RuntimeError(
+                                f"Tool {name} references unknown "
+                                f"tool-service '{service_ref}'"
+                            )
+
+                        service_config = tool_services[service_ref]
+                        request_queue = service_config.get("request-queue")
+                        response_queue = service_config.get("response-queue")
+                        if not request_queue or not response_queue:
+                            raise RuntimeError(
+                                f"Tool-service '{service_ref}' must define "
+                                f"'request-queue' and 'response-queue'"
+                            )
+
+                        config_params = service_config.get("config-params", [])
+                        config_values = {}
+                        for param in config_params:
+                            param_name = (
+                                param.get("name")
+                                if isinstance(param, dict) else param
+                            )
+                            if param_name in data:
+                                config_values[param_name] = data[param_name]
+                            elif (
+                                isinstance(param, dict)
+                                and param.get("required", False)
+                            ):
+                                raise RuntimeError(
+                                    f"Tool {name} missing required config "
+                                    f"param '{param_name}'"
+                                )
+
+                        config_args = data.get("arguments", [])
+                        arguments = [
+                            Argument(
+                                name=arg.get("name"),
+                                type=arg.get("type"),
+                                description=arg.get("description"),
+                            )
+                            for arg in config_args
+                        ]
+
+                        impl = functools.partial(
+                            ToolServiceImpl,
+                            request_queue=request_queue,
+                            response_queue=response_queue,
+                            config_values=config_values,
+                            arguments=arguments,
+                            processor=self,
+                        )
+                    else:
+                        raise RuntimeError(
+                            f"Tool type {impl_id} not known"
+                        )
+
+                    validate_tool_config(data)
+
+                    tools[name] = Tool(
+                        name=name,
+                        description=data.get("description"),
+                        implementation=impl,
+                        config=data,
+                        arguments=arguments,
+                    )
+
+            # Load additional context from agent config
+            additional = None
+            if self.config_key in config:
+                agent_config = config[self.config_key]
+                additional = agent_config.get("additional-context", None)
+
+            self.agent = AgentManager(
+                tools=tools,
+                additional_context=additional,
+            )
+
+            # Re-initialise meta-router with config
+            self.meta_router = MetaRouter(config=config)
+
+            logger.info(f"Loaded {len(tools)} tools")
+
+        except Exception as e:
+            logger.error(
+                f"on_tools_config Exception: {e}", exc_info=True
+            )
+            logger.error("Configuration reload failed")
+
+    async def _handle_subagent_completion(self, request, respond, next, flow):
+        """Handle a subagent completion by feeding it to the aggregator."""
+
+        correlation_id = request.correlation_id
+        subagent_goal = getattr(request, 'subagent_goal', '')
+        parent_session_id = getattr(request, 'parent_session_id', '')
+
+        # Extract the answer from the completion step
+        answer_text = ""
+        for step in request.history:
+            if getattr(step, 'step_type', '') == 'subagent-completion':
+                answer_text = step.observation
+                break
+
+        logger.debug(
+            f"Received subagent completion: "
+            f"correlation={correlation_id}, goal={subagent_goal}"
+        )
+
+        all_done = self.aggregator.record_completion(
+            correlation_id, subagent_goal, answer_text
+        )
+
+        if all_done is None:
+            logger.warning(
+                f"Unknown correlation_id {correlation_id} — "
+                f"possibly timed out or duplicate"
+            )
+            return
+
+        # Emit finding provenance for this subagent
+        template = self.aggregator.get_original_request(correlation_id)
+        if template and parent_session_id:
+            entry = self.aggregator.correlations.get(correlation_id)
+            finding_index = len(entry["results"]) - 1 if entry else 0
+            collection = getattr(template, 'collection', 'default')
+
+            subagent_session_id = getattr(request, 'session_id', '')
+
+            await self.supervisor_pattern.emit_finding_triples(
+                flow, parent_session_id, finding_index,
+                subagent_goal, answer_text,
+                template.user, collection,
+                respond, template.streaming,
+                subagent_session_id=subagent_session_id,
+            )
+
+        if all_done:
+            logger.info(
+                f"All subagents complete for {correlation_id}, "
+                f"dispatching synthesis"
+            )
+
+            if template is None:
+                logger.error(
+                    f"No template for correlation {correlation_id}"
+                )
+                return
+
+            synthesis_request = self.aggregator.build_synthesis_request(
+                correlation_id,
+                original_question=template.question,
+                user=template.user,
+                collection=getattr(template, 'collection', 'default'),
+            )
+
+            await next(synthesis_request)
+
+    async def agent_request(self, request, respond, next, flow):
+
+        try:
+
+            # Intercept subagent completion messages
+            correlation_id = getattr(request, 'correlation_id', '')
+            if correlation_id and request.history:
+                is_completion = any(
+                    getattr(h, 'step_type', '') == 'subagent-completion'
+                    for h in request.history
+                )
+                if is_completion:
+                    await self._handle_subagent_completion(
+                        request, respond, next, flow
+                    )
+                    return
+
+            pattern = getattr(request, 'pattern', '') or ''
+
+            # If no pattern set and this is the first iteration, route
+            if not pattern and not request.history:
+                context = UserAwareContext(flow, request.user)
+
+                if self.meta_router:
+                    pattern, task_type, framing = await self.meta_router.route(
+                        request.question, context,
+                    )
+                else:
+                    pattern = "react"
+                    task_type = "general"
+                    framing = ""
+
+                # Update request with routing decision
+                request.pattern = pattern
+                request.task_type = task_type
+                request.framing = framing
+
+                logger.info(
+                    f"Routed to pattern={pattern}, "
+                    f"task_type={task_type}"
+                )
+
+            # Dispatch to the selected pattern
+            if pattern == "plan-then-execute":
+                await self.plan_pattern.iterate(
+                    request, respond, next, flow,
+                )
+            elif pattern == "supervisor":
+                await self.supervisor_pattern.iterate(
+                    request, respond, next, flow,
+                )
+            else:
+                # Default to react
+                await self.react_pattern.iterate(
+                    request, respond, next, flow,
+                )
+
+        except Exception as e:
+
+            logger.error(
+                f"agent_request Exception: {e}", exc_info=True
+            )
+
+            logger.debug("Send error response...")
+
+            error_obj = Error(
+                type="agent-error",
+                message=str(e),
+            )
+
+            r = AgentResponse(
+                chunk_type="error",
+                content=str(e),
+                end_of_message=True,
+                end_of_dialog=True,
+                error=error_obj,
+            )
+
+            await respond(r)
+
+    @staticmethod
+    def add_args(parser):
+
+        AgentService.add_args(parser)
+
+        parser.add_argument(
+            '--max-iterations',
+            default=default_max_iterations,
+            help=f'Maximum number of react iterations '
+                 f'(default: {default_max_iterations})',
+        )
+
+        parser.add_argument(
+            '--config-type',
+            default="agent",
+            help='Configuration key for prompts (default: agent)',
+        )
+
+
+def run():
+    Processor.launch(default_ident, __doc__)

trustgraph-flow/trustgraph/agent/orchestrator/supervisor_pattern.py

@@ -0,0 +1,234 @@
+"""
+SupervisorPattern — decomposes a query into subagent goals, fans out,
+then synthesises results when all subagents complete.
+
+Phase 1 (decompose): LLM breaks the query into independent sub-goals.
+Fan-out: Each sub-goal is emitted as a new AgentRequest on the agent
+         request topic, carrying a correlation_id and parent_session_id.
+Phase 2 (synthesise): Triggered when the aggregator detects all
+         subagents have completed. The supervisor fetches results and
+         produces the final answer.
+"""
+
+import json
+import logging
+import uuid
+
+from ... schema import AgentRequest, AgentResponse, AgentStep
+
+from trustgraph.provenance import (
+    agent_finding_uri,
+    agent_decomposition_uri,
+    agent_synthesis_uri,
+)
+
+from . pattern_base import PatternBase
+
+logger = logging.getLogger(__name__)
+
+MAX_SUBAGENTS = 5
+
+
+class SupervisorPattern(PatternBase):
+    """
+    Supervisor pattern: decompose, fan-out, synthesise.
+
+    History tracks phase via AgentStep.step_type:
+      - "decompose": the decomposition step (subagent goals in arguments)
+      - "synthesise": triggered by aggregator with results in subagent_results
+    """
+
+    async def iterate(self, request, respond, next, flow):
+
+        streaming = getattr(request, 'streaming', False)
+        session_id = getattr(request, 'session_id', '') or str(uuid.uuid4())
+        collection = getattr(request, 'collection', 'default')
+        iteration_num = len(request.history) + 1
+        session_uri = self.processor.provenance_session_uri(session_id)
+
+        # Emit session provenance on first iteration
+        if iteration_num == 1:
+            await self.emit_session_triples(
+                flow, session_uri, request.question,
+                request.user, collection, respond, streaming,
+            )
+
+        logger.info(
+            f"SupervisorPattern iteration {iteration_num}: {request.question}"
+        )
+
+        # Check if this is a synthesis request (has subagent_results)
+        has_results = bool(
+            request.history
+            and any(
+                getattr(h, 'step_type', '') == 'synthesise'
+                and getattr(h, 'subagent_results', None)
+                for h in request.history
+            )
+        )
+
+        if has_results:
+            await self._synthesise(
+                request, respond, next, flow,
+                session_id, collection, streaming,
+                session_uri, iteration_num,
+            )
+        else:
+            await self._decompose_and_fanout(
+                request, respond, next, flow,
+                session_id, collection, streaming,
+                session_uri, iteration_num,
+            )
+
+    async def _decompose_and_fanout(self, request, respond, next, flow,
+                                    session_id, collection, streaming,
+                                    session_uri, iteration_num):
+        """Decompose the question into sub-goals and fan out subagents."""
+
+        decompose_msg_id = agent_decomposition_uri(session_id)
+        think = self.make_think_callback(
+            respond, streaming, message_id=decompose_msg_id,
+        )
+        framing = getattr(request, 'framing', '')
+
+        tools = self.filter_tools(self.processor.agent.tools, request)
+
+        context = self.make_context(
+            flow, request.user,
+            respond=respond, streaming=streaming,
+        )
+        client = context("prompt-request")
+
+        # Use the supervisor-decompose prompt template
+        goals = await client.prompt(
+            id="supervisor-decompose",
+            variables={
+                "question": request.question,
+                "framing": framing,
+                "max_subagents": MAX_SUBAGENTS,
+                "tools": [
+                    {"name": t.name, "description": t.description}
+                    for t in tools.values()
+                ],
+            },
+        )
+
+        # Validate result
+        if not isinstance(goals, list):
+            goals = []
+        goals = [g for g in goals if isinstance(g, str)]
+        goals = goals[:MAX_SUBAGENTS]
+
+        if not goals:
+            goals = [request.question]
+
+        await think(
+            f"Decomposed into {len(goals)} sub-goals: {goals}",
+            is_final=True,
+        )
+
+        # Generate correlation ID for this fan-out
+        correlation_id = str(uuid.uuid4())
+
+        # Emit decomposition provenance
+        await self.emit_decomposition_triples(
+            flow, session_id, session_uri, goals,
+            request.user, collection, respond, streaming,
+        )
+
+        # Fan out: emit a subagent request for each goal
+        for i, goal in enumerate(goals):
+            subagent_session = str(uuid.uuid4())
+            sub_request = AgentRequest(
+                question=goal,
+                state="",
+                group=getattr(request, 'group', []),
+                history=[],
+                user=request.user,
+                collection=collection,
+                streaming=False,  # Subagents don't stream
+                session_id=subagent_session,
+                conversation_id=getattr(request, 'conversation_id', ''),
+                pattern="react",  # Subagents use react by default
+                task_type=getattr(request, 'task_type', ''),
+                framing=getattr(request, 'framing', ''),
+                correlation_id=correlation_id,
+                parent_session_id=session_id,
+                subagent_goal=goal,
+                expected_siblings=len(goals),
+            )
+            await next(sub_request)
+            logger.info(f"Fan-out: emitted subagent {i} for goal: {goal}")
+
+        # Register with aggregator for fan-in tracking
+        self.processor.aggregator.register_fanout(
+            correlation_id=correlation_id,
+            parent_session_id=session_id,
+            expected_siblings=len(goals),
+            request_template=request,
+        )
+
+        logger.info(
+            f"Supervisor fan-out complete: {len(goals)} subagents, "
+            f"correlation_id={correlation_id}"
+        )
+
+    async def _synthesise(self, request, respond, next, flow,
+                          session_id, collection, streaming,
+                          session_uri, iteration_num):
+        """Synthesise final answer from subagent results."""
+
+        synthesis_msg_id = agent_synthesis_uri(session_id)
+        think = self.make_think_callback(
+            respond, streaming, message_id=synthesis_msg_id,
+        )
+        framing = getattr(request, 'framing', '')
+
+        # Collect subagent results from history
+        subagent_results = {}
+        for step in request.history:
+            results = getattr(step, 'subagent_results', None)
+            if results:
+                subagent_results.update(results)
+
+        if not subagent_results:
+            logger.warning("Synthesis called with no subagent results")
+            subagent_results = {"(no results)": "No subagent results available"}
+
+        context = self.make_context(
+            flow, request.user,
+            respond=respond, streaming=streaming,
+        )
+        client = context("prompt-request")
+
+        await think("Synthesising final answer from sub-agent results", is_final=True)
+
+        response_text = await self.prompt_as_answer(
+            client, "supervisor-synthesise",
+            variables={
+                "question": request.question,
+                "framing": framing,
+                "results": [
+                    {"goal": goal, "result": result}
+                    for goal, result in subagent_results.items()
+                ],
+            },
+            respond=respond,
+            streaming=streaming,
+            message_id=synthesis_msg_id,
+        )
+
+        # Emit synthesis provenance (links back to all findings)
+        finding_uris = [
+            agent_finding_uri(session_id, i)
+            for i in range(len(subagent_results))
+        ]
+        await self.emit_synthesis_triples(
+            flow, session_id, finding_uris,
+            response_text, request.user, collection, respond, streaming,
+        )
+
+        await self.send_final_response(
+            respond, streaming, response_text, already_streamed=streaming,
+            message_id=synthesis_msg_id,
+        )

trustgraph-flow/trustgraph/agent/react/agent_manager.py

@@ -16,37 +16,37 @@ class AgentManager:
 
     def parse_react_response(self, text):
         """Parse text-based ReAct response format.
-        
+
         Expected format:
         Thought: [reasoning about what to do next]
         Action: [tool_name]
         Args: {
             "param": "value"
         }
-        
+
         OR
-        
+
         Thought: [reasoning about the final answer]
         Final Answer: [the answer]
         """
         if not isinstance(text, str):
             raise ValueError(f"Expected string response, got {type(text)}")
-        
+
         # Remove any markdown code blocks that might wrap the response
         text = re.sub(r'^```[^\n]*\n', '', text.strip())
         text = re.sub(r'\n```$', '', text.strip())
-            
+
         lines = text.strip().split('\n')
-        
+
         thought = None
         action = None
         args = None
         final_answer = None
-        
+
         i = 0
         while i < len(lines):
             line = lines[i].strip()
-            
+
             # Parse Thought
             if line.startswith("Thought:"):
                 thought = line[8:].strip()
@@ -59,19 +59,19 @@ class AgentManager:
                     thought += " " + next_line
                     i += 1
                 continue
-            
+
             # Parse Final Answer
             if line.startswith("Final Answer:"):
                 final_answer = line[13:].strip()
                 # Handle multi-line final answers (including JSON)
                 i += 1
-                
+
                 # Check if the answer might be JSON
                 if final_answer.startswith('{') or (i < len(lines) and lines[i].strip().startswith('{')):
                     # Collect potential JSON answer
                     json_text = final_answer if final_answer.startswith('{') else ""
                     brace_count = json_text.count('{') - json_text.count('}')
-                    
+
                     while i < len(lines) and (brace_count > 0 or not json_text):
                         current_line = lines[i].strip()
                         if current_line.startswith(("Thought:", "Action:")) and brace_count == 0:
@@ -79,7 +79,7 @@ class AgentManager:
                         json_text += ("\n" if json_text else "") + current_line
                         brace_count += current_line.count('{') - current_line.count('}')
                         i += 1
-                    
+
                     # Try to parse as JSON
                     # try:
                     #     final_answer = json.loads(json_text)
@@ -95,13 +95,13 @@ class AgentManager:
                             break
                         final_answer += " " + next_line
                         i += 1
-                    
+
                 # If we have a final answer, return Final object
                 return Final(
                     thought=thought or "",
                     final=final_answer
                 )
-            
+
             # Parse Action
             if line.startswith("Action:"):
                 action = line[7:].strip()
@@ -112,7 +112,7 @@ class AgentManager:
 
                 while action and action[-1] == '"':
                     action = action[:-1]
-            
+
             # Parse Args
             if line.startswith("Args:"):
                 # Check if JSON starts on the same line
@@ -123,15 +123,15 @@ class AgentManager:
                 else:
                     args_text = ""
                     brace_count = 0
-                
+
                 # Collect all lines that form the JSON arguments
                 i += 1
                 started = bool(args_on_same_line and '{' in args_on_same_line)
-                
+
                 while i < len(lines) and (not started or brace_count > 0):
                     current_line = lines[i]
                     args_text += ("\n" if args_text else "") + current_line
-                    
+
                     # Count braces to determine when JSON is complete
                     for char in current_line:
                         if char == '{':
@@ -139,22 +139,22 @@ class AgentManager:
                             started = True
                         elif char == '}':
                             brace_count -= 1
-                    
+
                     # If we've started and braces are balanced, we're done
                     if started and brace_count == 0:
                         break
-                    
+
                     i += 1
-                
+
                 # Parse the JSON arguments
                 try:
                     args = json.loads(args_text.strip())
                 except json.JSONDecodeError as e:
                     logger.error(f"Failed to parse JSON arguments: {args_text}")
                     raise ValueError(f"Invalid JSON in Args: {e}")
-            
+
             i += 1
-        
+
         # If we have an action, return Action object
         if action:
             return Action(
@@ -163,11 +163,11 @@ class AgentManager:
                 arguments=args or {},
                 observation=""
             )
-        
+
         # If we only have a thought but no action or final answer
         if thought and not action and not final_answer:
             raise ValueError(f"Response has thought but no action or final answer: {text}")
-        
+
         raise ValueError(f"Could not parse response: {text}")
 
     async def reason(self, question, history, context, streaming=False, think=None, observe=None, answer=None):
@@ -176,15 +176,10 @@ class AgentManager:
 
         tools = self.tools
 
-        logger.debug("in reason")
-        logger.debug(f"tools: {tools}")
-
         tool_names = ",".join([
             t for t in self.tools.keys()
         ])
 
-        logger.debug(f"Tool names: {tool_names}")
-
         variables = {
             "question": question,
             "tools": [
@@ -218,17 +213,10 @@ class AgentManager:
 
         logger.debug(f"Variables: {json.dumps(variables, indent=4)}")
 
-        logger.info(f"prompt: {variables}")
-
-        logger.info(f"DEBUG: streaming={streaming}, think={think is not None}")
-
         # Streaming path - use StreamingReActParser
         if streaming and think:
-            logger.info("DEBUG: Entering streaming path")
             from .streaming_parser import StreamingReActParser
 
-            logger.info("DEBUG: Creating StreamingReActParser")
-
             # Collect chunks to send via async callbacks
             thought_chunks = []
             answer_chunks = []
@@ -238,24 +226,19 @@ class AgentManager:
                 on_thought_chunk=lambda chunk: thought_chunks.append(chunk),
                 on_answer_chunk=lambda chunk: answer_chunks.append(chunk),
             )
-            logger.info("DEBUG: StreamingReActParser created")
 
             # Create async chunk callback that feeds parser and sends collected chunks
             async def on_chunk(text, end_of_stream):
-                logger.info(f"DEBUG: on_chunk called with {len(text)} chars, end_of_stream={end_of_stream}")
 
                 # Track what we had before
                 prev_thought_count = len(thought_chunks)
                 prev_answer_count = len(answer_chunks)
 
                 # Feed the parser (synchronous)
-                logger.info(f"DEBUG: About to call parser.feed")
                 parser.feed(text)
-                logger.info(f"DEBUG: parser.feed returned")
 
                 # Send any new thought chunks
                 for i in range(prev_thought_count, len(thought_chunks)):
-                    logger.info(f"DEBUG: Sending thought chunk {i}")
                     # Mark last chunk as final if parser has moved out of THOUGHT state
                     is_last = (i == len(thought_chunks) - 1)
                     is_thought_complete = parser.state.value != "thought"
@@ -264,71 +247,52 @@ class AgentManager:
 
                 # Send any new answer chunks
                 for i in range(prev_answer_count, len(answer_chunks)):
-                    logger.info(f"DEBUG: Sending answer chunk {i}")
                     if answer:
                         await answer(answer_chunks[i])
                     else:
                         await think(answer_chunks[i])
 
-            logger.info("DEBUG: Getting prompt-request client from context")
             client = context("prompt-request")
-            logger.info(f"DEBUG: Got client: {client}")
 
-            logger.info("DEBUG: About to call agent_react with streaming=True")
             # Get streaming response
             response_text = await client.agent_react(
                 variables=variables,
                 streaming=True,
                 chunk_callback=on_chunk
             )
-            logger.info(f"DEBUG: agent_react returned, got {len(response_text) if response_text else 0} chars")
 
             # Finalize parser
-            logger.info("DEBUG: Finalizing parser")
             parser.finalize()
-            logger.info("DEBUG: Parser finalized")
 
             # Get result
-            logger.info("DEBUG: Getting result from parser")
             result = parser.get_result()
             if result is None:
                 raise RuntimeError("Parser failed to produce a result")
 
-            logger.info(f"Parsed result: {result}")
             return result
 
         else:
-            logger.info("DEBUG: Entering NON-streaming path")
             # Non-streaming path - get complete text and parse
-            logger.info("DEBUG: Getting prompt-request client from context")
             client = context("prompt-request")
-            logger.info(f"DEBUG: Got client: {client}")
 
-            logger.info("DEBUG: About to call agent_react with streaming=False")
             response_text = await client.agent_react(
                 variables=variables,
                 streaming=False
             )
-            logger.info(f"DEBUG: agent_react returned, got response")
 
             logger.debug(f"Response text:\n{response_text}")
 
-            logger.info(f"response: {response_text}")
-
             # Parse the text response
             try:
                 result = self.parse_react_response(response_text)
-                logger.info(f"Parsed result: {result}")
                 return result
             except ValueError as e:
                 logger.error(f"Failed to parse response: {e}")
-                # Try to provide a helpful error message
                 logger.error(f"Response was: {response_text}")
                 raise RuntimeError(f"Failed to parse agent response: {e}")
 
-    async def react(self, question, history, think, observe, context, streaming=False, answer=None):
-
-        logger.info(f"question: {question}")
+    async def react(self, question, history, think, observe, context,
+                    streaming=False, answer=None, on_action=None):
 
         act = await self.reason(
             question = question,
@@ -339,7 +303,6 @@ class AgentManager:
             observe = observe,
             answer = answer,
         )
-        logger.info(f"act: {act}")
 
         if isinstance(act, Final):
 
@@ -358,15 +321,14 @@ class AgentManager:
 
             logger.debug(f"ACTION: {act.name}")
 
-            logger.debug(f"Tools: {self.tools.keys()}")
-
             if act.name in self.tools:
                 action = self.tools[act.name]
             else:
-                logger.debug(f"Tools: {self.tools}")
                 raise RuntimeError(f"No action for {act.name}!")
 
-            logger.debug(f"TOOL>>> {act}")
+            # Notify caller before tool execution (for provenance)
+            if on_action:
+                await on_action(act)
 
             resp = await action.implementation(context).invoke(
                 **act.arguments
@@ -378,13 +340,8 @@ class AgentManager:
                 resp = str(resp)
                 resp = resp.strip()
 
-            logger.info(f"resp: {resp}")
-
             await observe(resp, is_final=True)
 
             act.observation = resp
 
-            logger.info(f"iter: {act}")
-
             return act
-

trustgraph-flow/trustgraph/agent/react/service.py

@@ -36,6 +36,7 @@ from trustgraph.provenance import (
     agent_final_uri,
     agent_session_triples,
     agent_iteration_triples,
+    agent_observation_triples,
     agent_final_triples,
     set_graph,
     GRAPH_RETRIEVAL,
@@ -80,7 +81,9 @@ class Processor(AgentService):
         # Track active tool service clients for cleanup
         self.tool_service_clients = {}
 
-        self.config_handlers.append(self.on_tools_config)
+        self.register_config_handler(
+            self.on_tools_config, types=["tool", "tool-service"]
+        )
 
         self.register_specification(
             TextCompletionClientSpec(
@@ -465,13 +468,13 @@ class Processor(AgentService):
                 logger.debug(f"Emitted session triples for {session_uri}")
 
                 # Send explain event for session
-                if streaming:
-                    await respond(AgentResponse(
-                        chunk_type="explain",
-                        content="",
-                        explain_id=session_uri,
-                        explain_graph=GRAPH_RETRIEVAL,
-                    ))
+                await respond(AgentResponse(
+                    chunk_type="explain",
+                    content="",
+                    explain_id=session_uri,
+                    explain_graph=GRAPH_RETRIEVAL,
+                    explain_triples=triples,
+                ))
 
             logger.info(f"Question: {request.question}")
 
@@ -480,32 +483,28 @@ class Processor(AgentService):
 
             logger.debug(f"History: {history}")
 
+            thought_msg_id = agent_thought_uri(session_id, iteration_num)
+            observation_msg_id = agent_observation_uri(session_id, iteration_num)
+
             async def think(x, is_final=False):
 
                 logger.debug(f"Think: {x} (is_final={is_final})")
 
                 if streaming:
-                    # Streaming format
                     r = AgentResponse(
                         chunk_type="thought",
                         content=x,
                         end_of_message=is_final,
                         end_of_dialog=False,
-                        # Legacy fields for backward compatibility
-                        answer=None,
-                        error=None,
-                        thought=x,
-                        observation=None,
+                        message_id=thought_msg_id,
                     )
                 else:
-                    # Non-streaming format
                     r = AgentResponse(
-                        answer=None,
-                        error=None,
-                        thought=x,
-                        observation=None,
+                        chunk_type="thought",
+                        content=x,
                         end_of_message=True,
                         end_of_dialog=False,
+                        message_id=thought_msg_id,
                     )
 
                 await respond(r)
@@ -515,57 +514,45 @@ class Processor(AgentService):
                 logger.debug(f"Observe: {x} (is_final={is_final})")
 
                 if streaming:
-                    # Streaming format
                     r = AgentResponse(
                         chunk_type="observation",
                         content=x,
                         end_of_message=is_final,
                         end_of_dialog=False,
-                        # Legacy fields for backward compatibility
-                        answer=None,
-                        error=None,
-                        thought=None,
-                        observation=x,
+                        message_id=observation_msg_id,
                     )
                 else:
-                    # Non-streaming format
                     r = AgentResponse(
-                        answer=None,
-                        error=None,
-                        thought=None,
-                        observation=x,
+                        chunk_type="observation",
+                        content=x,
                         end_of_message=True,
                         end_of_dialog=False,
+                        message_id=observation_msg_id,
                     )
 
                 await respond(r)
 
+            answer_msg_id = agent_final_uri(session_id)
+
             async def answer(x):
 
                 logger.debug(f"Answer: {x}")
 
                 if streaming:
-                    # Streaming format
                     r = AgentResponse(
                         chunk_type="answer",
                         content=x,
-                        end_of_message=False,  # More chunks may follow
+                        end_of_message=False,
                         end_of_dialog=False,
-                        # Legacy fields for backward compatibility
-                        answer=None,
-                        error=None,
-                        thought=None,
-                        observation=None,
+                        message_id=answer_msg_id,
                     )
                 else:
-                    # Non-streaming format - shouldn't normally be called
                     r = AgentResponse(
-                        answer=x,
-                        error=None,
-                        thought=None,
-                        observation=None,
+                        chunk_type="answer",
+                        content=x,
                         end_of_message=True,
                         end_of_dialog=False,
+                        message_id=answer_msg_id,
                     )
 
                 await respond(r)
@@ -577,8 +564,6 @@ class Processor(AgentService):
                 current_state=getattr(request, 'state', None)
             )
             
-            logger.info(f"Filtered from {len(self.agent.tools)} to {len(filtered_tools)} available tools")
-            
             # Create temporary agent with filtered tools
             temp_agent = AgentManager(
                 tools=filtered_tools,
@@ -593,6 +578,7 @@ class Processor(AgentService):
                 def __init__(self, flow, user):
                     self._flow = flow
                     self._user = user
+                    self.last_sub_explain_uri = None
 
                 def __call__(self, service_name):
                     client = self._flow(service_name)
@@ -601,14 +587,74 @@ class Processor(AgentService):
                         client._current_user = self._user
                     return client
 
+            # Callback: emit Analysis+ToolUse triples before tool executes
+            async def on_action(act_decision):
+                iter_uri = agent_iteration_uri(session_id, iteration_num)
+                if iteration_num > 1:
+                    iter_q_uri = None
+                    iter_prev_uri = agent_observation_uri(session_id, iteration_num - 1)
+                else:
+                    iter_q_uri = session_uri
+                    iter_prev_uri = None
+
+                # Save thought to librarian
+                t_doc_id = None
+                if act_decision.thought:
+                    t_doc_id = f"urn:trustgraph:agent:{session_id}/i{iteration_num}/thought"
+                    try:
+                        await self.save_answer_content(
+                            doc_id=t_doc_id,
+                            user=request.user,
+                            content=act_decision.thought,
+                            title=f"Agent Thought: {act_decision.name}",
+                        )
+                    except Exception as e:
+                        logger.warning(f"Failed to save thought to librarian: {e}")
+                        t_doc_id = None
+
+                t_entity_uri = agent_thought_uri(session_id, iteration_num)
+
+                iter_triples = set_graph(
+                    agent_iteration_triples(
+                        iter_uri,
+                        question_uri=iter_q_uri,
+                        previous_uri=iter_prev_uri,
+                        action=act_decision.name,
+                        arguments=act_decision.arguments,
+                        thought_uri=t_entity_uri if t_doc_id else None,
+                        thought_document_id=t_doc_id,
+                    ),
+                    GRAPH_RETRIEVAL
+                )
+                await flow("explainability").send(Triples(
+                    metadata=Metadata(
+                        id=iter_uri,
+                        user=request.user,
+                        collection=collection,
+                    ),
+                    triples=iter_triples,
+                ))
+                logger.debug(f"Emitted iteration triples for {iter_uri}")
+
+                await respond(AgentResponse(
+                    chunk_type="explain",
+                    content="",
+                    explain_id=iter_uri,
+                    explain_graph=GRAPH_RETRIEVAL,
+                    explain_triples=iter_triples,
+                ))
+
+            user_context = UserAwareContext(flow, request.user)
+
             act = await temp_agent.react(
                 question = request.question,
                 history = history,
                 think = think,
                 observe = observe,
                 answer = answer,
-                context = UserAwareContext(flow, request.user),
+                context = user_context,
                 streaming = streaming,
+                on_action = on_action,
             )
 
             logger.debug(f"Action: {act}")
@@ -624,10 +670,10 @@ class Processor(AgentService):
 
                 # Emit final answer provenance triples
                 final_uri = agent_final_uri(session_id)
-                # No iterations: link to question; otherwise: link to last iteration
+                # No iterations: link to question; otherwise: link to last observation
                 if iteration_num > 1:
                     final_question_uri = None
-                    final_previous_uri = agent_iteration_uri(session_id, iteration_num - 1)
+                    final_previous_uri = agent_observation_uri(session_id, iteration_num - 1)
                 else:
                     final_question_uri = session_uri
                     final_previous_uri = None
@@ -668,36 +714,30 @@ class Processor(AgentService):
                 logger.debug(f"Emitted final triples for {final_uri}")
 
                 # Send explain event for conclusion
-                if streaming:
-                    await respond(AgentResponse(
-                        chunk_type="explain",
-                        content="",
-                        explain_id=final_uri,
-                        explain_graph=GRAPH_RETRIEVAL,
-                    ))
+                await respond(AgentResponse(
+                    chunk_type="explain",
+                    content="",
+                    explain_id=final_uri,
+                    explain_graph=GRAPH_RETRIEVAL,
+                    explain_triples=final_triples,
+                ))
 
                 if streaming:
-                    # Streaming format - send end-of-dialog marker
-                    # Answer chunks were already sent via answer() callback during parsing
+                    # End-of-dialog marker — answer chunks already sent via callback
                     r = AgentResponse(
                         chunk_type="answer",
-                        content="",  # Empty content, just marking end of dialog
+                        content="",
                         end_of_message=True,
                         end_of_dialog=True,
-                        # Legacy fields set to None - answer already sent via streaming chunks
-                        answer=None,
-                        error=None,
-                        thought=None,
+                        message_id=answer_msg_id,
                     )
                 else:
-                    # Non-streaming format - send complete answer
                     r = AgentResponse(
-                        answer=act.final,
-                        error=None,
-                        thought=None,
-                        observation=None,
+                        chunk_type="answer",
+                        content=f,
                         end_of_message=True,
                         end_of_dialog=True,
+                        message_id=answer_msg_id,
                     )
 
                 await respond(r)
@@ -708,33 +748,15 @@ class Processor(AgentService):
 
             logger.debug("Send next...")
 
-            # Emit iteration provenance triples
+            # Emit standalone observation provenance (iteration was emitted in on_action)
             iteration_uri = agent_iteration_uri(session_id, iteration_num)
-            # First iteration links to question, subsequent to previous
-            if iteration_num > 1:
-                iter_question_uri = None
-                iter_previous_uri = agent_iteration_uri(session_id, iteration_num - 1)
-            else:
-                iter_question_uri = session_uri
-                iter_previous_uri = None
+            observation_entity_uri = agent_observation_uri(session_id, iteration_num)
 
-            # Save thought to librarian
-            thought_doc_id = None
-            if act.thought:
-                thought_doc_id = f"urn:trustgraph:agent:{session_id}/i{iteration_num}/thought"
-                try:
-                    await self.save_answer_content(
-                        doc_id=thought_doc_id,
-                        user=request.user,
-                        content=act.thought,
-                        title=f"Agent Thought: {act.name}",
-                    )
-                    logger.debug(f"Saved thought to librarian: {thought_doc_id}")
-                except Exception as e:
-                    logger.warning(f"Failed to save thought to librarian: {e}")
-                    thought_doc_id = None
+            # Derive from last sub-trace entity if available, else iteration
+            obs_parent_uri = iteration_uri
+            if user_context.last_sub_explain_uri:
+                obs_parent_uri = user_context.last_sub_explain_uri
 
-            # Save observation to librarian
             observation_doc_id = None
             if act.observation:
                 observation_doc_id = f"urn:trustgraph:agent:{session_id}/i{iteration_num}/observation"
@@ -743,48 +765,39 @@ class Processor(AgentService):
                         doc_id=observation_doc_id,
                         user=request.user,
                         content=act.observation,
-                        title=f"Agent Observation: {act.name}",
+                        title=f"Agent Observation",
                     )
                     logger.debug(f"Saved observation to librarian: {observation_doc_id}")
                 except Exception as e:
                     logger.warning(f"Failed to save observation to librarian: {e}")
                     observation_doc_id = None
 
-            thought_entity_uri = agent_thought_uri(session_id, iteration_num)
-            observation_entity_uri = agent_observation_uri(session_id, iteration_num)
-
-            iter_triples = set_graph(
-                agent_iteration_triples(
-                    iteration_uri,
-                    question_uri=iter_question_uri,
-                    previous_uri=iter_previous_uri,
-                    action=act.name,
-                    arguments=act.arguments,
-                    thought_uri=thought_entity_uri if thought_doc_id else None,
-                    thought_document_id=thought_doc_id,
-                    observation_uri=observation_entity_uri if observation_doc_id else None,
-                    observation_document_id=observation_doc_id,
+            obs_triples = set_graph(
+                agent_observation_triples(
+                    observation_entity_uri,
+                    obs_parent_uri,
+                    document_id=observation_doc_id,
                 ),
                 GRAPH_RETRIEVAL
             )
             await flow("explainability").send(Triples(
                 metadata=Metadata(
-                    id=iteration_uri,
+                    id=observation_entity_uri,
                     user=request.user,
                     collection=collection,
                 ),
-                triples=iter_triples,
+                triples=obs_triples,
             ))
-            logger.debug(f"Emitted iteration triples for {iteration_uri}")
+            logger.debug(f"Emitted observation triples for {observation_entity_uri}")
 
-            # Send explain event for iteration
-            if streaming:
-                await respond(AgentResponse(
-                    chunk_type="explain",
-                    content="",
-                    explain_id=iteration_uri,
-                    explain_graph=GRAPH_RETRIEVAL,
-                ))
+            # Send explain event for observation
+            await respond(AgentResponse(
+                chunk_type="explain",
+                content="",
+                explain_id=observation_entity_uri,
+                explain_graph=GRAPH_RETRIEVAL,
+                explain_triples=obs_triples,
+            ))
 
             history.append(act)
 
@@ -833,21 +846,13 @@ class Processor(AgentService):
             # Check if streaming was enabled (may not be set if error occurred early)
             streaming = getattr(request, 'streaming', False) if 'request' in locals() else False
 
-            if streaming:
-                # Streaming format
-                r = AgentResponse(
-                    chunk_type="error",
-                    content=str(e),
-                    end_of_message=True,
-                    end_of_dialog=True,
-                    # Legacy fields for backward compatibility
-                    error=error_obj,
-                )
-            else:
-                # Legacy format
-                r = AgentResponse(
-                    error=error_obj,
-                )
+            r = AgentResponse(
+                chunk_type="error",
+                content=str(e),
+                end_of_message=True,
+                end_of_dialog=True,
+                error=error_obj,
+            )
 
             await respond(r)
 

trustgraph-flow/trustgraph/agent/react/tools.py

@@ -12,7 +12,7 @@ class KnowledgeQueryImpl:
     def __init__(self, context, collection=None):
         self.context = context
         self.collection = collection
-    
+
     @staticmethod
     def get_arguments():
         return [
@@ -22,13 +22,41 @@ class KnowledgeQueryImpl:
                 description="The question to ask the knowledge base"
             )
         ]
-    
+
     async def invoke(self, **arguments):
         client = self.context("graph-rag-request")
         logger.debug("Graph RAG question...")
+
+        # Build explain_callback to forward sub-trace explain events
+        # to the agent's response stream
+        explain_callback = None
+        parent_uri = ""
+
+        respond = getattr(self.context, 'respond', None)
+        streaming = getattr(self.context, 'streaming', False)
+        current_uri = getattr(self.context, 'current_explain_uri', None)
+
+        if respond:
+            from ... schema import AgentResponse
+
+            async def explain_callback(explain_id, explain_graph, explain_triples=None):
+                self.context.last_sub_explain_uri = explain_id
+                await respond(AgentResponse(
+                    chunk_type="explain",
+                    content="",
+                    explain_id=explain_id,
+                    explain_graph=explain_graph,
+                    explain_triples=explain_triples or [],
+                ))
+
+        if current_uri:
+            parent_uri = current_uri
+
         return await client.rag(
             arguments.get("question"),
-            collection=self.collection if self.collection else "default"
+            collection=self.collection if self.collection else "default",
+            explain_callback=explain_callback,
+            parent_uri=parent_uri,
         )
 
 # This tool implementation knows how to do text completion.  This uses

trustgraph-flow/trustgraph/agent/tool_filter.py

@@ -34,17 +34,17 @@ def filter_tools_by_group_and_state(
     if current_state is None or current_state == "":
         current_state = "undefined"
         
-    logger.info(f"Filtering tools with groups={requested_groups}, state={current_state}")
-    
+    logger.debug(f"Filtering tools with groups={requested_groups}, state={current_state}")
+
     filtered_tools = {}
-    
+
     for tool_name, tool in tools.items():
         if _is_tool_available(tool, requested_groups, current_state):
             filtered_tools[tool_name] = tool
         else:
             logger.debug(f"Tool {tool_name} filtered out")
-            
-    logger.info(f"Filtered {len(tools)} tools to {len(filtered_tools)} available tools")
+
+    logger.debug(f"Filtered {len(tools)} tools to {len(filtered_tools)} available tools")
     return filtered_tools
 
 

trustgraph-flow/trustgraph/chunking/recursive/chunker.py

@@ -133,7 +133,7 @@ class Processor(ChunkingService):
             chunk_length = len(chunk.page_content)
 
             # Save chunk to librarian as child document
-            await self.save_child_document(
+            await self.librarian.save_child_document(
                 doc_id=chunk_doc_id,
                 parent_id=parent_doc_id,
                 user=v.metadata.user,

trustgraph-flow/trustgraph/chunking/token/chunker.py

@@ -131,7 +131,7 @@ class Processor(ChunkingService):
             chunk_length = len(chunk.page_content)
 
             # Save chunk to librarian as child document
-            await self.save_child_document(
+            await self.librarian.save_child_document(
                 doc_id=chunk_doc_id,
                 parent_id=parent_doc_id,
                 user=v.metadata.user,

trustgraph-flow/trustgraph/config/service/config.py

@@ -148,18 +148,7 @@ class Configuration:
 
     async def handle_delete(self, v):
 
-        # for k in v.keys:
-        #     if k.type not in self or k.key not in self[k.type]:
-        #         return ConfigResponse(
-        #             version = None,
-        #             values = None,
-        #             directory = None,
-        #             config = None,
-        #             error = Error(
-        #                 type = "key-error",
-        #                 message = f"Key error"
-        #             )
-        #         )
+        types = list(set(k.type for k in v.keys))
 
         for k in v.keys:
 
@@ -167,20 +156,22 @@ class Configuration:
 
         await self.inc_version()
 
-        await self.push()
+        await self.push(types=types)
 
         return ConfigResponse(
         )
 
     async def handle_put(self, v):
 
+        types = list(set(k.type for k in v.values))
+
         for k in v.values:
 
             await self.table_store.put_config(k.type, k.key, k.value)
 
         await self.inc_version()
 
-        await self.push()
+        await self.push(types=types)
 
         return ConfigResponse(
         )

trustgraph-flow/trustgraph/config/service/flow.py

@@ -126,12 +126,12 @@ class FlowConfig:
 
         await self.config.inc_version()
 
-        await self.config.push()
+        await self.config.push(types=["flow-blueprint"])
 
         return FlowResponse(
             error = None,
         )
-    
+
     async def handle_delete_blueprint(self, msg):
 
         logger.debug(f"Flow config message: {msg}")
@@ -140,7 +140,7 @@ class FlowConfig:
 
         await self.config.inc_version()
 
-        await self.config.push()
+        await self.config.push(types=["flow-blueprint"])
 
         return FlowResponse(
             error = None,
@@ -270,7 +270,7 @@ class FlowConfig:
 
         await self.config.inc_version()
 
-        await self.config.push()
+        await self.config.push(types=["active-flow", "flow"])
 
         return FlowResponse(
             error = None,
@@ -332,12 +332,12 @@ class FlowConfig:
 
         await self.config.inc_version()
 
-        await self.config.push()
+        await self.config.push(types=["active-flow", "flow"])
 
         return FlowResponse(
             error = None,
         )
-    
+
     async def handle(self, msg):
 
         logger.debug(f"Handling flow message: {msg.operation}")

trustgraph-flow/trustgraph/config/service/service.py

@@ -167,25 +167,22 @@ class Processor(AsyncProcessor):
 
     async def start(self):
 
-        await self.push()
+        await self.push()  # Startup poke: empty types = everything
         await self.config_request_consumer.start()
         await self.flow_request_consumer.start()
-        
-    async def push(self):
 
-        config = await self.config.get_config()
+    async def push(self, types=None):
+
         version = await self.config.get_version()
 
         resp = ConfigPush(
             version = version,
-            config = config,
+            types = types or [],
         )
 
         await self.config_push_producer.send(resp)
 
-        # Race condition, should make sure version & config sync
-
-        logger.info(f"Pushed configuration version {await self.config.get_version()}")
+        logger.info(f"Pushed config poke version {version}, types={resp.types}")
         
     async def on_config_request(self, msg, consumer, flow):
 

trustgraph-flow/trustgraph/cores/service.py

@@ -108,7 +108,7 @@ class Processor(AsyncProcessor):
             flow_config = self,
         )
 
-        self.register_config_handler(self.on_knowledge_config)
+        self.register_config_handler(self.on_knowledge_config, types=["flow"])
 
         self.flows = {}
 

trustgraph-flow/trustgraph/decoding/mistral_ocr/processor.py

@@ -9,20 +9,16 @@ for large documents.
 
 from pypdf import PdfWriter, PdfReader
 from io import BytesIO
-import asyncio
 import base64
 import uuid
 import os
 
 from mistralai import Mistral
-from mistralai.models import OCRResponse
 
 from ... schema import Document, TextDocument, Metadata
-from ... schema import LibrarianRequest, LibrarianResponse, DocumentMetadata
 from ... schema import librarian_request_queue, librarian_response_queue
 from ... schema import Triples
-from ... base import FlowProcessor, ConsumerSpec, ProducerSpec
-from ... base import Consumer, Producer, ConsumerMetrics, ProducerMetrics
+from ... base import FlowProcessor, ConsumerSpec, ProducerSpec, LibrarianClient
 
 from ... provenance import (
     document_uri, page_uri as make_page_uri, derived_entity_triples,
@@ -102,42 +98,10 @@ class Processor(FlowProcessor):
             )
         )
 
-        # Librarian client for fetching document content
-        librarian_request_q = params.get(
-            "librarian_request_queue", default_librarian_request_queue
+        # Librarian client
+        self.librarian = LibrarianClient(
+            id=id, backend=self.pubsub, taskgroup=self.taskgroup,
         )
-        librarian_response_q = params.get(
-            "librarian_response_queue", default_librarian_response_queue
-        )
-
-        librarian_request_metrics = ProducerMetrics(
-            processor = id, flow = None, name = "librarian-request"
-        )
-
-        self.librarian_request_producer = Producer(
-            backend = self.pubsub,
-            topic = librarian_request_q,
-            schema = LibrarianRequest,
-            metrics = librarian_request_metrics,
-        )
-
-        librarian_response_metrics = ConsumerMetrics(
-            processor = id, flow = None, name = "librarian-response"
-        )
-
-        self.librarian_response_consumer = Consumer(
-            taskgroup = self.taskgroup,
-            backend = self.pubsub,
-            flow = None,
-            topic = librarian_response_q,
-            subscriber = f"{id}-librarian",
-            schema = LibrarianResponse,
-            handler = self.on_librarian_response,
-            metrics = librarian_response_metrics,
-        )
-
-        # Pending librarian requests: request_id -> asyncio.Future
-        self.pending_requests = {}
 
         if api_key is None:
             raise RuntimeError("Mistral API key not specified")
@@ -151,132 +115,7 @@ class Processor(FlowProcessor):
 
     async def start(self):
         await super(Processor, self).start()
-        await self.librarian_request_producer.start()
-        await self.librarian_response_consumer.start()
-
-    async def on_librarian_response(self, msg, consumer, flow):
-        """Handle responses from the librarian service."""
-        response = msg.value()
-        request_id = msg.properties().get("id")
-
-        if request_id and request_id in self.pending_requests:
-            future = self.pending_requests.pop(request_id)
-            future.set_result(response)
-
-    async def fetch_document_metadata(self, document_id, user, timeout=120):
-        """
-        Fetch document metadata from librarian via Pulsar.
-        """
-        request_id = str(uuid.uuid4())
-
-        request = LibrarianRequest(
-            operation="get-document-metadata",
-            document_id=document_id,
-            user=user,
-        )
-
-        future = asyncio.get_event_loop().create_future()
-        self.pending_requests[request_id] = future
-
-        try:
-            await self.librarian_request_producer.send(
-                request, properties={"id": request_id}
-            )
-
-            response = await asyncio.wait_for(future, timeout=timeout)
-
-            if response.error:
-                raise RuntimeError(
-                    f"Librarian error: {response.error.type}: {response.error.message}"
-                )
-
-            return response.document_metadata
-
-        except asyncio.TimeoutError:
-            self.pending_requests.pop(request_id, None)
-            raise RuntimeError(f"Timeout fetching metadata for {document_id}")
-
-    async def fetch_document_content(self, document_id, user, timeout=120):
-        """
-        Fetch document content from librarian via Pulsar.
-        """
-        request_id = str(uuid.uuid4())
-
-        request = LibrarianRequest(
-            operation="get-document-content",
-            document_id=document_id,
-            user=user,
-        )
-
-        # Create future for response
-        future = asyncio.get_event_loop().create_future()
-        self.pending_requests[request_id] = future
-
-        try:
-            # Send request
-            await self.librarian_request_producer.send(
-                request, properties={"id": request_id}
-            )
-
-            # Wait for response
-            response = await asyncio.wait_for(future, timeout=timeout)
-
-            if response.error:
-                raise RuntimeError(
-                    f"Librarian error: {response.error.type}: {response.error.message}"
-                )
-
-            return response.content
-
-        except asyncio.TimeoutError:
-            self.pending_requests.pop(request_id, None)
-            raise RuntimeError(f"Timeout fetching document {document_id}")
-
-    async def save_child_document(self, doc_id, parent_id, user, content,
-                                   document_type="page", title=None, timeout=120):
-        """
-        Save a child document to the librarian.
-        """
-        request_id = str(uuid.uuid4())
-
-        doc_metadata = DocumentMetadata(
-            id=doc_id,
-            user=user,
-            kind="text/plain",
-            title=title or doc_id,
-            parent_id=parent_id,
-            document_type=document_type,
-        )
-
-        request = LibrarianRequest(
-            operation="add-child-document",
-            document_metadata=doc_metadata,
-            content=base64.b64encode(content).decode("utf-8"),
-        )
-
-        # Create future for response
-        future = asyncio.get_event_loop().create_future()
-        self.pending_requests[request_id] = future
-
-        try:
-            # Send request
-            await self.librarian_request_producer.send(
-                request, properties={"id": request_id}
-            )
-
-            # Wait for response
-            response = await asyncio.wait_for(future, timeout=timeout)
-
-            if response.error:
-                raise RuntimeError(
-                    f"Librarian error saving child document: {response.error.type}: {response.error.message}"
-                )
-
-            return doc_id
-
-        except asyncio.TimeoutError:
-            self.pending_requests.pop(request_id, None)
-            raise RuntimeError(f"Timeout saving child document {doc_id}")
+        await self.librarian.start()
 
     def ocr(self, blob):
         """
@@ -359,7 +198,7 @@ class Processor(FlowProcessor):
 
         # Check MIME type if fetching from librarian
         if v.document_id:
-            doc_meta = await self.fetch_document_metadata(
+            doc_meta = await self.librarian.fetch_document_metadata(
                 document_id=v.document_id,
                 user=v.metadata.user,
             )
@@ -374,7 +213,7 @@ class Processor(FlowProcessor):
         # Get PDF content - fetch from librarian or use inline data
         if v.document_id:
             logger.info(f"Fetching document {v.document_id} from librarian...")
-            content = await self.fetch_document_content(
+            content = await self.librarian.fetch_document_content(
                 document_id=v.document_id,
                 user=v.metadata.user,
             )
@@ -401,7 +240,7 @@ class Processor(FlowProcessor):
             page_content = markdown.encode("utf-8")
 
             # Save page as child document in librarian
-            await self.save_child_document(
+            await self.librarian.save_child_document(
                 doc_id=page_doc_id,
                 parent_id=source_doc_id,
                 user=v.metadata.user,

trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py

@@ -7,20 +7,16 @@ Supports both inline document data and fetching from librarian via Pulsar
 for large documents.
 """
 
-import asyncio
 import os
 import tempfile
 import base64
 import logging
-import uuid
 from langchain_community.document_loaders import PyPDFLoader
 
 from ... schema import Document, TextDocument, Metadata
-from ... schema import LibrarianRequest, LibrarianResponse, DocumentMetadata
 from ... schema import librarian_request_queue, librarian_response_queue
 from ... schema import Triples
-from ... base import FlowProcessor, ConsumerSpec, ProducerSpec
-from ... base import Consumer, Producer, ConsumerMetrics, ProducerMetrics
+from ... base import FlowProcessor, ConsumerSpec, ProducerSpec, LibrarianClient
 
 from ... provenance import (
     document_uri, page_uri as make_page_uri, derived_entity_triples,
@@ -74,187 +70,16 @@ class Processor(FlowProcessor):
             )
         )
 
-        # Librarian client for fetching document content
-        librarian_request_q = params.get(
-            "librarian_request_queue", default_librarian_request_queue
+        # Librarian client
+        self.librarian = LibrarianClient(
+            id=id, backend=self.pubsub, taskgroup=self.taskgroup,
         )
-        librarian_response_q = params.get(
-            "librarian_response_queue", default_librarian_response_queue
-        )
-
-        librarian_request_metrics = ProducerMetrics(
-            processor = id, flow = None, name = "librarian-request"
-        )
-
-        self.librarian_request_producer = Producer(
-            backend = self.pubsub,
-            topic = librarian_request_q,
-            schema = LibrarianRequest,
-            metrics = librarian_request_metrics,
-        )
-
-        librarian_response_metrics = ConsumerMetrics(
-            processor = id, flow = None, name = "librarian-response"
-        )
-
-        self.librarian_response_consumer = Consumer(
-            taskgroup = self.taskgroup,
-            backend = self.pubsub,
-            flow = None,
-            topic = librarian_response_q,
-            subscriber = f"{id}-librarian",
-            schema = LibrarianResponse,
-            handler = self.on_librarian_response,
-            metrics = librarian_response_metrics,
-        )
-
-        # Pending librarian requests: request_id -> asyncio.Future
-        self.pending_requests = {}
 
         logger.info("PDF decoder initialized")
 
     async def start(self):
         await super(Processor, self).start()
-        await self.librarian_request_producer.start()
-        await self.librarian_response_consumer.start()
-
-    async def on_librarian_response(self, msg, consumer, flow):
-        """Handle responses from the librarian service."""
-        response = msg.value()
-        request_id = msg.properties().get("id")
-
-        if request_id and request_id in self.pending_requests:
-            future = self.pending_requests.pop(request_id)
-            future.set_result(response)
-
-    async def fetch_document_metadata(self, document_id, user, timeout=120):
-        """
-        Fetch document metadata from librarian via Pulsar.
-        """
-        request_id = str(uuid.uuid4())
-
-        request = LibrarianRequest(
-            operation="get-document-metadata",
-            document_id=document_id,
-            user=user,
-        )
-
-        future = asyncio.get_event_loop().create_future()
-        self.pending_requests[request_id] = future
-
-        try:
-            await self.librarian_request_producer.send(
-                request, properties={"id": request_id}
-            )
-
-            response = await asyncio.wait_for(future, timeout=timeout)
-
-            if response.error:
-                raise RuntimeError(
-                    f"Librarian error: {response.error.type}: {response.error.message}"
-                )
-
-            return response.document_metadata
-
-        except asyncio.TimeoutError:
-            self.pending_requests.pop(request_id, None)
-            raise RuntimeError(f"Timeout fetching metadata for {document_id}")
-
-    async def fetch_document_content(self, document_id, user, timeout=120):
-        """
-        Fetch document content from librarian via Pulsar.
-        """
-        request_id = str(uuid.uuid4())
-
-        request = LibrarianRequest(
-            operation="get-document-content",
-            document_id=document_id,
-            user=user,
-        )
-
-        # Create future for response
-        future = asyncio.get_event_loop().create_future()
-        self.pending_requests[request_id] = future
-
-        try:
-            # Send request
-            await self.librarian_request_producer.send(
-                request, properties={"id": request_id}
-            )
-
-            # Wait for response
-            response = await asyncio.wait_for(future, timeout=timeout)
-
-            if response.error:
-                raise RuntimeError(
-                    f"Librarian error: {response.error.type}: {response.error.message}"
-                )
-
-            return response.content
-
-        except asyncio.TimeoutError:
-            self.pending_requests.pop(request_id, None)
-            raise RuntimeError(f"Timeout fetching document {document_id}")
-
-    async def save_child_document(self, doc_id, parent_id, user, content,
-                                   document_type="page", title=None, timeout=120):
-        """
-        Save a child document to the librarian.
-
-        Args:
-            doc_id: ID for the new child document
-            parent_id: ID of the parent document
-            user: User ID
-            content: Document content (bytes)
-            document_type: Type of document ("page", "chunk", etc.)
-            title: Optional title
-            timeout: Request timeout in seconds
-
-        Returns:
-            The document ID on success
-        """
-        import base64
-
-        request_id = str(uuid.uuid4())
-
-        doc_metadata = DocumentMetadata(
-            id=doc_id,
-            user=user,
-            kind="text/plain",
-            title=title or doc_id,
-            parent_id=parent_id,
-            document_type=document_type,
-        )
-
-        request = LibrarianRequest(
-            operation="add-child-document",
-            document_metadata=doc_metadata,
-            content=base64.b64encode(content).decode("utf-8"),
-        )
-
-        # Create future for response
-        future = asyncio.get_event_loop().create_future()
-        self.pending_requests[request_id] = future
-
-        try:
-            # Send request
-            await self.librarian_request_producer.send(
-                request, properties={"id": request_id}
-            )
-
-            # Wait for response
-            response = await asyncio.wait_for(future, timeout=timeout)
-
-            if response.error:
-                raise RuntimeError(
-                    f"Librarian error saving child document: {response.error.type}: {response.error.message}"
-                )
-
-            return doc_id
-
-        except asyncio.TimeoutError:
-            self.pending_requests.pop(request_id, None)
-            raise RuntimeError(f"Timeout saving child document {doc_id}")
+        await self.librarian.start()
 
     async def on_message(self, msg, consumer, flow):
 
@@ -266,7 +91,7 @@ class Processor(FlowProcessor):
 
         # Check MIME type if fetching from librarian
         if v.document_id:
-            doc_meta = await self.fetch_document_metadata(
+            doc_meta = await self.librarian.fetch_document_metadata(
                 document_id=v.document_id,
                 user=v.metadata.user,
             )
@@ -287,7 +112,7 @@ class Processor(FlowProcessor):
                 logger.info(f"Fetching document {v.document_id} from librarian...")
                 fp.close()
 
-                content = await self.fetch_document_content(
+                content = await self.librarian.fetch_document_content(
                     document_id=v.document_id,
                     user=v.metadata.user,
                 )
@@ -323,7 +148,7 @@ class Processor(FlowProcessor):
                 page_content = page.page_content.encode("utf-8")
 
                 # Save page as child document in librarian
-                await self.save_child_document(
+                await self.librarian.save_child_document(
                     doc_id=page_doc_id,
                     parent_id=source_doc_id,
                     user=v.metadata.user,

trustgraph-flow/trustgraph/embeddings/row_embeddings/embeddings.py

@@ -66,8 +66,8 @@ class Processor(CollectionConfigHandler, FlowProcessor):
         )
 
         # Register config handlers
-        self.register_config_handler(self.on_schema_config)
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_schema_config, types=["schema"])
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
         # Schema storage: name -> RowSchema
         self.schemas: Dict[str, RowSchema] = {}

trustgraph-flow/trustgraph/extract/kg/agent/extract.py

@@ -43,7 +43,7 @@ class Processor(FlowProcessor):
         self.template_id = template_id
         self.config_key = config_key
 
-        self.register_config_handler(self.on_prompt_config)
+        self.register_config_handler(self.on_prompt_config, types=["prompt"])
 
         self.register_specification(
             ConsumerSpec(

trustgraph-flow/trustgraph/extract/kg/ontology/extract.py

@@ -107,7 +107,7 @@ class Processor(FlowProcessor):
         )
 
         # Register config handler for ontology updates
-        self.register_config_handler(self.on_ontology_config)
+        self.register_config_handler(self.on_ontology_config, types=["ontology"])
 
         # Shared components (not flow-specific)
         self.ontology_loader = OntologyLoader()

trustgraph-flow/trustgraph/extract/kg/rows/processor.py

@@ -82,7 +82,7 @@ class Processor(FlowProcessor):
         )
 
         # Register config handler for schema updates
-        self.register_config_handler(self.on_schema_config)
+        self.register_config_handler(self.on_schema_config, types=["schema"])
 
         # Schema storage: name -> RowSchema
         self.schemas: Dict[str, RowSchema] = {}
@@ -145,7 +145,7 @@ class Processor(FlowProcessor):
         
         try:
             # Convert Pulsar RowSchema to JSON-serializable dict
-            schema_dict = row_schema_translator.from_pulsar(schema)
+            schema_dict = row_schema_translator.encode(schema)
             
             # Use prompt client to extract rows based on schema
             objects = await flow("prompt-request").extract_objects(

trustgraph-flow/trustgraph/gateway/config/receiver.py

@@ -1,37 +1,27 @@
 """
-API gateway.  Offers HTTP services which are translated to interaction on the
-Pulsar bus.
+API gateway config receiver.  Subscribes to config notify notifications and
+fetches full config via request/response to manage flow lifecycle.
 """
 
 module = "api-gateway"
 
-# FIXME: Subscribes to Pulsar unnecessarily, should only do it when there
-# are active listeners
-
-# FIXME: Connection errors in publishers / subscribers cause those threads
-# to fail and are not failed or retried
-
 import asyncio
-import argparse
-from aiohttp import web
-import logging
-import os
-import base64
 import uuid
-
-# Module logger
-logger = logging.getLogger(__name__)
+import logging
 import json
 
-import pulsar
-from prometheus_client import start_http_server
-
-from ... schema import ConfigPush, config_push_queue
-from ... base import Consumer
+from ... schema import ConfigPush, ConfigRequest, ConfigResponse
+from ... schema import config_push_queue, config_request_queue
+from ... schema import config_response_queue
+from ... base import Consumer, Producer
+from ... base.subscriber import Subscriber
+from ... base.request_response_spec import RequestResponse
+from ... base.metrics import ProducerMetrics, SubscriberMetrics
 
 logger = logging.getLogger("config.receiver")
 logger.setLevel(logging.INFO)
 
+
 class ConfigReceiver:
 
     def __init__(self, backend):
@@ -42,34 +32,137 @@ class ConfigReceiver:
 
         self.flows = {}
 
+        self.config_version = 0
+
     def add_handler(self, h):
         self.flow_handlers.append(h)
 
-    async def on_config(self, msg, proc, flow):
+    async def on_config_notify(self, msg, proc, flow):
 
         try:
 
             v = msg.value()
+            notify_version = v.version
+            notify_types = set(v.types)
 
-            logger.info(f"Config version: {v.version}")
+            # Skip if we already have this version or newer
+            if notify_version <= self.config_version:
+                logger.debug(
+                    f"Ignoring config notify v{notify_version}, "
+                    f"already at v{self.config_version}"
+                )
+                return
 
-            flows = v.config.get("flow", {})
+            # Gateway cares about flow config
+            if notify_types and "flow" not in notify_types and "active-flow" not in notify_types:
+                logger.debug(
+                    f"Ignoring config notify v{notify_version}, "
+                    f"no flow types in {notify_types}"
+                )
+                self.config_version = notify_version
+                return
 
-            wanted = list(flows.keys())
-            current = list(self.flows.keys())
+            logger.info(
+                f"Config notify v{notify_version}, fetching config..."
+            )
 
-            for k in wanted:
-                if k not in current:
-                    self.flows[k] = json.loads(flows[k])
-                    await self.start_flow(k, self.flows[k])
-
-            for k in current:
-                if k not in wanted:
-                    await self.stop_flow(k, self.flows[k])
-                    del self.flows[k]
+            await self.fetch_and_apply()
 
         except Exception as e:
-            logger.error(f"Config processing exception: {e}", exc_info=True)
+            logger.error(
+                f"Config notify processing exception: {e}", exc_info=True
+            )
+
+    def _create_config_client(self):
+        """Create a short-lived config request/response client."""
+        id = str(uuid.uuid4())
+
+        config_req_metrics = ProducerMetrics(
+            processor="api-gateway", flow=None,
+            name="config-request",
+        )
+        config_resp_metrics = SubscriberMetrics(
+            processor="api-gateway", flow=None,
+            name="config-response",
+        )
+
+        return RequestResponse(
+            backend=self.backend,
+            subscription=f"api-gateway--config--{id}",
+            consumer_name="api-gateway",
+            request_topic=config_request_queue,
+            request_schema=ConfigRequest,
+            request_metrics=config_req_metrics,
+            response_topic=config_response_queue,
+            response_schema=ConfigResponse,
+            response_metrics=config_resp_metrics,
+        )
+
+    async def fetch_and_apply(self, retry=False):
+        """Fetch full config and apply flow changes.
+        If retry=True, keeps retrying until successful."""
+
+        while True:
+
+            try:
+                logger.info("Fetching config from config service...")
+
+                client = self._create_config_client()
+                try:
+                    await client.start()
+                    resp = await client.request(
+                        ConfigRequest(operation="config"),
+                        timeout=10,
+                    )
+                finally:
+                    await client.stop()
+
+                logger.info(f"Config response received")
+
+                if resp.error:
+                    if retry:
+                        logger.warning(
+                            f"Config fetch error: {resp.error.message}, "
+                            f"retrying in 2s..."
+                        )
+                        await asyncio.sleep(2)
+                        continue
+                    logger.error(
+                        f"Config fetch error: {resp.error.message}"
+                    )
+                    return
+
+                self.config_version = resp.version
+                config = resp.config
+
+                flows = config.get("flow", {})
+
+                wanted = list(flows.keys())
+                current = list(self.flows.keys())
+
+                for k in wanted:
+                    if k not in current:
+                        self.flows[k] = json.loads(flows[k])
+                        await self.start_flow(k, self.flows[k])
+
+                for k in current:
+                    if k not in wanted:
+                        await self.stop_flow(k, self.flows[k])
+                        del self.flows[k]
+
+                return
+
+            except Exception as e:
+                if retry:
+                    logger.warning(
+                        f"Config fetch failed: {e}, retrying in 2s..."
+                    )
+                    await asyncio.sleep(2)
+                    continue
+                logger.error(
+                    f"Config fetch exception: {e}", exc_info=True
+                )
+                return
 
     async def start_flow(self, id, flow):
 
@@ -80,7 +173,9 @@ class ConfigReceiver:
             try:
                 await handler.start_flow(id, flow)
             except Exception as e:
-                logger.error(f"Config processing exception: {e}", exc_info=True)
+                logger.error(
+                    f"Config processing exception: {e}", exc_info=True
+                )
 
     async def stop_flow(self, id, flow):
 
@@ -91,32 +186,54 @@ class ConfigReceiver:
             try:
                 await handler.stop_flow(id, flow)
             except Exception as e:
-                logger.error(f"Config processing exception: {e}", exc_info=True)
+                logger.error(
+                    f"Config processing exception: {e}", exc_info=True
+                )
 
     async def config_loader(self):
 
-        async with asyncio.TaskGroup() as tg:
+        while True:
 
-            id = str(uuid.uuid4())
+            try:
 
-            self.config_cons = Consumer(
-                taskgroup = tg,
-                flow = None,
-                backend = self.backend,
-                subscriber = f"gateway-{id}",
-                topic = config_push_queue,
-                schema = ConfigPush,
-                handler = self.on_config,
-                start_of_messages = True,
-            )
+                async with asyncio.TaskGroup() as tg:
 
-            await self.config_cons.start()
+                    id = str(uuid.uuid4())
 
-            logger.debug("Waiting for config updates...")
+                    # Subscribe to notify queue
+                    self.config_cons = Consumer(
+                        taskgroup=tg,
+                        flow=None,
+                        backend=self.backend,
+                        subscriber=f"gateway-{id}",
+                        topic=config_push_queue,
+                        schema=ConfigPush,
+                        handler=self.on_config_notify,
+                        start_of_messages=False,
+                    )
 
-        logger.info("Config consumer finished")
+                    logger.info("Starting config notify consumer...")
+                    await self.config_cons.start()
+                    logger.info("Config notify consumer started")
+
+                    # Fetch current config (subscribe-then-fetch pattern)
+                    # Retry until config service is available
+                    await self.fetch_and_apply(retry=True)
+
+                    logger.info(
+                        "Config loader initialised, waiting for notifys..."
+                    )
+
+                logger.warning("Config consumer exited, restarting...")
+
+            except Exception as e:
+                logger.error(
+                    f"Config loader exception: {e}, restarting in 4s...",
+                    exc_info=True
+                )
+
+            await asyncio.sleep(4)
 
     async def start(self):
-        
-        asyncio.create_task(self.config_loader())
 
+        asyncio.create_task(self.config_loader())

trustgraph-flow/trustgraph/gateway/dispatch/agent.py

@@ -25,8 +25,8 @@ class AgentRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("agent")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/collection_management.py

@@ -28,9 +28,7 @@ class CollectionManagementRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("collection-management")
 
     def to_request(self, body):
-        print("REQUEST", body, flush=True)
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        print("RESPONSE", message, flush=True)
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/config.py

@@ -30,8 +30,8 @@ class ConfigRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("config")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/document_embeddings_import.py

@@ -44,7 +44,7 @@ class DocumentEmbeddingsImport:
     async def receive(self, msg):
 
         data = msg.json()
-        elt = self.translator.to_pulsar(data)
+        elt = self.translator.decode(data)
         await self.publisher.send(None, elt)
 
     async def run(self):

trustgraph-flow/trustgraph/gateway/dispatch/document_embeddings_query.py

@@ -25,7 +25,7 @@ class DocumentEmbeddingsQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("document-embeddings-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/document_load.py

@@ -23,5 +23,5 @@ class DocumentLoad(ServiceSender):
 
     def to_request(self, body):
         logger.info("Document received")
-        return self.translator.to_pulsar(body)
+        return self.translator.decode(body)
 

trustgraph-flow/trustgraph/gateway/dispatch/document_rag.py

@@ -25,8 +25,8 @@ class DocumentRagRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("document-rag")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/embeddings.py

@@ -25,8 +25,8 @@ class EmbeddingsRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("embeddings")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/flow.py

@@ -30,8 +30,8 @@ class FlowRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("flow")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/graph_embeddings_query.py

@@ -25,8 +25,8 @@ class GraphEmbeddingsQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("graph-embeddings-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/graph_rag.py

@@ -25,8 +25,8 @@ class GraphRagRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("graph-rag")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/knowledge.py

@@ -33,8 +33,8 @@ class KnowledgeRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("knowledge")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/librarian.py

@@ -40,8 +40,8 @@ class LibrarianRequestor(ServiceRequestor):
             body = body.copy()
             body["content"] = content
         
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/manager.py

@@ -22,6 +22,7 @@ from . document_rag import DocumentRagRequestor
 from . triples_query import TriplesQueryRequestor
 from . rows_query import RowsQueryRequestor
 from . nlp_query import NLPQueryRequestor
+from . sparql_query import SparqlQueryRequestor
 from . structured_query import StructuredQueryRequestor
 from . structured_diag import StructuredDiagRequestor
 from . embeddings import EmbeddingsRequestor
@@ -65,6 +66,7 @@ request_response_dispatchers = {
     "structured-query": StructuredQueryRequestor,
     "structured-diag": StructuredDiagRequestor,
     "row-embeddings": RowEmbeddingsQueryRequestor,
+    "sparql": SparqlQueryRequestor,
 }
 
 global_dispatchers = {

trustgraph-flow/trustgraph/gateway/dispatch/mcp_tool.py

@@ -25,8 +25,8 @@ class McpToolRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("tool")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/nlp_query.py

@@ -24,7 +24,7 @@ class NLPQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("nlp-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/prompt.py

@@ -27,8 +27,8 @@ class PromptRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("prompt")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/row_embeddings_query.py

@@ -25,7 +25,7 @@ class RowEmbeddingsQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("row-embeddings-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/rows_query.py

@@ -24,7 +24,7 @@ class RowsQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("rows-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/serialize.py

@@ -11,22 +11,22 @@ _triple_translator = TripleTranslator()
 
 def to_value(x):
     """Convert dict to Term. Delegates to TermTranslator."""
-    return _term_translator.to_pulsar(x)
+    return _term_translator.decode(x)
 
 
 def to_subgraph(x):
     """Convert list of dicts to list of Triples. Delegates to TripleTranslator."""
-    return [_triple_translator.to_pulsar(t) for t in x]
+    return [_triple_translator.decode(t) for t in x]
 
 
 def serialize_value(v):
     """Convert Term to dict. Delegates to TermTranslator."""
-    return _term_translator.from_pulsar(v)
+    return _term_translator.encode(v)
 
 
 def serialize_triple(t):
     """Convert Triple to dict. Delegates to TripleTranslator."""
-    return _triple_translator.from_pulsar(t)
+    return _triple_translator.encode(t)
 
 
 def serialize_subgraph(sg):

trustgraph-flow/trustgraph/gateway/dispatch/sparql_query.py

@@ -0,0 +1,30 @@
+from ... schema import SparqlQueryRequest, SparqlQueryResponse
+from ... messaging import TranslatorRegistry
+
+from . requestor import ServiceRequestor
+
+class SparqlQueryRequestor(ServiceRequestor):
+    def __init__(
+            self, backend, request_queue, response_queue, timeout,
+            consumer, subscriber,
+    ):
+
+        super(SparqlQueryRequestor, self).__init__(
+            backend=backend,
+            request_queue=request_queue,
+            response_queue=response_queue,
+            request_schema=SparqlQueryRequest,
+            response_schema=SparqlQueryResponse,
+            subscription = subscriber,
+            consumer_name = consumer,
+            timeout=timeout,
+        )
+
+        self.request_translator = TranslatorRegistry.get_request_translator("sparql-query")
+        self.response_translator = TranslatorRegistry.get_response_translator("sparql-query")
+
+    def to_request(self, body):
+        return self.request_translator.decode(body)
+
+    def from_response(self, message):
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/structured_diag.py

@@ -24,7 +24,7 @@ class StructuredDiagRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("structured-diag")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/structured_query.py

@@ -24,7 +24,7 @@ class StructuredQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("structured-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)

trustgraph-flow/trustgraph/gateway/dispatch/text_completion.py

@@ -25,8 +25,8 @@ class TextCompletionRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("text-completion")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/dispatch/text_load.py

@@ -23,5 +23,5 @@ class TextLoad(ServiceSender):
 
     def to_request(self, body):
         logger.info("Text document received")
-        return self.translator.to_pulsar(body)
+        return self.translator.decode(body)
 

trustgraph-flow/trustgraph/gateway/dispatch/triples_query.py

@@ -25,8 +25,8 @@ class TriplesQueryRequestor(ServiceRequestor):
         self.response_translator = TranslatorRegistry.get_response_translator("triples-query")
 
     def to_request(self, body):
-        return self.request_translator.to_pulsar(body)
+        return self.request_translator.decode(body)
 
     def from_response(self, message):
-        return self.response_translator.from_response_with_completion(message)
+        return self.response_translator.encode_with_completion(message)
 

trustgraph-flow/trustgraph/gateway/service.py

@@ -10,7 +10,7 @@ import logging
 import os
 
 from trustgraph.base.logging import setup_logging
-from trustgraph.base.pubsub import get_pubsub
+from trustgraph.base.pubsub import get_pubsub, add_pubsub_args
 
 from . auth import Authenticator
 from . config.receiver import ConfigReceiver
@@ -18,7 +18,6 @@ from . dispatch.manager import DispatcherManager
 
 from . endpoint.manager import EndpointManager
 
-import pulsar
 from prometheus_client import start_http_server
 
 # Import default queue names
@@ -168,30 +167,7 @@ def run():
         help='Service identifier for logging and metrics (default: api-gateway)',
     )
 
-    # Pub/sub backend selection
-    parser.add_argument(
-        '--pubsub-backend',
-        default=os.getenv('PUBSUB_BACKEND', 'pulsar'),
-        choices=['pulsar', 'mqtt'],
-        help='Pub/sub backend (default: pulsar, env: PUBSUB_BACKEND)',
-    )
-
-    parser.add_argument(
-        '-p', '--pulsar-host',
-        default=default_pulsar_host,
-        help=f'Pulsar host (default: {default_pulsar_host})',
-    )
-    
-    parser.add_argument(
-        '--pulsar-api-key',
-        default=default_pulsar_api_key,
-        help=f'Pulsar API key',
-    )
-
-    parser.add_argument(
-        '--pulsar-listener',
-        help=f'Pulsar listener (default: none)',
-    )
+    add_pubsub_args(parser)
 
     parser.add_argument(
         '-m', '--prometheus-url',

trustgraph-flow/trustgraph/librarian/service.py

@@ -246,7 +246,10 @@ class Processor(AsyncProcessor):
             taskgroup = self.taskgroup,
         )
 
-        self.register_config_handler(self.on_librarian_config)
+        self.register_config_handler(
+            self.on_librarian_config,
+            types=["flow", "active-flow"],
+        )
 
         self.flows = {}
 

trustgraph-flow/trustgraph/metering/counter.py

@@ -40,7 +40,7 @@ class Processor(FlowProcessor):
             }
         )
 
-        self.register_config_handler(self.on_cost_config)
+        self.register_config_handler(self.on_cost_config, types=["token-cost"])
 
         self.register_specification(
             ConsumerSpec(

trustgraph-flow/trustgraph/prompt/template/service.py

@@ -65,7 +65,7 @@ class Processor(FlowProcessor):
             )
         )
 
-        self.register_config_handler(self.on_prompt_config)
+        self.register_config_handler(self.on_prompt_config, types=["prompt"])
 
         # Null configuration, should reload quickly
         self.manager = PromptManager()

trustgraph-flow/trustgraph/query/rows/cassandra/service.py

@@ -84,7 +84,7 @@ class Processor(FlowProcessor):
         )
 
         # Register config handler for schema updates
-        self.register_config_handler(self.on_schema_config)
+        self.register_config_handler(self.on_schema_config, types=["schema"])
 
         # Schema storage: name -> RowSchema
         self.schemas: Dict[str, RowSchema] = {}

trustgraph-flow/trustgraph/query/sparql/__init__.py

@@ -0,0 +1 @@
+from . service import *

trustgraph-flow/trustgraph/query/sparql/__main__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from . service import run
+
+if __name__ == '__main__':
+    run()

trustgraph-flow/trustgraph/query/sparql/algebra.py

@@ -0,0 +1,541 @@
+"""
+SPARQL algebra evaluator.
+
+Recursively evaluates an rdflib SPARQL algebra tree by issuing triple
+pattern queries via TriplesClient (streaming) and performing in-memory
+joins, filters, and projections.
+"""
+
+import logging
+from collections import defaultdict
+
+from rdflib.term import Variable, URIRef, Literal, BNode
+from rdflib.plugins.sparql.parserutils import CompValue
+
+from ... schema import Term, Triple, IRI, LITERAL, BLANK
+from ... knowledge import Uri
+from ... knowledge import Literal as KgLiteral
+from . parser import rdflib_term_to_term
+from . solutions import (
+    hash_join, left_join, union, project, distinct,
+    order_by, slice_solutions, _term_key,
+)
+from . expressions import evaluate_expression, _effective_boolean
+
+logger = logging.getLogger(__name__)
+
+
+class EvaluationError(Exception):
+    """Raised when SPARQL evaluation fails."""
+    pass
+
+
+async def evaluate(node, triples_client, user, collection, limit=10000):
+    """
+    Evaluate a SPARQL algebra node.
+
+    Args:
+        node: rdflib CompValue algebra node
+        triples_client: TriplesClient instance for triple pattern queries
+        user: user/keyspace identifier
+        collection: collection identifier
+        limit: safety limit on results
+
+    Returns:
+        list of solutions (dicts mapping variable names to Term values)
+    """
+    if not isinstance(node, CompValue):
+        logger.warning(f"Expected CompValue, got {type(node)}: {node}")
+        return [{}]
+
+    name = node.name
+    handler = _HANDLERS.get(name)
+
+    if handler is None:
+        logger.warning(f"Unsupported algebra node: {name}")
+        return [{}]
+
+    return await handler(node, triples_client, user, collection, limit)
+
+
+# --- Node handlers ---
+
+async def _eval_select_query(node, tc, user, collection, limit):
+    """Evaluate a SelectQuery node."""
+    return await evaluate(node.p, tc, user, collection, limit)
+
+
+async def _eval_project(node, tc, user, collection, limit):
+    """Evaluate a Project node (SELECT variable projection)."""
+    solutions = await evaluate(node.p, tc, user, collection, limit)
+    variables = [str(v) for v in node.PV]
+    return project(solutions, variables)
+
+
+async def _eval_bgp(node, tc, user, collection, limit):
+    """
+    Evaluate a Basic Graph Pattern.
+
+    Issues streaming triple pattern queries and joins results. Patterns
+    are ordered by selectivity (more bound terms first) and evaluated
+    sequentially with bound-variable substitution.
+    """
+    triples = node.triples
+    if not triples:
+        return [{}]
+
+    # Sort patterns by selectivity: more bound terms = more selective
+    def selectivity(pattern):
+        return sum(1 for t in pattern if not isinstance(t, Variable))
+
+    sorted_patterns = sorted(
+        enumerate(triples), key=lambda x: -selectivity(x[1])
+    )
+
+    solutions = [{}]
+
+    for _, pattern in sorted_patterns:
+        s_tmpl, p_tmpl, o_tmpl = pattern
+
+        new_solutions = []
+
+        for sol in solutions:
+            # Substitute known bindings into the pattern
+            s_val = _resolve_term(s_tmpl, sol)
+            p_val = _resolve_term(p_tmpl, sol)
+            o_val = _resolve_term(o_tmpl, sol)
+
+            # Query the triples store
+            results = await _query_pattern(
+                tc, s_val, p_val, o_val, user, collection, limit
+            )
+
+            # Map results back to variable bindings,
+            # converting Uri/Literal to Term objects
+            for triple in results:
+                binding = dict(sol)
+                if isinstance(s_tmpl, Variable):
+                    binding[str(s_tmpl)] = _to_term(triple.s)
+                if isinstance(p_tmpl, Variable):
+                    binding[str(p_tmpl)] = _to_term(triple.p)
+                if isinstance(o_tmpl, Variable):
+                    binding[str(o_tmpl)] = _to_term(triple.o)
+                new_solutions.append(binding)
+
+        solutions = new_solutions
+
+        if not solutions:
+            break
+
+    return solutions[:limit]
+
+
+async def _eval_join(node, tc, user, collection, limit):
+    """Evaluate a Join node."""
+    left = await evaluate(node.p1, tc, user, collection, limit)
+    right = await evaluate(node.p2, tc, user, collection, limit)
+    return hash_join(left, right)[:limit]
+
+
+async def _eval_left_join(node, tc, user, collection, limit):
+    """Evaluate a LeftJoin node (OPTIONAL)."""
+    left_sols = await evaluate(node.p1, tc, user, collection, limit)
+    right_sols = await evaluate(node.p2, tc, user, collection, limit)
+
+    filter_fn = None
+    if hasattr(node, "expr") and node.expr is not None:
+        expr = node.expr
+        if not (isinstance(expr, CompValue) and expr.name == "TrueFilter"):
+            filter_fn = lambda sol: _effective_boolean(
+                evaluate_expression(expr, sol)
+            )
+
+    return left_join(left_sols, right_sols, filter_fn)[:limit]
+
+
+async def _eval_union(node, tc, user, collection, limit):
+    """Evaluate a Union node."""
+    left = await evaluate(node.p1, tc, user, collection, limit)
+    right = await evaluate(node.p2, tc, user, collection, limit)
+    return union(left, right)[:limit]
+
+
+async def _eval_filter(node, tc, user, collection, limit):
+    """Evaluate a Filter node."""
+    solutions = await evaluate(node.p, tc, user, collection, limit)
+    expr = node.expr
+    return [
+        sol for sol in solutions
+        if _effective_boolean(evaluate_expression(expr, sol))
+    ]
+
+
+async def _eval_distinct(node, tc, user, collection, limit):
+    """Evaluate a Distinct node."""
+    solutions = await evaluate(node.p, tc, user, collection, limit)
+    return distinct(solutions)
+
+
+async def _eval_reduced(node, tc, user, collection, limit):
+    """Evaluate a Reduced node (like Distinct but implementation-defined)."""
+    # Treat same as Distinct
+    solutions = await evaluate(node.p, tc, user, collection, limit)
+    return distinct(solutions)
+
+
+async def _eval_order_by(node, tc, user, collection, limit):
+    """Evaluate an OrderBy node."""
+    solutions = await evaluate(node.p, tc, user, collection, limit)
+
+    key_fns = []
+    for cond in node.expr:
+        if isinstance(cond, CompValue) and cond.name == "OrderCondition":
+            ascending = cond.order != "DESC"
+            expr = cond.expr
+            key_fns.append((
+                lambda sol, e=expr: evaluate_expression(e, sol),
+                ascending,
+            ))
+        else:
+            # Simple variable or expression
+            key_fns.append((
+                lambda sol, e=cond: evaluate_expression(e, sol),
+                True,
+            ))
+
+    return order_by(solutions, key_fns)
+
+
+async def _eval_slice(node, tc, user, collection, limit):
+    """Evaluate a Slice node (LIMIT/OFFSET)."""
+    # Pass tighter limit downstream if possible
+    inner_limit = limit
+    if node.length is not None:
+        offset = node.start or 0
+        inner_limit = min(limit, offset + node.length)
+
+    solutions = await evaluate(node.p, tc, user, collection, inner_limit)
+    return slice_solutions(solutions, node.start or 0, node.length)
+
+
+async def _eval_extend(node, tc, user, collection, limit):
+    """Evaluate an Extend node (BIND)."""
+    solutions = await evaluate(node.p, tc, user, collection, limit)
+    var_name = str(node.var)
+    expr = node.expr
+
+    result = []
+    for sol in solutions:
+        val = evaluate_expression(expr, sol)
+        new_sol = dict(sol)
+        if isinstance(val, Term):
+            new_sol[var_name] = val
+        elif isinstance(val, (int, float)):
+            new_sol[var_name] = Term(type=LITERAL, value=str(val))
+        elif isinstance(val, str):
+            new_sol[var_name] = Term(type=LITERAL, value=val)
+        elif isinstance(val, bool):
+            new_sol[var_name] = Term(
+                type=LITERAL, value=str(val).lower(),
+                datatype="http://www.w3.org/2001/XMLSchema#boolean"
+            )
+        elif val is not None:
+            new_sol[var_name] = Term(type=LITERAL, value=str(val))
+        result.append(new_sol)
+
+    return result
+
+
+async def _eval_group(node, tc, user, collection, limit):
+    """Evaluate a Group node (GROUP BY with aggregation)."""
+    solutions = await evaluate(node.p, tc, user, collection, limit)
+
+    # Extract grouping expressions
+    group_exprs = []
+    if hasattr(node, "expr") and node.expr:
+        for expr in node.expr:
+            if isinstance(expr, CompValue) and expr.name == "GroupAs":
+                group_exprs.append((expr.expr, str(expr.var) if hasattr(expr, "var") and expr.var else None))
+            elif isinstance(expr, Variable):
+                group_exprs.append((expr, str(expr)))
+            else:
+                group_exprs.append((expr, None))
+
+    # Group solutions
+    groups = defaultdict(list)
+    for sol in solutions:
+        key_parts = []
+        for expr, _ in group_exprs:
+            val = evaluate_expression(expr, sol)
+            key_parts.append(_term_key(val) if isinstance(val, Term) else val)
+        groups[tuple(key_parts)].append(sol)
+
+    if not group_exprs:
+        # No GROUP BY - entire result is one group
+        groups[()].extend(solutions)
+
+    # Build grouped solutions (one per group)
+    result = []
+    for key, group_sols in groups.items():
+        sol = {}
+        # Include group key variables
+        if group_sols:
+            for (expr, var_name), k in zip(group_exprs, key):
+                if var_name and group_sols:
+                    sol[var_name] = evaluate_expression(expr, group_sols[0])
+        sol["__group__"] = group_sols
+        result.append(sol)
+
+    return result
+
+
+async def _eval_aggregate_join(node, tc, user, collection, limit):
+    """Evaluate an AggregateJoin (aggregation functions after GROUP BY)."""
+    solutions = await evaluate(node.p, tc, user, collection, limit)
+
+    result = []
+    for sol in solutions:
+        group = sol.get("__group__", [sol])
+        new_sol = {k: v for k, v in sol.items() if k != "__group__"}
+
+        # Apply aggregate functions
+        if hasattr(node, "A") and node.A:
+            for agg in node.A:
+                var_name = str(agg.res)
+                agg_val = _compute_aggregate(agg, group)
+                new_sol[var_name] = agg_val
+
+        result.append(new_sol)
+
+    return result
+
+
+async def _eval_graph(node, tc, user, collection, limit):
+    """Evaluate a Graph node (GRAPH clause)."""
+    term = node.term
+
+    if isinstance(term, URIRef):
+        # GRAPH <uri> { ... } — fixed graph
+        # We'd need to pass graph to triples queries
+        # For now, evaluate inner pattern normally
+        logger.info(f"GRAPH <{term}> clause - graph filtering not yet wired")
+        return await evaluate(node.p, tc, user, collection, limit)
+    elif isinstance(term, Variable):
+        # GRAPH ?g { ... } — variable graph
+        logger.info(f"GRAPH ?{term} clause - variable graph not yet wired")
+        return await evaluate(node.p, tc, user, collection, limit)
+    else:
+        return await evaluate(node.p, tc, user, collection, limit)
+
+
+async def _eval_values(node, tc, user, collection, limit):
+    """Evaluate a VALUES clause (inline data)."""
+    variables = [str(v) for v in node.var]
+    solutions = []
+
+    for row in node.value:
+        sol = {}
+        for var_name, val in zip(variables, row):
+            if val is not None and str(val) != "UNDEF":
+                sol[var_name] = rdflib_term_to_term(val)
+        solutions.append(sol)
+
+    return solutions
+
+
+async def _eval_to_multiset(node, tc, user, collection, limit):
+    """Evaluate a ToMultiSet node (subquery)."""
+    return await evaluate(node.p, tc, user, collection, limit)
+
+
+# --- Aggregate computation ---
+
+def _compute_aggregate(agg, group):
+    """Compute a single aggregate function over a group of solutions."""
+    agg_name = agg.name if hasattr(agg, "name") else ""
+
+    # Get the expression to aggregate
+    expr = agg.vars if hasattr(agg, "vars") else None
+
+    if agg_name == "Aggregate_Count":
+        if hasattr(agg, "distinct") and agg.distinct:
+            vals = set()
+            for sol in group:
+                if expr:
+                    val = evaluate_expression(expr, sol)
+                    if val is not None:
+                        vals.add(_term_key(val) if isinstance(val, Term) else val)
+                else:
+                    vals.add(id(sol))
+            return Term(type=LITERAL, value=str(len(vals)),
+                       datatype="http://www.w3.org/2001/XMLSchema#integer")
+        return Term(type=LITERAL, value=str(len(group)),
+                   datatype="http://www.w3.org/2001/XMLSchema#integer")
+
+    if agg_name == "Aggregate_Sum":
+        total = 0
+        for sol in group:
+            val = evaluate_expression(expr, sol) if expr else None
+            num = _try_numeric(val)
+            if num is not None:
+                total += num
+        return Term(type=LITERAL, value=str(total),
+                   datatype="http://www.w3.org/2001/XMLSchema#decimal")
+
+    if agg_name == "Aggregate_Avg":
+        total = 0
+        count = 0
+        for sol in group:
+            val = evaluate_expression(expr, sol) if expr else None
+            num = _try_numeric(val)
+            if num is not None:
+                total += num
+                count += 1
+        avg = total / count if count > 0 else 0
+        return Term(type=LITERAL, value=str(avg),
+                   datatype="http://www.w3.org/2001/XMLSchema#decimal")
+
+    if agg_name == "Aggregate_Min":
+        min_val = None
+        for sol in group:
+            val = evaluate_expression(expr, sol) if expr else None
+            if val is not None:
+                cmp = _term_key(val) if isinstance(val, Term) else val
+                if min_val is None or cmp < min_val[0]:
+                    min_val = (cmp, val)
+        if min_val:
+            val = min_val[1]
+            if isinstance(val, Term):
+                return val
+            return Term(type=LITERAL, value=str(val))
+        return None
+
+    if agg_name == "Aggregate_Max":
+        max_val = None
+        for sol in group:
+            val = evaluate_expression(expr, sol) if expr else None
+            if val is not None:
+                cmp = _term_key(val) if isinstance(val, Term) else val
+                if max_val is None or cmp > max_val[0]:
+                    max_val = (cmp, val)
+        if max_val:
+            val = max_val[1]
+            if isinstance(val, Term):
+                return val
+            return Term(type=LITERAL, value=str(val))
+        return None
+
+    if agg_name == "Aggregate_GroupConcat":
+        separator = agg.separator if hasattr(agg, "separator") else " "
+        vals = []
+        for sol in group:
+            val = evaluate_expression(expr, sol) if expr else None
+            if val is not None:
+                if isinstance(val, Term):
+                    vals.append(val.value if val.type == LITERAL else val.iri)
+                else:
+                    vals.append(str(val))
+        return Term(type=LITERAL, value=separator.join(vals))
+
+    if agg_name == "Aggregate_Sample":
+        if group:
+            val = evaluate_expression(expr, group[0]) if expr else None
+            if isinstance(val, Term):
+                return val
+            if val is not None:
+                return Term(type=LITERAL, value=str(val))
+        return None
+
+    logger.warning(f"Unsupported aggregate: {agg_name}")
+    return None
+
+
+# --- Helper functions ---
+
+def _to_term(val):
+    """
+    Convert a value to a schema Term. Handles Uri and Literal from the
+    knowledge module (returned by TriplesClient) as well as plain strings.
+    """
+    if val is None:
+        return None
+    if isinstance(val, Term):
+        return val
+    if isinstance(val, Uri):
+        return Term(type=IRI, iri=str(val))
+    if isinstance(val, KgLiteral):
+        return Term(type=LITERAL, value=str(val))
+    if isinstance(val, str):
+        if val.startswith("http://") or val.startswith("https://") or val.startswith("urn:"):
+            return Term(type=IRI, iri=val)
+        return Term(type=LITERAL, value=val)
+    return Term(type=LITERAL, value=str(val))
+
+
+def _resolve_term(tmpl, solution):
+    """
+    Resolve a triple pattern term. If it's a variable and bound in the
+    solution, return the bound Term. Otherwise return None (wildcard)
+    for variables, or convert concrete terms.
+    """
+    if isinstance(tmpl, Variable):
+        name = str(tmpl)
+        if name in solution:
+            return solution[name]
+        return None
+    else:
+        return rdflib_term_to_term(tmpl)
+
+
+async def _query_pattern(tc, s, p, o, user, collection, limit):
+    """
+    Issue a streaming triple pattern query via TriplesClient.
+
+    Returns a list of Triple-like objects with s, p, o attributes.
+    """
+    results = await tc.query(
+        s=s, p=p, o=o,
+        limit=limit,
+        user=user,
+        collection=collection,
+    )
+    return results
+
+
+def _try_numeric(val):
+    """Try to convert a value to a number, return None on failure."""
+    if val is None:
+        return None
+    if isinstance(val, (int, float)):
+        return val
+    if isinstance(val, Term) and val.type == LITERAL:
+        try:
+            if "." in val.value:
+                return float(val.value)
+            return int(val.value)
+        except (ValueError, TypeError):
+            return None
+    return None
+
+
+# --- Handler registry ---
+
+_HANDLERS = {
+    "SelectQuery": _eval_select_query,
+    "Project": _eval_project,
+    "BGP": _eval_bgp,
+    "Join": _eval_join,
+    "LeftJoin": _eval_left_join,
+    "Union": _eval_union,
+    "Filter": _eval_filter,
+    "Distinct": _eval_distinct,
+    "Reduced": _eval_reduced,
+    "OrderBy": _eval_order_by,
+    "Slice": _eval_slice,
+    "Extend": _eval_extend,
+    "Group": _eval_group,
+    "AggregateJoin": _eval_aggregate_join,
+    "Graph": _eval_graph,
+    "values": _eval_values,
+    "ToMultiSet": _eval_to_multiset,
+}

trustgraph-flow/trustgraph/query/sparql/expressions.py

@@ -0,0 +1,481 @@
+"""
+SPARQL FILTER expression evaluator.
+
+Evaluates rdflib algebra expression nodes against a solution (variable
+binding) to produce a value or boolean result.
+"""
+
+import re
+import logging
+import operator
+
+from rdflib.term import Variable, URIRef, Literal, BNode
+from rdflib.plugins.sparql.parserutils import CompValue
+
+from ... schema import Term, IRI, LITERAL, BLANK
+from . parser import rdflib_term_to_term
+
+logger = logging.getLogger(__name__)
+
+
+class ExpressionError(Exception):
+    """Raised when a SPARQL expression cannot be evaluated."""
+    pass
+
+
+def evaluate_expression(expr, solution):
+    """
+    Evaluate a SPARQL expression against a solution binding.
+
+    Args:
+        expr: rdflib algebra expression node
+        solution: dict mapping variable names to Term values
+
+    Returns:
+        The result value (Term, bool, number, string, or None)
+    """
+    if expr is None:
+        return True
+
+    # rdflib Variable
+    if isinstance(expr, Variable):
+        name = str(expr)
+        return solution.get(name)
+
+    # rdflib concrete terms
+    if isinstance(expr, URIRef):
+        return Term(type=IRI, iri=str(expr))
+
+    if isinstance(expr, Literal):
+        return rdflib_term_to_term(expr)
+
+    if isinstance(expr, BNode):
+        return Term(type=BLANK, id=str(expr))
+
+    # Boolean constants
+    if isinstance(expr, bool):
+        return expr
+
+    # Numeric constants
+    if isinstance(expr, (int, float)):
+        return expr
+
+    # String constants
+    if isinstance(expr, str):
+        return expr
+
+    # CompValue nodes from rdflib algebra
+    if isinstance(expr, CompValue):
+        return _evaluate_comp_value(expr, solution)
+
+    # List/tuple (e.g. function arguments)
+    if isinstance(expr, (list, tuple)):
+        return [evaluate_expression(e, solution) for e in expr]
+
+    logger.warning(f"Unknown expression type: {type(expr)}: {expr}")
+    return None
+
+
+def _evaluate_comp_value(node, solution):
+    """Evaluate a CompValue expression node."""
+    name = node.name
+
+    # Relational expressions: =, !=, <, >, <=, >=
+    if name == "RelationalExpression":
+        return _eval_relational(node, solution)
+
+    # Conditional AND / OR
+    if name == "ConditionalAndExpression":
+        return _eval_conditional_and(node, solution)
+
+    if name == "ConditionalOrExpression":
+        return _eval_conditional_or(node, solution)
+
+    # Unary NOT
+    if name == "UnaryNot":
+        val = evaluate_expression(node.expr, solution)
+        return not _effective_boolean(val)
+
+    # Unary plus/minus
+    if name == "UnaryPlus":
+        return _to_numeric(evaluate_expression(node.expr, solution))
+
+    if name == "UnaryMinus":
+        val = _to_numeric(evaluate_expression(node.expr, solution))
+        return -val if val is not None else None
+
+    # Arithmetic
+    if name == "AdditiveExpression":
+        return _eval_additive(node, solution)
+
+    if name == "MultiplicativeExpression":
+        return _eval_multiplicative(node, solution)
+
+    # SPARQL built-in functions
+    if name.startswith("Builtin_"):
+        return _eval_builtin(name, node, solution)
+
+    # Function call
+    if name == "Function":
+        return _eval_function(node, solution)
+
+    # Exists / NotExists
+    if name == "Builtin_EXISTS":
+        # EXISTS requires graph pattern evaluation - not handled here
+        logger.warning("EXISTS not supported in filter expressions")
+        return True
+
+    if name == "Builtin_NOTEXISTS":
+        logger.warning("NOT EXISTS not supported in filter expressions")
+        return True
+
+    # TrueFilter (used with OPTIONAL)
+    if name == "TrueFilter":
+        return True
+
+    # IN / NOT IN
+    if name == "Builtin_IN":
+        return _eval_in(node, solution)
+
+    if name == "Builtin_NOTIN":
+        return not _eval_in(node, solution)
+
+    logger.warning(f"Unknown CompValue expression: {name}")
+    return None
+
+
+def _eval_relational(node, solution):
+    """Evaluate a relational expression (=, !=, <, >, <=, >=)."""
+    left = evaluate_expression(node.expr, solution)
+    right = evaluate_expression(node.other, solution)
+    op = node.op
+
+    if left is None or right is None:
+        return False
+
+    left_cmp = _comparable_value(left)
+    right_cmp = _comparable_value(right)
+
+    ops = {
+        "=": operator.eq, "==": operator.eq,
+        "!=": operator.ne,
+        "<": operator.lt,
+        ">": operator.gt,
+        "<=": operator.le,
+        ">=": operator.ge,
+    }
+
+    op_fn = ops.get(str(op))
+    if op_fn is None:
+        logger.warning(f"Unknown relational operator: {op}")
+        return False
+
+    try:
+        return op_fn(left_cmp, right_cmp)
+    except TypeError:
+        return False
+
+
+def _eval_conditional_and(node, solution):
+    """Evaluate AND expression."""
+    result = _effective_boolean(evaluate_expression(node.expr, solution))
+    if not result:
+        return False
+    for other in node.other:
+        result = _effective_boolean(evaluate_expression(other, solution))
+        if not result:
+            return False
+    return True
+
+
+def _eval_conditional_or(node, solution):
+    """Evaluate OR expression."""
+    result = _effective_boolean(evaluate_expression(node.expr, solution))
+    if result:
+        return True
+    for other in node.other:
+        result = _effective_boolean(evaluate_expression(other, solution))
+        if result:
+            return True
+    return False
+
+
+def _eval_additive(node, solution):
+    """Evaluate additive expression (a + b - c ...)."""
+    result = _to_numeric(evaluate_expression(node.expr, solution))
+    if result is None:
+        return None
+    for op, operand in zip(node.op, node.other):
+        val = _to_numeric(evaluate_expression(operand, solution))
+        if val is None:
+            return None
+        if str(op) == "+":
+            result = result + val
+        elif str(op) == "-":
+            result = result - val
+    return result
+
+
+def _eval_multiplicative(node, solution):
+    """Evaluate multiplicative expression (a * b / c ...)."""
+    result = _to_numeric(evaluate_expression(node.expr, solution))
+    if result is None:
+        return None
+    for op, operand in zip(node.op, node.other):
+        val = _to_numeric(evaluate_expression(operand, solution))
+        if val is None:
+            return None
+        if str(op) == "*":
+            result = result * val
+        elif str(op) == "/":
+            if val == 0:
+                return None
+            result = result / val
+    return result
+
+
+def _eval_builtin(name, node, solution):
+    """Evaluate SPARQL built-in functions."""
+    builtin = name[len("Builtin_"):]
+
+    if builtin == "BOUND":
+        var_name = str(node.arg)
+        return var_name in solution and solution[var_name] is not None
+
+    if builtin == "isIRI" or builtin == "isURI":
+        val = evaluate_expression(node.arg, solution)
+        return isinstance(val, Term) and val.type == IRI
+
+    if builtin == "isLITERAL":
+        val = evaluate_expression(node.arg, solution)
+        return isinstance(val, Term) and val.type == LITERAL
+
+    if builtin == "isBLANK":
+        val = evaluate_expression(node.arg, solution)
+        return isinstance(val, Term) and val.type == BLANK
+
+    if builtin == "STR":
+        val = evaluate_expression(node.arg, solution)
+        return Term(type=LITERAL, value=_to_string(val))
+
+    if builtin == "LANG":
+        val = evaluate_expression(node.arg, solution)
+        if isinstance(val, Term) and val.type == LITERAL:
+            return Term(type=LITERAL, value=val.language or "")
+        return Term(type=LITERAL, value="")
+
+    if builtin == "DATATYPE":
+        val = evaluate_expression(node.arg, solution)
+        if isinstance(val, Term) and val.type == LITERAL and val.datatype:
+            return Term(type=IRI, iri=val.datatype)
+        return Term(type=IRI, iri="http://www.w3.org/2001/XMLSchema#string")
+
+    if builtin == "REGEX":
+        text = _to_string(evaluate_expression(node.text, solution))
+        pattern = _to_string(evaluate_expression(node.pattern, solution))
+        flags_str = ""
+        if hasattr(node, "flags") and node.flags is not None:
+            flags_str = _to_string(evaluate_expression(node.flags, solution))
+
+        re_flags = 0
+        if "i" in flags_str:
+            re_flags |= re.IGNORECASE
+        if "m" in flags_str:
+            re_flags |= re.MULTILINE
+        if "s" in flags_str:
+            re_flags |= re.DOTALL
+
+        try:
+            return bool(re.search(pattern, text, re_flags))
+        except re.error:
+            return False
+
+    if builtin == "STRLEN":
+        val = _to_string(evaluate_expression(node.arg, solution))
+        return len(val)
+
+    if builtin == "UCASE":
+        val = _to_string(evaluate_expression(node.arg, solution))
+        return Term(type=LITERAL, value=val.upper())
+
+    if builtin == "LCASE":
+        val = _to_string(evaluate_expression(node.arg, solution))
+        return Term(type=LITERAL, value=val.lower())
+
+    if builtin == "CONTAINS":
+        string = _to_string(evaluate_expression(node.arg1, solution))
+        pattern = _to_string(evaluate_expression(node.arg2, solution))
+        return pattern in string
+
+    if builtin == "STRSTARTS":
+        string = _to_string(evaluate_expression(node.arg1, solution))
+        prefix = _to_string(evaluate_expression(node.arg2, solution))
+        return string.startswith(prefix)
+
+    if builtin == "STRENDS":
+        string = _to_string(evaluate_expression(node.arg1, solution))
+        suffix = _to_string(evaluate_expression(node.arg2, solution))
+        return string.endswith(suffix)
+
+    if builtin == "CONCAT":
+        args = [_to_string(evaluate_expression(a, solution)) for a in node.arg]
+        return Term(type=LITERAL, value="".join(args))
+
+    if builtin == "IF":
+        cond = _effective_boolean(evaluate_expression(node.arg1, solution))
+        if cond:
+            return evaluate_expression(node.arg2, solution)
+        else:
+            return evaluate_expression(node.arg3, solution)
+
+    if builtin == "COALESCE":
+        for arg in node.arg:
+            val = evaluate_expression(arg, solution)
+            if val is not None:
+                return val
+        return None
+
+    if builtin == "sameTerm":
+        left = evaluate_expression(node.arg1, solution)
+        right = evaluate_expression(node.arg2, solution)
+        if not isinstance(left, Term) or not isinstance(right, Term):
+            return False
+        from . solutions import _term_key
+        return _term_key(left) == _term_key(right)
+
+    logger.warning(f"Unsupported built-in function: {builtin}")
+    return None
+
+
+def _eval_function(node, solution):
+    """Evaluate a SPARQL function call."""
+    # Cast functions (xsd:integer, xsd:string, etc.)
+    iri = str(node.iri) if hasattr(node, "iri") else ""
+    args = [evaluate_expression(a, solution) for a in node.expr]
+
+    xsd = "http://www.w3.org/2001/XMLSchema#"
+    if iri == xsd + "integer":
+        try:
+            return int(_to_numeric(args[0]))
+        except (TypeError, ValueError):
+            return None
+    elif iri == xsd + "decimal" or iri == xsd + "double" or iri == xsd + "float":
+        try:
+            return float(_to_numeric(args[0]))
+        except (TypeError, ValueError):
+            return None
+    elif iri == xsd + "string":
+        return Term(type=LITERAL, value=_to_string(args[0]))
+    elif iri == xsd + "boolean":
+        return _effective_boolean(args[0])
+
+    logger.warning(f"Unsupported function: {iri}")
+    return None
+
+
+def _eval_in(node, solution):
+    """Evaluate IN expression."""
+    val = evaluate_expression(node.expr, solution)
+    for item in node.other:
+        other = evaluate_expression(item, solution)
+        if _comparable_value(val) == _comparable_value(other):
+            return True
+    return False
+
+
+# --- Value conversion helpers ---
+
+def _effective_boolean(val):
+    """Convert a value to its effective boolean value (EBV)."""
+    if isinstance(val, bool):
+        return val
+    if val is None:
+        return False
+    if isinstance(val, (int, float)):
+        return val != 0
+    if isinstance(val, str):
+        return len(val) > 0
+    if isinstance(val, Term):
+        if val.type == LITERAL:
+            v = val.value
+            if val.datatype == "http://www.w3.org/2001/XMLSchema#boolean":
+                return v.lower() in ("true", "1")
+            if val.datatype in (
+                "http://www.w3.org/2001/XMLSchema#integer",
+                "http://www.w3.org/2001/XMLSchema#decimal",
+                "http://www.w3.org/2001/XMLSchema#double",
+                "http://www.w3.org/2001/XMLSchema#float",
+            ):
+                try:
+                    return float(v) != 0
+                except ValueError:
+                    return False
+            return len(v) > 0
+        return True
+    return bool(val)
+
+
+def _to_string(val):
+    """Convert a value to a string."""
+    if val is None:
+        return ""
+    if isinstance(val, str):
+        return val
+    if isinstance(val, Term):
+        if val.type == IRI:
+            return val.iri
+        elif val.type == LITERAL:
+            return val.value
+        elif val.type == BLANK:
+            return val.id
+    return str(val)
+
+
+def _to_numeric(val):
+    """Convert a value to a number."""
+    if val is None:
+        return None
+    if isinstance(val, (int, float)):
+        return val
+    if isinstance(val, Term) and val.type == LITERAL:
+        try:
+            if "." in val.value:
+                return float(val.value)
+            return int(val.value)
+        except (ValueError, TypeError):
+            return None
+    if isinstance(val, str):
+        try:
+            if "." in val:
+                return float(val)
+            return int(val)
+        except (ValueError, TypeError):
+            return None
+    return None
+
+
+def _comparable_value(val):
+    """
+    Convert a value to a form suitable for comparison.
+    Returns a tuple (type, value) for consistent ordering.
+    """
+    if val is None:
+        return (0, "")
+    if isinstance(val, bool):
+        return (1, val)
+    if isinstance(val, (int, float)):
+        return (2, val)
+    if isinstance(val, str):
+        return (3, val)
+    if isinstance(val, Term):
+        if val.type == IRI:
+            return (4, val.iri)
+        elif val.type == LITERAL:
+            # Try numeric comparison for numeric types
+            num = _to_numeric(val)
+            if num is not None:
+                return (2, num)
+            return (3, val.value)
+        elif val.type == BLANK:
+            return (5, val.id)
+    return (6, str(val))

trustgraph-flow/trustgraph/query/sparql/parser.py

@@ -0,0 +1,139 @@
+"""
+SPARQL parser wrapping rdflib's SPARQL 1.1 parser and algebra compiler.
+Parses a SPARQL query string into an algebra tree for evaluation.
+"""
+
+import logging
+
+from rdflib.plugins.sparql import prepareQuery
+from rdflib.plugins.sparql.algebra import translateQuery
+from rdflib.plugins.sparql.parserutils import CompValue
+from rdflib.term import Variable, URIRef, Literal, BNode
+
+from ... schema import Term, Triple, IRI, LITERAL, BLANK
+
+logger = logging.getLogger(__name__)
+
+
+class ParseError(Exception):
+    """Raised when a SPARQL query cannot be parsed."""
+    pass
+
+
+class ParsedQuery:
+    """Result of parsing a SPARQL query string."""
+
+    def __init__(self, algebra, query_type, variables=None):
+        self.algebra = algebra
+        self.query_type = query_type        # "select", "ask", "construct", "describe"
+        self.variables = variables or []    # projected variable names (SELECT)
+
+
+def rdflib_term_to_term(t):
+    """Convert an rdflib term (URIRef, Literal, BNode) to a schema Term."""
+    if isinstance(t, URIRef):
+        return Term(type=IRI, iri=str(t))
+    elif isinstance(t, Literal):
+        term = Term(type=LITERAL, value=str(t))
+        if t.datatype:
+            term.datatype = str(t.datatype)
+        if t.language:
+            term.language = t.language
+        return term
+    elif isinstance(t, BNode):
+        return Term(type=BLANK, id=str(t))
+    else:
+        return Term(type=LITERAL, value=str(t))
+
+
+def term_to_rdflib(t):
+    """Convert a schema Term to an rdflib term."""
+    if t.type == IRI:
+        return URIRef(t.iri)
+    elif t.type == LITERAL:
+        kwargs = {}
+        if t.datatype:
+            kwargs["datatype"] = URIRef(t.datatype)
+        if t.language:
+            kwargs["lang"] = t.language
+        return Literal(t.value, **kwargs)
+    elif t.type == BLANK:
+        return BNode(t.id)
+    else:
+        return Literal(t.value)
+
+
+def parse_sparql(query_string):
+    """
+    Parse a SPARQL query string into a ParsedQuery.
+
+    Args:
+        query_string: SPARQL 1.1 query string
+
+    Returns:
+        ParsedQuery with algebra tree, query type, and projected variables
+
+    Raises:
+        ParseError: if the query cannot be parsed
+    """
+    try:
+        prepared = prepareQuery(query_string)
+    except Exception as e:
+        raise ParseError(f"SPARQL parse error: {e}") from e
+
+    algebra = prepared.algebra
+
+    # Determine query type and extract variables
+    query_type = _detect_query_type(algebra)
+    variables = _extract_variables(algebra, query_type)
+
+    return ParsedQuery(
+        algebra=algebra,
+        query_type=query_type,
+        variables=variables,
+    )
+
+
+def _detect_query_type(algebra):
+    """Detect the SPARQL query type from the algebra root."""
+    name = algebra.name
+
+    if name == "SelectQuery":
+        return "select"
+    elif name == "AskQuery":
+        return "ask"
+    elif name == "ConstructQuery":
+        return "construct"
+    elif name == "DescribeQuery":
+        return "describe"
+
+    # The top-level algebra node may be a modifier (Project, Slice, etc.)
+    # wrapping the actual query. Check for common patterns.
+    if name in ("Project", "Distinct", "Reduced", "OrderBy", "Slice"):
+        return "select"
+
+    logger.warning(f"Unknown algebra root type: {name}, assuming select")
+    return "select"
+
+
+def _extract_variables(algebra, query_type):
+    """Extract projected variable names from the algebra."""
+    if query_type != "select":
+        return []
+
+    # For SELECT queries, the Project node has PV (projected variables)
+    if hasattr(algebra, "PV"):
+        return [str(v) for v in algebra.PV]
+
+    # Walk down through modifiers to find Project
+    node = algebra
+    while hasattr(node, "p"):
+        node = node.p
+        if hasattr(node, "PV"):
+            return [str(v) for v in node.PV]
+
+    # Fallback: collect all variables from the algebra
+    if hasattr(algebra, "_vars"):
+        return [str(v) for v in algebra._vars]
+
+    return []

trustgraph-flow/trustgraph/query/sparql/service.py

@@ -0,0 +1,262 @@
+"""
+SPARQL query service. Accepts SPARQL queries, decomposes them into triple
+pattern lookups via the triples query pub/sub interface, performs in-memory
+joins/filters/projections, and returns SPARQL result bindings.
+"""
+
+import logging
+
+from ... schema import SparqlQueryRequest, SparqlQueryResponse
+from ... schema import SparqlBinding, Error, Term, Triple
+from ... base import FlowProcessor, ConsumerSpec, ProducerSpec
+from ... base import TriplesClientSpec
+
+from . parser import parse_sparql, ParseError
+from . algebra import evaluate, EvaluationError
+
+logger = logging.getLogger(__name__)
+
+default_ident = "sparql-query"
+default_concurrency = 10
+
+
+class Processor(FlowProcessor):
+
+    def __init__(self, **params):
+
+        id = params.get("id", default_ident)
+        concurrency = params.get("concurrency", default_concurrency)
+
+        super(Processor, self).__init__(
+            **params | {
+                "id": id,
+                "concurrency": concurrency,
+            }
+        )
+
+        self.register_specification(
+            ConsumerSpec(
+                name="request",
+                schema=SparqlQueryRequest,
+                handler=self.on_message,
+                concurrency=concurrency,
+            )
+        )
+
+        self.register_specification(
+            ProducerSpec(
+                name="response",
+                schema=SparqlQueryResponse,
+            )
+        )
+
+        self.register_specification(
+            TriplesClientSpec(
+                request_name="triples-request",
+                response_name="triples-response",
+            )
+        )
+
+    async def on_message(self, msg, consumer, flow):
+
+        try:
+
+            request = msg.value()
+            id = msg.properties()["id"]
+
+            logger.debug(f"Handling SPARQL query request {id}...")
+
+            response = await self.execute_sparql(request, flow)
+
+            if request.streaming and response.query_type == "select":
+                await self.send_streaming(response, flow, id, request)
+            else:
+                await flow("response").send(
+                    response, properties={"id": id}
+                )
+
+            logger.debug("SPARQL query request completed")
+
+        except Exception as e:
+
+            logger.error(
+                f"Exception in SPARQL query service: {e}", exc_info=True
+            )
+
+            r = SparqlQueryResponse(
+                error=Error(
+                    type="sparql-query-error",
+                    message=str(e),
+                ),
+            )
+
+            await flow("response").send(r, properties={"id": id})
+
+    async def send_streaming(self, response, flow, id, request):
+        """Send SELECT results in batches."""
+
+        bindings = response.bindings
+        batch_size = request.batch_size if request.batch_size > 0 else 20
+
+        for i in range(0, len(bindings), batch_size):
+            batch = bindings[i:i + batch_size]
+            is_final = (i + batch_size >= len(bindings))
+            r = SparqlQueryResponse(
+                query_type=response.query_type,
+                variables=response.variables,
+                bindings=batch,
+                is_final=is_final,
+            )
+            await flow("response").send(r, properties={"id": id})
+
+        # Handle empty results
+        if len(bindings) == 0:
+            r = SparqlQueryResponse(
+                query_type=response.query_type,
+                variables=response.variables,
+                bindings=[],
+                is_final=True,
+            )
+            await flow("response").send(r, properties={"id": id})
+
+    async def execute_sparql(self, request, flow):
+        """Parse and evaluate a SPARQL query."""
+
+        # Parse the SPARQL query
+        try:
+            parsed = parse_sparql(request.query)
+        except ParseError as e:
+            return SparqlQueryResponse(
+                error=Error(
+                    type="sparql-parse-error",
+                    message=str(e),
+                ),
+            )
+
+        # Get the triples client from the flow
+        triples_client = flow("triples-request")
+
+        # Evaluate the algebra
+        try:
+            solutions = await evaluate(
+                parsed.algebra,
+                triples_client,
+                user=request.user or "trustgraph",
+                collection=request.collection or "default",
+                limit=request.limit or 10000,
+            )
+        except EvaluationError as e:
+            return SparqlQueryResponse(
+                error=Error(
+                    type="sparql-evaluation-error",
+                    message=str(e),
+                ),
+            )
+
+        # Build response based on query type
+        if parsed.query_type == "select":
+            return self._build_select_response(parsed, solutions)
+        elif parsed.query_type == "ask":
+            return self._build_ask_response(solutions)
+        elif parsed.query_type == "construct":
+            return self._build_construct_response(parsed, solutions)
+        elif parsed.query_type == "describe":
+            return self._build_describe_response(parsed, solutions)
+        else:
+            return SparqlQueryResponse(
+                error=Error(
+                    type="sparql-unsupported",
+                    message=f"Unsupported query type: {parsed.query_type}",
+                ),
+            )
+
+    def _build_select_response(self, parsed, solutions):
+        """Build response for SELECT queries."""
+        variables = parsed.variables
+
+        bindings = []
+        for sol in solutions:
+            values = [sol.get(v) for v in variables]
+            bindings.append(SparqlBinding(values=values))
+
+        return SparqlQueryResponse(
+            query_type="select",
+            variables=variables,
+            bindings=bindings,
+        )
+
+    def _build_ask_response(self, solutions):
+        """Build response for ASK queries."""
+        return SparqlQueryResponse(
+            query_type="ask",
+            ask_result=len(solutions) > 0,
+        )
+
+    def _build_construct_response(self, parsed, solutions):
+        """Build response for CONSTRUCT queries."""
+        # CONSTRUCT template is in the algebra
+        template = []
+        if hasattr(parsed.algebra, "template"):
+            template = parsed.algebra.template
+
+        triples = []
+        seen = set()
+
+        for sol in solutions:
+            for s_tmpl, p_tmpl, o_tmpl in template:
+                from rdflib.term import Variable
+                from . parser import rdflib_term_to_term
+
+                s = self._resolve_construct_term(s_tmpl, sol)
+                p = self._resolve_construct_term(p_tmpl, sol)
+                o = self._resolve_construct_term(o_tmpl, sol)
+
+                if s is not None and p is not None and o is not None:
+                    key = (
+                        s.type, s.iri or s.value,
+                        p.type, p.iri or p.value,
+                        o.type, o.iri or o.value,
+                    )
+                    if key not in seen:
+                        seen.add(key)
+                        triples.append(Triple(s=s, p=p, o=o))
+
+        return SparqlQueryResponse(
+            query_type="construct",
+            triples=triples,
+        )
+
+    def _build_describe_response(self, parsed, solutions):
+        """Build response for DESCRIBE queries."""
+        # DESCRIBE returns all triples about the described resources
+        # For now, return empty - would need additional triples queries
+        return SparqlQueryResponse(
+            query_type="describe",
+            triples=[],
+        )
+
+    def _resolve_construct_term(self, tmpl, solution):
+        """Resolve a CONSTRUCT template term."""
+        from rdflib.term import Variable
+        from . parser import rdflib_term_to_term
+
+        if isinstance(tmpl, Variable):
+            return solution.get(str(tmpl))
+        else:
+            return rdflib_term_to_term(tmpl)
+
+    @staticmethod
+    def add_args(parser):
+        FlowProcessor.add_args(parser)
+
+        parser.add_argument(
+            '-c', '--concurrency',
+            type=int,
+            default=default_concurrency,
+            help=f'Number of concurrent requests '
+                 f'(default: {default_concurrency})'
+        )
+
+
+def run():
+    Processor.launch(default_ident, __doc__)

trustgraph-flow/trustgraph/query/sparql/solutions.py

@@ -0,0 +1,248 @@
+"""
+Solution sequence operations for SPARQL evaluation.
+
+A solution is a dict mapping variable names (str) to Term values.
+A solution sequence is a list of solutions.
+"""
+
+import logging
+from collections import defaultdict
+
+from ... schema import Term, IRI, LITERAL, BLANK
+
+logger = logging.getLogger(__name__)
+
+
+def _term_key(term):
+    """Create a hashable key from a Term for join/distinct operations."""
+    if term is None:
+        return None
+    if term.type == IRI:
+        return ("i", term.iri)
+    elif term.type == LITERAL:
+        return ("l", term.value, term.datatype, term.language)
+    elif term.type == BLANK:
+        return ("b", term.id)
+    else:
+        return ("?", str(term))
+
+
+def _solution_key(solution, variables):
+    """Create a hashable key from a solution for the given variables."""
+    return tuple(_term_key(solution.get(v)) for v in variables)
+
+
+def _terms_equal(a, b):
+    """Check if two Terms are equal."""
+    if a is None and b is None:
+        return True
+    if a is None or b is None:
+        return False
+    return _term_key(a) == _term_key(b)
+
+
+def _compatible(sol_a, sol_b):
+    """Check if two solutions are compatible (agree on shared variables)."""
+    shared = set(sol_a.keys()) & set(sol_b.keys())
+    return all(_terms_equal(sol_a[v], sol_b[v]) for v in shared)
+
+
+def _merge(sol_a, sol_b):
+    """Merge two compatible solutions into one."""
+    result = dict(sol_a)
+    result.update(sol_b)
+    return result
+
+
+def hash_join(left, right):
+    """
+    Inner join two solution sequences on shared variables.
+    Uses hash join for efficiency.
+    """
+    if not left or not right:
+        return []
+
+    left_vars = set()
+    for sol in left:
+        left_vars.update(sol.keys())
+
+    right_vars = set()
+    for sol in right:
+        right_vars.update(sol.keys())
+
+    shared = sorted(left_vars & right_vars)
+
+    if not shared:
+        # Cross product
+        return [_merge(l, r) for l in left for r in right]
+
+    # Build hash table on the smaller side
+    if len(left) <= len(right):
+        index = defaultdict(list)
+        for sol in left:
+            key = _solution_key(sol, shared)
+            index[key].append(sol)
+
+        results = []
+        for sol_r in right:
+            key = _solution_key(sol_r, shared)
+            for sol_l in index.get(key, []):
+                results.append(_merge(sol_l, sol_r))
+        return results
+    else:
+        index = defaultdict(list)
+        for sol in right:
+            key = _solution_key(sol, shared)
+            index[key].append(sol)
+
+        results = []
+        for sol_l in left:
+            key = _solution_key(sol_l, shared)
+            for sol_r in index.get(key, []):
+                results.append(_merge(sol_l, sol_r))
+        return results
+
+
+def left_join(left, right, filter_fn=None):
+    """
+    Left outer join (OPTIONAL semantics).
+    Every left solution is preserved. If it joins with right solutions
+    (and passes the optional filter), the merged solutions are included.
+    Otherwise the original left solution is kept.
+    """
+    if not left:
+        return []
+
+    if not right:
+        return list(left)
+
+    right_vars = set()
+    for sol in right:
+        right_vars.update(sol.keys())
+
+    left_vars = set()
+    for sol in left:
+        left_vars.update(sol.keys())
+
+    shared = sorted(left_vars & right_vars)
+
+    # Build hash table on right side
+    index = defaultdict(list)
+    for sol in right:
+        key = _solution_key(sol, shared) if shared else ()
+        index[key].append(sol)
+
+    results = []
+    for sol_l in left:
+        key = _solution_key(sol_l, shared) if shared else ()
+        matches = index.get(key, [])
+
+        matched = False
+        for sol_r in matches:
+            merged = _merge(sol_l, sol_r)
+            if filter_fn is None or filter_fn(merged):
+                results.append(merged)
+                matched = True
+
+        if not matched:
+            results.append(dict(sol_l))
+
+    return results
+
+
+def union(left, right):
+    """Union two solution sequences (concatenation)."""
+    return list(left) + list(right)
+
+
+def project(solutions, variables):
+    """Keep only the specified variables in each solution."""
+    return [
+        {v: sol[v] for v in variables if v in sol}
+        for sol in solutions
+    ]
+
+
+def distinct(solutions):
+    """Remove duplicate solutions."""
+    seen = set()
+    results = []
+    for sol in solutions:
+        key = tuple(sorted(
+            (k, _term_key(v)) for k, v in sol.items()
+        ))
+        if key not in seen:
+            seen.add(key)
+            results.append(sol)
+    return results
+
+
+def order_by(solutions, key_fns):
+    """
+    Sort solutions by the given key functions.
+
+    key_fns is a list of (fn, ascending) tuples where fn extracts
+    a comparable value from a solution.
+    """
+    if not key_fns:
+        return solutions
+
+    def sort_key(sol):
+        keys = []
+        for fn, ascending in key_fns:
+            val = fn(sol)
+            # Convert to comparable form
+            if val is None:
+                comparable = ("", "")
+            elif isinstance(val, Term):
+                comparable = _term_key(val)
+            else:
+                comparable = ("v", str(val))
+            keys.append(comparable)
+        return keys
+
+    # Handle ascending/descending
+    # For simplicity, sort ascending then reverse individual keys
+    # This works for single sort keys; for multiple mixed keys we
+    # need a wrapper
+    result = sorted(solutions, key=sort_key)
+
+    # If any key is descending, we need a more complex approach.
+    # Check if all are same direction for the simple case.
+    if key_fns and all(not asc for _, asc in key_fns):
+        result.reverse()
+    elif key_fns and not all(asc for _, asc in key_fns):
+        # Mixed ascending/descending - use negation wrapper
+        result = _mixed_sort(solutions, key_fns)
+
+    return result
+
+
+def _mixed_sort(solutions, key_fns):
+    """Sort with mixed ascending/descending keys."""
+    import functools
+
+    def compare(a, b):
+        for fn, ascending in key_fns:
+            va = fn(a)
+            vb = fn(b)
+            ka = _term_key(va) if isinstance(va, Term) else ("v", str(va)) if va is not None else ("", "")
+            kb = _term_key(vb) if isinstance(vb, Term) else ("v", str(vb)) if vb is not None else ("", "")
+
+            if ka < kb:
+                return -1 if ascending else 1
+            elif ka > kb:
+                return 1 if ascending else -1
+
+        return 0
+
+    return sorted(solutions, key=functools.cmp_to_key(compare))
+
+
+def slice_solutions(solutions, offset=0, limit=None):
+    """Apply OFFSET and LIMIT to a solution sequence."""
+    if offset:
+        solutions = solutions[offset:]
+    if limit is not None:
+        solutions = solutions[:limit]
+    return solutions

trustgraph-flow/trustgraph/retrieval/document_rag/rag.py

@@ -12,22 +12,18 @@ import uuid
 
 from ... schema import DocumentRagQuery, DocumentRagResponse, Error
 from ... schema import LibrarianRequest, LibrarianResponse, DocumentMetadata
-from ... schema import librarian_request_queue, librarian_response_queue
 from ... schema import Triples, Metadata
 from ... provenance import GRAPH_RETRIEVAL
 from . document_rag import DocumentRag
 from ... base import FlowProcessor, ConsumerSpec, ProducerSpec
 from ... base import PromptClientSpec, EmbeddingsClientSpec
 from ... base import DocumentEmbeddingsClientSpec
-from ... base import Consumer, Producer
-from ... base import ConsumerMetrics, ProducerMetrics
+from ... base import LibrarianClient
 
 # Module logger
 logger = logging.getLogger(__name__)
 
 default_ident = "document-rag"
-default_librarian_request_queue = librarian_request_queue
-default_librarian_response_queue = librarian_response_queue
 
 class Processor(FlowProcessor):
 
@@ -89,111 +85,26 @@ class Processor(FlowProcessor):
             )
         )
 
-        # Librarian client for fetching chunk content from Garage
-        librarian_request_q = params.get(
-            "librarian_request_queue", default_librarian_request_queue
-        )
-        librarian_response_q = params.get(
-            "librarian_response_queue", default_librarian_response_queue
-        )
-
-        librarian_request_metrics = ProducerMetrics(
-            processor=id, flow=None, name="librarian-request"
-        )
-
-        self.librarian_request_producer = Producer(
+        # Librarian client
+        self.librarian = LibrarianClient(
+            id=id,
             backend=self.pubsub,
-            topic=librarian_request_q,
-            schema=LibrarianRequest,
-            metrics=librarian_request_metrics,
-        )
-
-        librarian_response_metrics = ConsumerMetrics(
-            processor=id, flow=None, name="librarian-response"
-        )
-
-        self.librarian_response_consumer = Consumer(
             taskgroup=self.taskgroup,
-            backend=self.pubsub,
-            flow=None,
-            topic=librarian_response_q,
-            subscriber=f"{id}-librarian",
-            schema=LibrarianResponse,
-            handler=self.on_librarian_response,
-            metrics=librarian_response_metrics,
         )
 
-        # Pending librarian requests: request_id -> asyncio.Future
-        self.pending_requests = {}
-
     async def start(self):
         await super(Processor, self).start()
-        await self.librarian_request_producer.start()
-        await self.librarian_response_consumer.start()
-
-    async def on_librarian_response(self, msg, consumer, flow):
-        """Handle responses from the librarian service."""
-        response = msg.value()
-        request_id = msg.properties().get("id")
-
-        if request_id in self.pending_requests:
-            future = self.pending_requests.pop(request_id)
-            future.set_result(response)
+        await self.librarian.start()
 
     async def fetch_chunk_content(self, chunk_id, user, timeout=120):
-        """Fetch chunk content from librarian/Garage."""
-        import uuid
-        request_id = str(uuid.uuid4())
-
-        request = LibrarianRequest(
-            operation="get-document-content",
-            document_id=chunk_id,
-            user=user,
+        """Fetch chunk content from librarian. Chunks are small so
+        single request-response is fine."""
+        return await self.librarian.fetch_document_text(
+            document_id=chunk_id, user=user, timeout=timeout,
         )
 
-        # Create future for response
-        future = asyncio.get_event_loop().create_future()
-        self.pending_requests[request_id] = future
-
-        try:
-            # Send request
-            await self.librarian_request_producer.send(
-                request, properties={"id": request_id}
-            )
-
-            # Wait for response
-            response = await asyncio.wait_for(future, timeout=timeout)
-
-            if response.error:
-                raise RuntimeError(
-                    f"Librarian error: {response.error.type}: {response.error.message}"
-                )
-
-            # Content is base64 encoded
-            content = response.content
-            if isinstance(content, str):
-                content = content.encode('utf-8')
-            return base64.b64decode(content).decode("utf-8")
-
-        except asyncio.TimeoutError:
-            self.pending_requests.pop(request_id, None)
-            raise RuntimeError(f"Timeout fetching chunk {chunk_id}")
-
     async def save_answer_content(self, doc_id, user, content, title=None, timeout=120):
-        """
-        Save answer content to the librarian.
-
-        Args:
-            doc_id: ID for the answer document
-            user: User ID
-            content: Answer text content
-            title: Optional title
-            timeout: Request timeout in seconds
-
-        Returns:
-            The document ID on success
-        """
-        request_id = str(uuid.uuid4())
+        """Save answer content to the librarian."""
 
         doc_metadata = DocumentMetadata(
             id=doc_id,
@@ -211,29 +122,8 @@ class Processor(FlowProcessor):
             user=user,
         )
 
-        # Create future for response
-        future = asyncio.get_event_loop().create_future()
-        self.pending_requests[request_id] = future
-
-        try:
-            # Send request
-            await self.librarian_request_producer.send(
-                request, properties={"id": request_id}
-            )
-
-            # Wait for response
-            response = await asyncio.wait_for(future, timeout=timeout)
-
-            if response.error:
-                raise RuntimeError(
-                    f"Librarian error saving answer: {response.error.type}: {response.error.message}"
-                )
-
-            return doc_id
-
-        except asyncio.TimeoutError:
-            self.pending_requests.pop(request_id, None)
-            raise RuntimeError(f"Timeout saving answer document {doc_id}")
+        await self.librarian.request(request, timeout=timeout)
+        return doc_id
 
     async def on_request(self, msg, consumer, flow):
 
@@ -272,12 +162,13 @@ class Processor(FlowProcessor):
                     triples=triples,
                 ))
 
-                # Send explain ID and graph to response queue
+                # Send explain data to response queue
                 await flow("response").send(
                     DocumentRagResponse(
                         response=None,
                         explain_id=explain_id,
                         explain_graph=GRAPH_RETRIEVAL,
+                        explain_triples=triples,
                         message_type="explain",
                     ),
                     properties={"id": id}
@@ -390,4 +281,3 @@ class Processor(FlowProcessor):
 def run():
 
     Processor.launch(default_ident, __doc__)
-

trustgraph-flow/trustgraph/retrieval/graph_rag/graph_rag.py

@@ -10,6 +10,7 @@ from collections import OrderedDict
 from datetime import datetime
 
 from ... schema import Term, Triple as SchemaTriple, IRI, LITERAL, TRIPLE
+from ... knowledge import Uri, Literal
 
 # Provenance imports
 from trustgraph.provenance import (
@@ -46,6 +47,26 @@ def term_to_string(term):
     return term.iri or term.value or str(term)
 
 
+def to_term(val):
+    """Convert a Uri, Literal, or string to a schema Term.
+
+    The triples client returns Uri/Literal (str subclasses) rather than
+    Term objects. This converts them back so provenance quoted triples
+    preserve the correct type.
+    """
+    if isinstance(val, Term):
+        return val
+    if isinstance(val, Uri):
+        return Term(type=IRI, iri=str(val))
+    if isinstance(val, Literal):
+        return Term(type=LITERAL, value=str(val))
+    # Fallback: treat as IRI if it looks like one, otherwise literal
+    s = str(val)
+    if s.startswith(("http://", "https://", "urn:")):
+        return Term(type=IRI, iri=s)
+    return Term(type=LITERAL, value=s)
+
+
 def edge_id(s, p, o):
     """Generate an 8-character hash ID for an edge (s, p, o)."""
     edge_str = f"{s}|{p}|{o}"
@@ -258,10 +279,18 @@ class Query:
         return all_triples
 
     async def follow_edges_batch(self, entities, max_depth):
-        """Optimized iterative graph traversal with batching"""
+        """Optimized iterative graph traversal with batching.
+
+        Returns:
+            tuple: (subgraph, term_map) where subgraph is a set of
+                (str, str, str) tuples and term_map maps each string tuple
+                to its original (Term, Term, Term) for type-preserving
+                provenance.
+        """
         visited = set()
         current_level = set(entities)
         subgraph = set()
+        term_map = {}  # (str, str, str) -> (Term, Term, Term)
 
         for depth in range(max_depth):
             if not current_level or len(subgraph) >= self.max_subgraph_size:
@@ -282,6 +311,7 @@ class Query:
             for triple in triples:
                 triple_tuple = (str(triple.s), str(triple.p), str(triple.o))
                 subgraph.add(triple_tuple)
+                term_map[triple_tuple] = (to_term(triple.s), to_term(triple.p), to_term(triple.o))
 
                 # Collect entities for next level (only from s and o positions)
                 if depth < max_depth - 1:  # Don't collect for final depth
@@ -293,13 +323,13 @@ class Query:
 
                 # Stop if subgraph size limit reached
                 if len(subgraph) >= self.max_subgraph_size:
-                    return subgraph
+                    return subgraph, term_map
 
             # Update for next iteration
             visited.update(current_level)
             current_level = next_level
 
-        return subgraph
+        return subgraph, term_map
 
     async def follow_edges(self, ent, subgraph, path_length):
         """Legacy method - replaced by follow_edges_batch"""
@@ -311,7 +341,7 @@ class Query:
             return
 
         # For backward compatibility, convert to new approach
-        batch_result = await self.follow_edges_batch([ent], path_length)
+        batch_result, _ = await self.follow_edges_batch([ent], path_length)
         subgraph.update(batch_result)
 
     async def get_subgraph(self, query):
@@ -319,9 +349,10 @@ class Query:
         Get subgraph by extracting concepts, finding entities, and traversing.
 
         Returns:
-            tuple: (subgraph, entities, concepts) where subgraph is a list of
-                (s, p, o) tuples, entities is the seed entity list, and concepts
-                is the extracted concept list.
+            tuple: (subgraph, term_map, entities, concepts) where subgraph is
+                a list of (s, p, o) string tuples, term_map maps each string
+                tuple to its original (Term, Term, Term), entities is the seed
+                entity list, and concepts is the extracted concept list.
         """
 
         entities, concepts = await self.get_entities(query)
@@ -330,9 +361,9 @@ class Query:
             logger.debug("Getting subgraph...")
 
         # Use optimized batch traversal instead of sequential processing
-        subgraph = await self.follow_edges_batch(entities, self.max_path_length)
+        subgraph, term_map = await self.follow_edges_batch(entities, self.max_path_length)
 
-        return list(subgraph), entities, concepts
+        return list(subgraph), term_map, entities, concepts
 
     async def resolve_labels_batch(self, entities):
         """Resolve labels for multiple entities in parallel"""
@@ -353,7 +384,7 @@ class Query:
                 - entities: list of seed entity URI strings
                 - concepts: list of concept strings extracted from query
         """
-        subgraph, entities, concepts = await self.get_subgraph(query)
+        subgraph, term_map, entities, concepts = await self.get_subgraph(query)
 
         # Filter out label triples
         filtered_subgraph = [edge for edge in subgraph if edge[1] != LABEL]
@@ -377,7 +408,7 @@ class Query:
 
         # Apply labels to subgraph and build URI mapping
         labeled_edges = []
-        uri_map = {}  # Maps edge_id of labeled edge -> original URI triple
+        uri_map = {}  # Maps edge_id of labeled edge -> original Term triple
 
         for s, p, o in filtered_subgraph:
             labeled_triple = (
@@ -387,9 +418,9 @@ class Query:
             )
             labeled_edges.append(labeled_triple)
 
-            # Map from labeled edge ID to original URIs
+            # Map from labeled edge ID to original Terms (preserving types)
             labeled_eid = edge_id(labeled_triple[0], labeled_triple[1], labeled_triple[2])
-            uri_map[labeled_eid] = (s, p, o)
+            uri_map[labeled_eid] = term_map.get((s, p, o), (s, p, o))
 
         labeled_edges = labeled_edges[0:self.max_subgraph_size]
 
@@ -419,12 +450,14 @@ class Query:
         # Step 1: Find subgraphs containing these edges via tg:contains
         subgraph_tasks = []
         for s, p, o in edge_uris:
+            # s, p, o may be Term objects (preserving types) or strings
+            s_term = s if isinstance(s, Term) else Term(type=IRI, iri=s)
+            p_term = p if isinstance(p, Term) else Term(type=IRI, iri=p)
+            o_term = o if isinstance(o, Term) else Term(type=IRI, iri=o)
             quoted = Term(
                 type=TRIPLE,
                 triple=SchemaTriple(
-                    s=Term(type=IRI, iri=s),
-                    p=Term(type=IRI, iri=p),
-                    o=Term(type=IRI, iri=o),
+                    s=s_term, p=p_term, o=o_term,
                 )
             )
             subgraph_tasks.append(
@@ -555,6 +588,7 @@ class GraphRag:
             streaming = False,
             chunk_callback = None,
             explain_callback = None, save_answer_callback = None,
+            parent_uri = "",
     ):
         """
         Execute a GraphRAG query with real-time explainability tracking.
@@ -593,7 +627,10 @@ class GraphRag:
         # Emit question explainability immediately
         if explain_callback:
             q_triples = set_graph(
-                question_triples(q_uri, query, timestamp),
+                question_triples(
+                    q_uri, query, timestamp,
+                    parent_uri=parent_uri or None,
+                ),
                 GRAPH_RETRIEVAL
             )
             await explain_callback(q_triples, q_uri)

trustgraph-flow/trustgraph/retrieval/graph_rag/rag.py

@@ -253,12 +253,13 @@ class Processor(FlowProcessor):
                     triples=triples,
                 ))
 
-                # Send explain ID and graph to response queue
+                # Send explain data to response queue
                 await flow("response").send(
                     GraphRagResponse(
                         message_type="explain",
                         explain_id=explain_id,
                         explain_graph=GRAPH_RETRIEVAL,
+                        explain_triples=triples,
                     ),
                     properties={"id": id}
                 )
@@ -342,6 +343,7 @@ class Processor(FlowProcessor):
                     chunk_callback = send_chunk,
                     explain_callback = send_explainability,
                     save_answer_callback = save_answer,
+                    parent_uri = v.parent_uri,
                 )
 
             else:
@@ -355,6 +357,7 @@ class Processor(FlowProcessor):
                     edge_limit = edge_limit,
                     explain_callback = send_explainability,
                     save_answer_callback = save_answer,
+                    parent_uri = v.parent_uri,
                 )
 
                 # Send chunk with response

trustgraph-flow/trustgraph/retrieval/nlp_query/service.py

@@ -64,7 +64,7 @@ class Processor(FlowProcessor):
         )
         
         # Register config handler for schema updates
-        self.register_config_handler(self.on_schema_config)
+        self.register_config_handler(self.on_schema_config, types=["schema"])
         
         # Schema storage: name -> RowSchema
         self.schemas: Dict[str, RowSchema] = {}

trustgraph-flow/trustgraph/retrieval/structured_diag/service.py

@@ -70,7 +70,7 @@ class Processor(FlowProcessor):
         )
 
         # Register config handler for schema updates
-        self.register_config_handler(self.on_schema_config)
+        self.register_config_handler(self.on_schema_config, types=["schema"])
 
         # Schema storage: name -> RowSchema
         self.schemas: Dict[str, RowSchema] = {}

trustgraph-flow/trustgraph/storage/doc_embeddings/milvus/write.py

@@ -31,7 +31,7 @@ class Processor(CollectionConfigHandler, DocumentEmbeddingsStoreService):
         self.vecstore = DocVectors(store_uri)
 
         # Register for config push notifications
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
     async def store_document_embeddings(self, message):
 

trustgraph-flow/trustgraph/storage/doc_embeddings/pinecone/write.py

@@ -58,7 +58,7 @@ class Processor(CollectionConfigHandler, DocumentEmbeddingsStoreService):
         self.last_index_name = None
 
         # Register for config push notifications
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
     def create_index(self, index_name, dim):
 

trustgraph-flow/trustgraph/storage/doc_embeddings/qdrant/write.py

@@ -37,7 +37,7 @@ class Processor(CollectionConfigHandler, DocumentEmbeddingsStoreService):
         self.qdrant = QdrantClient(url=store_uri, api_key=api_key)
 
         # Register for config push notifications
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
     async def store_document_embeddings(self, message):
 

trustgraph-flow/trustgraph/storage/graph_embeddings/milvus/write.py

@@ -45,7 +45,7 @@ class Processor(CollectionConfigHandler, GraphEmbeddingsStoreService):
         self.vecstore = EntityVectors(store_uri)
 
         # Register for config push notifications
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
     async def store_graph_embeddings(self, message):
 

trustgraph-flow/trustgraph/storage/graph_embeddings/pinecone/write.py

@@ -72,7 +72,7 @@ class Processor(CollectionConfigHandler, GraphEmbeddingsStoreService):
         self.last_index_name = None
 
         # Register for config push notifications
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
     def create_index(self, index_name, dim):
 

trustgraph-flow/trustgraph/storage/graph_embeddings/qdrant/write.py

@@ -52,7 +52,7 @@ class Processor(CollectionConfigHandler, GraphEmbeddingsStoreService):
         self.qdrant = QdrantClient(url=store_uri, api_key=api_key)
 
         # Register for config push notifications
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
     async def store_graph_embeddings(self, message):
 

trustgraph-flow/trustgraph/storage/row_embeddings/qdrant/write.py

@@ -61,7 +61,7 @@ class Processor(CollectionConfigHandler, FlowProcessor):
         )
 
         # Register config handler for collection management
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
         # Cache of created Qdrant collections
         self.created_collections: Set[str] = set()

trustgraph-flow/trustgraph/storage/rows/cassandra/write.py

@@ -75,8 +75,8 @@ class Processor(CollectionConfigHandler, FlowProcessor):
         )
 
         # Register config handlers
-        self.register_config_handler(self.on_schema_config)
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_schema_config, types=["schema"])
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
         # Cache of known keyspaces and whether tables exist
         self.known_keyspaces: Set[str] = set()

trustgraph-flow/trustgraph/storage/triples/cassandra/write.py

@@ -3,7 +3,6 @@
 Graph writer.  Input is graph edge.  Writes edges to Cassandra graph.
 """
 
-import pulsar
 import base64
 import os
 import argparse
@@ -145,7 +144,7 @@ class Processor(CollectionConfigHandler, TriplesStoreService):
         self.tg = None
 
         # Register for config push notifications
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
     async def store_triples(self, message):
 

trustgraph-flow/trustgraph/storage/triples/falkordb/write.py

@@ -3,7 +3,6 @@
 Graph writer.  Input is graph edge.  Writes edges to FalkorDB graph.
 """
 
-import pulsar
 import base64
 import os
 import argparse
@@ -58,7 +57,7 @@ class Processor(CollectionConfigHandler, TriplesStoreService):
         self.io = FalkorDB.from_url(graph_url).select_graph(database)
 
         # Register for config push notifications
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
     def create_node(self, uri, user, collection):
 

trustgraph-flow/trustgraph/storage/triples/memgraph/write.py

@@ -3,7 +3,6 @@
 Graph writer.  Input is graph edge.  Writes edges to Memgraph.
 """
 
-import pulsar
 import base64
 import os
 import argparse
@@ -67,7 +66,7 @@ class Processor(CollectionConfigHandler, TriplesStoreService):
             self.create_indexes(session)
 
         # Register for config push notifications
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
     def create_indexes(self, session):
 

trustgraph-flow/trustgraph/storage/triples/neo4j/write.py

@@ -3,7 +3,6 @@
 Graph writer.  Input is graph edge.  Writes edges to Neo4j graph.
 """
 
-import pulsar
 import base64
 import os
 import argparse
@@ -67,7 +66,7 @@ class Processor(CollectionConfigHandler, TriplesStoreService):
             self.create_indexes(session)
 
         # Register for config push notifications
-        self.register_config_handler(self.on_collection_config)
+        self.register_config_handler(self.on_collection_config, types=["collection"])
 
     def create_indexes(self, session):