Expose LLM token usage across all service layers (#782)

Expose LLM token usage (in_token, out_token, model) across all
service layers

Propagate token counts from LLM services through the prompt,
text-completion, graph-RAG, document-RAG, and agent orchestrator
pipelines to the API gateway and Python SDK. All fields are Optional
— None means "not available", distinguishing from a real zero count.

Key changes:

- Schema: Add in_token/out_token/model to TextCompletionResponse,
  PromptResponse, GraphRagResponse, DocumentRagResponse,
  AgentResponse

- TextCompletionClient: New TextCompletionResult return type. Split
  into text_completion() (non-streaming) and
  text_completion_stream() (streaming with per-chunk handler
  callback)

- PromptClient: New PromptResult with response_type
  (text/json/jsonl), typed fields (text/object/objects), and token
  usage. All callers updated.

- RAG services: Accumulate token usage across all prompt calls
  (extract-concepts, edge-scoring, edge-reasoning,
  synthesis). Non-streaming path sends single combined response
  instead of chunk + end_of_session.

- Agent orchestrator: UsageTracker accumulates tokens across
  meta-router, pattern prompt calls, and react reasoning. Attached
  to end_of_dialog.

- Translators: Encode token fields when not None (is not None, not truthy)

- Python SDK: RAG and text-completion methods return
  TextCompletionResult (non-streaming) or RAGChunk/AgentAnswer with
  token fields (streaming)

- CLI: --show-usage flag on tg-invoke-llm, tg-invoke-prompt,
  tg-invoke-graph-rag, tg-invoke-document-rag, tg-invoke-agent

tests/unit/test_agent/test_meta_router.py

@@ -9,6 +9,7 @@ from unittest.mock import AsyncMock, MagicMock
 from trustgraph.agent.orchestrator.meta_router import (
     MetaRouter, DEFAULT_PATTERN, DEFAULT_TASK_TYPE,
 )
+from trustgraph.base import PromptResult
 
 
 def _make_config(patterns=None, task_types=None):
@@ -28,7 +29,9 @@ def _make_config(patterns=None, task_types=None):
 def _make_context(prompt_response):
     """Build a mock context that returns a mock prompt client."""
     client = AsyncMock()
-    client.prompt = AsyncMock(return_value=prompt_response)
+    client.prompt = AsyncMock(
+        return_value=PromptResult(response_type="text", text=prompt_response)
+    )
 
     def context(service_name):
         return client
@@ -274,8 +277,8 @@ class TestRoute:
             nonlocal call_count
             call_count += 1
             if call_count == 1:
-                return "research"  # task type
-            return "plan-then-execute"  # pattern
+                return PromptResult(response_type="text", text="research")
+            return PromptResult(response_type="text", text="plan-then-execute")
 
         client.prompt = mock_prompt
         context = lambda name: client

tests/unit/test_agent/test_orchestrator_provenance_integration.py

@@ -18,6 +18,7 @@ from dataclasses import dataclass, field
 from trustgraph.schema import (
     AgentRequest, AgentResponse, AgentStep, PlanStep,
 )
+from trustgraph.base import PromptResult
 
 from trustgraph.provenance.namespaces import (
     RDF_TYPE, PROV_ENTITY, PROV_WAS_DERIVED_FROM,
@@ -183,7 +184,7 @@ class TestReactPatternProvenance:
             )
 
             async def mock_react(question, history, think, observe, answer,
-                                 context, streaming, on_action):
+                                 context, streaming, on_action, **kwargs):
                 # Simulate the on_action callback before returning Final
                 if on_action:
                     await on_action(Action(
@@ -267,7 +268,7 @@ class TestReactPatternProvenance:
             MockAM.return_value = mock_am
 
             async def mock_react(question, history, think, observe, answer,
-                                 context, streaming, on_action):
+                                 context, streaming, on_action, **kwargs):
                 if on_action:
                     await on_action(action)
                 return action
@@ -309,7 +310,7 @@ class TestReactPatternProvenance:
             MockAM.return_value = mock_am
 
             async def mock_react(question, history, think, observe, answer,
-                                 context, streaming, on_action):
+                                 context, streaming, on_action, **kwargs):
                 if on_action:
                     await on_action(Action(
                         thought="done", name="final",
@@ -355,10 +356,13 @@ class TestPlanPatternProvenance:
 
         # Mock prompt client for plan creation
         mock_prompt_client = AsyncMock()
-        mock_prompt_client.prompt.return_value = [
-            {"goal": "Find information", "tool_hint": "knowledge-query", "depends_on": []},
-            {"goal": "Summarise findings", "tool_hint": "", "depends_on": [0]},
-        ]
+        mock_prompt_client.prompt.return_value = PromptResult(
+            response_type="jsonl",
+            objects=[
+                {"goal": "Find information", "tool_hint": "knowledge-query", "depends_on": []},
+                {"goal": "Summarise findings", "tool_hint": "", "depends_on": [0]},
+            ],
+        )
 
         def flow_factory(name):
             if name == "prompt-request":
@@ -418,10 +422,13 @@ class TestPlanPatternProvenance:
 
         # Mock prompt for step execution
         mock_prompt_client = AsyncMock()
-        mock_prompt_client.prompt.return_value = {
-            "tool": "knowledge-query",
-            "arguments": {"question": "quantum computing"},
-        }
+        mock_prompt_client.prompt.return_value = PromptResult(
+            response_type="json",
+            object={
+                "tool": "knowledge-query",
+                "arguments": {"question": "quantum computing"},
+            },
+        )
 
         def flow_factory(name):
             if name == "prompt-request":
@@ -475,7 +482,7 @@ class TestPlanPatternProvenance:
 
         # Mock prompt for synthesis
         mock_prompt_client = AsyncMock()
-        mock_prompt_client.prompt.return_value = "The synthesised answer."
+        mock_prompt_client.prompt.return_value = PromptResult(response_type="text", text="The synthesised answer.")
 
         def flow_factory(name):
             if name == "prompt-request":
@@ -542,10 +549,13 @@ class TestSupervisorPatternProvenance:
 
         # Mock prompt for decomposition
         mock_prompt_client = AsyncMock()
-        mock_prompt_client.prompt.return_value = [
-            "What is quantum computing?",
-            "What are qubits?",
-        ]
+        mock_prompt_client.prompt.return_value = PromptResult(
+            response_type="jsonl",
+            objects=[
+                "What is quantum computing?",
+                "What are qubits?",
+            ],
+        )
 
         def flow_factory(name):
             if name == "prompt-request":
@@ -590,7 +600,7 @@ class TestSupervisorPatternProvenance:
 
         # Mock prompt for synthesis
         mock_prompt_client = AsyncMock()
-        mock_prompt_client.prompt.return_value = "The combined answer."
+        mock_prompt_client.prompt.return_value = PromptResult(response_type="text", text="The combined answer.")
 
         def flow_factory(name):
             if name == "prompt-request":
@@ -639,7 +649,10 @@ class TestSupervisorPatternProvenance:
         flow = make_mock_flow()
 
         mock_prompt_client = AsyncMock()
-        mock_prompt_client.prompt.return_value = ["Goal A", "Goal B", "Goal C"]
+        mock_prompt_client.prompt.return_value = PromptResult(
+            response_type="jsonl",
+            objects=["Goal A", "Goal B", "Goal C"],
+        )
 
         def flow_factory(name):
             if name == "prompt-request":