apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-25 00:16:23 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Add unified explainability support and librarian storage for (#693)

Browse source 
Add unified explainability support and librarian storage for all retrieval engines

Implements consistent explainability/provenance tracking
across GraphRAG, DocumentRAG, and Agent retrieval
engines. All large content (answers, thoughts, observations)
is now stored in librarian rather than as inline literals in
the knowledge graph.

Explainability API:
- New explainability.py module with entity classes (Question,
  Exploration, Focus, Synthesis, Analysis, Conclusion) and
  ExplainabilityClient
- Quiescence-based eventual consistency handling for trace
  fetching
- Content fetching from librarian with retry logic

CLI updates:
- tg-invoke-graph-rag -x/--explainable flag returns
  explain_id
- tg-invoke-document-rag -x/--explainable flag returns
  explain_id
- tg-invoke-agent -x/--explainable flag returns explain_id
- tg-list-explain-traces uses new explainability API
- tg-show-explain-trace handles all three trace types

Agent provenance:
- Records session, iterations (think/act/observe), and conclusion
- Stores thoughts and observations in librarian with document
  references
- New predicates: tg:thoughtDocument, tg:observationDocument

DocumentRAG provenance:
- Records question, exploration (chunk retrieval), and synthesis
- Stores answers in librarian with document references

Schema changes:
- AgentResponse: added explain_id, explain_graph fields
- RetrievalResponse: added explain_id, explain_graph fields
- agent_iteration_triples: supports thought_document_id,
  observation_document_id

Update tests.
This commit is contained in:
cybermaggedon 2026-03-12 21:40:09 +00:00 • committed by GitHub
parent aecf00f040
commit 35128ff019
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 2736 additions and 846 deletions
Download patch file Download diff file Expand all files Collapse all files

33
trustgraph-base/trustgraph/api/__init__.py
View file

@ -59,7 +59,7 @@ from .flow import Flow, FlowInstance
from .async_flow import AsyncFlow, AsyncFlowInstance
# WebSocket clients
from .socket_client import SocketClient, SocketFlowInstance
from .socket_client import SocketClient, SocketFlowInstance, build_term
from .async_socket_client import AsyncSocketClient, AsyncSocketFlowInstance
# Bulk operation clients
@ -70,6 +70,21 @@ from .async_bulk_client import AsyncBulkClient
from .metrics import Metrics
from .async_metrics import AsyncMetrics
# Explainability
from .explainability import (
ExplainabilityClient,
ExplainEntity,
Question,
Exploration,
Focus,
Synthesis,
Analysis,
Conclusion,
EdgeSelection,
wire_triples_to_tuples,
extract_term_value,
)
# Types
from .types import (
Triple,
@ -85,6 +100,7 @@ from .types import (
AgentObservation,
AgentAnswer,
RAGChunk,
ProvenanceEvent,
)
# Exceptions
@ -124,6 +140,7 @@ __all__ = [
"SocketFlowInstance",
"AsyncSocketClient",
"AsyncSocketFlowInstance",
"build_term",
# Bulk operation clients
"BulkClient",
@ -133,6 +150,19 @@ __all__ = [
"Metrics",
"AsyncMetrics",
# Explainability
"ExplainabilityClient",
"ExplainEntity",
"Question",
"Exploration",
"Focus",
"Synthesis",
"Analysis",
"Conclusion",
"EdgeSelection",
"wire_triples_to_tuples",
"extract_term_value",
# Types
"Triple",
"Uri",
@ -147,6 +177,7 @@ __all__ = [
"AgentObservation",
"AgentAnswer",
"RAGChunk",
"ProvenanceEvent",
# Exceptions
"ProtocolException",

1132
trustgraph-base/trustgraph/api/explainability.py Normal file
View file

File diff suppressed because it is too large Load diff

422
trustgraph-base/trustgraph/api/socket_client.py
View file

@ -15,6 +15,63 @@ from . types import AgentThought, AgentObservation, AgentAnswer, RAGChunk, Strea
from . exceptions import ProtocolException, raise_from_error_dict
def build_term(value: Any, term_type: Optional[str] = None,
datatype: Optional[str] = None, language: Optional[str] = None) -> Optional[Dict[str, Any]]:
"""
Build wire-format Term dict from a value.
Auto-detection rules (when term_type is None):
- Already a dict with 't' key -> return as-is (already a Term)
- Starts with http://, https://, urn: -> IRI
- Wrapped in <> (e.g., <http://...>) -> IRI (angle brackets stripped)
- Anything else -> literal
Args:
value: The term value (string, dict, or None)
term_type: One of 'iri', 'literal', or None for auto-detect
datatype: Datatype for literal objects (e.g., xsd:integer)
language: Language tag for literal objects (e.g., en)
Returns:
dict: Wire-format Term dict, or None if value is None
"""
if value is None:
return None
# If already a Term dict, return as-is
if isinstance(value, dict) and "t" in value:
return value
# Convert to string for processing
value = str(value)
# Auto-detect type if not specified
if term_type is None:
if value.startswith("<") and value.endswith(">") and not value.startswith("<<"):
# Angle-bracket wrapped IRI: <http://...>
value = value[1:-1] # Strip < and >
term_type = "iri"
elif value.startswith(("http://", "https://", "urn:")):
term_type = "iri"
else:
term_type = "literal"
if term_type == "iri":
# Strip angle brackets if present
if value.startswith("<") and value.endswith(">"):
value = value[1:-1]
return {"t": "i", "i": value}
elif term_type == "literal":
result = {"t": "l", "v": value}
if datatype:
result["dt"] = datatype
if language:
result["ln"] = language
return result
else:
raise ValueError(f"Unknown term type: {term_type}")
class SocketClient:
"""
Synchronous WebSocket client for streaming operations.
@ -92,7 +149,8 @@ class SocketClient:
flow: Optional[str],
request: Dict[str, Any],
streaming: bool = False,
streaming_raw: bool = False
streaming_raw: bool = False,
include_provenance: bool = False
) -> Union[Dict[str, Any], Iterator[StreamingChunk], Iterator[Dict[str, Any]]]:
"""Synchronous wrapper around async WebSocket communication.
@ -119,7 +177,7 @@ class SocketClient:
return self._streaming_generator_raw(service, flow, request, loop)
elif streaming:
# Parsed streaming for agent/RAG chunk types
return self._streaming_generator(service, flow, request, loop)
return self._streaming_generator(service, flow, request, loop, include_provenance)
else:
# Non-streaming single response
return loop.run_until_complete(self._send_request_async(service, flow, request))
@ -129,10 +187,11 @@ class SocketClient:
service: str,
flow: Optional[str],
request: Dict[str, Any],
loop: asyncio.AbstractEventLoop
loop: asyncio.AbstractEventLoop,
include_provenance: bool = False
) -> Iterator[StreamingChunk]:
"""Generator that yields streaming chunks (for agent/RAG responses)"""
async_gen = self._send_request_async_streaming(service, flow, request)
async_gen = self._send_request_async_streaming(service, flow, request, include_provenance)
try:
while True:
@ -265,7 +324,8 @@ class SocketClient:
self,
service: str,
flow: Optional[str],
request: Dict[str, Any]
request: Dict[str, Any],
include_provenance: bool = False
) -> Iterator[StreamingChunk]:
"""Async implementation of WebSocket request (streaming)"""
# Generate unique request ID
@ -309,8 +369,8 @@ class SocketClient:
raise_from_error_dict(resp["error"])
# Parse different chunk types
chunk = self._parse_chunk(resp)
if chunk is not None: # Skip provenance messages in streaming
chunk = self._parse_chunk(resp, include_provenance=include_provenance)
if chunk is not None: # Skip provenance messages unless include_provenance
yield chunk
# Check if this is the final message
@ -325,14 +385,26 @@ class SocketClient:
chunk_type = resp.get("chunk_type")
message_type = resp.get("message_type")
# Handle new GraphRAG message format with message_type
if message_type == "provenance":
# Handle GraphRAG/DocRAG message format with message_type
if message_type == "explain":
if include_provenance:
# Return provenance event for explainability
return ProvenanceEvent(provenance_id=resp.get("provenance_id", ""))
return ProvenanceEvent(
explain_id=resp.get("explain_id", ""),
explain_graph=resp.get("explain_graph", "")
)
# Provenance messages are not yielded to user - they're metadata
return None
# Handle Agent message format with chunk_type="explain"
if chunk_type == "explain":
if include_provenance:
return ProvenanceEvent(
explain_id=resp.get("explain_id", ""),
explain_graph=resp.get("explain_graph", "")
)
return None
if chunk_type == "thought":
return AgentThought(
content=resp.get("content", ""),
@ -477,6 +549,95 @@ class SocketFlowInstance:
# regardless of streaming flag, so always use the streaming code path
return self.client._send_request_sync("agent", self.flow_id, request, streaming=True)
def agent_explain(
self,
question: str,
user: str,
collection: str,
state: Optional[Dict[str, Any]] = None,
group: Optional[str] = None,
history: Optional[List[Dict[str, Any]]] = None,
**kwargs: Any
) -> Iterator[Union[StreamingChunk, ProvenanceEvent]]:
"""
Execute an agent operation with explainability support.
Streams both content chunks (AgentThought, AgentObservation, AgentAnswer)
and provenance events (ProvenanceEvent). Provenance events contain URIs
that can be fetched using ExplainabilityClient to get detailed information
about the agent's reasoning process.
Agent trace consists of:
- Session: The initial question and session metadata
- Iterations: Each thought/action/observation cycle
- Conclusion: The final answer
Args:
question: User question or instruction
user: User identifier
collection: Collection identifier for provenance storage
state: Optional state dictionary for stateful conversations
group: Optional group identifier for multi-user contexts
history: Optional conversation history as list of message dicts
**kwargs: Additional parameters passed to the agent service
Yields:
Union[StreamingChunk, ProvenanceEvent]: Agent chunks and provenance events
Example:
```python
from trustgraph.api import Api, ExplainabilityClient, ProvenanceEvent
from trustgraph.api import AgentThought, AgentObservation, AgentAnswer
socket = api.socket()
flow = socket.flow("default")
explain_client = ExplainabilityClient(flow)
provenance_ids = []
for item in flow.agent_explain(
question="What is the capital of France?",
user="trustgraph",
collection="default"
):
if isinstance(item, AgentThought):
print(f"[Thought] {item.content}")
elif isinstance(item, AgentObservation):
print(f"[Observation] {item.content}")
elif isinstance(item, AgentAnswer):
print(f"[Answer] {item.content}")
elif isinstance(item, ProvenanceEvent):
provenance_ids.append(item.explain_id)
# Fetch session trace after completion
if provenance_ids:
trace = explain_client.fetch_agent_trace(
provenance_ids[0], # Session URI is first
graph="urn:graph:retrieval",
user="trustgraph",
collection="default"
)
```
"""
request = {
"question": question,
"user": user,
"collection": collection,
"streaming": True # Always streaming for explain
}
if state is not None:
request["state"] = state
if group is not None:
request["group"] = group
if history is not None:
request["history"] = history
request.update(kwargs)
# Use streaming with provenance enabled
return self.client._send_request_sync(
"agent", self.flow_id, request,
streaming=True, include_provenance=True
)
def text_completion(self, system: str, prompt: str, streaming: bool = False, **kwargs) -> Union[str, Iterator[str]]:
"""
Execute text completion with optional streaming.
@ -596,6 +757,86 @@ class SocketFlowInstance:
else:
return result.get("response", "")
def graph_rag_explain(
self,
query: str,
user: str,
collection: str,
max_subgraph_size: int = 1000,
max_subgraph_count: int = 5,
max_entity_distance: int = 3,
**kwargs: Any
) -> Iterator[Union[RAGChunk, ProvenanceEvent]]:
"""
Execute graph-based RAG query with explainability support.
Streams both content chunks (RAGChunk) and provenance events (ProvenanceEvent).
Provenance events contain URIs that can be fetched using ExplainabilityClient
to get detailed information about how the response was generated.
Args:
query: Natural language query
user: User/keyspace identifier
collection: Collection identifier
max_subgraph_size: Maximum total triples in subgraph (default: 1000)
max_subgraph_count: Maximum number of subgraphs (default: 5)
max_entity_distance: Maximum traversal depth (default: 3)
**kwargs: Additional parameters passed to the service
Yields:
Union[RAGChunk, ProvenanceEvent]: Content chunks and provenance events
Example:
```python
from trustgraph.api import Api, ExplainabilityClient, RAGChunk, ProvenanceEvent
socket = api.socket()
flow = socket.flow("default")
explain_client = ExplainabilityClient(flow)
provenance_ids = []
response_text = ""
for item in flow.graph_rag_explain(
query="Tell me about Marie Curie",
user="trustgraph",
collection="scientists"
):
if isinstance(item, RAGChunk):
response_text += item.content
print(item.content, end='', flush=True)
elif isinstance(item, ProvenanceEvent):
provenance_ids.append(item.provenance_id)
# Fetch explainability details
for prov_id in provenance_ids:
entity = explain_client.fetch_entity(
prov_id,
graph="urn:graph:retrieval",
user="trustgraph",
collection="scientists"
)
print(f"Entity: {entity}")
```
"""
request = {
"query": query,
"user": user,
"collection": collection,
"max-subgraph-size": max_subgraph_size,
"max-subgraph-count": max_subgraph_count,
"max-entity-distance": max_entity_distance,
"streaming": True,
"explainable": True, # Enable explainability mode
}
request.update(kwargs)
# Use streaming with provenance events included
return self.client._send_request_sync(
"graph-rag", self.flow_id, request,
streaming=True, include_provenance=True
)
def document_rag(
self,
query: str,
@ -654,6 +895,79 @@ class SocketFlowInstance:
else:
return result.get("response", "")
def document_rag_explain(
self,
query: str,
user: str,
collection: str,
doc_limit: int = 10,
**kwargs: Any
) -> Iterator[Union[RAGChunk, ProvenanceEvent]]:
"""
Execute document-based RAG query with explainability support.
Streams both content chunks (RAGChunk) and provenance events (ProvenanceEvent).
Provenance events contain URIs that can be fetched using ExplainabilityClient
to get detailed information about how the response was generated.
Document RAG trace consists of:
- Question: The user's query
- Exploration: Chunks retrieved from document store (chunk_count)
- Synthesis: The generated answer
Args:
query: Natural language query
user: User/keyspace identifier
collection: Collection identifier
doc_limit: Maximum document chunks to retrieve (default: 10)
**kwargs: Additional parameters passed to the service
Yields:
Union[RAGChunk, ProvenanceEvent]: Content chunks and provenance events
Example:
```python
from trustgraph.api import Api, ExplainabilityClient, RAGChunk, ProvenanceEvent
socket = api.socket()
flow = socket.flow("default")
explain_client = ExplainabilityClient(flow)
for item in flow.document_rag_explain(
query="Summarize the key findings",
user="trustgraph",
collection="research-papers",
doc_limit=5
):
if isinstance(item, RAGChunk):
print(item.content, end='', flush=True)
elif isinstance(item, ProvenanceEvent):
# Fetch entity details
entity = explain_client.fetch_entity(
item.explain_id,
graph=item.explain_graph,
user="trustgraph",
collection="research-papers"
)
print(f"Event: {entity}", file=sys.stderr)
```
"""
request = {
"query": query,
"user": user,
"collection": collection,
"doc-limit": doc_limit,
"streaming": True,
"explainable": True,
}
request.update(kwargs)
# Use streaming with provenance events included
return self.client._send_request_sync(
"document-rag", self.flow_id, request,
streaming=True, include_provenance=True
)
def _rag_generator(self, result: Iterator[StreamingChunk]) -> Iterator[str]:
"""Generator for RAG streaming (graph-rag and document-rag)"""
for chunk in result:
@ -831,28 +1145,30 @@ class SocketFlowInstance:
def triples_query(
self,
s: Optional[str] = None,
p: Optional[str] = None,
o: Optional[str] = None,
s: Optional[Union[str, Dict[str, Any]]] = None,
p: Optional[Union[str, Dict[str, Any]]] = None,
o: Optional[Union[str, Dict[str, Any]]] = None,
g: Optional[str] = None,
user: Optional[str] = None,
collection: Optional[str] = None,
limit: int = 100,
**kwargs: Any
) -> Dict[str, Any]:
) -> List[Dict[str, Any]]:
"""
Query knowledge graph triples using pattern matching.
Args:
s: Subject URI (optional, use None for wildcard)
p: Predicate URI (optional, use None for wildcard)
o: Object URI or Literal (optional, use None for wildcard)
s: Subject filter - URI string, Term dict, or None for wildcard
p: Predicate filter - URI string, Term dict, or None for wildcard
o: Object filter - URI/literal string, Term dict, or None for wildcard
g: Named graph filter - URI string or None for all graphs
user: User/keyspace identifier (optional)
collection: Collection identifier (optional)
limit: Maximum results to return (default: 100)
**kwargs: Additional parameters passed to the service
Returns:
dict: Query results with matching triples
List[Dict]: List of matching triples in wire format
Example:
```python
@ -860,33 +1176,54 @@ class SocketFlowInstance:
flow = socket.flow("default")
# Find all triples about a specific subject
result = flow.triples_query(
triples = flow.triples_query(
s="http://example.org/person/marie-curie",
user="trustgraph",
collection="scientists"
)
# Query with named graph filter
triples = flow.triples_query(
s="urn:trustgraph:session:abc123",
g="urn:graph:retrieval",
user="trustgraph",
collection="default"
)
```
"""
request = {"limit": limit}
if s is not None:
request["s"] = str(s)
if p is not None:
request["p"] = str(p)
if o is not None:
request["o"] = str(o)
# Build Term dicts for s/p/o (auto-converts strings)
s_term = build_term(s)
p_term = build_term(p)
o_term = build_term(o)
if s_term is not None:
request["s"] = s_term
if p_term is not None:
request["p"] = p_term
if o_term is not None:
request["o"] = o_term
if g is not None:
request["g"] = g
if user is not None:
request["user"] = user
if collection is not None:
request["collection"] = collection
request.update(kwargs)
return self.client._send_request_sync("triples", self.flow_id, request, False)
result = self.client._send_request_sync("triples", self.flow_id, request, False)
# Return the triples list from the response
if isinstance(result, dict) and "response" in result:
return result["response"]
return result
def triples_query_stream(
self,
s: Optional[str] = None,
p: Optional[str] = None,
o: Optional[str] = None,
s: Optional[Union[str, Dict[str, Any]]] = None,
p: Optional[Union[str, Dict[str, Any]]] = None,
o: Optional[Union[str, Dict[str, Any]]] = None,
g: Optional[str] = None,
user: Optional[str] = None,
collection: Optional[str] = None,
limit: int = 100,
@ -900,9 +1237,10 @@ class SocketFlowInstance:
and memory overhead for large result sets.
Args:
s: Subject URI (optional, use None for wildcard)
p: Predicate URI (optional, use None for wildcard)
o: Object URI or Literal (optional, use None for wildcard)
s: Subject filter - URI string, Term dict, or None for wildcard
p: Predicate filter - URI string, Term dict, or None for wildcard
o: Object filter - URI/literal string, Term dict, or None for wildcard
g: Named graph filter - URI string or None for all graphs
user: User/keyspace identifier (optional)
collection: Collection identifier (optional)
limit: Maximum results to return (default: 100)
@ -930,12 +1268,20 @@ class SocketFlowInstance:
"streaming": True,
"batch-size": batch_size,
}
if s is not None:
request["s"] = str(s)
if p is not None:
request["p"] = str(p)
if o is not None:
request["o"] = str(o)
# Build Term dicts for s/p/o (auto-converts strings)
s_term = build_term(s)
p_term = build_term(p)
o_term = build_term(o)
if s_term is not None:
request["s"] = s_term
if p_term is not None:
request["p"] = p_term
if o_term is not None:
request["o"] = o_term
if g is not None:
request["g"] = g
if user is not None:
request["user"] = user
if collection is not None:

28
trustgraph-base/trustgraph/api/types.py
View file

@ -212,19 +212,21 @@ class ProvenanceEvent:
Each event represents a provenance node created during query processing.
Attributes:
provenance_id: URI of the provenance node (e.g., urn:trustgraph:session:abc123)
event_type: Type of provenance event (session, retrieval, selection, answer)
explain_id: URI of the provenance node (e.g., urn:trustgraph:question:abc123)
explain_graph: Named graph where provenance triples are stored (e.g., urn:graph:retrieval)
event_type: Type of provenance event (question, exploration, focus, synthesis)
"""
provenance_id: str
event_type: str = "" # Derived from provenance_id (session, retrieval, selection, answer)
explain_id: str
explain_graph: str = ""
event_type: str = "" # Derived from explain_id
def __post_init__(self):
# Extract event type from provenance_id
if "session" in self.provenance_id:
self.event_type = "session"
elif "retrieval" in self.provenance_id:
self.event_type = "retrieval"
elif "selection" in self.provenance_id:
self.event_type = "selection"
elif "answer" in self.provenance_id:
self.event_type = "answer"
# Extract event type from explain_id
if "question" in self.explain_id:
self.event_type = "question"
elif "exploration" in self.explain_id:
self.event_type = "exploration"
elif "focus" in self.explain_id:
self.event_type = "focus"
elif "synthesis" in self.explain_id:
self.event_type = "synthesis"

9
trustgraph-base/trustgraph/messaging/translators/agent.py
View file

@ -59,6 +59,15 @@ class AgentResponseTranslator(MessageTranslator):
result["end_of_message"] = getattr(obj, "end_of_message", False)
result["end_of_dialog"] = getattr(obj, "end_of_dialog", False)
# Include explainability fields if present
explain_id = getattr(obj, "explain_id", None)
if explain_id:
result["explain_id"] = explain_id
explain_graph = getattr(obj, "explain_graph", None)
if explain_graph is not None:
result["explain_graph"] = explain_graph
# Always include error if present
if hasattr(obj, 'error') and obj.error and obj.error.message:
result["error"] = {"message": obj.error.message, "code": obj.error.code}

15
trustgraph-base/trustgraph/messaging/translators/retrieval.py
View file

@ -34,7 +34,12 @@ class DocumentRagResponseTranslator(MessageTranslator):
def from_pulsar(self, obj: DocumentRagResponse) -> Dict[str, Any]:
result = {}
# Include response content (even if empty string)
# Include message_type for distinguishing chunk vs explain messages
message_type = getattr(obj, "message_type", "")
if message_type:
result["message_type"] = message_type
# Include response content for chunk messages
if obj.response is not None:
result["response"] = obj.response
@ -48,9 +53,12 @@ class DocumentRagResponseTranslator(MessageTranslator):
if explain_graph is not None:
result["explain_graph"] = explain_graph
# Include end_of_stream flag
# Include end_of_stream flag (LLM stream complete)
result["end_of_stream"] = getattr(obj, "end_of_stream", False)
# Include end_of_session flag (entire session complete)
result["end_of_session"] = getattr(obj, "end_of_session", False)
# Always include error if present
if hasattr(obj, 'error') and obj.error and obj.error.message:
result["error"] = {"message": obj.error.message, "type": obj.error.type}
@ -59,7 +67,8 @@ class DocumentRagResponseTranslator(MessageTranslator):
def from_response_with_completion(self, obj: DocumentRagResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
is_final = getattr(obj, 'end_of_stream', False)
# Session is complete when end_of_session is True
is_final = getattr(obj, 'end_of_session', False)
return self.from_pulsar(obj), is_final

8
trustgraph-base/trustgraph/provenance/__init__.py
View file

@ -82,6 +82,10 @@ from . namespaces import (
TG_GRAPH_RAG_QUESTION, TG_DOC_RAG_QUESTION, TG_AGENT_QUESTION,
# Agent provenance predicates
TG_THOUGHT, TG_ACTION, TG_ARGUMENTS, TG_OBSERVATION, TG_ANSWER,
# Agent document references
TG_THOUGHT_DOCUMENT, TG_OBSERVATION_DOCUMENT,
# Document reference predicate
TG_DOCUMENT,
# Named graphs
GRAPH_DEFAULT, GRAPH_SOURCE, GRAPH_RETRIEVAL,
)
@ -165,6 +169,10 @@ __all__ = [
"TG_GRAPH_RAG_QUESTION", "TG_DOC_RAG_QUESTION", "TG_AGENT_QUESTION",
# Agent provenance predicates
"TG_THOUGHT", "TG_ACTION", "TG_ARGUMENTS", "TG_OBSERVATION", "TG_ANSWER",
# Agent document references
"TG_THOUGHT_DOCUMENT", "TG_OBSERVATION_DOCUMENT",
# Document reference predicate
"TG_DOCUMENT",
# Named graphs
"GRAPH_DEFAULT", "GRAPH_SOURCE", "GRAPH_RETRIEVAL",
# Triple builders

57
trustgraph-base/trustgraph/provenance/agent.py
View file

@ -17,7 +17,8 @@ from . namespaces import (
RDF_TYPE, RDFS_LABEL,
PROV_ACTIVITY, PROV_ENTITY, PROV_WAS_DERIVED_FROM, PROV_STARTED_AT_TIME,
TG_QUERY, TG_THOUGHT, TG_ACTION, TG_ARGUMENTS, TG_OBSERVATION, TG_ANSWER,
TG_QUESTION, TG_ANALYSIS, TG_CONCLUSION,
TG_QUESTION, TG_ANALYSIS, TG_CONCLUSION, TG_DOCUMENT,
TG_THOUGHT_DOCUMENT, TG_OBSERVATION_DOCUMENT,
TG_AGENT_QUESTION,
)
@ -73,10 +74,12 @@ def agent_session_triples(
def agent_iteration_triples(
iteration_uri: str,
parent_uri: str,
thought: str,
action: str,
arguments: Dict[str, Any],
observation: str,
thought: str = "",
action: str = "",
arguments: Dict[str, Any] = None,
observation: str = "",
thought_document_id: Optional[str] = None,
observation_document_id: Optional[str] = None,
) -> List[Triple]:
"""
Build triples for one agent iteration (Analysis - think/act/observe cycle).
@ -85,36 +88,53 @@ def agent_iteration_triples(
- Entity declaration with tg:Analysis type
- wasDerivedFrom link to parent (previous iteration or session)
- Thought, action, arguments, and observation data
- Document references for thought/observation when stored in librarian
Args:
iteration_uri: URI of this iteration (from agent_iteration_uri)
parent_uri: URI of the parent (previous iteration or session)
thought: The agent's reasoning/thought
thought: The agent's reasoning/thought (used if thought_document_id not provided)
action: The tool/action name
arguments: Arguments passed to the tool (will be JSON-encoded)
observation: The result/observation from the tool
observation: The result/observation from the tool (used if observation_document_id not provided)
thought_document_id: Optional document URI for thought in librarian (preferred)
observation_document_id: Optional document URI for observation in librarian (preferred)
Returns:
List of Triple objects
"""
if arguments is None:
arguments = {}
triples = [
_triple(iteration_uri, RDF_TYPE, _iri(PROV_ENTITY)),
_triple(iteration_uri, RDF_TYPE, _iri(TG_ANALYSIS)),
_triple(iteration_uri, RDFS_LABEL, _literal(f"Analysis: {action}")),
_triple(iteration_uri, PROV_WAS_DERIVED_FROM, _iri(parent_uri)),
_triple(iteration_uri, TG_THOUGHT, _literal(thought)),
_triple(iteration_uri, TG_ACTION, _literal(action)),
_triple(iteration_uri, TG_ARGUMENTS, _literal(json.dumps(arguments))),
_triple(iteration_uri, TG_OBSERVATION, _literal(observation)),
]
# Thought: use document reference or inline
if thought_document_id:
triples.append(_triple(iteration_uri, TG_THOUGHT_DOCUMENT, _iri(thought_document_id)))
elif thought:
triples.append(_triple(iteration_uri, TG_THOUGHT, _literal(thought)))
# Observation: use document reference or inline
if observation_document_id:
triples.append(_triple(iteration_uri, TG_OBSERVATION_DOCUMENT, _iri(observation_document_id)))
elif observation:
triples.append(_triple(iteration_uri, TG_OBSERVATION, _literal(observation)))
return triples
def agent_final_triples(
final_uri: str,
parent_uri: str,
answer: str,
answer: str = "",
document_id: Optional[str] = None,
) -> List[Triple]:
"""
Build triples for an agent final answer (Conclusion).
@ -122,20 +142,29 @@ def agent_final_triples(
Creates:
- Entity declaration with tg:Conclusion type
- wasDerivedFrom link to parent (last iteration or session)
- The answer text
- Either document reference (if document_id provided) or inline answer
Args:
final_uri: URI of the final answer (from agent_final_uri)
parent_uri: URI of the parent (last iteration or session if no iterations)
answer: The final answer text
answer: The final answer text (used if document_id not provided)
document_id: Optional document URI in librarian (preferred)
Returns:
List of Triple objects
"""
return [
triples = [
_triple(final_uri, RDF_TYPE, _iri(PROV_ENTITY)),
_triple(final_uri, RDF_TYPE, _iri(TG_CONCLUSION)),
_triple(final_uri, RDFS_LABEL, _literal("Conclusion")),
_triple(final_uri, PROV_WAS_DERIVED_FROM, _iri(parent_uri)),
_triple(final_uri, TG_ANSWER, _literal(answer)),
]
if document_id:
# Store reference to document in librarian (as IRI)
triples.append(_triple(final_uri, TG_DOCUMENT, _iri(document_id)))
elif answer:
# Fallback: store inline answer
triples.append(_triple(final_uri, TG_ANSWER, _literal(answer)))
return triples

4
trustgraph-base/trustgraph/provenance/namespaces.py
View file

@ -92,6 +92,10 @@ TG_ARGUMENTS = TG + "arguments"
TG_OBSERVATION = TG + "observation"
TG_ANSWER = TG + "answer"
# Agent document references (for librarian storage)
TG_THOUGHT_DOCUMENT = TG + "thoughtDocument"
TG_OBSERVATION_DOCUMENT = TG + "observationDocument"
# Named graph URIs for RDF datasets
# These separate different types of data while keeping them in the same collection
GRAPH_DEFAULT = "" # Core knowledge facts (triples extracted from documents)

6
trustgraph-base/trustgraph/schema/services/agent.py
View file

@ -30,11 +30,15 @@ class AgentRequest:
@dataclass
class AgentResponse:
# Streaming-first design
chunk_type: str = "" # "thought", "action", "observation", "answer", "error"
chunk_type: str = "" # "thought", "action", "observation", "answer", "explain", "error"
content: str = "" # The actual content (interpretation depends on chunk_type)
end_of_message: bool = False # Current chunk type (thought/action/etc.) is complete
end_of_dialog: bool = False # Entire agent dialog is complete
# Explainability fields
explain_id: str | None = None # Provenance URI (announced as created)
explain_graph: str | None = None # Named graph where explain was stored
# Legacy fields (deprecated but kept for backward compatibility)
answer: str = ""
error: Error | None = None

4
trustgraph-base/trustgraph/schema/services/retrieval.py
View file

@ -43,6 +43,8 @@ class DocumentRagQuery:
class DocumentRagResponse:
error: Error | None = None
response: str | None = ""
end_of_stream: bool = False
end_of_stream: bool = False # LLM response stream complete
explain_id: str | None = None # Single explain URI (announced as created)
explain_graph: str | None = None # Named graph where explain was stored (e.g., urn:graph:retrieval)
message_type: str = "" # "chunk" or "explain"
end_of_session: bool = False # Entire session complete