mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-04-25 08:26:21 +02:00
7a6197d8c3
Implements full explainability pipeline for GraphRAG queries, enabling
traceability from answers back to source documents.
Renamed throughout for clarity:
- provenance_callback → explain_callback
- provenance_id → explain_id
- provenance_collection → explain_collection
- message_type "provenance" → "explain"
- Queue name "provenance" → "explainability"
GraphRAG queries now emit explainability events as they execute:
1. Session - query text and timestamp
2. Retrieval - edges retrieved from subgraph
3. Selection - selected edges with LLM reasoning (JSONL with id +
reasoning)
4. Answer - reference to synthesized response
Events stream via explain_callback during query(), enabling
real-time UX.
- Answers stored in librarian service (not inline in graph - too large)
- Document ID as URN: urn:trustgraph:answer:{session_id}
- Graph stores tg:document reference (IRI) to librarian document
- Added librarian producer/consumer to graph-rag service
- get_labelgraph() now returns (labeled_edges, uri_map)
- uri_map maps edge_id(label_s, label_p, label_o) →
(uri_s, uri_p, uri_o)
- Explainability data stores original URIs, not labels
- Enables tracing edges back to reifying statements via tg:reifies
- Added serialize_triple() to query service (matches storage format)
- get_term_value() now handles TRIPLE type terms
- Enables querying by quoted triple in object position:
?stmt tg:reifies <<s p o>>
- Displays real-time explainability events during query
- Resolves rdfs:label for edge components (s, p, o)
- Traces source chain via prov:wasDerivedFrom to root document
- Output: "Source: Chunk 1 → Page 2 → Document Title"
- Label caching to avoid repeated queries
GraphRagResponse:
- explain_id: str | None
- explain_collection: str | None
- message_type: str ("chunk" or "explain")
- end_of_session: bool
trustgraph-base/trustgraph/provenance/:
- namespaces.py - Added TG_DOCUMENT predicate
- triples.py - answer_triples() supports document_id reference
- uris.py - Added edge_selection_uri()
trustgraph-base/trustgraph/schema/services/retrieval.py:
- GraphRagResponse with explain_id, explain_collection, end_of_session
trustgraph-flow/trustgraph/retrieval/graph_rag/:
- graph_rag.py - URI preservation, streaming answer accumulation
- rag.py - Librarian integration, real-time explain emission
trustgraph-flow/trustgraph/query/triples/cassandra/service.py:
- Quoted triple serialization for query matching
trustgraph-cli/trustgraph/cli/invoke_graph_rag.py:
- Full explainability display with label resolution and source tracing
134 lines
3.8 KiB
Python
134 lines
3.8 KiB
Python
"""
|
|
Provenance module for extraction-time provenance support.
|
|
|
|
Provides helpers for:
|
|
- URI generation for documents, pages, chunks, activities, statements
|
|
- PROV-O triple building for provenance metadata
|
|
- Vocabulary bootstrap for per-collection initialization
|
|
|
|
Usage example:
|
|
|
|
from trustgraph.provenance import (
|
|
document_uri, page_uri, chunk_uri_from_page,
|
|
document_triples, derived_entity_triples,
|
|
get_vocabulary_triples,
|
|
)
|
|
|
|
# Generate URIs
|
|
doc_uri = document_uri("my-doc-123")
|
|
page_uri = page_uri("my-doc-123", page_number=1)
|
|
|
|
# Build provenance triples
|
|
triples = document_triples(
|
|
doc_uri,
|
|
title="My Document",
|
|
mime_type="application/pdf",
|
|
page_count=10,
|
|
)
|
|
|
|
# Get vocabulary bootstrap triples (once per collection)
|
|
vocab_triples = get_vocabulary_triples()
|
|
"""
|
|
|
|
# URI generation
|
|
from . uris import (
|
|
TRUSTGRAPH_BASE,
|
|
document_uri,
|
|
page_uri,
|
|
chunk_uri_from_page,
|
|
chunk_uri_from_doc,
|
|
activity_uri,
|
|
statement_uri,
|
|
agent_uri,
|
|
# Query-time provenance URIs
|
|
query_session_uri,
|
|
retrieval_uri,
|
|
selection_uri,
|
|
answer_uri,
|
|
)
|
|
|
|
# Namespace constants
|
|
from . namespaces import (
|
|
# PROV-O
|
|
PROV, PROV_ENTITY, PROV_ACTIVITY, PROV_AGENT,
|
|
PROV_WAS_DERIVED_FROM, PROV_WAS_GENERATED_BY,
|
|
PROV_USED, PROV_WAS_ASSOCIATED_WITH, PROV_STARTED_AT_TIME,
|
|
# Dublin Core
|
|
DC, DC_TITLE, DC_SOURCE, DC_DATE, DC_CREATOR,
|
|
# RDF/RDFS
|
|
RDF, RDF_TYPE, RDFS, RDFS_LABEL,
|
|
# TrustGraph
|
|
TG, TG_REIFIES, TG_PAGE_COUNT, TG_MIME_TYPE, TG_PAGE_NUMBER,
|
|
TG_CHUNK_INDEX, TG_CHAR_OFFSET, TG_CHAR_LENGTH,
|
|
TG_CHUNK_SIZE, TG_CHUNK_OVERLAP, TG_COMPONENT_VERSION,
|
|
TG_LLM_MODEL, TG_ONTOLOGY, TG_EMBEDDING_MODEL,
|
|
TG_SOURCE_TEXT, TG_SOURCE_CHAR_OFFSET, TG_SOURCE_CHAR_LENGTH,
|
|
# Query-time provenance predicates
|
|
TG_QUERY, TG_EDGE_COUNT, TG_SELECTED_EDGE, TG_REASONING, TG_CONTENT,
|
|
)
|
|
|
|
# Triple builders
|
|
from . triples import (
|
|
document_triples,
|
|
derived_entity_triples,
|
|
triple_provenance_triples,
|
|
# Query-time provenance triple builders
|
|
query_session_triples,
|
|
retrieval_triples,
|
|
selection_triples,
|
|
answer_triples,
|
|
)
|
|
|
|
# Vocabulary bootstrap
|
|
from . vocabulary import (
|
|
get_vocabulary_triples,
|
|
PROV_CLASS_LABELS,
|
|
PROV_PREDICATE_LABELS,
|
|
DC_PREDICATE_LABELS,
|
|
TG_PREDICATE_LABELS,
|
|
)
|
|
|
|
__all__ = [
|
|
# URIs
|
|
"TRUSTGRAPH_BASE",
|
|
"document_uri",
|
|
"page_uri",
|
|
"chunk_uri_from_page",
|
|
"chunk_uri_from_doc",
|
|
"activity_uri",
|
|
"statement_uri",
|
|
"agent_uri",
|
|
# Query-time provenance URIs
|
|
"query_session_uri",
|
|
"retrieval_uri",
|
|
"selection_uri",
|
|
"answer_uri",
|
|
# Namespaces
|
|
"PROV", "PROV_ENTITY", "PROV_ACTIVITY", "PROV_AGENT",
|
|
"PROV_WAS_DERIVED_FROM", "PROV_WAS_GENERATED_BY",
|
|
"PROV_USED", "PROV_WAS_ASSOCIATED_WITH", "PROV_STARTED_AT_TIME",
|
|
"DC", "DC_TITLE", "DC_SOURCE", "DC_DATE", "DC_CREATOR",
|
|
"RDF", "RDF_TYPE", "RDFS", "RDFS_LABEL",
|
|
"TG", "TG_REIFIES", "TG_PAGE_COUNT", "TG_MIME_TYPE", "TG_PAGE_NUMBER",
|
|
"TG_CHUNK_INDEX", "TG_CHAR_OFFSET", "TG_CHAR_LENGTH",
|
|
"TG_CHUNK_SIZE", "TG_CHUNK_OVERLAP", "TG_COMPONENT_VERSION",
|
|
"TG_LLM_MODEL", "TG_ONTOLOGY", "TG_EMBEDDING_MODEL",
|
|
"TG_SOURCE_TEXT", "TG_SOURCE_CHAR_OFFSET", "TG_SOURCE_CHAR_LENGTH",
|
|
# Query-time provenance predicates
|
|
"TG_QUERY", "TG_EDGE_COUNT", "TG_SELECTED_EDGE", "TG_REASONING", "TG_CONTENT",
|
|
# Triple builders
|
|
"document_triples",
|
|
"derived_entity_triples",
|
|
"triple_provenance_triples",
|
|
# Query-time provenance triple builders
|
|
"query_session_triples",
|
|
"retrieval_triples",
|
|
"selection_triples",
|
|
"answer_triples",
|
|
# Vocabulary
|
|
"get_vocabulary_triples",
|
|
"PROV_CLASS_LABELS",
|
|
"PROV_PREDICATE_LABELS",
|
|
"DC_PREDICATE_LABELS",
|
|
"TG_PREDICATE_LABELS",
|
|
]
|