diff --git a/trustgraph-base/trustgraph/provenance/triples.py b/trustgraph-base/trustgraph/provenance/triples.py index 732f2fb7..cbb0e420 100644 --- a/trustgraph-base/trustgraph/provenance/triples.py +++ b/trustgraph-base/trustgraph/provenance/triples.py @@ -5,7 +5,7 @@ Helper functions to build PROV-O triples for extraction-time provenance. from datetime import datetime from typing import List, Optional -from .. schema import Triple, Term, IRI, LITERAL +from .. schema import Triple, Term, IRI, LITERAL, TRIPLE from . namespaces import ( RDF_TYPE, RDFS_LABEL, @@ -182,9 +182,7 @@ def derived_entity_triples( def triple_provenance_triples( stmt_uri: str, - subject_uri: str, - predicate_uri: str, - object_term: Term, + extracted_triple: Triple, chunk_uri: str, component_name: str, component_version: str, @@ -196,15 +194,13 @@ def triple_provenance_triples( Build provenance triples for an extracted knowledge triple using reification. Creates: - - Statement object that reifies the triple + - Reification triple: stmt_uri tg:reifies <> - wasDerivedFrom link to source chunk - Activity and agent metadata Args: stmt_uri: URI for the reified statement - subject_uri: Subject of the extracted triple - predicate_uri: Predicate of the extracted triple - object_term: Object of the extracted triple (Term) + extracted_triple: The extracted Triple to reify chunk_uri: URI of source chunk component_name: Name of extractor component component_version: Version of the component @@ -213,7 +209,7 @@ def triple_provenance_triples( timestamp: ISO timestamp Returns: - List of Triple objects for the provenance (not the triple itself) + List of Triple objects for the provenance (including reification) """ if timestamp is None: timestamp = datetime.utcnow().isoformat() + "Z" @@ -221,12 +217,17 @@ def triple_provenance_triples( act_uri = activity_uri() agt_uri = agent_uri(component_name) - # Note: The actual reification (tg:reifies pointing at the edge) requires - # RDF 1.2 triple term support. This builds the surrounding provenance. - # The actual reification link must be handled by the knowledge extractor - # using the graph store's reification API. + # Create the quoted triple term (RDF-star reification) + triple_term = Term(type=TRIPLE, triple=extracted_triple) triples = [ + # Reification: stmt_uri tg:reifies <> + Triple( + s=_iri(stmt_uri), + p=_iri(TG_REIFIES), + o=triple_term + ), + # Statement provenance _triple(stmt_uri, PROV_WAS_DERIVED_FROM, _iri(chunk_uri)), _triple(stmt_uri, PROV_WAS_GENERATED_BY, _iri(act_uri)), diff --git a/trustgraph-flow/trustgraph/extract/kg/definitions/extract.py b/trustgraph-flow/trustgraph/extract/kg/definitions/extract.py index 578f519d..b91d30ed 100755 --- a/trustgraph-flow/trustgraph/extract/kg/definitions/extract.py +++ b/trustgraph-flow/trustgraph/extract/kg/definitions/extract.py @@ -18,7 +18,10 @@ from .... schema import PromptRequest, PromptResponse from .... rdf import TRUSTGRAPH_ENTITIES, DEFINITION, RDF_LABEL, SUBJECT_OF from .... base import FlowProcessor, ConsumerSpec, ProducerSpec -from .... base import PromptClientSpec +from .... base import PromptClientSpec, ParameterSpec + +from .... provenance import statement_uri, triple_provenance_triples +from .... flow_version import __version__ as COMPONENT_VERSION DEFINITION_VALUE = Term(type=IRI, iri=DEFINITION) RDF_LABEL_VALUE = Term(type=IRI, iri=RDF_LABEL) @@ -75,6 +78,10 @@ class Processor(FlowProcessor): ) ) + # Optional flow parameters for provenance + self.register_specification(ParameterSpec("llm-model")) + self.register_specification(ParameterSpec("ontology")) + def to_uri(self, text): part = text.replace(" ", "-").lower().encode("utf-8") @@ -132,6 +139,10 @@ class Processor(FlowProcessor): chunk_doc_id = v.document_id if v.document_id else v.metadata.id chunk_uri = v.metadata.id # The URI form for the chunk + # Get optional provenance parameters + llm_model = flow("llm-model") + ontology_uri = flow("ontology") + # Note: Document metadata is now emitted once by librarian at processing # initiation, so we don't need to duplicate it here. @@ -157,9 +168,24 @@ class Processor(FlowProcessor): o=Term(type=LITERAL, value=s), )) - triples.append(Triple( + # The definition triple - this is the main extracted fact + definition_triple = Triple( s=s_value, p=DEFINITION_VALUE, o=o_value - )) + ) + triples.append(definition_triple) + + # Generate provenance for the definition triple (reification) + stmt_uri = statement_uri() + prov_triples = triple_provenance_triples( + stmt_uri=stmt_uri, + extracted_triple=definition_triple, + chunk_uri=chunk_uri, + component_name=default_ident, + component_version=COMPONENT_VERSION, + llm_model=llm_model, + ontology_uri=ontology_uri, + ) + triples.extend(prov_triples) # Link entity to chunk (not top-level document) triples.append(Triple( diff --git a/trustgraph-flow/trustgraph/extract/kg/relationships/extract.py b/trustgraph-flow/trustgraph/extract/kg/relationships/extract.py index 1a719500..7bda91eb 100755 --- a/trustgraph-flow/trustgraph/extract/kg/relationships/extract.py +++ b/trustgraph-flow/trustgraph/extract/kg/relationships/extract.py @@ -18,7 +18,10 @@ from .... schema import PromptRequest, PromptResponse from .... rdf import RDF_LABEL, TRUSTGRAPH_ENTITIES, SUBJECT_OF from .... base import FlowProcessor, ConsumerSpec, ProducerSpec -from .... base import PromptClientSpec +from .... base import PromptClientSpec, ParameterSpec + +from .... provenance import statement_uri, triple_provenance_triples +from .... flow_version import __version__ as COMPONENT_VERSION RDF_LABEL_VALUE = Term(type=IRI, iri=RDF_LABEL) SUBJECT_OF_VALUE = Term(type=IRI, iri=SUBJECT_OF) @@ -65,6 +68,10 @@ class Processor(FlowProcessor): ) ) + # Optional flow parameters for provenance + self.register_specification(ParameterSpec("llm-model")) + self.register_specification(ParameterSpec("ontology")) + def to_uri(self, text): part = text.replace(" ", "-").lower().encode("utf-8") @@ -113,6 +120,10 @@ class Processor(FlowProcessor): chunk_doc_id = v.document_id if v.document_id else v.metadata.id chunk_uri = v.metadata.id # The URI form for the chunk + # Get optional provenance parameters + llm_model = flow("llm-model") + ontology_uri = flow("ontology") + # Note: Document metadata is now emitted once by librarian at processing # initiation, so we don't need to duplicate it here. @@ -142,11 +153,26 @@ class Processor(FlowProcessor): else: o_value = Term(type=LITERAL, value=str(o)) - triples.append(Triple( + # The relationship triple - this is the main extracted fact + relationship_triple = Triple( s=s_value, p=p_value, o=o_value - )) + ) + triples.append(relationship_triple) + + # Generate provenance for the relationship triple (reification) + stmt_uri = statement_uri() + prov_triples = triple_provenance_triples( + stmt_uri=stmt_uri, + extracted_triple=relationship_triple, + chunk_uri=chunk_uri, + component_name=default_ident, + component_version=COMPONENT_VERSION, + llm_model=llm_model, + ontology_uri=ontology_uri, + ) + triples.extend(prov_triples) # Label for s triples.append(Triple(