mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-16 08:41:03 +02:00
Fix/extraction prov (#662)
Quoted triple fixes, including...
1. Updated triple_provenance_triples() in triples.py:
- Now accepts a Triple object directly
- Creates the reification triple using TRIPLE term type: stmt_uri tg:reifies
<<extracted_triple>>
- Includes it in the returned provenance triples
2. Updated definitions extractor:
- Added imports for provenance functions and component version
- Added ParameterSpec for optional llm-model and ontology flow parameters
- For each definition triple, generates provenance with reification
3. Updated relationships extractor:
- Same changes as definitions extractor
This commit is contained in:
parent
cd5580be59
commit
2b9232917c
19 changed files with 361 additions and 72 deletions
|
|
@ -28,6 +28,18 @@ RDF_TYPE = RDF + "type"
|
|||
RDFS = "http://www.w3.org/2000/01/rdf-schema#"
|
||||
RDFS_LABEL = RDFS + "label"
|
||||
|
||||
# Schema.org namespace
|
||||
SCHEMA = "https://schema.org/"
|
||||
SCHEMA_SUBJECT_OF = SCHEMA + "subjectOf"
|
||||
SCHEMA_DIGITAL_DOCUMENT = SCHEMA + "DigitalDocument"
|
||||
SCHEMA_DESCRIPTION = SCHEMA + "description"
|
||||
SCHEMA_KEYWORDS = SCHEMA + "keywords"
|
||||
SCHEMA_NAME = SCHEMA + "name"
|
||||
|
||||
# SKOS namespace
|
||||
SKOS = "http://www.w3.org/2004/02/skos/core#"
|
||||
SKOS_DEFINITION = SKOS + "definition"
|
||||
|
||||
# TrustGraph namespace for custom predicates
|
||||
TG = "https://trustgraph.ai/ns/"
|
||||
TG_REIFIES = TG + "reifies"
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Helper functions to build PROV-O triples for extraction-time provenance.
|
|||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
from .. schema import Triple, Term, IRI, LITERAL
|
||||
from .. schema import Triple, Term, IRI, LITERAL, TRIPLE
|
||||
|
||||
from . namespaces import (
|
||||
RDF_TYPE, RDFS_LABEL,
|
||||
|
|
@ -145,6 +145,7 @@ def derived_entity_triples(
|
|||
|
||||
# Activity declaration
|
||||
_triple(act_uri, RDF_TYPE, _iri(PROV_ACTIVITY)),
|
||||
_triple(act_uri, RDFS_LABEL, _literal(f"{component_name} extraction")),
|
||||
_triple(act_uri, PROV_USED, _iri(parent_uri)),
|
||||
_triple(act_uri, PROV_WAS_ASSOCIATED_WITH, _iri(agt_uri)),
|
||||
_triple(act_uri, PROV_STARTED_AT_TIME, _literal(timestamp)),
|
||||
|
|
@ -181,9 +182,7 @@ def derived_entity_triples(
|
|||
|
||||
def triple_provenance_triples(
|
||||
stmt_uri: str,
|
||||
subject_uri: str,
|
||||
predicate_uri: str,
|
||||
object_term: Term,
|
||||
extracted_triple: Triple,
|
||||
chunk_uri: str,
|
||||
component_name: str,
|
||||
component_version: str,
|
||||
|
|
@ -195,15 +194,13 @@ def triple_provenance_triples(
|
|||
Build provenance triples for an extracted knowledge triple using reification.
|
||||
|
||||
Creates:
|
||||
- Statement object that reifies the triple
|
||||
- Reification triple: stmt_uri tg:reifies <<extracted_triple>>
|
||||
- wasDerivedFrom link to source chunk
|
||||
- Activity and agent metadata
|
||||
|
||||
Args:
|
||||
stmt_uri: URI for the reified statement
|
||||
subject_uri: Subject of the extracted triple
|
||||
predicate_uri: Predicate of the extracted triple
|
||||
object_term: Object of the extracted triple (Term)
|
||||
extracted_triple: The extracted Triple to reify
|
||||
chunk_uri: URI of source chunk
|
||||
component_name: Name of extractor component
|
||||
component_version: Version of the component
|
||||
|
|
@ -212,7 +209,7 @@ def triple_provenance_triples(
|
|||
timestamp: ISO timestamp
|
||||
|
||||
Returns:
|
||||
List of Triple objects for the provenance (not the triple itself)
|
||||
List of Triple objects for the provenance (including reification)
|
||||
"""
|
||||
if timestamp is None:
|
||||
timestamp = datetime.utcnow().isoformat() + "Z"
|
||||
|
|
@ -220,18 +217,24 @@ def triple_provenance_triples(
|
|||
act_uri = activity_uri()
|
||||
agt_uri = agent_uri(component_name)
|
||||
|
||||
# Note: The actual reification (tg:reifies pointing at the edge) requires
|
||||
# RDF 1.2 triple term support. This builds the surrounding provenance.
|
||||
# The actual reification link must be handled by the knowledge extractor
|
||||
# using the graph store's reification API.
|
||||
# Create the quoted triple term (RDF-star reification)
|
||||
triple_term = Term(type=TRIPLE, triple=extracted_triple)
|
||||
|
||||
triples = [
|
||||
# Reification: stmt_uri tg:reifies <<s p o>>
|
||||
Triple(
|
||||
s=_iri(stmt_uri),
|
||||
p=_iri(TG_REIFIES),
|
||||
o=triple_term
|
||||
),
|
||||
|
||||
# Statement provenance
|
||||
_triple(stmt_uri, PROV_WAS_DERIVED_FROM, _iri(chunk_uri)),
|
||||
_triple(stmt_uri, PROV_WAS_GENERATED_BY, _iri(act_uri)),
|
||||
|
||||
# Activity
|
||||
_triple(act_uri, RDF_TYPE, _iri(PROV_ACTIVITY)),
|
||||
_triple(act_uri, RDFS_LABEL, _literal(f"{component_name} extraction")),
|
||||
_triple(act_uri, PROV_USED, _iri(chunk_uri)),
|
||||
_triple(act_uri, PROV_WAS_ASSOCIATED_WITH, _iri(agt_uri)),
|
||||
_triple(act_uri, PROV_STARTED_AT_TIME, _literal(timestamp)),
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
"""
|
||||
URI generation for provenance entities.
|
||||
|
||||
URI patterns:
|
||||
- Document: https://trustgraph.ai/doc/{doc_id}
|
||||
- Page: https://trustgraph.ai/page/{doc_id}/p{page_number}
|
||||
- Chunk: https://trustgraph.ai/chunk/{doc_id}/p{page}/c{chunk} (from page)
|
||||
https://trustgraph.ai/chunk/{doc_id}/c{chunk} (from text doc)
|
||||
Document IDs are already IRIs (e.g., https://trustgraph.ai/doc/abc123).
|
||||
Child entities (pages, chunks) append path segments to the parent IRI:
|
||||
- Document: {doc_iri} (as provided)
|
||||
- Page: {doc_iri}/p{page_number}
|
||||
- Chunk: {page_iri}/c{chunk_index} (from page)
|
||||
{doc_iri}/c{chunk_index} (from text doc)
|
||||
- Activity: https://trustgraph.ai/activity/{uuid}
|
||||
- Statement: https://trustgraph.ai/stmt/{uuid}
|
||||
"""
|
||||
|
|
@ -13,7 +14,7 @@ URI patterns:
|
|||
import uuid
|
||||
import urllib.parse
|
||||
|
||||
# Base URI prefix
|
||||
# Base URI prefix for generated URIs (activities, statements, agents)
|
||||
TRUSTGRAPH_BASE = "https://trustgraph.ai"
|
||||
|
||||
|
||||
|
|
@ -22,24 +23,24 @@ def _encode_id(id_str: str) -> str:
|
|||
return urllib.parse.quote(str(id_str), safe='')
|
||||
|
||||
|
||||
def document_uri(doc_id: str) -> str:
|
||||
"""Generate URI for a source document."""
|
||||
return f"{TRUSTGRAPH_BASE}/doc/{_encode_id(doc_id)}"
|
||||
def document_uri(doc_iri: str) -> str:
|
||||
"""Return the document IRI as-is (already a full URI)."""
|
||||
return doc_iri
|
||||
|
||||
|
||||
def page_uri(doc_id: str, page_number: int) -> str:
|
||||
"""Generate URI for a page extracted from a document."""
|
||||
return f"{TRUSTGRAPH_BASE}/page/{_encode_id(doc_id)}/p{page_number}"
|
||||
def page_uri(doc_iri: str, page_number: int) -> str:
|
||||
"""Generate URI for a page by appending to document IRI."""
|
||||
return f"{doc_iri}/p{page_number}"
|
||||
|
||||
|
||||
def chunk_uri_from_page(doc_id: str, page_number: int, chunk_index: int) -> str:
|
||||
def chunk_uri_from_page(doc_iri: str, page_number: int, chunk_index: int) -> str:
|
||||
"""Generate URI for a chunk extracted from a page."""
|
||||
return f"{TRUSTGRAPH_BASE}/chunk/{_encode_id(doc_id)}/p{page_number}/c{chunk_index}"
|
||||
return f"{doc_iri}/p{page_number}/c{chunk_index}"
|
||||
|
||||
|
||||
def chunk_uri_from_doc(doc_id: str, chunk_index: int) -> str:
|
||||
def chunk_uri_from_doc(doc_iri: str, chunk_index: int) -> str:
|
||||
"""Generate URI for a chunk extracted directly from a text document."""
|
||||
return f"{TRUSTGRAPH_BASE}/chunk/{_encode_id(doc_id)}/c{chunk_index}"
|
||||
return f"{doc_iri}/c{chunk_index}"
|
||||
|
||||
|
||||
def activity_uri(activity_id: str = None) -> str:
|
||||
|
|
|
|||
|
|
@ -16,6 +16,9 @@ from . namespaces import (
|
|||
PROV_WAS_DERIVED_FROM, PROV_WAS_GENERATED_BY,
|
||||
PROV_USED, PROV_WAS_ASSOCIATED_WITH, PROV_STARTED_AT_TIME,
|
||||
DC_TITLE, DC_SOURCE, DC_DATE, DC_CREATOR,
|
||||
SCHEMA_SUBJECT_OF, SCHEMA_DIGITAL_DOCUMENT, SCHEMA_DESCRIPTION,
|
||||
SCHEMA_KEYWORDS, SCHEMA_NAME,
|
||||
SKOS_DEFINITION,
|
||||
TG_REIFIES, TG_PAGE_COUNT, TG_MIME_TYPE, TG_PAGE_NUMBER,
|
||||
TG_CHUNK_INDEX, TG_CHAR_OFFSET, TG_CHAR_LENGTH,
|
||||
TG_CHUNK_SIZE, TG_CHUNK_OVERLAP, TG_COMPONENT_VERSION,
|
||||
|
|
@ -57,6 +60,20 @@ DC_PREDICATE_LABELS = [
|
|||
_label_triple(DC_CREATOR, "creator"),
|
||||
]
|
||||
|
||||
# Schema.org labels
|
||||
SCHEMA_LABELS = [
|
||||
_label_triple(SCHEMA_SUBJECT_OF, "subject of"),
|
||||
_label_triple(SCHEMA_DIGITAL_DOCUMENT, "Digital Document"),
|
||||
_label_triple(SCHEMA_DESCRIPTION, "description"),
|
||||
_label_triple(SCHEMA_KEYWORDS, "keywords"),
|
||||
_label_triple(SCHEMA_NAME, "name"),
|
||||
]
|
||||
|
||||
# SKOS labels
|
||||
SKOS_LABELS = [
|
||||
_label_triple(SKOS_DEFINITION, "definition"),
|
||||
]
|
||||
|
||||
# TrustGraph predicate labels
|
||||
TG_PREDICATE_LABELS = [
|
||||
_label_triple(TG_REIFIES, "reifies"),
|
||||
|
|
@ -97,5 +114,7 @@ def get_vocabulary_triples() -> List[Triple]:
|
|||
PROV_CLASS_LABELS +
|
||||
PROV_PREDICATE_LABELS +
|
||||
DC_PREDICATE_LABELS +
|
||||
SCHEMA_LABELS +
|
||||
SKOS_LABELS +
|
||||
TG_PREDICATE_LABELS
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue