apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-23 04:01:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Following the schema change, Value -> Term in the flow package

Browse source
This commit is contained in:
Cyber MacGeddon 2026-01-26 23:09:31 +00:00
parent 7c1c52b39f
commit d1d73ae38e
9 changed files with 194 additions and 160 deletions
Download patch file Download diff file Expand all files Collapse all files

76
trustgraph-flow/trustgraph/extract/kg/agent/extract.py
View file

@ -3,7 +3,7 @@ import json
import urllib.parse import urllib.parse
import logging import logging
from ....schema import Chunk, Triple, Triples, Metadata, Value from ....schema import Chunk, Triple, Triples, Metadata, Term, IRI, LITERAL
from ....schema import EntityContext, EntityContexts from ....schema import EntityContext, EntityContexts
from ....rdf import TRUSTGRAPH_ENTITIES, RDF_LABEL, SUBJECT_OF, DEFINITION from ....rdf import TRUSTGRAPH_ENTITIES, RDF_LABEL, SUBJECT_OF, DEFINITION
@ -253,32 +253,32 @@ class Processor(FlowProcessor):
for defn in definitions: for defn in definitions:
entity_uri = self.to_uri(defn["entity"]) entity_uri = self.to_uri(defn["entity"])
# Add entity label # Add entity label
triples.append(Triple( triples.append(Triple(
s = Value(value=entity_uri, is_uri=True), s = Term(type=IRI, iri=entity_uri),
p = Value(value=RDF_LABEL, is_uri=True), p = Term(type=IRI, iri=RDF_LABEL),
o = Value(value=defn["entity"], is_uri=False), o = Term(type=LITERAL, value=defn["entity"]),
)) ))
# Add definition # Add definition
triples.append(Triple( triples.append(Triple(
s = Value(value=entity_uri, is_uri=True), s = Term(type=IRI, iri=entity_uri),
p = Value(value=DEFINITION, is_uri=True), p = Term(type=IRI, iri=DEFINITION),
o = Value(value=defn["definition"], is_uri=False), o = Term(type=LITERAL, value=defn["definition"]),
)) ))
# Add subject-of relationship to document # Add subject-of relationship to document
if metadata.id: if metadata.id:
triples.append(Triple( triples.append(Triple(
s = Value(value=entity_uri, is_uri=True), s = Term(type=IRI, iri=entity_uri),
p = Value(value=SUBJECT_OF, is_uri=True), p = Term(type=IRI, iri=SUBJECT_OF),
o = Value(value=metadata.id, is_uri=True), o = Term(type=IRI, iri=metadata.id),
)) ))
# Create entity context for embeddings # Create entity context for embeddings
entity_contexts.append(EntityContext( entity_contexts.append(EntityContext(
entity=Value(value=entity_uri, is_uri=True), entity=Term(type=IRI, iri=entity_uri),
context=defn["definition"] context=defn["definition"]
)) ))
@ -288,61 +288,61 @@ class Processor(FlowProcessor):
subject_uri = self.to_uri(rel["subject"]) subject_uri = self.to_uri(rel["subject"])
predicate_uri = self.to_uri(rel["predicate"]) predicate_uri = self.to_uri(rel["predicate"])
subject_value = Value(value=subject_uri, is_uri=True) subject_value = Term(type=IRI, iri=subject_uri)
predicate_value = Value(value=predicate_uri, is_uri=True) predicate_value = Term(type=IRI, iri=predicate_uri)
if rel.get("object-entity", True): if rel.get("object-entity", True):
object_uri = self.to_uri(rel["object"]) object_uri = self.to_uri(rel["object"])
object_value = Value(value=object_uri, is_uri=True) object_value = Term(type=IRI, iri=object_uri)
else: else:
object_value = Value(value=rel["object"], is_uri=False) object_value = Term(type=LITERAL, value=rel["object"])
# Add subject and predicate labels # Add subject and predicate labels
triples.append(Triple( triples.append(Triple(
s = subject_value, s = subject_value,
p = Value(value=RDF_LABEL, is_uri=True), p = Term(type=IRI, iri=RDF_LABEL),
o = Value(value=rel["subject"], is_uri=False), o = Term(type=LITERAL, value=rel["subject"]),
)) ))
triples.append(Triple( triples.append(Triple(
s = predicate_value, s = predicate_value,
p = Value(value=RDF_LABEL, is_uri=True), p = Term(type=IRI, iri=RDF_LABEL),
o = Value(value=rel["predicate"], is_uri=False), o = Term(type=LITERAL, value=rel["predicate"]),
)) ))
# Handle object (entity vs literal) # Handle object (entity vs literal)
if rel.get("object-entity", True): if rel.get("object-entity", True):
triples.append(Triple( triples.append(Triple(
s = object_value, s = object_value,
p = Value(value=RDF_LABEL, is_uri=True), p = Term(type=IRI, iri=RDF_LABEL),
o = Value(value=rel["object"], is_uri=True), o = Term(type=LITERAL, value=rel["object"]),
)) ))
# Add the main relationship triple # Add the main relationship triple
triples.append(Triple( triples.append(Triple(
s = subject_value, s = subject_value,
p = predicate_value, p = predicate_value,
o = object_value o = object_value
)) ))
# Add subject-of relationships to document # Add subject-of relationships to document
if metadata.id: if metadata.id:
triples.append(Triple( triples.append(Triple(
s = subject_value, s = subject_value,
p = Value(value=SUBJECT_OF, is_uri=True), p = Term(type=IRI, iri=SUBJECT_OF),
o = Value(value=metadata.id, is_uri=True), o = Term(type=IRI, iri=metadata.id),
)) ))
triples.append(Triple( triples.append(Triple(
s = predicate_value, s = predicate_value,
p = Value(value=SUBJECT_OF, is_uri=True), p = Term(type=IRI, iri=SUBJECT_OF),
o = Value(value=metadata.id, is_uri=True), o = Term(type=IRI, iri=metadata.id),
)) ))
if rel.get("object-entity", True): if rel.get("object-entity", True):
triples.append(Triple( triples.append(Triple(
s = object_value, s = object_value,
p = Value(value=SUBJECT_OF, is_uri=True), p = Term(type=IRI, iri=SUBJECT_OF),
o = Value(value=metadata.id, is_uri=True), o = Term(type=IRI, iri=metadata.id),
)) ))
return triples, entity_contexts return triples, entity_contexts

16
trustgraph-flow/trustgraph/extract/kg/definitions/extract.py
View file

@ -9,7 +9,7 @@ import json
import urllib.parse import urllib.parse
import logging import logging
from .... schema import Chunk, Triple, Triples, Metadata, Value from .... schema import Chunk, Triple, Triples, Metadata, Term, IRI, LITERAL
# Module logger # Module logger
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -20,9 +20,9 @@ from .... rdf import TRUSTGRAPH_ENTITIES, DEFINITION, RDF_LABEL, SUBJECT_OF
from .... base import FlowProcessor, ConsumerSpec, ProducerSpec from .... base import FlowProcessor, ConsumerSpec, ProducerSpec
from .... base import PromptClientSpec from .... base import PromptClientSpec
DEFINITION_VALUE = Value(value=DEFINITION, is_uri=True) DEFINITION_VALUE = Term(type=IRI, iri=DEFINITION)
RDF_LABEL_VALUE = Value(value=RDF_LABEL, is_uri=True) RDF_LABEL_VALUE = Term(type=IRI, iri=RDF_LABEL)
SUBJECT_OF_VALUE = Value(value=SUBJECT_OF, is_uri=True) SUBJECT_OF_VALUE = Term(type=IRI, iri=SUBJECT_OF)
default_ident = "kg-extract-definitions" default_ident = "kg-extract-definitions"
default_concurrency = 1 default_concurrency = 1
@ -142,13 +142,13 @@ class Processor(FlowProcessor):
s_uri = self.to_uri(s) s_uri = self.to_uri(s)
s_value = Value(value=str(s_uri), is_uri=True) s_value = Term(type=IRI, iri=str(s_uri))
o_value = Value(value=str(o), is_uri=False) o_value = Term(type=LITERAL, value=str(o))
triples.append(Triple( triples.append(Triple(
s=s_value, s=s_value,
p=RDF_LABEL_VALUE, p=RDF_LABEL_VALUE,
o=Value(value=s, is_uri=False), o=Term(type=LITERAL, value=s),
)) ))
triples.append(Triple( triples.append(Triple(
@ -158,7 +158,7 @@ class Processor(FlowProcessor):
triples.append(Triple( triples.append(Triple(
s=s_value, s=s_value,
p=SUBJECT_OF_VALUE, p=SUBJECT_OF_VALUE,
o=Value(value=v.metadata.id, is_uri=True) o=Term(type=IRI, iri=v.metadata.id)
)) ))
ec = EntityContext( ec = EntityContext(

112
trustgraph-flow/trustgraph/extract/kg/ontology/extract.py
View file

@ -8,7 +8,7 @@ import logging
import asyncio import asyncio
from typing import List, Dict, Any, Optional from typing import List, Dict, Any, Optional
from .... schema import Chunk, Triple, Triples, Metadata, Value from .... schema import Chunk, Triple, Triples, Metadata, Term, IRI, LITERAL
from .... schema import EntityContext, EntityContexts from .... schema import EntityContext, EntityContexts
from .... schema import PromptRequest, PromptResponse from .... schema import PromptRequest, PromptResponse
from .... rdf import TRUSTGRAPH_ENTITIES, RDF_TYPE, RDF_LABEL, DEFINITION from .... rdf import TRUSTGRAPH_ENTITIES, RDF_TYPE, RDF_LABEL, DEFINITION
@ -39,6 +39,14 @@ URI_PREFIXES = {
} }
def make_term(v, is_uri):
"""Helper to create Term from value and is_uri flag."""
if is_uri:
return Term(type=IRI, iri=v)
else:
return Term(type=LITERAL, value=v)
class Processor(FlowProcessor): class Processor(FlowProcessor):
"""Main OntoRAG extraction processor.""" """Main OntoRAG extraction processor."""
@ -446,9 +454,9 @@ class Processor(FlowProcessor):
is_object_uri = False is_object_uri = False
# Create Triple object with expanded URIs # Create Triple object with expanded URIs
s_value = Value(value=subject_uri, is_uri=True) s_value = make_term(subject_uri, is_uri=True)
p_value = Value(value=predicate_uri, is_uri=True) p_value = make_term(predicate_uri, is_uri=True)
o_value = Value(value=object_uri, is_uri=is_object_uri) o_value = make_term(object_uri, is_uri=is_object_uri)
validated_triples.append(Triple( validated_triples.append(Triple(
s=s_value, s=s_value,
@ -609,9 +617,9 @@ class Processor(FlowProcessor):
# rdf:type owl:Class # rdf:type owl:Class
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=class_uri, is_uri=True), s=make_term(class_uri, is_uri=True),
p=Value(value="http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True), p=make_term("http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True),
o=Value(value="http://www.w3.org/2002/07/owl#Class", is_uri=True) o=make_term("http://www.w3.org/2002/07/owl#Class", is_uri=True)
)) ))
# rdfs:label (stored as 'labels' in OntologyClass.__dict__) # rdfs:label (stored as 'labels' in OntologyClass.__dict__)
@ -620,18 +628,18 @@ class Processor(FlowProcessor):
if isinstance(labels, list) and labels: if isinstance(labels, list) and labels:
label_val = labels[0].get('value', class_id) if isinstance(labels[0], dict) else str(labels[0]) label_val = labels[0].get('value', class_id) if isinstance(labels[0], dict) else str(labels[0])
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=class_uri, is_uri=True), s=make_term(class_uri, is_uri=True),
p=Value(value=RDF_LABEL, is_uri=True), p=make_term(RDF_LABEL, is_uri=True),
o=Value(value=label_val, is_uri=False) o=make_term(label_val, is_uri=False)
)) ))
# rdfs:comment (stored as 'comment' in OntologyClass.__dict__) # rdfs:comment (stored as 'comment' in OntologyClass.__dict__)
if isinstance(class_def, dict) and 'comment' in class_def and class_def['comment']: if isinstance(class_def, dict) and 'comment' in class_def and class_def['comment']:
comment = class_def['comment'] comment = class_def['comment']
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=class_uri, is_uri=True), s=make_term(class_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True), p=make_term("http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True),
o=Value(value=comment, is_uri=False) o=make_term(comment, is_uri=False)
)) ))
# rdfs:subClassOf (stored as 'subclass_of' in OntologyClass.__dict__) # rdfs:subClassOf (stored as 'subclass_of' in OntologyClass.__dict__)
@ -648,9 +656,9 @@ class Processor(FlowProcessor):
parent_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{parent}" parent_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{parent}"
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=class_uri, is_uri=True), s=make_term(class_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#subClassOf", is_uri=True), p=make_term("http://www.w3.org/2000/01/rdf-schema#subClassOf", is_uri=True),
o=Value(value=parent_uri, is_uri=True) o=make_term(parent_uri, is_uri=True)
)) ))
# Generate triples for object properties # Generate triples for object properties
@ -663,9 +671,9 @@ class Processor(FlowProcessor):
# rdf:type owl:ObjectProperty # rdf:type owl:ObjectProperty
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True), s=make_term(prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True), p=make_term("http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True),
o=Value(value="http://www.w3.org/2002/07/owl#ObjectProperty", is_uri=True) o=make_term("http://www.w3.org/2002/07/owl#ObjectProperty", is_uri=True)
)) ))
# rdfs:label (stored as 'labels' in OntologyProperty.__dict__) # rdfs:label (stored as 'labels' in OntologyProperty.__dict__)
@ -674,18 +682,18 @@ class Processor(FlowProcessor):
if isinstance(labels, list) and labels: if isinstance(labels, list) and labels:
label_val = labels[0].get('value', prop_id) if isinstance(labels[0], dict) else str(labels[0]) label_val = labels[0].get('value', prop_id) if isinstance(labels[0], dict) else str(labels[0])
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True), s=make_term(prop_uri, is_uri=True),
p=Value(value=RDF_LABEL, is_uri=True), p=make_term(RDF_LABEL, is_uri=True),
o=Value(value=label_val, is_uri=False) o=make_term(label_val, is_uri=False)
)) ))
# rdfs:comment (stored as 'comment' in OntologyProperty.__dict__) # rdfs:comment (stored as 'comment' in OntologyProperty.__dict__)
if isinstance(prop_def, dict) and 'comment' in prop_def and prop_def['comment']: if isinstance(prop_def, dict) and 'comment' in prop_def and prop_def['comment']:
comment = prop_def['comment'] comment = prop_def['comment']
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True), s=make_term(prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True), p=make_term("http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True),
o=Value(value=comment, is_uri=False) o=make_term(comment, is_uri=False)
)) ))
# rdfs:domain (stored as 'domain' in OntologyProperty.__dict__) # rdfs:domain (stored as 'domain' in OntologyProperty.__dict__)
@ -702,9 +710,9 @@ class Processor(FlowProcessor):
domain_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{domain}" domain_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{domain}"
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True), s=make_term(prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#domain", is_uri=True), p=make_term("http://www.w3.org/2000/01/rdf-schema#domain", is_uri=True),
o=Value(value=domain_uri, is_uri=True) o=make_term(domain_uri, is_uri=True)
)) ))
# rdfs:range (stored as 'range' in OntologyProperty.__dict__) # rdfs:range (stored as 'range' in OntologyProperty.__dict__)
@ -721,9 +729,9 @@ class Processor(FlowProcessor):
range_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{range_val}" range_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{range_val}"
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True), s=make_term(prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#range", is_uri=True), p=make_term("http://www.w3.org/2000/01/rdf-schema#range", is_uri=True),
o=Value(value=range_uri, is_uri=True) o=make_term(range_uri, is_uri=True)
)) ))
# Generate triples for datatype properties # Generate triples for datatype properties
@ -736,9 +744,9 @@ class Processor(FlowProcessor):
# rdf:type owl:DatatypeProperty # rdf:type owl:DatatypeProperty
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True), s=make_term(prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True), p=make_term("http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True),
o=Value(value="http://www.w3.org/2002/07/owl#DatatypeProperty", is_uri=True) o=make_term("http://www.w3.org/2002/07/owl#DatatypeProperty", is_uri=True)
)) ))
# rdfs:label (stored as 'labels' in OntologyProperty.__dict__) # rdfs:label (stored as 'labels' in OntologyProperty.__dict__)
@ -747,18 +755,18 @@ class Processor(FlowProcessor):
if isinstance(labels, list) and labels: if isinstance(labels, list) and labels:
label_val = labels[0].get('value', prop_id) if isinstance(labels[0], dict) else str(labels[0]) label_val = labels[0].get('value', prop_id) if isinstance(labels[0], dict) else str(labels[0])
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True), s=make_term(prop_uri, is_uri=True),
p=Value(value=RDF_LABEL, is_uri=True), p=make_term(RDF_LABEL, is_uri=True),
o=Value(value=label_val, is_uri=False) o=make_term(label_val, is_uri=False)
)) ))
# rdfs:comment (stored as 'comment' in OntologyProperty.__dict__) # rdfs:comment (stored as 'comment' in OntologyProperty.__dict__)
if isinstance(prop_def, dict) and 'comment' in prop_def and prop_def['comment']: if isinstance(prop_def, dict) and 'comment' in prop_def and prop_def['comment']:
comment = prop_def['comment'] comment = prop_def['comment']
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True), s=make_term(prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True), p=make_term("http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True),
o=Value(value=comment, is_uri=False) o=make_term(comment, is_uri=False)
)) ))
# rdfs:domain (stored as 'domain' in OntologyProperty.__dict__) # rdfs:domain (stored as 'domain' in OntologyProperty.__dict__)
@ -775,9 +783,9 @@ class Processor(FlowProcessor):
domain_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{domain}" domain_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{domain}"
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True), s=make_term(prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#domain", is_uri=True), p=make_term("http://www.w3.org/2000/01/rdf-schema#domain", is_uri=True),
o=Value(value=domain_uri, is_uri=True) o=make_term(domain_uri, is_uri=True)
)) ))
# rdfs:range (datatype) # rdfs:range (datatype)
@ -790,9 +798,9 @@ class Processor(FlowProcessor):
range_uri = range_val range_uri = range_val
ontology_triples.append(Triple( ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True), s=make_term(prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#range", is_uri=True), p=make_term("http://www.w3.org/2000/01/rdf-schema#range", is_uri=True),
o=Value(value=range_uri, is_uri=True) o=make_term(range_uri, is_uri=True)
)) ))
logger.info(f"Generated {len(ontology_triples)} triples describing ontology elements") logger.info(f"Generated {len(ontology_triples)} triples describing ontology elements")
@ -814,9 +822,9 @@ class Processor(FlowProcessor):
entity_data = {} # subject_uri -> {labels: [], definitions: []} entity_data = {} # subject_uri -> {labels: [], definitions: []}
for triple in triples: for triple in triples:
subject_uri = triple.s.value subject_uri = triple.s.iri if triple.s.type == IRI else triple.s.value
predicate_uri = triple.p.value predicate_uri = triple.p.iri if triple.p.type == IRI else triple.p.value
object_val = triple.o.value object_val = triple.o.value if triple.o.type == LITERAL else triple.o.iri
# Initialize entity data if not exists # Initialize entity data if not exists
if subject_uri not in entity_data: if subject_uri not in entity_data:
@ -824,12 +832,12 @@ class Processor(FlowProcessor):
# Collect labels (rdfs:label) # Collect labels (rdfs:label)
if predicate_uri == RDF_LABEL: if predicate_uri == RDF_LABEL:
if not triple.o.is_uri: # Labels are literals if triple.o.type == LITERAL: # Labels are literals
entity_data[subject_uri]['labels'].append(object_val) entity_data[subject_uri]['labels'].append(object_val)
# Collect definitions (skos:definition, schema:description) # Collect definitions (skos:definition, schema:description)
elif predicate_uri == DEFINITION or predicate_uri == "https://schema.org/description": elif predicate_uri == DEFINITION or predicate_uri == "https://schema.org/description":
if not triple.o.is_uri: if triple.o.type == LITERAL:
entity_data[subject_uri]['definitions'].append(object_val) entity_data[subject_uri]['definitions'].append(object_val)
# Build EntityContext objects # Build EntityContext objects
@ -848,7 +856,7 @@ class Processor(FlowProcessor):
if context_parts: if context_parts:
context_text = ". ".join(context_parts) context_text = ". ".join(context_parts)
entity_contexts.append(EntityContext( entity_contexts.append(EntityContext(
entity=Value(value=subject_uri, is_uri=True), entity=make_term(subject_uri, is_uri=True),
context=context_text context=context_text
)) ))

26
trustgraph-flow/trustgraph/extract/kg/ontology/triple_converter.py
View file

@ -8,7 +8,7 @@ with full URIs and correct is_uri flags.
import logging import logging
from typing import List, Optional from typing import List, Optional
from .... schema import Triple, Value from .... schema import Triple, Term, IRI, LITERAL
from .... rdf import RDF_TYPE, RDF_LABEL from .... rdf import RDF_TYPE, RDF_LABEL
from .simplified_parser import Entity, Relationship, Attribute, ExtractionResult from .simplified_parser import Entity, Relationship, Attribute, ExtractionResult
@ -87,17 +87,17 @@ class TripleConverter:
# Generate type triple: entity rdf:type ClassURI # Generate type triple: entity rdf:type ClassURI
type_triple = Triple( type_triple = Triple(
s=Value(value=entity_uri, is_uri=True), s=Term(type=IRI, iri=entity_uri),
p=Value(value=RDF_TYPE, is_uri=True), p=Term(type=IRI, iri=RDF_TYPE),
o=Value(value=class_uri, is_uri=True) o=Term(type=IRI, iri=class_uri)
) )
triples.append(type_triple) triples.append(type_triple)
# Generate label triple: entity rdfs:label "entity name" # Generate label triple: entity rdfs:label "entity name"
label_triple = Triple( label_triple = Triple(
s=Value(value=entity_uri, is_uri=True), s=Term(type=IRI, iri=entity_uri),
p=Value(value=RDF_LABEL, is_uri=True), p=Term(type=IRI, iri=RDF_LABEL),
o=Value(value=entity.entity, is_uri=False) # Literal! o=Term(type=LITERAL, value=entity.entity) # Literal!
) )
triples.append(label_triple) triples.append(label_triple)
@ -131,9 +131,9 @@ class TripleConverter:
# Generate triple: subject property object # Generate triple: subject property object
return Triple( return Triple(
s=Value(value=subject_uri, is_uri=True), s=Term(type=IRI, iri=subject_uri),
p=Value(value=property_uri, is_uri=True), p=Term(type=IRI, iri=property_uri),
o=Value(value=object_uri, is_uri=True) o=Term(type=IRI, iri=object_uri)
) )
def convert_attribute(self, attribute: Attribute) -> Optional[Triple]: def convert_attribute(self, attribute: Attribute) -> Optional[Triple]:
@ -159,9 +159,9 @@ class TripleConverter:
# Generate triple: entity property "literal value" # Generate triple: entity property "literal value"
return Triple( return Triple(
s=Value(value=entity_uri, is_uri=True), s=Term(type=IRI, iri=entity_uri),
p=Value(value=property_uri, is_uri=True), p=Term(type=IRI, iri=property_uri),
o=Value(value=attribute.value, is_uri=False) # Literal! o=Term(type=LITERAL, value=attribute.value) # Literal!
) )
def _get_class_uri(self, class_id: str) -> Optional[str]: def _get_class_uri(self, class_id: str) -> Optional[str]:

26
trustgraph-flow/trustgraph/extract/kg/relationships/extract.py
View file

@ -13,15 +13,15 @@ import urllib.parse
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from .... schema import Chunk, Triple, Triples from .... schema import Chunk, Triple, Triples
from .... schema import Metadata, Value from .... schema import Metadata, Term, IRI, LITERAL
from .... schema import PromptRequest, PromptResponse from .... schema import PromptRequest, PromptResponse
from .... rdf import RDF_LABEL, TRUSTGRAPH_ENTITIES, SUBJECT_OF from .... rdf import RDF_LABEL, TRUSTGRAPH_ENTITIES, SUBJECT_OF
from .... base import FlowProcessor, ConsumerSpec, ProducerSpec from .... base import FlowProcessor, ConsumerSpec, ProducerSpec
from .... base import PromptClientSpec from .... base import PromptClientSpec
RDF_LABEL_VALUE = Value(value=RDF_LABEL, is_uri=True) RDF_LABEL_VALUE = Term(type=IRI, iri=RDF_LABEL)
SUBJECT_OF_VALUE = Value(value=SUBJECT_OF, is_uri=True) SUBJECT_OF_VALUE = Term(type=IRI, iri=SUBJECT_OF)
default_ident = "kg-extract-relationships" default_ident = "kg-extract-relationships"
default_concurrency = 1 default_concurrency = 1
@ -127,16 +127,16 @@ class Processor(FlowProcessor):
if o is None: continue if o is None: continue
s_uri = self.to_uri(s) s_uri = self.to_uri(s)
s_value = Value(value=str(s_uri), is_uri=True) s_value = Term(type=IRI, iri=str(s_uri))
p_uri = self.to_uri(p) p_uri = self.to_uri(p)
p_value = Value(value=str(p_uri), is_uri=True) p_value = Term(type=IRI, iri=str(p_uri))
if rel["object-entity"]: if rel["object-entity"]:
o_uri = self.to_uri(o) o_uri = self.to_uri(o)
o_value = Value(value=str(o_uri), is_uri=True) o_value = Term(type=IRI, iri=str(o_uri))
else: else:
o_value = Value(value=str(o), is_uri=False) o_value = Term(type=LITERAL, value=str(o))
triples.append(Triple( triples.append(Triple(
s=s_value, s=s_value,
@ -148,14 +148,14 @@ class Processor(FlowProcessor):
triples.append(Triple( triples.append(Triple(
s=s_value, s=s_value,
p=RDF_LABEL_VALUE, p=RDF_LABEL_VALUE,
o=Value(value=str(s), is_uri=False) o=Term(type=LITERAL, value=str(s))
)) ))
# Label for p # Label for p
triples.append(Triple( triples.append(Triple(
s=p_value, s=p_value,
p=RDF_LABEL_VALUE, p=RDF_LABEL_VALUE,
o=Value(value=str(p), is_uri=False) o=Term(type=LITERAL, value=str(p))
)) ))
if rel["object-entity"]: if rel["object-entity"]:
@ -163,14 +163,14 @@ class Processor(FlowProcessor):
triples.append(Triple( triples.append(Triple(
s=o_value, s=o_value,
p=RDF_LABEL_VALUE, p=RDF_LABEL_VALUE,
o=Value(value=str(o), is_uri=False) o=Term(type=LITERAL, value=str(o))
)) ))
# 'Subject of' for s # 'Subject of' for s
triples.append(Triple( triples.append(Triple(
s=s_value, s=s_value,
p=SUBJECT_OF_VALUE, p=SUBJECT_OF_VALUE,
o=Value(value=v.metadata.id, is_uri=True) o=Term(type=IRI, iri=v.metadata.id)
)) ))
if rel["object-entity"]: if rel["object-entity"]:
@ -178,7 +178,7 @@ class Processor(FlowProcessor):
triples.append(Triple( triples.append(Triple(
s=o_value, s=o_value,
p=SUBJECT_OF_VALUE, p=SUBJECT_OF_VALUE,
o=Value(value=v.metadata.id, is_uri=True) o=Term(type=IRI, iri=v.metadata.id)
)) ))
await self.emit_triples( await self.emit_triples(

8
trustgraph-flow/trustgraph/extract/kg/topics/extract.py
View file

@ -11,7 +11,7 @@ import logging
# Module logger # Module logger
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from .... schema import Chunk, Triple, Triples, Metadata, Value from .... schema import Chunk, Triple, Triples, Metadata, Term, IRI, LITERAL
from .... schema import chunk_ingest_queue, triples_store_queue from .... schema import chunk_ingest_queue, triples_store_queue
from .... schema import prompt_request_queue from .... schema import prompt_request_queue
from .... schema import prompt_response_queue from .... schema import prompt_response_queue
@ -20,7 +20,7 @@ from .... clients.prompt_client import PromptClient
from .... rdf import TRUSTGRAPH_ENTITIES, DEFINITION from .... rdf import TRUSTGRAPH_ENTITIES, DEFINITION
from .... base import ConsumerProducer from .... base import ConsumerProducer
DEFINITION_VALUE = Value(value=DEFINITION, is_uri=True) DEFINITION_VALUE = Term(type=IRI, iri=DEFINITION)
module = "kg-extract-topics" module = "kg-extract-topics"
@ -106,8 +106,8 @@ class Processor(ConsumerProducer):
s_uri = self.to_uri(s) s_uri = self.to_uri(s)
s_value = Value(value=str(s_uri), is_uri=True) s_value = Term(type=IRI, iri=str(s_uri))
o_value = Value(value=str(o), is_uri=False) o_value = Term(type=LITERAL, value=str(o))
await self.emit_edge( await self.emit_edge(
v.metadata, s_value, DEFINITION_VALUE, o_value v.metadata, s_value, DEFINITION_VALUE, o_value

2
trustgraph-flow/trustgraph/tables/config.py
View file

@ -1,6 +1,6 @@
from .. schema import KnowledgeResponse, Triple, Triples, EntityEmbeddings from .. schema import KnowledgeResponse, Triple, Triples, EntityEmbeddings
from .. schema import Metadata, Value, GraphEmbeddings from .. schema import Metadata, GraphEmbeddings
from cassandra.cluster import Cluster from cassandra.cluster import Cluster
from cassandra.auth import PlainTextAuthProvider from cassandra.auth import PlainTextAuthProvider

52
trustgraph-flow/trustgraph/tables/knowledge.py
View file

@ -1,8 +1,24 @@
from .. schema import KnowledgeResponse, Triple, Triples, EntityEmbeddings from .. schema import KnowledgeResponse, Triple, Triples, EntityEmbeddings
from .. schema import Metadata, Value, GraphEmbeddings from .. schema import Metadata, Term, IRI, LITERAL, GraphEmbeddings
from cassandra.cluster import Cluster from cassandra.cluster import Cluster
def term_to_tuple(term):
"""Convert Term to (value, is_uri) tuple for database storage."""
if term.type == IRI:
return (term.iri, True)
else: # LITERAL
return (term.value, False)
def tuple_to_term(value, is_uri):
"""Convert (value, is_uri) tuple from database to Term."""
if is_uri:
return Term(type=IRI, iri=value)
else:
return Term(type=LITERAL, value=value)
from cassandra.auth import PlainTextAuthProvider from cassandra.auth import PlainTextAuthProvider
from ssl import SSLContext, PROTOCOL_TLSv1_2 from ssl import SSLContext, PROTOCOL_TLSv1_2
@ -205,8 +221,7 @@ class KnowledgeTableStore:
if m.metadata.metadata: if m.metadata.metadata:
metadata = [ metadata = [
( (
v.s.value, v.s.is_uri, v.p.value, v.p.is_uri, *term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o)
v.o.value, v.o.is_uri
) )
for v in m.metadata.metadata for v in m.metadata.metadata
] ]
@ -215,8 +230,7 @@ class KnowledgeTableStore:
triples = [ triples = [
( (
v.s.value, v.s.is_uri, v.p.value, v.p.is_uri, *term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o)
v.o.value, v.o.is_uri
) )
for v in m.triples for v in m.triples
] ]
@ -248,8 +262,7 @@ class KnowledgeTableStore:
if m.metadata.metadata: if m.metadata.metadata:
metadata = [ metadata = [
( (
v.s.value, v.s.is_uri, v.p.value, v.p.is_uri, *term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o)
v.o.value, v.o.is_uri
) )
for v in m.metadata.metadata for v in m.metadata.metadata
] ]
@ -258,7 +271,7 @@ class KnowledgeTableStore:
entities = [ entities = [
( (
(v.entity.value, v.entity.is_uri), term_to_tuple(v.entity),
v.vectors v.vectors
) )
for v in m.entities for v in m.entities
@ -291,8 +304,7 @@ class KnowledgeTableStore:
if m.metadata.metadata: if m.metadata.metadata:
metadata = [ metadata = [
( (
v.s.value, v.s.is_uri, v.p.value, v.p.is_uri, *term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o)
v.o.value, v.o.is_uri
) )
for v in m.metadata.metadata for v in m.metadata.metadata
] ]
@ -414,9 +426,9 @@ class KnowledgeTableStore:
if row[2]: if row[2]:
metadata = [ metadata = [
Triple( Triple(
s = Value(value = elt[0], is_uri = elt[1]), s = tuple_to_term(elt[0], elt[1]),
p = Value(value = elt[2], is_uri = elt[3]), p = tuple_to_term(elt[2], elt[3]),
o = Value(value = elt[4], is_uri = elt[5]), o = tuple_to_term(elt[4], elt[5]),
) )
for elt in row[2] for elt in row[2]
] ]
@ -425,9 +437,9 @@ class KnowledgeTableStore:
triples = [ triples = [
Triple( Triple(
s = Value(value = elt[0], is_uri = elt[1]), s = tuple_to_term(elt[0], elt[1]),
p = Value(value = elt[2], is_uri = elt[3]), p = tuple_to_term(elt[2], elt[3]),
o = Value(value = elt[4], is_uri = elt[5]), o = tuple_to_term(elt[4], elt[5]),
) )
for elt in row[3] for elt in row[3]
] ]
@ -470,9 +482,9 @@ class KnowledgeTableStore:
if row[2]: if row[2]:
metadata = [ metadata = [
Triple( Triple(
s = Value(value = elt[0], is_uri = elt[1]), s = tuple_to_term(elt[0], elt[1]),
p = Value(value = elt[2], is_uri = elt[3]), p = tuple_to_term(elt[2], elt[3]),
o = Value(value = elt[4], is_uri = elt[5]), o = tuple_to_term(elt[4], elt[5]),
) )
for elt in row[2] for elt in row[2]
] ]
@ -481,7 +493,7 @@ class KnowledgeTableStore:
entities = [ entities = [
EntityEmbeddings( EntityEmbeddings(
entity = Value(value = ent[0][0], is_uri = ent[0][1]), entity = tuple_to_term(ent[0][0], ent[0][1]),
vectors = ent[1] vectors = ent[1]
) )
for ent in row[3] for ent in row[3]

36
trustgraph-flow/trustgraph/tables/library.py
View file

@ -1,8 +1,24 @@
from .. schema import LibrarianRequest, LibrarianResponse from .. schema import LibrarianRequest, LibrarianResponse
from .. schema import DocumentMetadata, ProcessingMetadata from .. schema import DocumentMetadata, ProcessingMetadata
from .. schema import Error, Triple, Value from .. schema import Error, Triple, Term, IRI, LITERAL
from .. knowledge import hash from .. knowledge import hash
def term_to_tuple(term):
"""Convert Term to (value, is_uri) tuple for database storage."""
if term.type == IRI:
return (term.iri, True)
else: # LITERAL
return (term.value, False)
def tuple_to_term(value, is_uri):
"""Convert (value, is_uri) tuple from database to Term."""
if is_uri:
return Term(type=IRI, iri=value)
else:
return Term(type=LITERAL, value=value)
from .. exceptions import RequestError from .. exceptions import RequestError
from cassandra.cluster import Cluster from cassandra.cluster import Cluster
@ -215,8 +231,7 @@ class LibraryTableStore:
metadata = [ metadata = [
( (
v.s.value, v.s.is_uri, v.p.value, v.p.is_uri, *term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o)
v.o.value, v.o.is_uri
) )
for v in document.metadata for v in document.metadata
] ]
@ -249,8 +264,7 @@ class LibraryTableStore:
metadata = [ metadata = [
( (
v.s.value, v.s.is_uri, v.p.value, v.p.is_uri, *term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o)
v.o.value, v.o.is_uri
) )
for v in document.metadata for v in document.metadata
] ]
@ -331,9 +345,9 @@ class LibraryTableStore:
comments = row[4], comments = row[4],
metadata = [ metadata = [
Triple( Triple(
s=Value(value=m[0], is_uri=m[1]), s=tuple_to_term(m[0], m[1]),
p=Value(value=m[2], is_uri=m[3]), p=tuple_to_term(m[2], m[3]),
o=Value(value=m[4], is_uri=m[5]) o=tuple_to_term(m[4], m[5])
) )
for m in row[5] for m in row[5]
], ],
@ -376,9 +390,9 @@ class LibraryTableStore:
comments = row[3], comments = row[3],
metadata = [ metadata = [
Triple( Triple(
s=Value(value=m[0], is_uri=m[1]), s=tuple_to_term(m[0], m[1]),
p=Value(value=m[2], is_uri=m[3]), p=tuple_to_term(m[2], m[3]),
o=Value(value=m[4], is_uri=m[5]) o=tuple_to_term(m[4], m[5])
) )
for m in row[4] for m in row[4]
], ],