diff --git a/trustgraph-flow/trustgraph/extract/kg/agent/extract.py b/trustgraph-flow/trustgraph/extract/kg/agent/extract.py index 58230a41..d9057909 100644 --- a/trustgraph-flow/trustgraph/extract/kg/agent/extract.py +++ b/trustgraph-flow/trustgraph/extract/kg/agent/extract.py @@ -3,7 +3,7 @@ import json import urllib.parse import logging -from ....schema import Chunk, Triple, Triples, Metadata, Value +from ....schema import Chunk, Triple, Triples, Metadata, Term, IRI, LITERAL from ....schema import EntityContext, EntityContexts from ....rdf import TRUSTGRAPH_ENTITIES, RDF_LABEL, SUBJECT_OF, DEFINITION @@ -253,32 +253,32 @@ class Processor(FlowProcessor): for defn in definitions: entity_uri = self.to_uri(defn["entity"]) - + # Add entity label triples.append(Triple( - s = Value(value=entity_uri, is_uri=True), - p = Value(value=RDF_LABEL, is_uri=True), - o = Value(value=defn["entity"], is_uri=False), + s = Term(type=IRI, iri=entity_uri), + p = Term(type=IRI, iri=RDF_LABEL), + o = Term(type=LITERAL, value=defn["entity"]), )) - + # Add definition triples.append(Triple( - s = Value(value=entity_uri, is_uri=True), - p = Value(value=DEFINITION, is_uri=True), - o = Value(value=defn["definition"], is_uri=False), + s = Term(type=IRI, iri=entity_uri), + p = Term(type=IRI, iri=DEFINITION), + o = Term(type=LITERAL, value=defn["definition"]), )) - + # Add subject-of relationship to document if metadata.id: triples.append(Triple( - s = Value(value=entity_uri, is_uri=True), - p = Value(value=SUBJECT_OF, is_uri=True), - o = Value(value=metadata.id, is_uri=True), + s = Term(type=IRI, iri=entity_uri), + p = Term(type=IRI, iri=SUBJECT_OF), + o = Term(type=IRI, iri=metadata.id), )) - + # Create entity context for embeddings entity_contexts.append(EntityContext( - entity=Value(value=entity_uri, is_uri=True), + entity=Term(type=IRI, iri=entity_uri), context=defn["definition"] )) @@ -288,61 +288,61 @@ class Processor(FlowProcessor): subject_uri = self.to_uri(rel["subject"]) predicate_uri = self.to_uri(rel["predicate"]) - subject_value = Value(value=subject_uri, is_uri=True) - predicate_value = Value(value=predicate_uri, is_uri=True) + subject_value = Term(type=IRI, iri=subject_uri) + predicate_value = Term(type=IRI, iri=predicate_uri) if rel.get("object-entity", True): object_uri = self.to_uri(rel["object"]) - object_value = Value(value=object_uri, is_uri=True) + object_value = Term(type=IRI, iri=object_uri) else: - object_value = Value(value=rel["object"], is_uri=False) - + object_value = Term(type=LITERAL, value=rel["object"]) + # Add subject and predicate labels triples.append(Triple( s = subject_value, - p = Value(value=RDF_LABEL, is_uri=True), - o = Value(value=rel["subject"], is_uri=False), + p = Term(type=IRI, iri=RDF_LABEL), + o = Term(type=LITERAL, value=rel["subject"]), )) - + triples.append(Triple( s = predicate_value, - p = Value(value=RDF_LABEL, is_uri=True), - o = Value(value=rel["predicate"], is_uri=False), + p = Term(type=IRI, iri=RDF_LABEL), + o = Term(type=LITERAL, value=rel["predicate"]), )) - + # Handle object (entity vs literal) if rel.get("object-entity", True): triples.append(Triple( s = object_value, - p = Value(value=RDF_LABEL, is_uri=True), - o = Value(value=rel["object"], is_uri=True), + p = Term(type=IRI, iri=RDF_LABEL), + o = Term(type=LITERAL, value=rel["object"]), )) - + # Add the main relationship triple triples.append(Triple( s = subject_value, p = predicate_value, o = object_value )) - + # Add subject-of relationships to document if metadata.id: triples.append(Triple( s = subject_value, - p = Value(value=SUBJECT_OF, is_uri=True), - o = Value(value=metadata.id, is_uri=True), + p = Term(type=IRI, iri=SUBJECT_OF), + o = Term(type=IRI, iri=metadata.id), )) - + triples.append(Triple( s = predicate_value, - p = Value(value=SUBJECT_OF, is_uri=True), - o = Value(value=metadata.id, is_uri=True), + p = Term(type=IRI, iri=SUBJECT_OF), + o = Term(type=IRI, iri=metadata.id), )) - + if rel.get("object-entity", True): triples.append(Triple( s = object_value, - p = Value(value=SUBJECT_OF, is_uri=True), - o = Value(value=metadata.id, is_uri=True), + p = Term(type=IRI, iri=SUBJECT_OF), + o = Term(type=IRI, iri=metadata.id), )) return triples, entity_contexts diff --git a/trustgraph-flow/trustgraph/extract/kg/definitions/extract.py b/trustgraph-flow/trustgraph/extract/kg/definitions/extract.py index 1d414b7e..8c278c25 100755 --- a/trustgraph-flow/trustgraph/extract/kg/definitions/extract.py +++ b/trustgraph-flow/trustgraph/extract/kg/definitions/extract.py @@ -9,7 +9,7 @@ import json import urllib.parse import logging -from .... schema import Chunk, Triple, Triples, Metadata, Value +from .... schema import Chunk, Triple, Triples, Metadata, Term, IRI, LITERAL # Module logger logger = logging.getLogger(__name__) @@ -20,9 +20,9 @@ from .... rdf import TRUSTGRAPH_ENTITIES, DEFINITION, RDF_LABEL, SUBJECT_OF from .... base import FlowProcessor, ConsumerSpec, ProducerSpec from .... base import PromptClientSpec -DEFINITION_VALUE = Value(value=DEFINITION, is_uri=True) -RDF_LABEL_VALUE = Value(value=RDF_LABEL, is_uri=True) -SUBJECT_OF_VALUE = Value(value=SUBJECT_OF, is_uri=True) +DEFINITION_VALUE = Term(type=IRI, iri=DEFINITION) +RDF_LABEL_VALUE = Term(type=IRI, iri=RDF_LABEL) +SUBJECT_OF_VALUE = Term(type=IRI, iri=SUBJECT_OF) default_ident = "kg-extract-definitions" default_concurrency = 1 @@ -142,13 +142,13 @@ class Processor(FlowProcessor): s_uri = self.to_uri(s) - s_value = Value(value=str(s_uri), is_uri=True) - o_value = Value(value=str(o), is_uri=False) + s_value = Term(type=IRI, iri=str(s_uri)) + o_value = Term(type=LITERAL, value=str(o)) triples.append(Triple( s=s_value, p=RDF_LABEL_VALUE, - o=Value(value=s, is_uri=False), + o=Term(type=LITERAL, value=s), )) triples.append(Triple( @@ -158,7 +158,7 @@ class Processor(FlowProcessor): triples.append(Triple( s=s_value, p=SUBJECT_OF_VALUE, - o=Value(value=v.metadata.id, is_uri=True) + o=Term(type=IRI, iri=v.metadata.id) )) ec = EntityContext( diff --git a/trustgraph-flow/trustgraph/extract/kg/ontology/extract.py b/trustgraph-flow/trustgraph/extract/kg/ontology/extract.py index 335f07d2..adcbe0a8 100644 --- a/trustgraph-flow/trustgraph/extract/kg/ontology/extract.py +++ b/trustgraph-flow/trustgraph/extract/kg/ontology/extract.py @@ -8,7 +8,7 @@ import logging import asyncio from typing import List, Dict, Any, Optional -from .... schema import Chunk, Triple, Triples, Metadata, Value +from .... schema import Chunk, Triple, Triples, Metadata, Term, IRI, LITERAL from .... schema import EntityContext, EntityContexts from .... schema import PromptRequest, PromptResponse from .... rdf import TRUSTGRAPH_ENTITIES, RDF_TYPE, RDF_LABEL, DEFINITION @@ -39,6 +39,14 @@ URI_PREFIXES = { } +def make_term(v, is_uri): + """Helper to create Term from value and is_uri flag.""" + if is_uri: + return Term(type=IRI, iri=v) + else: + return Term(type=LITERAL, value=v) + + class Processor(FlowProcessor): """Main OntoRAG extraction processor.""" @@ -446,9 +454,9 @@ class Processor(FlowProcessor): is_object_uri = False # Create Triple object with expanded URIs - s_value = Value(value=subject_uri, is_uri=True) - p_value = Value(value=predicate_uri, is_uri=True) - o_value = Value(value=object_uri, is_uri=is_object_uri) + s_value = make_term(subject_uri, is_uri=True) + p_value = make_term(predicate_uri, is_uri=True) + o_value = make_term(object_uri, is_uri=is_object_uri) validated_triples.append(Triple( s=s_value, @@ -609,9 +617,9 @@ class Processor(FlowProcessor): # rdf:type owl:Class ontology_triples.append(Triple( - s=Value(value=class_uri, is_uri=True), - p=Value(value="http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True), - o=Value(value="http://www.w3.org/2002/07/owl#Class", is_uri=True) + s=make_term(class_uri, is_uri=True), + p=make_term("http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True), + o=make_term("http://www.w3.org/2002/07/owl#Class", is_uri=True) )) # rdfs:label (stored as 'labels' in OntologyClass.__dict__) @@ -620,18 +628,18 @@ class Processor(FlowProcessor): if isinstance(labels, list) and labels: label_val = labels[0].get('value', class_id) if isinstance(labels[0], dict) else str(labels[0]) ontology_triples.append(Triple( - s=Value(value=class_uri, is_uri=True), - p=Value(value=RDF_LABEL, is_uri=True), - o=Value(value=label_val, is_uri=False) + s=make_term(class_uri, is_uri=True), + p=make_term(RDF_LABEL, is_uri=True), + o=make_term(label_val, is_uri=False) )) # rdfs:comment (stored as 'comment' in OntologyClass.__dict__) if isinstance(class_def, dict) and 'comment' in class_def and class_def['comment']: comment = class_def['comment'] ontology_triples.append(Triple( - s=Value(value=class_uri, is_uri=True), - p=Value(value="http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True), - o=Value(value=comment, is_uri=False) + s=make_term(class_uri, is_uri=True), + p=make_term("http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True), + o=make_term(comment, is_uri=False) )) # rdfs:subClassOf (stored as 'subclass_of' in OntologyClass.__dict__) @@ -648,9 +656,9 @@ class Processor(FlowProcessor): parent_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{parent}" ontology_triples.append(Triple( - s=Value(value=class_uri, is_uri=True), - p=Value(value="http://www.w3.org/2000/01/rdf-schema#subClassOf", is_uri=True), - o=Value(value=parent_uri, is_uri=True) + s=make_term(class_uri, is_uri=True), + p=make_term("http://www.w3.org/2000/01/rdf-schema#subClassOf", is_uri=True), + o=make_term(parent_uri, is_uri=True) )) # Generate triples for object properties @@ -663,9 +671,9 @@ class Processor(FlowProcessor): # rdf:type owl:ObjectProperty ontology_triples.append(Triple( - s=Value(value=prop_uri, is_uri=True), - p=Value(value="http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True), - o=Value(value="http://www.w3.org/2002/07/owl#ObjectProperty", is_uri=True) + s=make_term(prop_uri, is_uri=True), + p=make_term("http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True), + o=make_term("http://www.w3.org/2002/07/owl#ObjectProperty", is_uri=True) )) # rdfs:label (stored as 'labels' in OntologyProperty.__dict__) @@ -674,18 +682,18 @@ class Processor(FlowProcessor): if isinstance(labels, list) and labels: label_val = labels[0].get('value', prop_id) if isinstance(labels[0], dict) else str(labels[0]) ontology_triples.append(Triple( - s=Value(value=prop_uri, is_uri=True), - p=Value(value=RDF_LABEL, is_uri=True), - o=Value(value=label_val, is_uri=False) + s=make_term(prop_uri, is_uri=True), + p=make_term(RDF_LABEL, is_uri=True), + o=make_term(label_val, is_uri=False) )) # rdfs:comment (stored as 'comment' in OntologyProperty.__dict__) if isinstance(prop_def, dict) and 'comment' in prop_def and prop_def['comment']: comment = prop_def['comment'] ontology_triples.append(Triple( - s=Value(value=prop_uri, is_uri=True), - p=Value(value="http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True), - o=Value(value=comment, is_uri=False) + s=make_term(prop_uri, is_uri=True), + p=make_term("http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True), + o=make_term(comment, is_uri=False) )) # rdfs:domain (stored as 'domain' in OntologyProperty.__dict__) @@ -702,9 +710,9 @@ class Processor(FlowProcessor): domain_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{domain}" ontology_triples.append(Triple( - s=Value(value=prop_uri, is_uri=True), - p=Value(value="http://www.w3.org/2000/01/rdf-schema#domain", is_uri=True), - o=Value(value=domain_uri, is_uri=True) + s=make_term(prop_uri, is_uri=True), + p=make_term("http://www.w3.org/2000/01/rdf-schema#domain", is_uri=True), + o=make_term(domain_uri, is_uri=True) )) # rdfs:range (stored as 'range' in OntologyProperty.__dict__) @@ -721,9 +729,9 @@ class Processor(FlowProcessor): range_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{range_val}" ontology_triples.append(Triple( - s=Value(value=prop_uri, is_uri=True), - p=Value(value="http://www.w3.org/2000/01/rdf-schema#range", is_uri=True), - o=Value(value=range_uri, is_uri=True) + s=make_term(prop_uri, is_uri=True), + p=make_term("http://www.w3.org/2000/01/rdf-schema#range", is_uri=True), + o=make_term(range_uri, is_uri=True) )) # Generate triples for datatype properties @@ -736,9 +744,9 @@ class Processor(FlowProcessor): # rdf:type owl:DatatypeProperty ontology_triples.append(Triple( - s=Value(value=prop_uri, is_uri=True), - p=Value(value="http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True), - o=Value(value="http://www.w3.org/2002/07/owl#DatatypeProperty", is_uri=True) + s=make_term(prop_uri, is_uri=True), + p=make_term("http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True), + o=make_term("http://www.w3.org/2002/07/owl#DatatypeProperty", is_uri=True) )) # rdfs:label (stored as 'labels' in OntologyProperty.__dict__) @@ -747,18 +755,18 @@ class Processor(FlowProcessor): if isinstance(labels, list) and labels: label_val = labels[0].get('value', prop_id) if isinstance(labels[0], dict) else str(labels[0]) ontology_triples.append(Triple( - s=Value(value=prop_uri, is_uri=True), - p=Value(value=RDF_LABEL, is_uri=True), - o=Value(value=label_val, is_uri=False) + s=make_term(prop_uri, is_uri=True), + p=make_term(RDF_LABEL, is_uri=True), + o=make_term(label_val, is_uri=False) )) # rdfs:comment (stored as 'comment' in OntologyProperty.__dict__) if isinstance(prop_def, dict) and 'comment' in prop_def and prop_def['comment']: comment = prop_def['comment'] ontology_triples.append(Triple( - s=Value(value=prop_uri, is_uri=True), - p=Value(value="http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True), - o=Value(value=comment, is_uri=False) + s=make_term(prop_uri, is_uri=True), + p=make_term("http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True), + o=make_term(comment, is_uri=False) )) # rdfs:domain (stored as 'domain' in OntologyProperty.__dict__) @@ -775,9 +783,9 @@ class Processor(FlowProcessor): domain_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{domain}" ontology_triples.append(Triple( - s=Value(value=prop_uri, is_uri=True), - p=Value(value="http://www.w3.org/2000/01/rdf-schema#domain", is_uri=True), - o=Value(value=domain_uri, is_uri=True) + s=make_term(prop_uri, is_uri=True), + p=make_term("http://www.w3.org/2000/01/rdf-schema#domain", is_uri=True), + o=make_term(domain_uri, is_uri=True) )) # rdfs:range (datatype) @@ -790,9 +798,9 @@ class Processor(FlowProcessor): range_uri = range_val ontology_triples.append(Triple( - s=Value(value=prop_uri, is_uri=True), - p=Value(value="http://www.w3.org/2000/01/rdf-schema#range", is_uri=True), - o=Value(value=range_uri, is_uri=True) + s=make_term(prop_uri, is_uri=True), + p=make_term("http://www.w3.org/2000/01/rdf-schema#range", is_uri=True), + o=make_term(range_uri, is_uri=True) )) logger.info(f"Generated {len(ontology_triples)} triples describing ontology elements") @@ -814,9 +822,9 @@ class Processor(FlowProcessor): entity_data = {} # subject_uri -> {labels: [], definitions: []} for triple in triples: - subject_uri = triple.s.value - predicate_uri = triple.p.value - object_val = triple.o.value + subject_uri = triple.s.iri if triple.s.type == IRI else triple.s.value + predicate_uri = triple.p.iri if triple.p.type == IRI else triple.p.value + object_val = triple.o.value if triple.o.type == LITERAL else triple.o.iri # Initialize entity data if not exists if subject_uri not in entity_data: @@ -824,12 +832,12 @@ class Processor(FlowProcessor): # Collect labels (rdfs:label) if predicate_uri == RDF_LABEL: - if not triple.o.is_uri: # Labels are literals + if triple.o.type == LITERAL: # Labels are literals entity_data[subject_uri]['labels'].append(object_val) # Collect definitions (skos:definition, schema:description) elif predicate_uri == DEFINITION or predicate_uri == "https://schema.org/description": - if not triple.o.is_uri: + if triple.o.type == LITERAL: entity_data[subject_uri]['definitions'].append(object_val) # Build EntityContext objects @@ -848,7 +856,7 @@ class Processor(FlowProcessor): if context_parts: context_text = ". ".join(context_parts) entity_contexts.append(EntityContext( - entity=Value(value=subject_uri, is_uri=True), + entity=make_term(subject_uri, is_uri=True), context=context_text )) diff --git a/trustgraph-flow/trustgraph/extract/kg/ontology/triple_converter.py b/trustgraph-flow/trustgraph/extract/kg/ontology/triple_converter.py index 2eb43b19..06fff4f4 100644 --- a/trustgraph-flow/trustgraph/extract/kg/ontology/triple_converter.py +++ b/trustgraph-flow/trustgraph/extract/kg/ontology/triple_converter.py @@ -8,7 +8,7 @@ with full URIs and correct is_uri flags. import logging from typing import List, Optional -from .... schema import Triple, Value +from .... schema import Triple, Term, IRI, LITERAL from .... rdf import RDF_TYPE, RDF_LABEL from .simplified_parser import Entity, Relationship, Attribute, ExtractionResult @@ -87,17 +87,17 @@ class TripleConverter: # Generate type triple: entity rdf:type ClassURI type_triple = Triple( - s=Value(value=entity_uri, is_uri=True), - p=Value(value=RDF_TYPE, is_uri=True), - o=Value(value=class_uri, is_uri=True) + s=Term(type=IRI, iri=entity_uri), + p=Term(type=IRI, iri=RDF_TYPE), + o=Term(type=IRI, iri=class_uri) ) triples.append(type_triple) # Generate label triple: entity rdfs:label "entity name" label_triple = Triple( - s=Value(value=entity_uri, is_uri=True), - p=Value(value=RDF_LABEL, is_uri=True), - o=Value(value=entity.entity, is_uri=False) # Literal! + s=Term(type=IRI, iri=entity_uri), + p=Term(type=IRI, iri=RDF_LABEL), + o=Term(type=LITERAL, value=entity.entity) # Literal! ) triples.append(label_triple) @@ -131,9 +131,9 @@ class TripleConverter: # Generate triple: subject property object return Triple( - s=Value(value=subject_uri, is_uri=True), - p=Value(value=property_uri, is_uri=True), - o=Value(value=object_uri, is_uri=True) + s=Term(type=IRI, iri=subject_uri), + p=Term(type=IRI, iri=property_uri), + o=Term(type=IRI, iri=object_uri) ) def convert_attribute(self, attribute: Attribute) -> Optional[Triple]: @@ -159,9 +159,9 @@ class TripleConverter: # Generate triple: entity property "literal value" return Triple( - s=Value(value=entity_uri, is_uri=True), - p=Value(value=property_uri, is_uri=True), - o=Value(value=attribute.value, is_uri=False) # Literal! + s=Term(type=IRI, iri=entity_uri), + p=Term(type=IRI, iri=property_uri), + o=Term(type=LITERAL, value=attribute.value) # Literal! ) def _get_class_uri(self, class_id: str) -> Optional[str]: diff --git a/trustgraph-flow/trustgraph/extract/kg/relationships/extract.py b/trustgraph-flow/trustgraph/extract/kg/relationships/extract.py index 6d461997..be0bd705 100755 --- a/trustgraph-flow/trustgraph/extract/kg/relationships/extract.py +++ b/trustgraph-flow/trustgraph/extract/kg/relationships/extract.py @@ -13,15 +13,15 @@ import urllib.parse logger = logging.getLogger(__name__) from .... schema import Chunk, Triple, Triples -from .... schema import Metadata, Value +from .... schema import Metadata, Term, IRI, LITERAL from .... schema import PromptRequest, PromptResponse from .... rdf import RDF_LABEL, TRUSTGRAPH_ENTITIES, SUBJECT_OF from .... base import FlowProcessor, ConsumerSpec, ProducerSpec from .... base import PromptClientSpec -RDF_LABEL_VALUE = Value(value=RDF_LABEL, is_uri=True) -SUBJECT_OF_VALUE = Value(value=SUBJECT_OF, is_uri=True) +RDF_LABEL_VALUE = Term(type=IRI, iri=RDF_LABEL) +SUBJECT_OF_VALUE = Term(type=IRI, iri=SUBJECT_OF) default_ident = "kg-extract-relationships" default_concurrency = 1 @@ -127,16 +127,16 @@ class Processor(FlowProcessor): if o is None: continue s_uri = self.to_uri(s) - s_value = Value(value=str(s_uri), is_uri=True) + s_value = Term(type=IRI, iri=str(s_uri)) p_uri = self.to_uri(p) - p_value = Value(value=str(p_uri), is_uri=True) + p_value = Term(type=IRI, iri=str(p_uri)) - if rel["object-entity"]: + if rel["object-entity"]: o_uri = self.to_uri(o) - o_value = Value(value=str(o_uri), is_uri=True) + o_value = Term(type=IRI, iri=str(o_uri)) else: - o_value = Value(value=str(o), is_uri=False) + o_value = Term(type=LITERAL, value=str(o)) triples.append(Triple( s=s_value, @@ -148,14 +148,14 @@ class Processor(FlowProcessor): triples.append(Triple( s=s_value, p=RDF_LABEL_VALUE, - o=Value(value=str(s), is_uri=False) + o=Term(type=LITERAL, value=str(s)) )) # Label for p triples.append(Triple( s=p_value, p=RDF_LABEL_VALUE, - o=Value(value=str(p), is_uri=False) + o=Term(type=LITERAL, value=str(p)) )) if rel["object-entity"]: @@ -163,14 +163,14 @@ class Processor(FlowProcessor): triples.append(Triple( s=o_value, p=RDF_LABEL_VALUE, - o=Value(value=str(o), is_uri=False) + o=Term(type=LITERAL, value=str(o)) )) # 'Subject of' for s triples.append(Triple( s=s_value, p=SUBJECT_OF_VALUE, - o=Value(value=v.metadata.id, is_uri=True) + o=Term(type=IRI, iri=v.metadata.id) )) if rel["object-entity"]: @@ -178,7 +178,7 @@ class Processor(FlowProcessor): triples.append(Triple( s=o_value, p=SUBJECT_OF_VALUE, - o=Value(value=v.metadata.id, is_uri=True) + o=Term(type=IRI, iri=v.metadata.id) )) await self.emit_triples( diff --git a/trustgraph-flow/trustgraph/extract/kg/topics/extract.py b/trustgraph-flow/trustgraph/extract/kg/topics/extract.py index 129cc64c..206d14d0 100755 --- a/trustgraph-flow/trustgraph/extract/kg/topics/extract.py +++ b/trustgraph-flow/trustgraph/extract/kg/topics/extract.py @@ -11,7 +11,7 @@ import logging # Module logger logger = logging.getLogger(__name__) -from .... schema import Chunk, Triple, Triples, Metadata, Value +from .... schema import Chunk, Triple, Triples, Metadata, Term, IRI, LITERAL from .... schema import chunk_ingest_queue, triples_store_queue from .... schema import prompt_request_queue from .... schema import prompt_response_queue @@ -20,7 +20,7 @@ from .... clients.prompt_client import PromptClient from .... rdf import TRUSTGRAPH_ENTITIES, DEFINITION from .... base import ConsumerProducer -DEFINITION_VALUE = Value(value=DEFINITION, is_uri=True) +DEFINITION_VALUE = Term(type=IRI, iri=DEFINITION) module = "kg-extract-topics" @@ -106,8 +106,8 @@ class Processor(ConsumerProducer): s_uri = self.to_uri(s) - s_value = Value(value=str(s_uri), is_uri=True) - o_value = Value(value=str(o), is_uri=False) + s_value = Term(type=IRI, iri=str(s_uri)) + o_value = Term(type=LITERAL, value=str(o)) await self.emit_edge( v.metadata, s_value, DEFINITION_VALUE, o_value diff --git a/trustgraph-flow/trustgraph/tables/config.py b/trustgraph-flow/trustgraph/tables/config.py index f98929e1..fb9ea0a7 100644 --- a/trustgraph-flow/trustgraph/tables/config.py +++ b/trustgraph-flow/trustgraph/tables/config.py @@ -1,6 +1,6 @@ from .. schema import KnowledgeResponse, Triple, Triples, EntityEmbeddings -from .. schema import Metadata, Value, GraphEmbeddings +from .. schema import Metadata, GraphEmbeddings from cassandra.cluster import Cluster from cassandra.auth import PlainTextAuthProvider diff --git a/trustgraph-flow/trustgraph/tables/knowledge.py b/trustgraph-flow/trustgraph/tables/knowledge.py index 1ee61088..0e5d4e1d 100644 --- a/trustgraph-flow/trustgraph/tables/knowledge.py +++ b/trustgraph-flow/trustgraph/tables/knowledge.py @@ -1,8 +1,24 @@ from .. schema import KnowledgeResponse, Triple, Triples, EntityEmbeddings -from .. schema import Metadata, Value, GraphEmbeddings +from .. schema import Metadata, Term, IRI, LITERAL, GraphEmbeddings from cassandra.cluster import Cluster + + +def term_to_tuple(term): + """Convert Term to (value, is_uri) tuple for database storage.""" + if term.type == IRI: + return (term.iri, True) + else: # LITERAL + return (term.value, False) + + +def tuple_to_term(value, is_uri): + """Convert (value, is_uri) tuple from database to Term.""" + if is_uri: + return Term(type=IRI, iri=value) + else: + return Term(type=LITERAL, value=value) from cassandra.auth import PlainTextAuthProvider from ssl import SSLContext, PROTOCOL_TLSv1_2 @@ -205,8 +221,7 @@ class KnowledgeTableStore: if m.metadata.metadata: metadata = [ ( - v.s.value, v.s.is_uri, v.p.value, v.p.is_uri, - v.o.value, v.o.is_uri + *term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o) ) for v in m.metadata.metadata ] @@ -215,8 +230,7 @@ class KnowledgeTableStore: triples = [ ( - v.s.value, v.s.is_uri, v.p.value, v.p.is_uri, - v.o.value, v.o.is_uri + *term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o) ) for v in m.triples ] @@ -248,8 +262,7 @@ class KnowledgeTableStore: if m.metadata.metadata: metadata = [ ( - v.s.value, v.s.is_uri, v.p.value, v.p.is_uri, - v.o.value, v.o.is_uri + *term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o) ) for v in m.metadata.metadata ] @@ -258,7 +271,7 @@ class KnowledgeTableStore: entities = [ ( - (v.entity.value, v.entity.is_uri), + term_to_tuple(v.entity), v.vectors ) for v in m.entities @@ -291,8 +304,7 @@ class KnowledgeTableStore: if m.metadata.metadata: metadata = [ ( - v.s.value, v.s.is_uri, v.p.value, v.p.is_uri, - v.o.value, v.o.is_uri + *term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o) ) for v in m.metadata.metadata ] @@ -414,9 +426,9 @@ class KnowledgeTableStore: if row[2]: metadata = [ Triple( - s = Value(value = elt[0], is_uri = elt[1]), - p = Value(value = elt[2], is_uri = elt[3]), - o = Value(value = elt[4], is_uri = elt[5]), + s = tuple_to_term(elt[0], elt[1]), + p = tuple_to_term(elt[2], elt[3]), + o = tuple_to_term(elt[4], elt[5]), ) for elt in row[2] ] @@ -425,9 +437,9 @@ class KnowledgeTableStore: triples = [ Triple( - s = Value(value = elt[0], is_uri = elt[1]), - p = Value(value = elt[2], is_uri = elt[3]), - o = Value(value = elt[4], is_uri = elt[5]), + s = tuple_to_term(elt[0], elt[1]), + p = tuple_to_term(elt[2], elt[3]), + o = tuple_to_term(elt[4], elt[5]), ) for elt in row[3] ] @@ -470,9 +482,9 @@ class KnowledgeTableStore: if row[2]: metadata = [ Triple( - s = Value(value = elt[0], is_uri = elt[1]), - p = Value(value = elt[2], is_uri = elt[3]), - o = Value(value = elt[4], is_uri = elt[5]), + s = tuple_to_term(elt[0], elt[1]), + p = tuple_to_term(elt[2], elt[3]), + o = tuple_to_term(elt[4], elt[5]), ) for elt in row[2] ] @@ -481,7 +493,7 @@ class KnowledgeTableStore: entities = [ EntityEmbeddings( - entity = Value(value = ent[0][0], is_uri = ent[0][1]), + entity = tuple_to_term(ent[0][0], ent[0][1]), vectors = ent[1] ) for ent in row[3] diff --git a/trustgraph-flow/trustgraph/tables/library.py b/trustgraph-flow/trustgraph/tables/library.py index 0a7c6081..8bbe2bad 100644 --- a/trustgraph-flow/trustgraph/tables/library.py +++ b/trustgraph-flow/trustgraph/tables/library.py @@ -1,8 +1,24 @@ from .. schema import LibrarianRequest, LibrarianResponse from .. schema import DocumentMetadata, ProcessingMetadata -from .. schema import Error, Triple, Value +from .. schema import Error, Triple, Term, IRI, LITERAL from .. knowledge import hash + + +def term_to_tuple(term): + """Convert Term to (value, is_uri) tuple for database storage.""" + if term.type == IRI: + return (term.iri, True) + else: # LITERAL + return (term.value, False) + + +def tuple_to_term(value, is_uri): + """Convert (value, is_uri) tuple from database to Term.""" + if is_uri: + return Term(type=IRI, iri=value) + else: + return Term(type=LITERAL, value=value) from .. exceptions import RequestError from cassandra.cluster import Cluster @@ -215,8 +231,7 @@ class LibraryTableStore: metadata = [ ( - v.s.value, v.s.is_uri, v.p.value, v.p.is_uri, - v.o.value, v.o.is_uri + *term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o) ) for v in document.metadata ] @@ -249,8 +264,7 @@ class LibraryTableStore: metadata = [ ( - v.s.value, v.s.is_uri, v.p.value, v.p.is_uri, - v.o.value, v.o.is_uri + *term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o) ) for v in document.metadata ] @@ -331,9 +345,9 @@ class LibraryTableStore: comments = row[4], metadata = [ Triple( - s=Value(value=m[0], is_uri=m[1]), - p=Value(value=m[2], is_uri=m[3]), - o=Value(value=m[4], is_uri=m[5]) + s=tuple_to_term(m[0], m[1]), + p=tuple_to_term(m[2], m[3]), + o=tuple_to_term(m[4], m[5]) ) for m in row[5] ], @@ -376,9 +390,9 @@ class LibraryTableStore: comments = row[3], metadata = [ Triple( - s=Value(value=m[0], is_uri=m[1]), - p=Value(value=m[2], is_uri=m[3]), - o=Value(value=m[4], is_uri=m[5]) + s=tuple_to_term(m[0], m[1]), + p=tuple_to_term(m[2], m[3]), + o=tuple_to_term(m[4], m[5]) ) for m in row[4] ],