diff --git a/trustgraph-flow/trustgraph/extract/kg/definitions/extract.py b/trustgraph-flow/trustgraph/extract/kg/definitions/extract.py index 9e3e7344..f7cdea99 100755 --- a/trustgraph-flow/trustgraph/extract/kg/definitions/extract.py +++ b/trustgraph-flow/trustgraph/extract/kg/definitions/extract.py @@ -69,11 +69,11 @@ class Processor(ConsumerProducer): return self.prompt.request_definitions(chunk) - def emit_edge(self, metadata, s, p, o): + def emit_edges(self, metadata, triples): t = Triples( metadata=metadata, - triples=[Triple(s=s, p=p, o=o)], + triples=triples, ) self.producer.send(t) @@ -88,6 +88,13 @@ class Processor(ConsumerProducer): defs = self.get_definitions(chunk) + triples = [] + + # FIXME: Putting metadata into triples store is duplicated in + # relationships extractor too + for t in v.metadata.metadata: + triples.append(t) + for defn in defs: s = defn.name @@ -104,7 +111,19 @@ class Processor(ConsumerProducer): s_value = Value(value=str(s_uri), is_uri=True) o_value = Value(value=str(o), is_uri=False) - self.emit_edge(v.metadata, s_value, DEFINITION_VALUE, o_value) + triples.append(Triple( + s=s_value, p=DEFINITION_VALUE, o=o_value + )) + + self.emit_edges( + Metadata( + id=v.metadata.id, + metadata=[], + user=v.metadata.user, + collection=v.metadata.collection, + ), + triples + ) except Exception as e: print("Exception: ", e, flush=True) diff --git a/trustgraph-flow/trustgraph/extract/kg/relationships/extract.py b/trustgraph-flow/trustgraph/extract/kg/relationships/extract.py index 0cade2f9..d63549e2 100755 --- a/trustgraph-flow/trustgraph/extract/kg/relationships/extract.py +++ b/trustgraph-flow/trustgraph/extract/kg/relationships/extract.py @@ -92,11 +92,11 @@ class Processor(ConsumerProducer): return self.prompt.request_relationships(chunk) - def emit_edge(self, metadata, s, p, o): + def emit_edges(self, metadata, triples): t = Triples( metadata=metadata, - triples=[Triple(s=s, p=p, o=o)], + triples=triples, ) self.producer.send(t) @@ -116,6 +116,13 @@ class Processor(ConsumerProducer): rels = self.get_relationships(chunk) + triples = [] + + # FIXME: Putting metadata into triples store is duplicated in + # relationships extractor too + for t in v.metadata.metadata: + triples.append(t) + for rel in rels: s = rel.s @@ -142,43 +149,49 @@ class Processor(ConsumerProducer): else: o_value = Value(value=str(o), is_uri=False) - self.emit_edge( - v.metadata, - s_value, - p_value, - o_value - ) + triples.append(Triple( + s=s_value, + p=p_value, + o=o_value + )) # Label for s - self.emit_edge( - v.metadata, - s_value, - RDF_LABEL_VALUE, - Value(value=str(s), is_uri=False) - ) + triples.append(Triple( + s=s_value, + p=RDF_LABEL_VALUE, + o=Value(value=str(s), is_uri=False) + )) # Label for p - self.emit_edge( - v.metadata, - p_value, - RDF_LABEL_VALUE, - Value(value=str(p), is_uri=False) - ) + triples.append(Triple( + s=p_value, + p=RDF_LABEL_VALUE, + o=Value(value=str(p), is_uri=False) + )) if rel.o_entity: # Label for o - self.emit_edge( - v.metadata, - o_value, - RDF_LABEL_VALUE, - Value(value=str(o), is_uri=False) - ) + triples.append(Triple( + s=o_value, + p=RDF_LABEL_VALUE, + o=Value(value=str(o), is_uri=False) + )) self.emit_vec(v.metadata, s_value, v.vectors) self.emit_vec(v.metadata, p_value, v.vectors) if rel.o_entity: self.emit_vec(v.metadata, o_value, v.vectors) + self.emit_edges( + Metadata( + id=v.metadata.id, + metadata=[], + user=v.metadata.user, + collection=v.metadata.collection, + ), + triples + ) + except Exception as e: print("Exception: ", e, flush=True)