mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-02 02:58:10 +02:00
Increase storage test coverage (#435)
* Fixing storage and adding tests * PR pipeline only runs quick tests
This commit is contained in:
cybermaggedon
GitHub
parent
4daa54abaf
commit
f37decea2b
33 changed files with 7606 additions and 754 deletions
|
|
@ -9,28 +9,19 @@ from neo4j import GraphDatabase
|
|||
|
||||
from .... schema import TriplesQueryRequest, TriplesQueryResponse, Error
|
||||
from .... schema import Value, Triple
|
||||
from .... schema import triples_request_queue
|
||||
from .... schema import triples_response_queue
|
||||
from .... base import ConsumerProducer
|
||||
from .... base import TriplesQueryService
|
||||
|
||||
module = "triples-query"
|
||||
|
||||
default_input_queue = triples_request_queue
|
||||
default_output_queue = triples_response_queue
|
||||
default_subscriber = module
|
||||
default_ident = "triples-query"
|
||||
|
||||
default_graph_host = 'bolt://memgraph:7687'
|
||||
default_username = 'memgraph'
|
||||
default_password = 'password'
|
||||
default_database = 'memgraph'
|
||||
|
||||
class Processor(ConsumerProducer):
|
||||
class Processor(TriplesQueryService):
|
||||
|
||||
def __init__(self, **params):
|
||||
|
||||
input_queue = params.get("input_queue", default_input_queue)
|
||||
output_queue = params.get("output_queue", default_output_queue)
|
||||
subscriber = params.get("subscriber", default_subscriber)
|
||||
graph_host = params.get("graph_host", default_graph_host)
|
||||
username = params.get("username", default_username)
|
||||
password = params.get("password", default_password)
|
||||
|
|
@ -38,12 +29,9 @@ class Processor(ConsumerProducer):
|
|||
|
||||
super(Processor, self).__init__(
|
||||
**params | {
|
||||
"input_queue": input_queue,
|
||||
"output_queue": output_queue,
|
||||
"subscriber": subscriber,
|
||||
"input_schema": TriplesQueryRequest,
|
||||
"output_schema": TriplesQueryResponse,
|
||||
"graph_host": graph_host,
|
||||
"username": username,
|
||||
"database": database,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -58,46 +46,39 @@ class Processor(ConsumerProducer):
|
|||
else:
|
||||
return Value(value=ent, is_uri=False)
|
||||
|
||||
async def handle(self, msg):
|
||||
async def query_triples(self, query):
|
||||
|
||||
try:
|
||||
|
||||
v = msg.value()
|
||||
|
||||
# Sender-produced ID
|
||||
id = msg.properties()["id"]
|
||||
|
||||
print(f"Handling input {id}...", flush=True)
|
||||
|
||||
triples = []
|
||||
|
||||
if v.s is not None:
|
||||
if v.p is not None:
|
||||
if v.o is not None:
|
||||
if query.s is not None:
|
||||
if query.p is not None:
|
||||
if query.o is not None:
|
||||
|
||||
# SPO
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Literal {value: $value}) "
|
||||
"RETURN $src as src "
|
||||
"LIMIT " + str(v.limit),
|
||||
src=v.s.value, rel=v.p.value, value=v.o.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
src=query.s.value, rel=query.p.value, value=query.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
triples.append((v.s.value, v.p.value, v.o.value))
|
||||
triples.append((query.s.value, query.p.value, query.o.value))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Node {uri: $uri}) "
|
||||
"RETURN $src as src "
|
||||
"LIMIT " + str(v.limit),
|
||||
src=v.s.value, rel=v.p.value, uri=v.o.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
src=query.s.value, rel=query.p.value, uri=query.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
triples.append((v.s.value, v.p.value, v.o.value))
|
||||
triples.append((query.s.value, query.p.value, query.o.value))
|
||||
|
||||
else:
|
||||
|
||||
|
|
@ -106,56 +87,56 @@ class Processor(ConsumerProducer):
|
|||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Literal) "
|
||||
"RETURN dest.value as dest "
|
||||
"LIMIT " + str(v.limit),
|
||||
src=v.s.value, rel=v.p.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
src=query.s.value, rel=query.p.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((v.s.value, v.p.value, data["dest"]))
|
||||
triples.append((query.s.value, query.p.value, data["dest"]))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Node) "
|
||||
"RETURN dest.uri as dest "
|
||||
"LIMIT " + str(v.limit),
|
||||
src=v.s.value, rel=v.p.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
src=query.s.value, rel=query.p.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((v.s.value, v.p.value, data["dest"]))
|
||||
triples.append((query.s.value, query.p.value, data["dest"]))
|
||||
|
||||
else:
|
||||
|
||||
if v.o is not None:
|
||||
if query.o is not None:
|
||||
|
||||
# SO
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Literal {value: $value}) "
|
||||
"RETURN rel.uri as rel "
|
||||
"LIMIT " + str(v.limit),
|
||||
src=v.s.value, value=v.o.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
src=query.s.value, value=query.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((v.s.value, data["rel"], v.o.value))
|
||||
triples.append((query.s.value, data["rel"], query.o.value))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Node {uri: $uri}) "
|
||||
"RETURN rel.uri as rel "
|
||||
"LIMIT " + str(v.limit),
|
||||
src=v.s.value, uri=v.o.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
src=query.s.value, uri=query.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((v.s.value, data["rel"], v.o.value))
|
||||
triples.append((query.s.value, data["rel"], query.o.value))
|
||||
|
||||
else:
|
||||
|
||||
|
|
@ -164,59 +145,59 @@ class Processor(ConsumerProducer):
|
|||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Literal) "
|
||||
"RETURN rel.uri as rel, dest.value as dest "
|
||||
"LIMIT " + str(v.limit),
|
||||
src=v.s.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
src=query.s.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((v.s.value, data["rel"], data["dest"]))
|
||||
triples.append((query.s.value, data["rel"], data["dest"]))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Node) "
|
||||
"RETURN rel.uri as rel, dest.uri as dest "
|
||||
"LIMIT " + str(v.limit),
|
||||
src=v.s.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
src=query.s.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((v.s.value, data["rel"], data["dest"]))
|
||||
triples.append((query.s.value, data["rel"], data["dest"]))
|
||||
|
||||
|
||||
else:
|
||||
|
||||
if v.p is not None:
|
||||
if query.p is not None:
|
||||
|
||||
if v.o is not None:
|
||||
if query.o is not None:
|
||||
|
||||
# PO
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Literal {value: $value}) "
|
||||
"RETURN src.uri as src "
|
||||
"LIMIT " + str(v.limit),
|
||||
uri=v.p.value, value=v.o.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
uri=query.p.value, value=query.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], v.p.value, v.o.value))
|
||||
triples.append((data["src"], query.p.value, query.o.value))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Node {uri: $uri}) "
|
||||
"MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Node {uri: $dest}) "
|
||||
"RETURN src.uri as src "
|
||||
"LIMIT " + str(v.limit),
|
||||
uri=v.p.value, dest=v.o.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
uri=query.p.value, dest=query.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], v.p.value, v.o.value))
|
||||
triples.append((data["src"], query.p.value, query.o.value))
|
||||
|
||||
else:
|
||||
|
||||
|
|
@ -225,56 +206,56 @@ class Processor(ConsumerProducer):
|
|||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Literal) "
|
||||
"RETURN src.uri as src, dest.value as dest "
|
||||
"LIMIT " + str(v.limit),
|
||||
uri=v.p.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
uri=query.p.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], v.p.value, data["dest"]))
|
||||
triples.append((data["src"], query.p.value, data["dest"]))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Node) "
|
||||
"RETURN src.uri as src, dest.uri as dest "
|
||||
"LIMIT " + str(v.limit),
|
||||
uri=v.p.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
uri=query.p.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], v.p.value, data["dest"]))
|
||||
triples.append((data["src"], query.p.value, data["dest"]))
|
||||
|
||||
else:
|
||||
|
||||
if v.o is not None:
|
||||
if query.o is not None:
|
||||
|
||||
# O
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel]->(dest:Literal {value: $value}) "
|
||||
"RETURN src.uri as src, rel.uri as rel "
|
||||
"LIMIT " + str(v.limit),
|
||||
value=v.o.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
value=query.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], data["rel"], v.o.value))
|
||||
triples.append((data["src"], data["rel"], query.o.value))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel]->(dest:Node {uri: $uri}) "
|
||||
"RETURN src.uri as src, rel.uri as rel "
|
||||
"LIMIT " + str(v.limit),
|
||||
uri=v.o.value,
|
||||
"LIMIT " + str(query.limit),
|
||||
uri=query.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], data["rel"], v.o.value))
|
||||
triples.append((data["src"], data["rel"], query.o.value))
|
||||
|
||||
else:
|
||||
|
||||
|
|
@ -283,7 +264,7 @@ class Processor(ConsumerProducer):
|
|||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel]->(dest:Literal) "
|
||||
"RETURN src.uri as src, rel.uri as rel, dest.value as dest "
|
||||
"LIMIT " + str(v.limit),
|
||||
"LIMIT " + str(query.limit),
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
|
|
@ -294,7 +275,7 @@ class Processor(ConsumerProducer):
|
|||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel]->(dest:Node) "
|
||||
"RETURN src.uri as src, rel.uri as rel, dest.uri as dest "
|
||||
"LIMIT " + str(v.limit),
|
||||
"LIMIT " + str(query.limit),
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
|
|
@ -308,40 +289,22 @@ class Processor(ConsumerProducer):
|
|||
p=self.create_value(t[1]),
|
||||
o=self.create_value(t[2])
|
||||
)
|
||||
for t in triples[:v.limit]
|
||||
for t in triples[:query.limit]
|
||||
]
|
||||
|
||||
print("Send response...", flush=True)
|
||||
r = TriplesQueryResponse(triples=triples, error=None)
|
||||
await self.send(r, properties={"id": id})
|
||||
|
||||
print("Done.", flush=True)
|
||||
return triples
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print(f"Exception: {e}")
|
||||
|
||||
print("Send error response...", flush=True)
|
||||
|
||||
r = TriplesQueryResponse(
|
||||
error=Error(
|
||||
type = "llm-error",
|
||||
message = str(e),
|
||||
),
|
||||
response=None,
|
||||
)
|
||||
|
||||
await self.send(r, properties={"id": id})
|
||||
|
||||
self.consumer.acknowledge(msg)
|
||||
print(f"Exception: {e}")
|
||||
raise e
|
||||
|
||||
@staticmethod
|
||||
def add_args(parser):
|
||||
|
||||
ConsumerProducer.add_args(
|
||||
parser, default_input_queue, default_subscriber,
|
||||
default_output_queue,
|
||||
)
|
||||
TriplesQueryService.add_args(parser)
|
||||
|
||||
parser.add_argument(
|
||||
'-g', '--graph-host',
|
||||
|
|
@ -369,5 +332,5 @@ class Processor(ConsumerProducer):
|
|||
|
||||
def run():
|
||||
|
||||
Processor.launch(module, __doc__)
|
||||
Processor.launch(default_ident, __doc__)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue