From c1ec3d6abb58ad209efc65dcd33f5df3c370d1f1 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Thu, 28 Nov 2024 19:18:03 +0000 Subject: [PATCH] Add database override to bolt output, default is neo4j --- templates/components/memgraph.jsonnet | 81 +++++++++++++++++++ templates/stores/memgraph.jsonnet | 65 +++++++++++++++ .../trustgraph/query/triples/neo4j/service.py | 12 ++- .../trustgraph/storage/triples/neo4j/write.py | 12 ++- 4 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 templates/components/memgraph.jsonnet create mode 100644 templates/stores/memgraph.jsonnet diff --git a/templates/components/memgraph.jsonnet b/templates/components/memgraph.jsonnet new file mode 100644 index 00000000..5ec0a76e --- /dev/null +++ b/templates/components/memgraph.jsonnet @@ -0,0 +1,81 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local memgraph = import "stores/memgraph.jsonnet"; + +memgraph + { + + "memgraph-url":: "bolt://memgraph:7687", + "memgraph-database":: "memgraph", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph) + .with_command([ + "triples-write-neo4j", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8080, 8080, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph) + .with_command([ + "triples-query-neo4j", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8080, 8080, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/templates/stores/memgraph.jsonnet b/templates/stores/memgraph.jsonnet new file mode 100644 index 00000000..8f8b6216 --- /dev/null +++ b/templates/stores/memgraph.jsonnet @@ -0,0 +1,65 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "memgraph" +: { + + create:: function(engine) + + local container = + engine.container("memgraph") + .with_image(images.memgraph_mage) + .with_limits("1.0", "1000M") + .with_reservations("0.5", "1000M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + local containerSet = engine.containers( + "memgraph", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "memgraph-lab" +: { + + create:: function(engine) + + local container = + engine.container("lab") + .with_image(images.memgraph_lab) + .with_environment({ + QUICK_CONNECT_MG_HOST: "memgraph", + QUICK_CONNECT_MG_PORT: "7687", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3010, 3000, "http"); + + local containerSet = engine.containers( + "lab", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3010, 3010, "http"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/trustgraph-flow/trustgraph/query/triples/neo4j/service.py b/trustgraph-flow/trustgraph/query/triples/neo4j/service.py index 9038f76d..2caa0193 100755 --- a/trustgraph-flow/trustgraph/query/triples/neo4j/service.py +++ b/trustgraph-flow/trustgraph/query/triples/neo4j/service.py @@ -21,6 +21,7 @@ default_subscriber = module default_graph_host = 'bolt://neo4j:7687' default_username = 'neo4j' default_password = 'password' +default_database = 'neo4j' class Processor(ConsumerProducer): @@ -31,7 +32,8 @@ class Processor(ConsumerProducer): subscriber = params.get("subscriber", default_subscriber) graph_host = params.get("graph_host", default_graph_host) username = params.get("username", default_username) - password = params.get("passowrd", default_password) + password = params.get("password", default_password) + database = params.get("database", default_database) super(Processor, self).__init__( **params | { @@ -44,7 +46,7 @@ class Processor(ConsumerProducer): } ) - self.db = "neo4j" + self.db = database self.io = GraphDatabase.driver(graph_host, auth=(username, password)) @@ -342,6 +344,12 @@ class Processor(ConsumerProducer): help=f'Neo4j password (default: {default_password})' ) + parser.add_argument( + '--database', + default=default_database, + help=f'Neo4j database (default: {default_database})' + ) + def run(): Processor.start(module, __doc__) diff --git a/trustgraph-flow/trustgraph/storage/triples/neo4j/write.py b/trustgraph-flow/trustgraph/storage/triples/neo4j/write.py index 82302e96..929333e5 100755 --- a/trustgraph-flow/trustgraph/storage/triples/neo4j/write.py +++ b/trustgraph-flow/trustgraph/storage/triples/neo4j/write.py @@ -24,6 +24,7 @@ default_subscriber = module default_graph_host = 'bolt://neo4j:7687' default_username = 'neo4j' default_password = 'password' +default_database = 'neo4j' class Processor(Consumer): @@ -33,7 +34,8 @@ class Processor(Consumer): subscriber = params.get("subscriber", default_subscriber) graph_host = params.get("graph_host", default_graph_host) username = params.get("username", default_username) - password = params.get("passowrd", default_password) + password = params.get("password", default_password) + database = params.get("database", default_database) super(Processor, self).__init__( **params | { @@ -44,7 +46,7 @@ class Processor(Consumer): } ) - self.db = "neo4j" + self.db = database self.io = GraphDatabase.driver(graph_host, auth=(username, password)) @@ -152,6 +154,12 @@ class Processor(Consumer): help=f'Neo4j password (default: {default_password})' ) + parser.add_argument( + '--database', + default=default_database, + help=f'Neo4j database (default: {default_database})' + ) + def run(): Processor.start(module, __doc__)