From d3e213f19404711872dafde6a1b7729d31ece65a Mon Sep 17 00:00:00 2001 From: cybermaggedon Date: Wed, 14 Aug 2024 09:06:33 +0100 Subject: [PATCH] Add Neo4j support (#9) - Add triples-write-neo4j and triples-query-neo4j to interact with neo4j - Add docker-compose-openai-neo4j to demo Neo4j working --- Makefile | 6 +- docker-compose-azure.yaml | 30 +- docker-compose-bedrock.yaml | 30 +- docker-compose-claude.yaml | 30 +- docker-compose-cohere.yaml | 30 +- docker-compose-mix.yaml | 30 +- docker-compose-ollama.yaml | 30 +- docker-compose-openai-neo4j.yaml | 255 ++++++++++++++ docker-compose-openai.yaml | 30 +- docker-compose-vertexai.yaml | 30 +- processing-just-llm.yaml | 45 --- processing.yaml | 45 --- scripts/graph-show | 25 +- scripts/triples-query-neo4j | 6 + scripts/triples-write-neo4j | 6 + setup.py | 5 +- templates/components/images.jsonnet | 1 + templates/components/neo4j.jsonnet | 25 ++ templates/components/version.jsonnet | 1 - templates/docker-compose-openai-neo4j.jsonnet | 68 ++++ tests/test-triples | 6 +- trustgraph/query/triples/neo4j/__init__.py | 3 + trustgraph/query/triples/neo4j/__main__.py | 7 + trustgraph/query/triples/neo4j/service.py | 328 ++++++++++++++++++ trustgraph/storage/triples/neo4j/__init__.py | 3 + trustgraph/storage/triples/neo4j/__main__.py | 7 + trustgraph/storage/triples/neo4j/write.py | 156 +++++++++ 27 files changed, 1008 insertions(+), 230 deletions(-) create mode 100644 docker-compose-openai-neo4j.yaml delete mode 100644 processing-just-llm.yaml delete mode 100644 processing.yaml create mode 100755 scripts/triples-query-neo4j create mode 100755 scripts/triples-write-neo4j create mode 100644 templates/components/neo4j.jsonnet delete mode 100644 templates/components/version.jsonnet create mode 100644 templates/docker-compose-openai-neo4j.jsonnet create mode 100644 trustgraph/query/triples/neo4j/__init__.py create mode 100755 trustgraph/query/triples/neo4j/__main__.py create mode 100755 trustgraph/query/triples/neo4j/service.py create mode 100644 trustgraph/storage/triples/neo4j/__init__.py create mode 100755 trustgraph/storage/triples/neo4j/__main__.py create mode 100755 trustgraph/storage/triples/neo4j/write.py diff --git a/Makefile b/Makefile index 25af5451..6951087b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # VERSION=$(shell git describe | sed 's/^v//') -VERSION=0.6.0 +VERSION=0.6.1 DOCKER=podman @@ -31,9 +31,9 @@ clean: set-version: # sed -i 's/trustgraph-flow:[0-9]*\.[0-9]*\.[0-9]*/trustgraph-flow:'${VERSION}'/' docker-compose*.yaml - echo '"${VERSION}"' > templates/version.jsonnet + echo '"${VERSION}"' > templates/components/version.jsonnet -TEMPLATES=azure bedrock claude cohere mix ollama openai vertexai +TEMPLATES=azure bedrock claude cohere mix ollama openai vertexai openai-neo4j DCS=$(foreach template,${TEMPLATES},${template:%=docker-compose-%.yaml}) update-templates: set-version ${DCS} diff --git a/docker-compose-azure.yaml b/docker-compose-azure.yaml index b1f21c64..98a25510 100644 --- a/docker-compose-azure.yaml +++ b/docker-compose-azure.yaml @@ -11,14 +11,14 @@ - "chunker-recursive" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "embeddings": "command": - "embeddings-hf" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "etcd": "command": @@ -60,7 +60,7 @@ - "non-persistent://tg/request/prompt-rag" - "--prompt-response-queue" - "non-persistent://tg/response/prompt-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "init-pulsar": "command": @@ -77,14 +77,14 @@ - "kg-extract-definitions" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "kg-extract-relationships": "command": - "kg-extract-relationships" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "milvus": "command": @@ -122,7 +122,7 @@ - "pdf-decoder" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prometheus": "image": "docker.io/prom/prometheus:v2.53.1" @@ -141,7 +141,7 @@ - "non-persistent://tg/request/text-completion" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prompt-rag": "command": @@ -156,7 +156,7 @@ - "non-persistent://tg/request/text-completion-rag" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -183,7 +183,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "query-triples": "command": @@ -192,7 +192,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -201,7 +201,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-triples": "command": @@ -210,7 +210,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion": "command": @@ -221,7 +221,7 @@ - "${AZURE_TOKEN}" - "-e" - "${AZURE_ENDPOINT}" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion-rag": "command": @@ -236,14 +236,14 @@ - "non-persistent://tg/request/text-completion-rag" - "-o" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "vectorize": "command": - "embeddings-vectorize" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/docker-compose-bedrock.yaml b/docker-compose-bedrock.yaml index ab7b82c7..fa6edfa4 100644 --- a/docker-compose-bedrock.yaml +++ b/docker-compose-bedrock.yaml @@ -15,14 +15,14 @@ - "2000" - "--chunk-overlap" - "100" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "embeddings": "command": - "embeddings-hf" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "etcd": "command": @@ -64,7 +64,7 @@ - "non-persistent://tg/request/prompt-rag" - "--prompt-response-queue" - "non-persistent://tg/response/prompt-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "init-pulsar": "command": @@ -81,14 +81,14 @@ - "kg-extract-definitions" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "kg-extract-relationships": "command": - "kg-extract-relationships" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "milvus": "command": @@ -126,7 +126,7 @@ - "pdf-decoder" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prometheus": "image": "docker.io/prom/prometheus:v2.53.1" @@ -145,7 +145,7 @@ - "non-persistent://tg/request/text-completion" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prompt-rag": "command": @@ -160,7 +160,7 @@ - "non-persistent://tg/request/text-completion-rag" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -187,7 +187,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "query-triples": "command": @@ -196,7 +196,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -205,7 +205,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-triples": "command": @@ -214,7 +214,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion": "command": @@ -227,7 +227,7 @@ - "${AWS_SECRET_KEY}" - "-r" - "us-west-2" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion-rag": "command": @@ -244,14 +244,14 @@ - "non-persistent://tg/request/text-completion-rag" - "-o" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "vectorize": "command": - "embeddings-vectorize" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/docker-compose-claude.yaml b/docker-compose-claude.yaml index a7498f41..635cd368 100644 --- a/docker-compose-claude.yaml +++ b/docker-compose-claude.yaml @@ -11,14 +11,14 @@ - "chunker-recursive" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "embeddings": "command": - "embeddings-hf" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "etcd": "command": @@ -60,7 +60,7 @@ - "non-persistent://tg/request/prompt-rag" - "--prompt-response-queue" - "non-persistent://tg/response/prompt-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "init-pulsar": "command": @@ -77,14 +77,14 @@ - "kg-extract-definitions" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "kg-extract-relationships": "command": - "kg-extract-relationships" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "milvus": "command": @@ -122,7 +122,7 @@ - "pdf-decoder" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prometheus": "image": "docker.io/prom/prometheus:v2.53.1" @@ -141,7 +141,7 @@ - "non-persistent://tg/request/text-completion" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prompt-rag": "command": @@ -156,7 +156,7 @@ - "non-persistent://tg/request/text-completion-rag" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -183,7 +183,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "query-triples": "command": @@ -192,7 +192,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -201,7 +201,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-triples": "command": @@ -210,7 +210,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion": "command": @@ -219,7 +219,7 @@ - "pulsar://pulsar:6650" - "-k" - "${CLAUDE_KEY}" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion-rag": "command": @@ -232,14 +232,14 @@ - "non-persistent://tg/request/text-completion-rag" - "-o" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "vectorize": "command": - "embeddings-vectorize" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/docker-compose-cohere.yaml b/docker-compose-cohere.yaml index b2d3f4cb..2f01fc0f 100644 --- a/docker-compose-cohere.yaml +++ b/docker-compose-cohere.yaml @@ -15,14 +15,14 @@ - "1000" - "--chunk-overlap" - "50" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "embeddings": "command": - "embeddings-hf" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "etcd": "command": @@ -64,7 +64,7 @@ - "non-persistent://tg/request/prompt-rag" - "--prompt-response-queue" - "non-persistent://tg/response/prompt-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "init-pulsar": "command": @@ -81,14 +81,14 @@ - "kg-extract-definitions" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "kg-extract-relationships": "command": - "kg-extract-relationships" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "milvus": "command": @@ -126,7 +126,7 @@ - "pdf-decoder" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prometheus": "image": "docker.io/prom/prometheus:v2.53.1" @@ -145,7 +145,7 @@ - "non-persistent://tg/request/text-completion" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prompt-rag": "command": @@ -160,7 +160,7 @@ - "non-persistent://tg/request/text-completion-rag" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -187,7 +187,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "query-triples": "command": @@ -196,7 +196,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -205,7 +205,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-triples": "command": @@ -214,7 +214,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion": "command": @@ -223,7 +223,7 @@ - "pulsar://pulsar:6650" - "-k" - "${COHERE_KEY}" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion-rag": "command": @@ -236,14 +236,14 @@ - "non-persistent://tg/request/text-completion-rag" - "-o" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "vectorize": "command": - "embeddings-vectorize" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/docker-compose-mix.yaml b/docker-compose-mix.yaml index 7090b1ad..0bcd76ad 100644 --- a/docker-compose-mix.yaml +++ b/docker-compose-mix.yaml @@ -15,14 +15,14 @@ - "4000" - "--chunk-overlap" - "120" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "embeddings": "command": - "embeddings-hf" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "etcd": "command": @@ -64,7 +64,7 @@ - "non-persistent://tg/request/prompt-rag" - "--prompt-response-queue" - "non-persistent://tg/response/prompt-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "init-pulsar": "command": @@ -81,14 +81,14 @@ - "kg-extract-definitions" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "kg-extract-relationships": "command": - "kg-extract-relationships" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "milvus": "command": @@ -126,7 +126,7 @@ - "pdf-decoder" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prometheus": "image": "docker.io/prom/prometheus:v2.53.1" @@ -145,7 +145,7 @@ - "non-persistent://tg/request/text-completion" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prompt-rag": "command": @@ -160,7 +160,7 @@ - "non-persistent://tg/request/text-completion-rag" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -187,7 +187,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "query-triples": "command": @@ -196,7 +196,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -205,7 +205,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-triples": "command": @@ -214,7 +214,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion": "command": @@ -225,7 +225,7 @@ - "${COHERE_KEY}" - "-m" - "c4ai-aya-23-35b" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion-rag": "command": @@ -240,14 +240,14 @@ - "non-persistent://tg/response/text-completion-rag-response" - "-m" - "c4ai-aya-23-8b" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "vectorize": "command": - "embeddings-vectorize" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/docker-compose-ollama.yaml b/docker-compose-ollama.yaml index dc7abf9e..3450c875 100644 --- a/docker-compose-ollama.yaml +++ b/docker-compose-ollama.yaml @@ -11,14 +11,14 @@ - "chunker-recursive" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "embeddings": "command": - "embeddings-hf" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "etcd": "command": @@ -60,7 +60,7 @@ - "non-persistent://tg/request/prompt-rag" - "--prompt-response-queue" - "non-persistent://tg/response/prompt-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "init-pulsar": "command": @@ -77,14 +77,14 @@ - "kg-extract-definitions" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "kg-extract-relationships": "command": - "kg-extract-relationships" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "milvus": "command": @@ -122,7 +122,7 @@ - "pdf-decoder" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prometheus": "image": "docker.io/prom/prometheus:v2.53.1" @@ -141,7 +141,7 @@ - "non-persistent://tg/request/text-completion" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prompt-rag": "command": @@ -156,7 +156,7 @@ - "non-persistent://tg/request/text-completion-rag" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -183,7 +183,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "query-triples": "command": @@ -192,7 +192,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -201,7 +201,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-triples": "command": @@ -210,7 +210,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion": "command": @@ -219,7 +219,7 @@ - "pulsar://pulsar:6650" - "-r" - "${OLLAMA_HOST}" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion-rag": "command": @@ -232,14 +232,14 @@ - "non-persistent://tg/request/text-completion-rag" - "-o" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "vectorize": "command": - "embeddings-vectorize" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/docker-compose-openai-neo4j.yaml b/docker-compose-openai-neo4j.yaml new file mode 100644 index 00000000..4ef1c177 --- /dev/null +++ b/docker-compose-openai-neo4j.yaml @@ -0,0 +1,255 @@ +"services": + "chunker": + "command": + - "chunker-recursive" + - "-p" + - "pulsar://pulsar:6650" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "embeddings": + "command": + - "embeddings-hf" + - "-p" + - "pulsar://pulsar:6650" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "etcd": + "command": + - "etcd" + - "-advertise-client-urls=http://127.0.0.1:2379" + - "-listen-client-urls" + - "http://0.0.0.0:2379" + - "--data-dir" + - "/etcd" + "environment": + "ETCD_AUTO_COMPACTION_MODE": "revision" + "ETCD_AUTO_COMPACTION_RETENTION": "1000" + "ETCD_QUOTA_BACKEND_BYTES": "4294967296" + "ETCD_SNAPSHOT_COUNT": "50000" + "image": "quay.io/coreos/etcd:v3.5.5" + "ports": + - "2379:2379" + "restart": "on-failure:100" + "volumes": + - "etcd:/etcd" + "grafana": + "environment": + "GF_ORG_NAME": "trustgraph.ai" + "image": "docker.io/grafana/grafana:10.0.0" + "ports": + - "3000:3000" + "restart": "on-failure:100" + "volumes": + - "grafana-storage:/var/lib/grafana" + - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml" + - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml" + - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json" + "graph-rag": + "command": + - "graph-rag" + - "-p" + - "pulsar://pulsar:6650" + - "--prompt-request-queue" + - "non-persistent://tg/request/prompt-rag" + - "--prompt-response-queue" + - "non-persistent://tg/response/prompt-rag-response" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "init-pulsar": + "command": + - "sh" + - "-c" + - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response" + "depends_on": + "pulsar": + "condition": "service_started" + "image": "docker.io/apachepulsar/pulsar:3.3.0" + "restart": "on-failure:100" + "kg-extract-definitions": + "command": + - "kg-extract-definitions" + - "-p" + - "pulsar://pulsar:6650" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "kg-extract-relationships": + "command": + - "kg-extract-relationships" + - "-p" + - "pulsar://pulsar:6650" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "milvus": + "command": + - "milvus" + - "run" + - "standalone" + "environment": + "ETCD_ENDPOINTS": "etcd:2379" + "MINIO_ADDRESS": "minio:9000" + "image": "docker.io/milvusdb/milvus:v2.4.5" + "ports": + - "9091:9091" + - "19530:19530" + "restart": "on-failure:100" + "volumes": + - "milvus:/var/lib/milvus" + "minio": + "command": + - "minio" + - "server" + - "/minio_data" + - "--console-address" + - ":9001" + "environment": + "MINIO_ROOT_PASSWORD": "minioadmin" + "MINIO_ROOT_USER": "minioadmin" + "image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z" + "ports": + - "9001:9001" + "restart": "on-failure:100" + "volumes": + - "minio-data:/minio_data" + "neo4j": + "environment": + "NEO4J_AUTH": "neo4j/password" + "image": "docker.io/neo4j:5.22.0-community-bullseye" + "ports": + - "7474:7474" + - "7687:7687" + "restart": "on-failure:100" + "volumes": + - "neo4j:/data" + "pdf-decoder": + "command": + - "pdf-decoder" + - "-p" + - "pulsar://pulsar:6650" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "prometheus": + "image": "docker.io/prom/prometheus:v2.53.1" + "ports": + - "9090:9090" + "restart": "on-failure:100" + "volumes": + - "./prometheus:/etc/prometheus" + - "prometheus-data:/prometheus" + "prompt": + "command": + - "prompt-generic" + - "-p" + - "pulsar://pulsar:6650" + - "--text-completion-request-queue" + - "non-persistent://tg/request/text-completion" + - "--text-completion-response-queue" + - "non-persistent://tg/response/text-completion-response" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "prompt-rag": + "command": + - "prompt-generic" + - "-p" + - "pulsar://pulsar:6650" + - "-i" + - "non-persistent://tg/request/prompt-rag" + - "-o" + - "non-persistent://tg/response/prompt-rag-response" + - "--text-completion-request-queue" + - "non-persistent://tg/request/text-completion-rag" + - "--text-completion-response-queue" + - "non-persistent://tg/response/text-completion-rag-response" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "pulsar": + "command": "bin/pulsar standalone" + "image": "docker.io/apachepulsar/pulsar:3.3.0" + "ports": + - "6650:6650" + - "8080:8080" + "restart": "on-failure:100" + "volumes": + - "pulsar-conf:/pulsar/conf" + - "pulsar-data:/pulsar/data" + "pulsar-manager": + "environment": + "SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties" + "image": "docker.io/apachepulsar/pulsar-manager:v0.3.0" + "ports": + - "9527:9527" + - "7750:7750" + "restart": "on-failure:100" + "query-graph-embeddings": + "command": + - "ge-query-milvus" + - "-p" + - "pulsar://pulsar:6650" + - "-t" + - "http://milvus:19530" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "query-triples": + "command": + - "triples-query-neo4j" + - "-p" + - "pulsar://pulsar:6650" + - "-g" + - "bolt://neo4j:7687" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "store-graph-embeddings": + "command": + - "ge-write-milvus" + - "-p" + - "pulsar://pulsar:6650" + - "-t" + - "http://milvus:19530" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "store-triples": + "command": + - "triples-write-neo4j" + - "-p" + - "pulsar://pulsar:6650" + - "-g" + - "bolt://neo4j:7687" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "text-completion": + "command": + - "text-completion-openai" + - "-p" + - "pulsar://pulsar:6650" + - "-k" + - "${OPENAI_KEY}" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "text-completion-rag": + "command": + - "text-completion-openai" + - "-p" + - "pulsar://pulsar:6650" + - "-k" + - "${OPENAI_KEY}" + - "-i" + - "non-persistent://tg/request/text-completion-rag" + - "-o" + - "non-persistent://tg/response/text-completion-rag-response" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" + "vectorize": + "command": + - "embeddings-vectorize" + - "-p" + - "pulsar://pulsar:6650" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" + "restart": "on-failure:100" +"volumes": + "etcd": {} + "grafana-storage": {} + "milvus": {} + "minio-data": {} + "neo4j": {} + "prometheus-data": {} + "pulsar-conf": {} + "pulsar-data": {} diff --git a/docker-compose-openai.yaml b/docker-compose-openai.yaml index 56c1d869..8a6e03ee 100644 --- a/docker-compose-openai.yaml +++ b/docker-compose-openai.yaml @@ -11,14 +11,14 @@ - "chunker-recursive" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "embeddings": "command": - "embeddings-hf" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "etcd": "command": @@ -60,7 +60,7 @@ - "non-persistent://tg/request/prompt-rag" - "--prompt-response-queue" - "non-persistent://tg/response/prompt-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "init-pulsar": "command": @@ -77,14 +77,14 @@ - "kg-extract-definitions" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "kg-extract-relationships": "command": - "kg-extract-relationships" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "milvus": "command": @@ -122,7 +122,7 @@ - "pdf-decoder" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prometheus": "image": "docker.io/prom/prometheus:v2.53.1" @@ -141,7 +141,7 @@ - "non-persistent://tg/request/text-completion" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prompt-rag": "command": @@ -156,7 +156,7 @@ - "non-persistent://tg/request/text-completion-rag" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -183,7 +183,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "query-triples": "command": @@ -192,7 +192,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -201,7 +201,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-triples": "command": @@ -210,7 +210,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion": "command": @@ -219,7 +219,7 @@ - "pulsar://pulsar:6650" - "-k" - "${OPENAI_KEY}" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion-rag": "command": @@ -232,14 +232,14 @@ - "non-persistent://tg/request/text-completion-rag" - "-o" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "vectorize": "command": - "embeddings-vectorize" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/docker-compose-vertexai.yaml b/docker-compose-vertexai.yaml index 2c468ba9..424e6942 100644 --- a/docker-compose-vertexai.yaml +++ b/docker-compose-vertexai.yaml @@ -11,14 +11,14 @@ - "chunker-recursive" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "embeddings": "command": - "embeddings-hf" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "etcd": "command": @@ -60,7 +60,7 @@ - "non-persistent://tg/request/prompt-rag" - "--prompt-response-queue" - "non-persistent://tg/response/prompt-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "init-pulsar": "command": @@ -77,14 +77,14 @@ - "kg-extract-definitions" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "kg-extract-relationships": "command": - "kg-extract-relationships" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "milvus": "command": @@ -122,7 +122,7 @@ - "pdf-decoder" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prometheus": "image": "docker.io/prom/prometheus:v2.53.1" @@ -141,7 +141,7 @@ - "non-persistent://tg/request/text-completion" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "prompt-rag": "command": @@ -156,7 +156,7 @@ - "non-persistent://tg/request/text-completion-rag" - "--text-completion-response-queue" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -183,7 +183,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "query-triples": "command": @@ -192,7 +192,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -201,7 +201,7 @@ - "pulsar://pulsar:6650" - "-t" - "http://milvus:19530" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "store-triples": "command": @@ -210,7 +210,7 @@ - "pulsar://pulsar:6650" - "-g" - "cassandra" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "text-completion": "command": @@ -221,7 +221,7 @@ - "/vertexai/private.json" - "-r" - "us-west1" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "volumes": - "./vertexai:/vertexai" @@ -238,7 +238,7 @@ - "non-persistent://tg/request/text-completion-rag" - "-o" - "non-persistent://tg/response/text-completion-rag-response" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "volumes": - "./vertexai:/vertexai" @@ -247,7 +247,7 @@ - "embeddings-vectorize" - "-p" - "pulsar://pulsar:6650" - "image": "docker.io/trustgraph/trustgraph-flow:0.6.0" + "image": "docker.io/trustgraph/trustgraph-flow:0.6.1" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/processing-just-llm.yaml b/processing-just-llm.yaml deleted file mode 100644 index 817d6594..00000000 --- a/processing-just-llm.yaml +++ /dev/null @@ -1,45 +0,0 @@ - -services: - - pdf-decoder: - module: trustgraph.decoder.pdf - - chunker: - module: trustgraph.chunker.recursive - - vectorize: - module: trustgraph.embeddings.vectorize - -# embeddings: -# module: trustgraph.embeddings.hf - - # embeddings: - # module: trustgraph.embeddings.ollama - # parameters: - # ollama: http://ollama:11434 - - embeddings: - module: trustgraph.embeddings.hf - parameters: - model: mixedbread-ai/mxbai-embed-large-v1 - - kg-extract-definitions: - module: trustgraph.kg.extract_definitions - - kg-extract-relationships: - module: trustgraph.kg.extract_relationships - - vector-write: - module: trustgraph.vector.milvus_write - - graph-write: - module: trustgraph.graph.cassandra_write - - llm: - module: trustgraph.llm.ollama_text - parameters: - ollama: http://ollama:11434 - - graph-rag: - module: trustgraph.rag.graph - diff --git a/processing.yaml b/processing.yaml deleted file mode 100644 index 817d6594..00000000 --- a/processing.yaml +++ /dev/null @@ -1,45 +0,0 @@ - -services: - - pdf-decoder: - module: trustgraph.decoder.pdf - - chunker: - module: trustgraph.chunker.recursive - - vectorize: - module: trustgraph.embeddings.vectorize - -# embeddings: -# module: trustgraph.embeddings.hf - - # embeddings: - # module: trustgraph.embeddings.ollama - # parameters: - # ollama: http://ollama:11434 - - embeddings: - module: trustgraph.embeddings.hf - parameters: - model: mixedbread-ai/mxbai-embed-large-v1 - - kg-extract-definitions: - module: trustgraph.kg.extract_definitions - - kg-extract-relationships: - module: trustgraph.kg.extract_relationships - - vector-write: - module: trustgraph.vector.milvus_write - - graph-write: - module: trustgraph.graph.cassandra_write - - llm: - module: trustgraph.llm.ollama_text - parameters: - ollama: http://ollama:11434 - - graph-rag: - module: trustgraph.rag.graph - diff --git a/scripts/graph-show b/scripts/graph-show index 4340758f..f92c277e 100755 --- a/scripts/graph-show +++ b/scripts/graph-show @@ -1,22 +1,23 @@ #!/usr/bin/env python3 """ -Connects to the trustgraph graph hosts and dumps all graph edges. +Connects to the graph query service and dumps all graph edges. """ import argparse -import time +import os +from trustgraph.triples_query_client import TriplesQueryClient -from trustgraph.direct.cassandra import TrustGraph +default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650') -def show_graph(graph_hosts): +def show_graph(pulsar): - t = TrustGraph(hosts=graph_hosts) + tq = TriplesQueryClient(pulsar_host="pulsar://localhost:6650") - rows = t.get_all(limit=100_000_000) - for s, p, o in rows: - print(s, p, o) + rows = tq.request(None, None, None, limit=10_000_000) + for row in rows: + print(row.s.value, row.p.value, row.o.value) def main(): @@ -26,16 +27,16 @@ def main(): ) parser.add_argument( - '-g', '--graph-hosts', - default="localhost", - help=f'Graph host (default: localhost)', + '-p', '--pulsar-host', + default=default_pulsar_host, + help=f'Pulsar host (default: {default_pulsar_host})', ) args = parser.parse_args() try: - show_graph(graph_hosts=args.graph_hosts.split(",")) + show_graph(args.pulsar_host) except Exception as e: diff --git a/scripts/triples-query-neo4j b/scripts/triples-query-neo4j new file mode 100755 index 00000000..05d97b10 --- /dev/null +++ b/scripts/triples-query-neo4j @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from trustgraph.query.triples.neo4j import run + +run() + diff --git a/scripts/triples-write-neo4j b/scripts/triples-write-neo4j new file mode 100755 index 00000000..58786d44 --- /dev/null +++ b/scripts/triples-write-neo4j @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from trustgraph.storage.triples.neo4j import run + +run() + diff --git a/setup.py b/setup.py index 41cdf5e3..2963cc61 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import os with open("README.md", "r") as fh: long_description = fh.read() -version = "0.6.0" +version = "0.6.1" setuptools.setup( name="trustgraph", @@ -48,6 +48,7 @@ setuptools.setup( "cohere", "boto3", "openai", + "neo4j", ], scripts=[ "scripts/chunker-recursive", @@ -82,5 +83,7 @@ setuptools.setup( "scripts/triples-dump-parquet", "scripts/triples-query-cassandra", "scripts/triples-write-cassandra", + "scripts/triples-query-neo4j", + "scripts/triples-write-neo4j", ] ) diff --git a/templates/components/images.jsonnet b/templates/components/images.jsonnet index f5f94983..33b7b65d 100644 --- a/templates/components/images.jsonnet +++ b/templates/components/images.jsonnet @@ -1,6 +1,7 @@ local version = import "version.jsonnet"; { cassandra: "docker.io/cassandra:4.1.5", + neo4j: "docker.io/neo4j:5.22.0-community-bullseye", pulsar: "docker.io/apachepulsar/pulsar:3.3.0", pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.3.0", etcd: "quay.io/coreos/etcd:v3.5.5", diff --git a/templates/components/neo4j.jsonnet b/templates/components/neo4j.jsonnet new file mode 100644 index 00000000..47f42394 --- /dev/null +++ b/templates/components/neo4j.jsonnet @@ -0,0 +1,25 @@ +local base = import "base.jsonnet"; +local images = import "images.jsonnet"; +{ + volumes +: { + neo4j: {}, + }, + services +: { + neo4j: base + { + image: images.neo4j, + ports: [ + "7474:7474", + "7687:7687", + ], + environment: { + NEO4J_AUTH: "neo4j/password", +// NEO4J_server_bolt_listen__address: "0.0.0.0:7687", +// NEO4J_server_default__listen__address: "0.0.0.0", +// NEO4J_server_http_listen__address: "0.0.0.0:7474", + }, + volumes: [ + "neo4j:/data" + ], + } + }, +} diff --git a/templates/components/version.jsonnet b/templates/components/version.jsonnet deleted file mode 100644 index 675381e4..00000000 --- a/templates/components/version.jsonnet +++ /dev/null @@ -1 +0,0 @@ -"0.6.0" diff --git a/templates/docker-compose-openai-neo4j.jsonnet b/templates/docker-compose-openai-neo4j.jsonnet new file mode 100644 index 00000000..391dcbf9 --- /dev/null +++ b/templates/docker-compose-openai-neo4j.jsonnet @@ -0,0 +1,68 @@ + +local base = import "components/base.jsonnet"; +local url = import "components/url.jsonnet"; +local images = import "components/images.jsonnet"; + +local neo4j = import "components/neo4j.jsonnet"; +local pulsar = import "components/pulsar.jsonnet"; +local milvus = import "components/milvus.jsonnet"; +local grafana = import "components/grafana.jsonnet"; +local trustgraph = import "components/trustgraph.jsonnet"; + +local config = neo4j + pulsar + milvus + grafana + trustgraph + { + + services: std.mergePatch(super.services, { + + "text-completion": base + { + image: images.trustgraph, + command: [ + "text-completion-openai", + "-p", + url.pulsar, + "-k", + "${OPENAI_KEY}", + ], + }, + + "text-completion-rag": base + { + image: images.trustgraph, + command: [ + "text-completion-openai", + "-p", + url.pulsar, + "-k", + "${OPENAI_KEY}", + "-i", + "non-persistent://tg/request/text-completion-rag", + "-o", + "non-persistent://tg/response/text-completion-rag-response", + ], + }, + + "query-triples": { + image: images.trustgraph, + command: [ + "triples-query-neo4j", + "-p", + url.pulsar, + "-g", + "bolt://neo4j:7687", + ], + }, + + "store-triples": { + image: images.trustgraph, + command: [ + "triples-write-neo4j", + "-p", + url.pulsar, + "-g", + "bolt://neo4j:7687", + ], + } + + }) +}; + +std.manifestYamlDoc(config) + diff --git a/tests/test-triples b/tests/test-triples index 9762d672..1f486948 100755 --- a/tests/test-triples +++ b/tests/test-triples @@ -29,7 +29,7 @@ print("-- p ---------------------------") resp = tq.request(None, "http://trustgraph.ai/e/landed", None, limit) dump(resp) -print("-- p ---------------------------") +print("-- o ---------------------------") resp = tq.request(None, None, "President", limit) dump(resp) @@ -53,8 +53,8 @@ dump(resp) print("-- po ---------------------------") resp = tq.request( - "http://trustgraph.ai/e/shuttle", "http://trustgraph.ai/e/landed", - None, + None, "http://trustgraph.ai/e/landed", + "on the concrete runway at Kennedy Space Center", limit ) dump(resp) diff --git a/trustgraph/query/triples/neo4j/__init__.py b/trustgraph/query/triples/neo4j/__init__.py new file mode 100644 index 00000000..ba844705 --- /dev/null +++ b/trustgraph/query/triples/neo4j/__init__.py @@ -0,0 +1,3 @@ + +from . service import * + diff --git a/trustgraph/query/triples/neo4j/__main__.py b/trustgraph/query/triples/neo4j/__main__.py new file mode 100755 index 00000000..89684e3e --- /dev/null +++ b/trustgraph/query/triples/neo4j/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from . hf import run + +if __name__ == '__main__': + run() + diff --git a/trustgraph/query/triples/neo4j/service.py b/trustgraph/query/triples/neo4j/service.py new file mode 100755 index 00000000..5fca61c3 --- /dev/null +++ b/trustgraph/query/triples/neo4j/service.py @@ -0,0 +1,328 @@ + +""" +Triples query service. Input is a (s, p, o) triple, some values may be +null. Output is a list of triples. +""" + +from neo4j import GraphDatabase + +from .... schema import TriplesQueryRequest, TriplesQueryResponse +from .... schema import Value, Triple +from .... schema import triples_request_queue +from .... schema import triples_response_queue +from .... base import ConsumerProducer + +module = ".".join(__name__.split(".")[1:-1]) + +default_input_queue = triples_request_queue +default_output_queue = triples_response_queue +default_subscriber = module + +default_graph_host = 'bolt://neo4j:7687' +default_username = 'neo4j' +default_password = 'password' + +class Processor(ConsumerProducer): + + def __init__(self, **params): + + input_queue = params.get("input_queue", default_input_queue) + output_queue = params.get("output_queue", default_output_queue) + subscriber = params.get("subscriber", default_subscriber) + graph_host = params.get("graph_host", default_graph_host) + username = params.get("username", default_username) + password = params.get("passowrd", default_password) + + super(Processor, self).__init__( + **params | { + "input_queue": input_queue, + "output_queue": output_queue, + "subscriber": subscriber, + "input_schema": TriplesQueryRequest, + "output_schema": TriplesQueryResponse, + "graph_host": graph_host, + } + ) + + self.db = "neo4j" + + self.io = GraphDatabase.driver(graph_host, auth=(username, password)) + + def create_value(self, ent): + + if ent.startswith("http://") or ent.startswith("https://"): + return Value(value=ent, is_uri=True) + else: + return Value(value=ent, is_uri=False) + + def handle(self, msg): + + v = msg.value() + + # Sender-produced ID + id = msg.properties()["id"] + + print(f"Handling input {id}...", flush=True) + + triples = [] + + if v.s is not None: + if v.p is not None: + if v.o is not None: + + # SPO + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Literal {value: $value}) " + "RETURN $src as src", + src=v.s.value, rel=v.p.value, value=v.o.value, + database_=self.db, + ) + + for rec in records: + triples.append((v.s.value, v.p.value, v.o.value)) + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Node {uri: $uri}) " + "RETURN $src as src", + src=v.s.value, rel=v.p.value, uri=v.o.value, + database_=self.db, + ) + + for rec in records: + triples.append((v.s.value, v.p.value, v.o.value)) + + else: + + # SP + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Literal) " + "RETURN dest.value as dest", + src=v.s.value, rel=v.p.value, + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((v.s.value, v.p.value, data["dest"])) + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Node) " + "RETURN dest.uri as dest", + src=v.s.value, rel=v.p.value, + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((v.s.value, v.p.value, data["dest"])) + + else: + + if v.o is not None: + + # SO + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Literal {value: $value}) " + "RETURN rel.uri as rel", + src=v.s.value, value=v.o.value, + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((v.s.value, data["rel"], v.o.value)) + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Node {uri: $uri}) " + "RETURN rel.uri as rel", + src=v.s.value, uri=v.o.value, + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((v.s.value, data["rel"], v.o.value)) + + else: + + # S + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Literal) " + "RETURN rel.uri as rel, dest.value as dest", + src=v.s.value, + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((v.s.value, data["rel"], data["dest"])) + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Node) " + "RETURN rel.uri as rel, dest.uri as dest", + src=v.s.value, + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((v.s.value, data["rel"], data["dest"])) + + + else: + + if v.p is not None: + + if v.o is not None: + + # PO + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Literal {value: $value}) " + "RETURN src.uri as src", + uri=v.p.value, value=v.o.value, + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((data["src"], v.p.value, v.o.value)) + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Node {uri: $uri}) " + "RETURN src.uri as src", + uri=v.p.value, dest=v.o.value, + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((data["src"], v.p.value, v.o.value)) + + else: + + # P + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Literal) " + "RETURN src.uri as src, dest.value as dest", + uri=v.p.value, + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((data["src"], v.p.value, data["dest"])) + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Node) " + "RETURN src.uri as src, dest.uri as dest", + uri=v.p.value, + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((data["src"], v.p.value, data["dest"])) + + else: + + if v.o is not None: + + # O + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node)-[rel:Rel]->(dest:Literal {value: $value}) " + "RETURN src.uri as src, rel.uri as rel", + value=v.o.value, + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((data["src"], data["rel"], v.o.value)) + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node)-[rel:Rel]->(dest:Node {uri: $uri}) " + "RETURN src.uri as src, rel.uri as rel", + uri=v.o.value, + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((data["src"], data["rel"], v.o.value)) + + else: + + # * + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node)-[rel:Rel]->(dest:Literal) " + "RETURN src.uri as src, rel.uri as rel, dest.value as dest", + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((data["src"], data["rel"], data["dest"])) + + records, summary, keys = self.io.execute_query( + "MATCH (src:Node)-[rel:Rel]->(dest:Node) " + "RETURN src.uri as src, rel.uri as rel, dest.uri as dest", + database_=self.db, + ) + + for rec in records: + data = rec.data() + triples.append((data["src"], data["rel"], data["dest"])) + + triples = [ + Triple( + s=self.create_value(t[0]), + p=self.create_value(t[1]), + o=self.create_value(t[2]) + ) + for t in triples + ] + + print("Send response...", flush=True) + r = TriplesQueryResponse(triples=triples) + self.producer.send(r, properties={"id": id}) + + print("Done.", flush=True) + + @staticmethod + def add_args(parser): + + ConsumerProducer.add_args( + parser, default_input_queue, default_subscriber, + default_output_queue, + ) + + parser.add_argument( + '-g', '--graph-host', + default=default_graph_host, + help=f'Graph host (default: {default_graph_host})' + ) + + parser.add_argument( + '--username', + default=default_username, + help=f'Neo4j username (default: {default_username})' + ) + + parser.add_argument( + '--password', + default=default_password, + help=f'Neo4j password (default: {default_password})' + ) + +def run(): + + Processor.start(module, __doc__) + diff --git a/trustgraph/storage/triples/neo4j/__init__.py b/trustgraph/storage/triples/neo4j/__init__.py new file mode 100644 index 00000000..d891d55f --- /dev/null +++ b/trustgraph/storage/triples/neo4j/__init__.py @@ -0,0 +1,3 @@ + +from . write import * + diff --git a/trustgraph/storage/triples/neo4j/__main__.py b/trustgraph/storage/triples/neo4j/__main__.py new file mode 100755 index 00000000..c05d8c6d --- /dev/null +++ b/trustgraph/storage/triples/neo4j/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from . write import run + +if __name__ == '__main__': + run() + diff --git a/trustgraph/storage/triples/neo4j/write.py b/trustgraph/storage/triples/neo4j/write.py new file mode 100755 index 00000000..d20ecdd2 --- /dev/null +++ b/trustgraph/storage/triples/neo4j/write.py @@ -0,0 +1,156 @@ + +""" +Graph writer. Input is graph edge. Writes edges to Cassandra graph. +""" + +import pulsar +import base64 +import os +import argparse +import time + +from neo4j import GraphDatabase + +from .... schema import Triple +from .... schema import triples_store_queue +from .... log_level import LogLevel +from .... base import Consumer + +module = ".".join(__name__.split(".")[1:-1]) + +default_input_queue = triples_store_queue +default_subscriber = module + +default_graph_host = 'bolt://neo4j:7687' +default_username = 'neo4j' +default_password = 'password' + +class Processor(Consumer): + + def __init__(self, **params): + + input_queue = params.get("input_queue", default_input_queue) + subscriber = params.get("subscriber", default_subscriber) + graph_host = params.get("graph_host", default_graph_host) + username = params.get("username", default_username) + password = params.get("passowrd", default_password) + + super(Processor, self).__init__( + **params | { + "input_queue": input_queue, + "subscriber": subscriber, + "input_schema": Triple, + "graph_host": graph_host, + } + ) + + self.db = "neo4j" + + self.io = GraphDatabase.driver(graph_host, auth=(username, password)) + + def create_node(self, uri): + + print("Create node", uri) + + summary = self.io.execute_query( + "MERGE (n:Node {uri: $uri})", + uri=uri, + database_=self.db, + ).summary + + print("Created {nodes_created} nodes in {time} ms.".format( + nodes_created=summary.counters.nodes_created, + time=summary.result_available_after + )) + + def create_literal(self, value): + + print("Create literal", value) + + summary = self.io.execute_query( + "MERGE (n:Literal {value: $value})", + value=value, + database_=self.db, + ).summary + + print("Created {nodes_created} nodes in {time} ms.".format( + nodes_created=summary.counters.nodes_created, + time=summary.result_available_after + )) + + def relate_node(self, src, uri, dest): + + print("Create node rel", src, uri, dest) + + summary = self.io.execute_query( + "MATCH (src:Node {uri: $src}) " + "MATCH (dest:Node {uri: $dest}) " + "MERGE (src)-[:Rel {uri: $uri}]->(dest)", + src=src, dest=dest, uri=uri, + database_=self.db, + ).summary + + print("Created {nodes_created} nodes in {time} ms.".format( + nodes_created=summary.counters.nodes_created, + time=summary.result_available_after + )) + + def relate_literal(self, src, uri, dest): + + print("Create literal rel", src, uri, dest) + + summary = self.io.execute_query( + "MATCH (src:Node {uri: $src}) " + "MATCH (dest:Literal {value: $dest}) " + "MERGE (src)-[:Rel {uri: $uri}]->(dest)", + src=src, dest=dest, uri=uri, + database_=self.db, + ).summary + + print("Created {nodes_created} nodes in {time} ms.".format( + nodes_created=summary.counters.nodes_created, + time=summary.result_available_after + )) + + def handle(self, msg): + + v = msg.value() + + self.create_node(v.s.value) + + if v.o.is_uri: + self.create_node(v.o.value) + self.relate_node(v.s.value, v.p.value, v.o.value) + else: + self.create_literal(v.o.value) + self.relate_literal(v.s.value, v.p.value, v.o.value) + + @staticmethod + def add_args(parser): + + Consumer.add_args( + parser, default_input_queue, default_subscriber, + ) + + parser.add_argument( + '-g', '--graph_host', + default=default_graph_host, + help=f'Graph host (default: {default_graph_host})' + ) + + parser.add_argument( + '--username', + default=default_username, + help=f'Neo4j username (default: {default_username})' + ) + + parser.add_argument( + '--password', + default=default_password, + help=f'Neo4j password (default: {default_password})' + ) + +def run(): + + Processor.start(module, __doc__) +