mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-04-25 00:16:23 +02:00
Add Neo4j support (#9)
- Add triples-write-neo4j and triples-query-neo4j to interact with neo4j - Add docker-compose-openai-neo4j to demo Neo4j working
This commit is contained in:
cybermaggedon
GitHub
parent
2f72fceaa2
commit
d3e213f194
27 changed files with 1008 additions and 230 deletions
6
Makefile
6
Makefile
|
|
@ -1,6 +1,6 @@
|
|||
|
||||
# VERSION=$(shell git describe | sed 's/^v//')
|
||||
VERSION=0.6.0
|
||||
VERSION=0.6.1
|
||||
|
||||
DOCKER=podman
|
||||
|
||||
|
|
@ -31,9 +31,9 @@ clean:
|
|||
|
||||
set-version:
|
||||
# sed -i 's/trustgraph-flow:[0-9]*\.[0-9]*\.[0-9]*/trustgraph-flow:'${VERSION}'/' docker-compose*.yaml
|
||||
echo '"${VERSION}"' > templates/version.jsonnet
|
||||
echo '"${VERSION}"' > templates/components/version.jsonnet
|
||||
|
||||
TEMPLATES=azure bedrock claude cohere mix ollama openai vertexai
|
||||
TEMPLATES=azure bedrock claude cohere mix ollama openai vertexai openai-neo4j
|
||||
DCS=$(foreach template,${TEMPLATES},${template:%=docker-compose-%.yaml})
|
||||
|
||||
update-templates: set-version ${DCS}
|
||||
|
|
|
|||
|
|
@ -11,14 +11,14 @@
|
|||
- "chunker-recursive"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"embeddings":
|
||||
"command":
|
||||
- "embeddings-hf"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"etcd":
|
||||
"command":
|
||||
|
|
@ -60,7 +60,7 @@
|
|||
- "non-persistent://tg/request/prompt-rag"
|
||||
- "--prompt-response-queue"
|
||||
- "non-persistent://tg/response/prompt-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"init-pulsar":
|
||||
"command":
|
||||
|
|
@ -77,14 +77,14 @@
|
|||
- "kg-extract-definitions"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"kg-extract-relationships":
|
||||
"command":
|
||||
- "kg-extract-relationships"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"milvus":
|
||||
"command":
|
||||
|
|
@ -122,7 +122,7 @@
|
|||
- "pdf-decoder"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prometheus":
|
||||
"image": "docker.io/prom/prometheus:v2.53.1"
|
||||
|
|
@ -141,7 +141,7 @@
|
|||
- "non-persistent://tg/request/text-completion"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prompt-rag":
|
||||
"command":
|
||||
|
|
@ -156,7 +156,7 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"pulsar":
|
||||
"command": "bin/pulsar standalone"
|
||||
|
|
@ -183,7 +183,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"query-triples":
|
||||
"command":
|
||||
|
|
@ -192,7 +192,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-graph-embeddings":
|
||||
"command":
|
||||
|
|
@ -201,7 +201,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-triples":
|
||||
"command":
|
||||
|
|
@ -210,7 +210,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion":
|
||||
"command":
|
||||
|
|
@ -221,7 +221,7 @@
|
|||
- "${AZURE_TOKEN}"
|
||||
- "-e"
|
||||
- "${AZURE_ENDPOINT}"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion-rag":
|
||||
"command":
|
||||
|
|
@ -236,14 +236,14 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "-o"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"vectorize":
|
||||
"command":
|
||||
- "embeddings-vectorize"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
"cassandra": {}
|
||||
|
|
|
|||
|
|
@ -15,14 +15,14 @@
|
|||
- "2000"
|
||||
- "--chunk-overlap"
|
||||
- "100"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"embeddings":
|
||||
"command":
|
||||
- "embeddings-hf"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"etcd":
|
||||
"command":
|
||||
|
|
@ -64,7 +64,7 @@
|
|||
- "non-persistent://tg/request/prompt-rag"
|
||||
- "--prompt-response-queue"
|
||||
- "non-persistent://tg/response/prompt-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"init-pulsar":
|
||||
"command":
|
||||
|
|
@ -81,14 +81,14 @@
|
|||
- "kg-extract-definitions"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"kg-extract-relationships":
|
||||
"command":
|
||||
- "kg-extract-relationships"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"milvus":
|
||||
"command":
|
||||
|
|
@ -126,7 +126,7 @@
|
|||
- "pdf-decoder"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prometheus":
|
||||
"image": "docker.io/prom/prometheus:v2.53.1"
|
||||
|
|
@ -145,7 +145,7 @@
|
|||
- "non-persistent://tg/request/text-completion"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prompt-rag":
|
||||
"command":
|
||||
|
|
@ -160,7 +160,7 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"pulsar":
|
||||
"command": "bin/pulsar standalone"
|
||||
|
|
@ -187,7 +187,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"query-triples":
|
||||
"command":
|
||||
|
|
@ -196,7 +196,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-graph-embeddings":
|
||||
"command":
|
||||
|
|
@ -205,7 +205,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-triples":
|
||||
"command":
|
||||
|
|
@ -214,7 +214,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion":
|
||||
"command":
|
||||
|
|
@ -227,7 +227,7 @@
|
|||
- "${AWS_SECRET_KEY}"
|
||||
- "-r"
|
||||
- "us-west-2"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion-rag":
|
||||
"command":
|
||||
|
|
@ -244,14 +244,14 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "-o"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"vectorize":
|
||||
"command":
|
||||
- "embeddings-vectorize"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
"cassandra": {}
|
||||
|
|
|
|||
|
|
@ -11,14 +11,14 @@
|
|||
- "chunker-recursive"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"embeddings":
|
||||
"command":
|
||||
- "embeddings-hf"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"etcd":
|
||||
"command":
|
||||
|
|
@ -60,7 +60,7 @@
|
|||
- "non-persistent://tg/request/prompt-rag"
|
||||
- "--prompt-response-queue"
|
||||
- "non-persistent://tg/response/prompt-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"init-pulsar":
|
||||
"command":
|
||||
|
|
@ -77,14 +77,14 @@
|
|||
- "kg-extract-definitions"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"kg-extract-relationships":
|
||||
"command":
|
||||
- "kg-extract-relationships"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"milvus":
|
||||
"command":
|
||||
|
|
@ -122,7 +122,7 @@
|
|||
- "pdf-decoder"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prometheus":
|
||||
"image": "docker.io/prom/prometheus:v2.53.1"
|
||||
|
|
@ -141,7 +141,7 @@
|
|||
- "non-persistent://tg/request/text-completion"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prompt-rag":
|
||||
"command":
|
||||
|
|
@ -156,7 +156,7 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"pulsar":
|
||||
"command": "bin/pulsar standalone"
|
||||
|
|
@ -183,7 +183,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"query-triples":
|
||||
"command":
|
||||
|
|
@ -192,7 +192,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-graph-embeddings":
|
||||
"command":
|
||||
|
|
@ -201,7 +201,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-triples":
|
||||
"command":
|
||||
|
|
@ -210,7 +210,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion":
|
||||
"command":
|
||||
|
|
@ -219,7 +219,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-k"
|
||||
- "${CLAUDE_KEY}"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion-rag":
|
||||
"command":
|
||||
|
|
@ -232,14 +232,14 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "-o"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"vectorize":
|
||||
"command":
|
||||
- "embeddings-vectorize"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
"cassandra": {}
|
||||
|
|
|
|||
|
|
@ -15,14 +15,14 @@
|
|||
- "1000"
|
||||
- "--chunk-overlap"
|
||||
- "50"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"embeddings":
|
||||
"command":
|
||||
- "embeddings-hf"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"etcd":
|
||||
"command":
|
||||
|
|
@ -64,7 +64,7 @@
|
|||
- "non-persistent://tg/request/prompt-rag"
|
||||
- "--prompt-response-queue"
|
||||
- "non-persistent://tg/response/prompt-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"init-pulsar":
|
||||
"command":
|
||||
|
|
@ -81,14 +81,14 @@
|
|||
- "kg-extract-definitions"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"kg-extract-relationships":
|
||||
"command":
|
||||
- "kg-extract-relationships"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"milvus":
|
||||
"command":
|
||||
|
|
@ -126,7 +126,7 @@
|
|||
- "pdf-decoder"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prometheus":
|
||||
"image": "docker.io/prom/prometheus:v2.53.1"
|
||||
|
|
@ -145,7 +145,7 @@
|
|||
- "non-persistent://tg/request/text-completion"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prompt-rag":
|
||||
"command":
|
||||
|
|
@ -160,7 +160,7 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"pulsar":
|
||||
"command": "bin/pulsar standalone"
|
||||
|
|
@ -187,7 +187,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"query-triples":
|
||||
"command":
|
||||
|
|
@ -196,7 +196,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-graph-embeddings":
|
||||
"command":
|
||||
|
|
@ -205,7 +205,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-triples":
|
||||
"command":
|
||||
|
|
@ -214,7 +214,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion":
|
||||
"command":
|
||||
|
|
@ -223,7 +223,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-k"
|
||||
- "${COHERE_KEY}"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion-rag":
|
||||
"command":
|
||||
|
|
@ -236,14 +236,14 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "-o"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"vectorize":
|
||||
"command":
|
||||
- "embeddings-vectorize"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
"cassandra": {}
|
||||
|
|
|
|||
|
|
@ -15,14 +15,14 @@
|
|||
- "4000"
|
||||
- "--chunk-overlap"
|
||||
- "120"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"embeddings":
|
||||
"command":
|
||||
- "embeddings-hf"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"etcd":
|
||||
"command":
|
||||
|
|
@ -64,7 +64,7 @@
|
|||
- "non-persistent://tg/request/prompt-rag"
|
||||
- "--prompt-response-queue"
|
||||
- "non-persistent://tg/response/prompt-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"init-pulsar":
|
||||
"command":
|
||||
|
|
@ -81,14 +81,14 @@
|
|||
- "kg-extract-definitions"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"kg-extract-relationships":
|
||||
"command":
|
||||
- "kg-extract-relationships"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"milvus":
|
||||
"command":
|
||||
|
|
@ -126,7 +126,7 @@
|
|||
- "pdf-decoder"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prometheus":
|
||||
"image": "docker.io/prom/prometheus:v2.53.1"
|
||||
|
|
@ -145,7 +145,7 @@
|
|||
- "non-persistent://tg/request/text-completion"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prompt-rag":
|
||||
"command":
|
||||
|
|
@ -160,7 +160,7 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"pulsar":
|
||||
"command": "bin/pulsar standalone"
|
||||
|
|
@ -187,7 +187,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"query-triples":
|
||||
"command":
|
||||
|
|
@ -196,7 +196,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-graph-embeddings":
|
||||
"command":
|
||||
|
|
@ -205,7 +205,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-triples":
|
||||
"command":
|
||||
|
|
@ -214,7 +214,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion":
|
||||
"command":
|
||||
|
|
@ -225,7 +225,7 @@
|
|||
- "${COHERE_KEY}"
|
||||
- "-m"
|
||||
- "c4ai-aya-23-35b"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion-rag":
|
||||
"command":
|
||||
|
|
@ -240,14 +240,14 @@
|
|||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
- "-m"
|
||||
- "c4ai-aya-23-8b"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"vectorize":
|
||||
"command":
|
||||
- "embeddings-vectorize"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
"cassandra": {}
|
||||
|
|
|
|||
|
|
@ -11,14 +11,14 @@
|
|||
- "chunker-recursive"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"embeddings":
|
||||
"command":
|
||||
- "embeddings-hf"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"etcd":
|
||||
"command":
|
||||
|
|
@ -60,7 +60,7 @@
|
|||
- "non-persistent://tg/request/prompt-rag"
|
||||
- "--prompt-response-queue"
|
||||
- "non-persistent://tg/response/prompt-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"init-pulsar":
|
||||
"command":
|
||||
|
|
@ -77,14 +77,14 @@
|
|||
- "kg-extract-definitions"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"kg-extract-relationships":
|
||||
"command":
|
||||
- "kg-extract-relationships"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"milvus":
|
||||
"command":
|
||||
|
|
@ -122,7 +122,7 @@
|
|||
- "pdf-decoder"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prometheus":
|
||||
"image": "docker.io/prom/prometheus:v2.53.1"
|
||||
|
|
@ -141,7 +141,7 @@
|
|||
- "non-persistent://tg/request/text-completion"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prompt-rag":
|
||||
"command":
|
||||
|
|
@ -156,7 +156,7 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"pulsar":
|
||||
"command": "bin/pulsar standalone"
|
||||
|
|
@ -183,7 +183,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"query-triples":
|
||||
"command":
|
||||
|
|
@ -192,7 +192,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-graph-embeddings":
|
||||
"command":
|
||||
|
|
@ -201,7 +201,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-triples":
|
||||
"command":
|
||||
|
|
@ -210,7 +210,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion":
|
||||
"command":
|
||||
|
|
@ -219,7 +219,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-r"
|
||||
- "${OLLAMA_HOST}"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion-rag":
|
||||
"command":
|
||||
|
|
@ -232,14 +232,14 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "-o"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"vectorize":
|
||||
"command":
|
||||
- "embeddings-vectorize"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
"cassandra": {}
|
||||
|
|
|
|||
255
docker-compose-openai-neo4j.yaml
Normal file
255
docker-compose-openai-neo4j.yaml
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
"services":
|
||||
"chunker":
|
||||
"command":
|
||||
- "chunker-recursive"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"embeddings":
|
||||
"command":
|
||||
- "embeddings-hf"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"etcd":
|
||||
"command":
|
||||
- "etcd"
|
||||
- "-advertise-client-urls=http://127.0.0.1:2379"
|
||||
- "-listen-client-urls"
|
||||
- "http://0.0.0.0:2379"
|
||||
- "--data-dir"
|
||||
- "/etcd"
|
||||
"environment":
|
||||
"ETCD_AUTO_COMPACTION_MODE": "revision"
|
||||
"ETCD_AUTO_COMPACTION_RETENTION": "1000"
|
||||
"ETCD_QUOTA_BACKEND_BYTES": "4294967296"
|
||||
"ETCD_SNAPSHOT_COUNT": "50000"
|
||||
"image": "quay.io/coreos/etcd:v3.5.5"
|
||||
"ports":
|
||||
- "2379:2379"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
- "etcd:/etcd"
|
||||
"grafana":
|
||||
"environment":
|
||||
"GF_ORG_NAME": "trustgraph.ai"
|
||||
"image": "docker.io/grafana/grafana:10.0.0"
|
||||
"ports":
|
||||
- "3000:3000"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
- "grafana-storage:/var/lib/grafana"
|
||||
- "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
|
||||
- "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
|
||||
- "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
|
||||
"graph-rag":
|
||||
"command":
|
||||
- "graph-rag"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
- "--prompt-request-queue"
|
||||
- "non-persistent://tg/request/prompt-rag"
|
||||
- "--prompt-response-queue"
|
||||
- "non-persistent://tg/response/prompt-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"init-pulsar":
|
||||
"command":
|
||||
- "sh"
|
||||
- "-c"
|
||||
- "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
|
||||
"depends_on":
|
||||
"pulsar":
|
||||
"condition": "service_started"
|
||||
"image": "docker.io/apachepulsar/pulsar:3.3.0"
|
||||
"restart": "on-failure:100"
|
||||
"kg-extract-definitions":
|
||||
"command":
|
||||
- "kg-extract-definitions"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"kg-extract-relationships":
|
||||
"command":
|
||||
- "kg-extract-relationships"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"milvus":
|
||||
"command":
|
||||
- "milvus"
|
||||
- "run"
|
||||
- "standalone"
|
||||
"environment":
|
||||
"ETCD_ENDPOINTS": "etcd:2379"
|
||||
"MINIO_ADDRESS": "minio:9000"
|
||||
"image": "docker.io/milvusdb/milvus:v2.4.5"
|
||||
"ports":
|
||||
- "9091:9091"
|
||||
- "19530:19530"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
- "milvus:/var/lib/milvus"
|
||||
"minio":
|
||||
"command":
|
||||
- "minio"
|
||||
- "server"
|
||||
- "/minio_data"
|
||||
- "--console-address"
|
||||
- ":9001"
|
||||
"environment":
|
||||
"MINIO_ROOT_PASSWORD": "minioadmin"
|
||||
"MINIO_ROOT_USER": "minioadmin"
|
||||
"image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z"
|
||||
"ports":
|
||||
- "9001:9001"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
- "minio-data:/minio_data"
|
||||
"neo4j":
|
||||
"environment":
|
||||
"NEO4J_AUTH": "neo4j/password"
|
||||
"image": "docker.io/neo4j:5.22.0-community-bullseye"
|
||||
"ports":
|
||||
- "7474:7474"
|
||||
- "7687:7687"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
- "neo4j:/data"
|
||||
"pdf-decoder":
|
||||
"command":
|
||||
- "pdf-decoder"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prometheus":
|
||||
"image": "docker.io/prom/prometheus:v2.53.1"
|
||||
"ports":
|
||||
- "9090:9090"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
- "./prometheus:/etc/prometheus"
|
||||
- "prometheus-data:/prometheus"
|
||||
"prompt":
|
||||
"command":
|
||||
- "prompt-generic"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
- "--text-completion-request-queue"
|
||||
- "non-persistent://tg/request/text-completion"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prompt-rag":
|
||||
"command":
|
||||
- "prompt-generic"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
- "-i"
|
||||
- "non-persistent://tg/request/prompt-rag"
|
||||
- "-o"
|
||||
- "non-persistent://tg/response/prompt-rag-response"
|
||||
- "--text-completion-request-queue"
|
||||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"pulsar":
|
||||
"command": "bin/pulsar standalone"
|
||||
"image": "docker.io/apachepulsar/pulsar:3.3.0"
|
||||
"ports":
|
||||
- "6650:6650"
|
||||
- "8080:8080"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
- "pulsar-conf:/pulsar/conf"
|
||||
- "pulsar-data:/pulsar/data"
|
||||
"pulsar-manager":
|
||||
"environment":
|
||||
"SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties"
|
||||
"image": "docker.io/apachepulsar/pulsar-manager:v0.3.0"
|
||||
"ports":
|
||||
- "9527:9527"
|
||||
- "7750:7750"
|
||||
"restart": "on-failure:100"
|
||||
"query-graph-embeddings":
|
||||
"command":
|
||||
- "ge-query-milvus"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"query-triples":
|
||||
"command":
|
||||
- "triples-query-neo4j"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "bolt://neo4j:7687"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-graph-embeddings":
|
||||
"command":
|
||||
- "ge-write-milvus"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-triples":
|
||||
"command":
|
||||
- "triples-write-neo4j"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "bolt://neo4j:7687"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion":
|
||||
"command":
|
||||
- "text-completion-openai"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
- "-k"
|
||||
- "${OPENAI_KEY}"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion-rag":
|
||||
"command":
|
||||
- "text-completion-openai"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
- "-k"
|
||||
- "${OPENAI_KEY}"
|
||||
- "-i"
|
||||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "-o"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"vectorize":
|
||||
"command":
|
||||
- "embeddings-vectorize"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
"etcd": {}
|
||||
"grafana-storage": {}
|
||||
"milvus": {}
|
||||
"minio-data": {}
|
||||
"neo4j": {}
|
||||
"prometheus-data": {}
|
||||
"pulsar-conf": {}
|
||||
"pulsar-data": {}
|
||||
|
|
@ -11,14 +11,14 @@
|
|||
- "chunker-recursive"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"embeddings":
|
||||
"command":
|
||||
- "embeddings-hf"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"etcd":
|
||||
"command":
|
||||
|
|
@ -60,7 +60,7 @@
|
|||
- "non-persistent://tg/request/prompt-rag"
|
||||
- "--prompt-response-queue"
|
||||
- "non-persistent://tg/response/prompt-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"init-pulsar":
|
||||
"command":
|
||||
|
|
@ -77,14 +77,14 @@
|
|||
- "kg-extract-definitions"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"kg-extract-relationships":
|
||||
"command":
|
||||
- "kg-extract-relationships"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"milvus":
|
||||
"command":
|
||||
|
|
@ -122,7 +122,7 @@
|
|||
- "pdf-decoder"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prometheus":
|
||||
"image": "docker.io/prom/prometheus:v2.53.1"
|
||||
|
|
@ -141,7 +141,7 @@
|
|||
- "non-persistent://tg/request/text-completion"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prompt-rag":
|
||||
"command":
|
||||
|
|
@ -156,7 +156,7 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"pulsar":
|
||||
"command": "bin/pulsar standalone"
|
||||
|
|
@ -183,7 +183,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"query-triples":
|
||||
"command":
|
||||
|
|
@ -192,7 +192,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-graph-embeddings":
|
||||
"command":
|
||||
|
|
@ -201,7 +201,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-triples":
|
||||
"command":
|
||||
|
|
@ -210,7 +210,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion":
|
||||
"command":
|
||||
|
|
@ -219,7 +219,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-k"
|
||||
- "${OPENAI_KEY}"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion-rag":
|
||||
"command":
|
||||
|
|
@ -232,14 +232,14 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "-o"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"vectorize":
|
||||
"command":
|
||||
- "embeddings-vectorize"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
"cassandra": {}
|
||||
|
|
|
|||
|
|
@ -11,14 +11,14 @@
|
|||
- "chunker-recursive"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"embeddings":
|
||||
"command":
|
||||
- "embeddings-hf"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"etcd":
|
||||
"command":
|
||||
|
|
@ -60,7 +60,7 @@
|
|||
- "non-persistent://tg/request/prompt-rag"
|
||||
- "--prompt-response-queue"
|
||||
- "non-persistent://tg/response/prompt-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"init-pulsar":
|
||||
"command":
|
||||
|
|
@ -77,14 +77,14 @@
|
|||
- "kg-extract-definitions"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"kg-extract-relationships":
|
||||
"command":
|
||||
- "kg-extract-relationships"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"milvus":
|
||||
"command":
|
||||
|
|
@ -122,7 +122,7 @@
|
|||
- "pdf-decoder"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prometheus":
|
||||
"image": "docker.io/prom/prometheus:v2.53.1"
|
||||
|
|
@ -141,7 +141,7 @@
|
|||
- "non-persistent://tg/request/text-completion"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"prompt-rag":
|
||||
"command":
|
||||
|
|
@ -156,7 +156,7 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "--text-completion-response-queue"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"pulsar":
|
||||
"command": "bin/pulsar standalone"
|
||||
|
|
@ -183,7 +183,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"query-triples":
|
||||
"command":
|
||||
|
|
@ -192,7 +192,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-graph-embeddings":
|
||||
"command":
|
||||
|
|
@ -201,7 +201,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-t"
|
||||
- "http://milvus:19530"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"store-triples":
|
||||
"command":
|
||||
|
|
@ -210,7 +210,7 @@
|
|||
- "pulsar://pulsar:6650"
|
||||
- "-g"
|
||||
- "cassandra"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"text-completion":
|
||||
"command":
|
||||
|
|
@ -221,7 +221,7 @@
|
|||
- "/vertexai/private.json"
|
||||
- "-r"
|
||||
- "us-west1"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
- "./vertexai:/vertexai"
|
||||
|
|
@ -238,7 +238,7 @@
|
|||
- "non-persistent://tg/request/text-completion-rag"
|
||||
- "-o"
|
||||
- "non-persistent://tg/response/text-completion-rag-response"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
- "./vertexai:/vertexai"
|
||||
|
|
@ -247,7 +247,7 @@
|
|||
- "embeddings-vectorize"
|
||||
- "-p"
|
||||
- "pulsar://pulsar:6650"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
|
||||
"image": "docker.io/trustgraph/trustgraph-flow:0.6.1"
|
||||
"restart": "on-failure:100"
|
||||
"volumes":
|
||||
"cassandra": {}
|
||||
|
|
|
|||
|
|
@ -1,45 +0,0 @@
|
|||
|
||||
services:
|
||||
|
||||
pdf-decoder:
|
||||
module: trustgraph.decoder.pdf
|
||||
|
||||
chunker:
|
||||
module: trustgraph.chunker.recursive
|
||||
|
||||
vectorize:
|
||||
module: trustgraph.embeddings.vectorize
|
||||
|
||||
# embeddings:
|
||||
# module: trustgraph.embeddings.hf
|
||||
|
||||
# embeddings:
|
||||
# module: trustgraph.embeddings.ollama
|
||||
# parameters:
|
||||
# ollama: http://ollama:11434
|
||||
|
||||
embeddings:
|
||||
module: trustgraph.embeddings.hf
|
||||
parameters:
|
||||
model: mixedbread-ai/mxbai-embed-large-v1
|
||||
|
||||
kg-extract-definitions:
|
||||
module: trustgraph.kg.extract_definitions
|
||||
|
||||
kg-extract-relationships:
|
||||
module: trustgraph.kg.extract_relationships
|
||||
|
||||
vector-write:
|
||||
module: trustgraph.vector.milvus_write
|
||||
|
||||
graph-write:
|
||||
module: trustgraph.graph.cassandra_write
|
||||
|
||||
llm:
|
||||
module: trustgraph.llm.ollama_text
|
||||
parameters:
|
||||
ollama: http://ollama:11434
|
||||
|
||||
graph-rag:
|
||||
module: trustgraph.rag.graph
|
||||
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
|
||||
services:
|
||||
|
||||
pdf-decoder:
|
||||
module: trustgraph.decoder.pdf
|
||||
|
||||
chunker:
|
||||
module: trustgraph.chunker.recursive
|
||||
|
||||
vectorize:
|
||||
module: trustgraph.embeddings.vectorize
|
||||
|
||||
# embeddings:
|
||||
# module: trustgraph.embeddings.hf
|
||||
|
||||
# embeddings:
|
||||
# module: trustgraph.embeddings.ollama
|
||||
# parameters:
|
||||
# ollama: http://ollama:11434
|
||||
|
||||
embeddings:
|
||||
module: trustgraph.embeddings.hf
|
||||
parameters:
|
||||
model: mixedbread-ai/mxbai-embed-large-v1
|
||||
|
||||
kg-extract-definitions:
|
||||
module: trustgraph.kg.extract_definitions
|
||||
|
||||
kg-extract-relationships:
|
||||
module: trustgraph.kg.extract_relationships
|
||||
|
||||
vector-write:
|
||||
module: trustgraph.vector.milvus_write
|
||||
|
||||
graph-write:
|
||||
module: trustgraph.graph.cassandra_write
|
||||
|
||||
llm:
|
||||
module: trustgraph.llm.ollama_text
|
||||
parameters:
|
||||
ollama: http://ollama:11434
|
||||
|
||||
graph-rag:
|
||||
module: trustgraph.rag.graph
|
||||
|
||||
|
|
@ -1,22 +1,23 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Connects to the trustgraph graph hosts and dumps all graph edges.
|
||||
Connects to the graph query service and dumps all graph edges.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import time
|
||||
import os
|
||||
from trustgraph.triples_query_client import TriplesQueryClient
|
||||
|
||||
from trustgraph.direct.cassandra import TrustGraph
|
||||
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
|
||||
|
||||
def show_graph(graph_hosts):
|
||||
def show_graph(pulsar):
|
||||
|
||||
t = TrustGraph(hosts=graph_hosts)
|
||||
tq = TriplesQueryClient(pulsar_host="pulsar://localhost:6650")
|
||||
|
||||
rows = t.get_all(limit=100_000_000)
|
||||
for s, p, o in rows:
|
||||
print(s, p, o)
|
||||
rows = tq.request(None, None, None, limit=10_000_000)
|
||||
|
||||
for row in rows:
|
||||
print(row.s.value, row.p.value, row.o.value)
|
||||
|
||||
def main():
|
||||
|
||||
|
|
@ -26,16 +27,16 @@ def main():
|
|||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-g', '--graph-hosts',
|
||||
default="localhost",
|
||||
help=f'Graph host (default: localhost)',
|
||||
'-p', '--pulsar-host',
|
||||
default=default_pulsar_host,
|
||||
help=f'Pulsar host (default: {default_pulsar_host})',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
show_graph(graph_hosts=args.graph_hosts.split(","))
|
||||
show_graph(args.pulsar_host)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
|
|
|
|||
6
scripts/triples-query-neo4j
Executable file
6
scripts/triples-query-neo4j
Executable file
|
|
@ -0,0 +1,6 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from trustgraph.query.triples.neo4j import run
|
||||
|
||||
run()
|
||||
|
||||
6
scripts/triples-write-neo4j
Executable file
6
scripts/triples-write-neo4j
Executable file
|
|
@ -0,0 +1,6 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from trustgraph.storage.triples.neo4j import run
|
||||
|
||||
run()
|
||||
|
||||
5
setup.py
5
setup.py
|
|
@ -4,7 +4,7 @@ import os
|
|||
with open("README.md", "r") as fh:
|
||||
long_description = fh.read()
|
||||
|
||||
version = "0.6.0"
|
||||
version = "0.6.1"
|
||||
|
||||
setuptools.setup(
|
||||
name="trustgraph",
|
||||
|
|
@ -48,6 +48,7 @@ setuptools.setup(
|
|||
"cohere",
|
||||
"boto3",
|
||||
"openai",
|
||||
"neo4j",
|
||||
],
|
||||
scripts=[
|
||||
"scripts/chunker-recursive",
|
||||
|
|
@ -82,5 +83,7 @@ setuptools.setup(
|
|||
"scripts/triples-dump-parquet",
|
||||
"scripts/triples-query-cassandra",
|
||||
"scripts/triples-write-cassandra",
|
||||
"scripts/triples-query-neo4j",
|
||||
"scripts/triples-write-neo4j",
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
local version = import "version.jsonnet";
|
||||
{
|
||||
cassandra: "docker.io/cassandra:4.1.5",
|
||||
neo4j: "docker.io/neo4j:5.22.0-community-bullseye",
|
||||
pulsar: "docker.io/apachepulsar/pulsar:3.3.0",
|
||||
pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.3.0",
|
||||
etcd: "quay.io/coreos/etcd:v3.5.5",
|
||||
|
|
|
|||
25
templates/components/neo4j.jsonnet
Normal file
25
templates/components/neo4j.jsonnet
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
local base = import "base.jsonnet";
|
||||
local images = import "images.jsonnet";
|
||||
{
|
||||
volumes +: {
|
||||
neo4j: {},
|
||||
},
|
||||
services +: {
|
||||
neo4j: base + {
|
||||
image: images.neo4j,
|
||||
ports: [
|
||||
"7474:7474",
|
||||
"7687:7687",
|
||||
],
|
||||
environment: {
|
||||
NEO4J_AUTH: "neo4j/password",
|
||||
// NEO4J_server_bolt_listen__address: "0.0.0.0:7687",
|
||||
// NEO4J_server_default__listen__address: "0.0.0.0",
|
||||
// NEO4J_server_http_listen__address: "0.0.0.0:7474",
|
||||
},
|
||||
volumes: [
|
||||
"neo4j:/data"
|
||||
],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
|
@ -1 +0,0 @@
|
|||
"0.6.0"
|
||||
68
templates/docker-compose-openai-neo4j.jsonnet
Normal file
68
templates/docker-compose-openai-neo4j.jsonnet
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
|
||||
local base = import "components/base.jsonnet";
|
||||
local url = import "components/url.jsonnet";
|
||||
local images = import "components/images.jsonnet";
|
||||
|
||||
local neo4j = import "components/neo4j.jsonnet";
|
||||
local pulsar = import "components/pulsar.jsonnet";
|
||||
local milvus = import "components/milvus.jsonnet";
|
||||
local grafana = import "components/grafana.jsonnet";
|
||||
local trustgraph = import "components/trustgraph.jsonnet";
|
||||
|
||||
local config = neo4j + pulsar + milvus + grafana + trustgraph + {
|
||||
|
||||
services: std.mergePatch(super.services, {
|
||||
|
||||
"text-completion": base + {
|
||||
image: images.trustgraph,
|
||||
command: [
|
||||
"text-completion-openai",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-k",
|
||||
"${OPENAI_KEY}",
|
||||
],
|
||||
},
|
||||
|
||||
"text-completion-rag": base + {
|
||||
image: images.trustgraph,
|
||||
command: [
|
||||
"text-completion-openai",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-k",
|
||||
"${OPENAI_KEY}",
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag-response",
|
||||
],
|
||||
},
|
||||
|
||||
"query-triples": {
|
||||
image: images.trustgraph,
|
||||
command: [
|
||||
"triples-query-neo4j",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-g",
|
||||
"bolt://neo4j:7687",
|
||||
],
|
||||
},
|
||||
|
||||
"store-triples": {
|
||||
image: images.trustgraph,
|
||||
command: [
|
||||
"triples-write-neo4j",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-g",
|
||||
"bolt://neo4j:7687",
|
||||
],
|
||||
}
|
||||
|
||||
})
|
||||
};
|
||||
|
||||
std.manifestYamlDoc(config)
|
||||
|
||||
|
|
@ -29,7 +29,7 @@ print("-- p ---------------------------")
|
|||
resp = tq.request(None, "http://trustgraph.ai/e/landed", None, limit)
|
||||
dump(resp)
|
||||
|
||||
print("-- p ---------------------------")
|
||||
print("-- o ---------------------------")
|
||||
|
||||
resp = tq.request(None, None, "President", limit)
|
||||
dump(resp)
|
||||
|
|
@ -53,8 +53,8 @@ dump(resp)
|
|||
print("-- po ---------------------------")
|
||||
|
||||
resp = tq.request(
|
||||
"http://trustgraph.ai/e/shuttle", "http://trustgraph.ai/e/landed",
|
||||
None,
|
||||
None, "http://trustgraph.ai/e/landed",
|
||||
"on the concrete runway at Kennedy Space Center",
|
||||
limit
|
||||
)
|
||||
dump(resp)
|
||||
|
|
|
|||
3
trustgraph/query/triples/neo4j/__init__.py
Normal file
3
trustgraph/query/triples/neo4j/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
|
||||
from . service import *
|
||||
|
||||
7
trustgraph/query/triples/neo4j/__main__.py
Executable file
7
trustgraph/query/triples/neo4j/__main__.py
Executable file
|
|
@ -0,0 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from . hf import run
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
|
||||
328
trustgraph/query/triples/neo4j/service.py
Executable file
328
trustgraph/query/triples/neo4j/service.py
Executable file
|
|
@ -0,0 +1,328 @@
|
|||
|
||||
"""
|
||||
Triples query service. Input is a (s, p, o) triple, some values may be
|
||||
null. Output is a list of triples.
|
||||
"""
|
||||
|
||||
from neo4j import GraphDatabase
|
||||
|
||||
from .... schema import TriplesQueryRequest, TriplesQueryResponse
|
||||
from .... schema import Value, Triple
|
||||
from .... schema import triples_request_queue
|
||||
from .... schema import triples_response_queue
|
||||
from .... base import ConsumerProducer
|
||||
|
||||
module = ".".join(__name__.split(".")[1:-1])
|
||||
|
||||
default_input_queue = triples_request_queue
|
||||
default_output_queue = triples_response_queue
|
||||
default_subscriber = module
|
||||
|
||||
default_graph_host = 'bolt://neo4j:7687'
|
||||
default_username = 'neo4j'
|
||||
default_password = 'password'
|
||||
|
||||
class Processor(ConsumerProducer):
|
||||
|
||||
def __init__(self, **params):
|
||||
|
||||
input_queue = params.get("input_queue", default_input_queue)
|
||||
output_queue = params.get("output_queue", default_output_queue)
|
||||
subscriber = params.get("subscriber", default_subscriber)
|
||||
graph_host = params.get("graph_host", default_graph_host)
|
||||
username = params.get("username", default_username)
|
||||
password = params.get("passowrd", default_password)
|
||||
|
||||
super(Processor, self).__init__(
|
||||
**params | {
|
||||
"input_queue": input_queue,
|
||||
"output_queue": output_queue,
|
||||
"subscriber": subscriber,
|
||||
"input_schema": TriplesQueryRequest,
|
||||
"output_schema": TriplesQueryResponse,
|
||||
"graph_host": graph_host,
|
||||
}
|
||||
)
|
||||
|
||||
self.db = "neo4j"
|
||||
|
||||
self.io = GraphDatabase.driver(graph_host, auth=(username, password))
|
||||
|
||||
def create_value(self, ent):
|
||||
|
||||
if ent.startswith("http://") or ent.startswith("https://"):
|
||||
return Value(value=ent, is_uri=True)
|
||||
else:
|
||||
return Value(value=ent, is_uri=False)
|
||||
|
||||
def handle(self, msg):
|
||||
|
||||
v = msg.value()
|
||||
|
||||
# Sender-produced ID
|
||||
id = msg.properties()["id"]
|
||||
|
||||
print(f"Handling input {id}...", flush=True)
|
||||
|
||||
triples = []
|
||||
|
||||
if v.s is not None:
|
||||
if v.p is not None:
|
||||
if v.o is not None:
|
||||
|
||||
# SPO
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Literal {value: $value}) "
|
||||
"RETURN $src as src",
|
||||
src=v.s.value, rel=v.p.value, value=v.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
triples.append((v.s.value, v.p.value, v.o.value))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Node {uri: $uri}) "
|
||||
"RETURN $src as src",
|
||||
src=v.s.value, rel=v.p.value, uri=v.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
triples.append((v.s.value, v.p.value, v.o.value))
|
||||
|
||||
else:
|
||||
|
||||
# SP
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Literal) "
|
||||
"RETURN dest.value as dest",
|
||||
src=v.s.value, rel=v.p.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((v.s.value, v.p.value, data["dest"]))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Node) "
|
||||
"RETURN dest.uri as dest",
|
||||
src=v.s.value, rel=v.p.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((v.s.value, v.p.value, data["dest"]))
|
||||
|
||||
else:
|
||||
|
||||
if v.o is not None:
|
||||
|
||||
# SO
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Literal {value: $value}) "
|
||||
"RETURN rel.uri as rel",
|
||||
src=v.s.value, value=v.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((v.s.value, data["rel"], v.o.value))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Node {uri: $uri}) "
|
||||
"RETURN rel.uri as rel",
|
||||
src=v.s.value, uri=v.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((v.s.value, data["rel"], v.o.value))
|
||||
|
||||
else:
|
||||
|
||||
# S
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Literal) "
|
||||
"RETURN rel.uri as rel, dest.value as dest",
|
||||
src=v.s.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((v.s.value, data["rel"], data["dest"]))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Node) "
|
||||
"RETURN rel.uri as rel, dest.uri as dest",
|
||||
src=v.s.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((v.s.value, data["rel"], data["dest"]))
|
||||
|
||||
|
||||
else:
|
||||
|
||||
if v.p is not None:
|
||||
|
||||
if v.o is not None:
|
||||
|
||||
# PO
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Literal {value: $value}) "
|
||||
"RETURN src.uri as src",
|
||||
uri=v.p.value, value=v.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], v.p.value, v.o.value))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Node {uri: $uri}) "
|
||||
"RETURN src.uri as src",
|
||||
uri=v.p.value, dest=v.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], v.p.value, v.o.value))
|
||||
|
||||
else:
|
||||
|
||||
# P
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Literal) "
|
||||
"RETURN src.uri as src, dest.value as dest",
|
||||
uri=v.p.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], v.p.value, data["dest"]))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Node) "
|
||||
"RETURN src.uri as src, dest.uri as dest",
|
||||
uri=v.p.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], v.p.value, data["dest"]))
|
||||
|
||||
else:
|
||||
|
||||
if v.o is not None:
|
||||
|
||||
# O
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel]->(dest:Literal {value: $value}) "
|
||||
"RETURN src.uri as src, rel.uri as rel",
|
||||
value=v.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], data["rel"], v.o.value))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel]->(dest:Node {uri: $uri}) "
|
||||
"RETURN src.uri as src, rel.uri as rel",
|
||||
uri=v.o.value,
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], data["rel"], v.o.value))
|
||||
|
||||
else:
|
||||
|
||||
# *
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel]->(dest:Literal) "
|
||||
"RETURN src.uri as src, rel.uri as rel, dest.value as dest",
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], data["rel"], data["dest"]))
|
||||
|
||||
records, summary, keys = self.io.execute_query(
|
||||
"MATCH (src:Node)-[rel:Rel]->(dest:Node) "
|
||||
"RETURN src.uri as src, rel.uri as rel, dest.uri as dest",
|
||||
database_=self.db,
|
||||
)
|
||||
|
||||
for rec in records:
|
||||
data = rec.data()
|
||||
triples.append((data["src"], data["rel"], data["dest"]))
|
||||
|
||||
triples = [
|
||||
Triple(
|
||||
s=self.create_value(t[0]),
|
||||
p=self.create_value(t[1]),
|
||||
o=self.create_value(t[2])
|
||||
)
|
||||
for t in triples
|
||||
]
|
||||
|
||||
print("Send response...", flush=True)
|
||||
r = TriplesQueryResponse(triples=triples)
|
||||
self.producer.send(r, properties={"id": id})
|
||||
|
||||
print("Done.", flush=True)
|
||||
|
||||
@staticmethod
|
||||
def add_args(parser):
|
||||
|
||||
ConsumerProducer.add_args(
|
||||
parser, default_input_queue, default_subscriber,
|
||||
default_output_queue,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-g', '--graph-host',
|
||||
default=default_graph_host,
|
||||
help=f'Graph host (default: {default_graph_host})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--username',
|
||||
default=default_username,
|
||||
help=f'Neo4j username (default: {default_username})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--password',
|
||||
default=default_password,
|
||||
help=f'Neo4j password (default: {default_password})'
|
||||
)
|
||||
|
||||
def run():
|
||||
|
||||
Processor.start(module, __doc__)
|
||||
|
||||
3
trustgraph/storage/triples/neo4j/__init__.py
Normal file
3
trustgraph/storage/triples/neo4j/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
|
||||
from . write import *
|
||||
|
||||
7
trustgraph/storage/triples/neo4j/__main__.py
Executable file
7
trustgraph/storage/triples/neo4j/__main__.py
Executable file
|
|
@ -0,0 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from . write import run
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
|
||||
156
trustgraph/storage/triples/neo4j/write.py
Executable file
156
trustgraph/storage/triples/neo4j/write.py
Executable file
|
|
@ -0,0 +1,156 @@
|
|||
|
||||
"""
|
||||
Graph writer. Input is graph edge. Writes edges to Cassandra graph.
|
||||
"""
|
||||
|
||||
import pulsar
|
||||
import base64
|
||||
import os
|
||||
import argparse
|
||||
import time
|
||||
|
||||
from neo4j import GraphDatabase
|
||||
|
||||
from .... schema import Triple
|
||||
from .... schema import triples_store_queue
|
||||
from .... log_level import LogLevel
|
||||
from .... base import Consumer
|
||||
|
||||
module = ".".join(__name__.split(".")[1:-1])
|
||||
|
||||
default_input_queue = triples_store_queue
|
||||
default_subscriber = module
|
||||
|
||||
default_graph_host = 'bolt://neo4j:7687'
|
||||
default_username = 'neo4j'
|
||||
default_password = 'password'
|
||||
|
||||
class Processor(Consumer):
|
||||
|
||||
def __init__(self, **params):
|
||||
|
||||
input_queue = params.get("input_queue", default_input_queue)
|
||||
subscriber = params.get("subscriber", default_subscriber)
|
||||
graph_host = params.get("graph_host", default_graph_host)
|
||||
username = params.get("username", default_username)
|
||||
password = params.get("passowrd", default_password)
|
||||
|
||||
super(Processor, self).__init__(
|
||||
**params | {
|
||||
"input_queue": input_queue,
|
||||
"subscriber": subscriber,
|
||||
"input_schema": Triple,
|
||||
"graph_host": graph_host,
|
||||
}
|
||||
)
|
||||
|
||||
self.db = "neo4j"
|
||||
|
||||
self.io = GraphDatabase.driver(graph_host, auth=(username, password))
|
||||
|
||||
def create_node(self, uri):
|
||||
|
||||
print("Create node", uri)
|
||||
|
||||
summary = self.io.execute_query(
|
||||
"MERGE (n:Node {uri: $uri})",
|
||||
uri=uri,
|
||||
database_=self.db,
|
||||
).summary
|
||||
|
||||
print("Created {nodes_created} nodes in {time} ms.".format(
|
||||
nodes_created=summary.counters.nodes_created,
|
||||
time=summary.result_available_after
|
||||
))
|
||||
|
||||
def create_literal(self, value):
|
||||
|
||||
print("Create literal", value)
|
||||
|
||||
summary = self.io.execute_query(
|
||||
"MERGE (n:Literal {value: $value})",
|
||||
value=value,
|
||||
database_=self.db,
|
||||
).summary
|
||||
|
||||
print("Created {nodes_created} nodes in {time} ms.".format(
|
||||
nodes_created=summary.counters.nodes_created,
|
||||
time=summary.result_available_after
|
||||
))
|
||||
|
||||
def relate_node(self, src, uri, dest):
|
||||
|
||||
print("Create node rel", src, uri, dest)
|
||||
|
||||
summary = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src}) "
|
||||
"MATCH (dest:Node {uri: $dest}) "
|
||||
"MERGE (src)-[:Rel {uri: $uri}]->(dest)",
|
||||
src=src, dest=dest, uri=uri,
|
||||
database_=self.db,
|
||||
).summary
|
||||
|
||||
print("Created {nodes_created} nodes in {time} ms.".format(
|
||||
nodes_created=summary.counters.nodes_created,
|
||||
time=summary.result_available_after
|
||||
))
|
||||
|
||||
def relate_literal(self, src, uri, dest):
|
||||
|
||||
print("Create literal rel", src, uri, dest)
|
||||
|
||||
summary = self.io.execute_query(
|
||||
"MATCH (src:Node {uri: $src}) "
|
||||
"MATCH (dest:Literal {value: $dest}) "
|
||||
"MERGE (src)-[:Rel {uri: $uri}]->(dest)",
|
||||
src=src, dest=dest, uri=uri,
|
||||
database_=self.db,
|
||||
).summary
|
||||
|
||||
print("Created {nodes_created} nodes in {time} ms.".format(
|
||||
nodes_created=summary.counters.nodes_created,
|
||||
time=summary.result_available_after
|
||||
))
|
||||
|
||||
def handle(self, msg):
|
||||
|
||||
v = msg.value()
|
||||
|
||||
self.create_node(v.s.value)
|
||||
|
||||
if v.o.is_uri:
|
||||
self.create_node(v.o.value)
|
||||
self.relate_node(v.s.value, v.p.value, v.o.value)
|
||||
else:
|
||||
self.create_literal(v.o.value)
|
||||
self.relate_literal(v.s.value, v.p.value, v.o.value)
|
||||
|
||||
@staticmethod
|
||||
def add_args(parser):
|
||||
|
||||
Consumer.add_args(
|
||||
parser, default_input_queue, default_subscriber,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-g', '--graph_host',
|
||||
default=default_graph_host,
|
||||
help=f'Graph host (default: {default_graph_host})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--username',
|
||||
default=default_username,
|
||||
help=f'Neo4j username (default: {default_username})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--password',
|
||||
default=default_password,
|
||||
help=f'Neo4j password (default: {default_password})'
|
||||
)
|
||||
|
||||
def run():
|
||||
|
||||
Processor.start(module, __doc__)
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue