apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-30 10:56:23 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Add token chunker (#14)

Browse source
This commit is contained in:
cybermaggedon 2024-08-21 16:51:33 +01:00 • committed by GitHub
parent 0e2db095e3
commit 7113d04f21
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 253 additions and 138 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -1,6 +1,6 @@
# VERSION=$(shell git describe | sed 's/^v//') # VERSION=$(shell git describe | sed 's/^v//')
VERSION=0.6.7 VERSION=0.6.8
DOCKER=podman DOCKER=podman

30
docker-compose-azure.yaml
View file

@ -11,14 +11,14 @@
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"embeddings": "embeddings":
"command": "command":
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"etcd": "etcd":
"command": "command":
@ -60,7 +60,7 @@
- "non-persistent://tg/request/prompt-rag" - "non-persistent://tg/request/prompt-rag"
- "--prompt-response-queue" - "--prompt-response-queue"
- "non-persistent://tg/response/prompt-rag-response" - "non-persistent://tg/response/prompt-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"init-pulsar": "init-pulsar":
"command": "command":
@ -77,14 +77,14 @@
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"kg-extract-relationships": "kg-extract-relationships":
"command": "command":
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"milvus": "milvus":
"command": "command":
@ -122,7 +122,7 @@
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prometheus": "prometheus":
"image": "docker.io/prom/prometheus:v2.53.1" "image": "docker.io/prom/prometheus:v2.53.1"
@ -141,7 +141,7 @@
- "non-persistent://tg/request/text-completion" - "non-persistent://tg/request/text-completion"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-response" - "non-persistent://tg/response/text-completion-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prompt-rag": "prompt-rag":
"command": "command":
@ -156,7 +156,7 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"pulsar": "pulsar":
"command": "bin/pulsar standalone" "command": "bin/pulsar standalone"
@ -183,7 +183,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"query-triples": "query-triples":
"command": "command":
@ -192,7 +192,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-graph-embeddings": "store-graph-embeddings":
"command": "command":
@ -201,7 +201,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-triples": "store-triples":
"command": "command":
@ -210,7 +210,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion": "text-completion":
"command": "command":
@ -221,7 +221,7 @@
- "${AZURE_TOKEN}" - "${AZURE_TOKEN}"
- "-e" - "-e"
- "${AZURE_ENDPOINT}" - "${AZURE_ENDPOINT}"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion-rag": "text-completion-rag":
"command": "command":
@ -236,14 +236,14 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "-o" - "-o"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"vectorize": "vectorize":
"command": "command":
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"volumes": "volumes":
"cassandra": {} "cassandra": {}

30
docker-compose-bedrock.yaml
View file

@ -15,14 +15,14 @@
- "2000" - "2000"
- "--chunk-overlap" - "--chunk-overlap"
- "100" - "100"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"embeddings": "embeddings":
"command": "command":
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"etcd": "etcd":
"command": "command":
@ -64,7 +64,7 @@
- "non-persistent://tg/request/prompt-rag" - "non-persistent://tg/request/prompt-rag"
- "--prompt-response-queue" - "--prompt-response-queue"
- "non-persistent://tg/response/prompt-rag-response" - "non-persistent://tg/response/prompt-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"init-pulsar": "init-pulsar":
"command": "command":
@ -81,14 +81,14 @@
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"kg-extract-relationships": "kg-extract-relationships":
"command": "command":
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"milvus": "milvus":
"command": "command":
@ -126,7 +126,7 @@
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prometheus": "prometheus":
"image": "docker.io/prom/prometheus:v2.53.1" "image": "docker.io/prom/prometheus:v2.53.1"
@ -145,7 +145,7 @@
- "non-persistent://tg/request/text-completion" - "non-persistent://tg/request/text-completion"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-response" - "non-persistent://tg/response/text-completion-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prompt-rag": "prompt-rag":
"command": "command":
@ -160,7 +160,7 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"pulsar": "pulsar":
"command": "bin/pulsar standalone" "command": "bin/pulsar standalone"
@ -187,7 +187,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"query-triples": "query-triples":
"command": "command":
@ -196,7 +196,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-graph-embeddings": "store-graph-embeddings":
"command": "command":
@ -205,7 +205,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-triples": "store-triples":
"command": "command":
@ -214,7 +214,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion": "text-completion":
"command": "command":
@ -227,7 +227,7 @@
- "${AWS_SECRET_KEY}" - "${AWS_SECRET_KEY}"
- "-r" - "-r"
- "us-west-2" - "us-west-2"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion-rag": "text-completion-rag":
"command": "command":
@ -244,14 +244,14 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "-o" - "-o"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"vectorize": "vectorize":
"command": "command":
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"volumes": "volumes":
"cassandra": {} "cassandra": {}

30
docker-compose-claude.yaml
View file

@ -11,14 +11,14 @@
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"embeddings": "embeddings":
"command": "command":
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"etcd": "etcd":
"command": "command":
@ -60,7 +60,7 @@
- "non-persistent://tg/request/prompt-rag" - "non-persistent://tg/request/prompt-rag"
- "--prompt-response-queue" - "--prompt-response-queue"
- "non-persistent://tg/response/prompt-rag-response" - "non-persistent://tg/response/prompt-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"init-pulsar": "init-pulsar":
"command": "command":
@ -77,14 +77,14 @@
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"kg-extract-relationships": "kg-extract-relationships":
"command": "command":
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"milvus": "milvus":
"command": "command":
@ -122,7 +122,7 @@
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prometheus": "prometheus":
"image": "docker.io/prom/prometheus:v2.53.1" "image": "docker.io/prom/prometheus:v2.53.1"
@ -141,7 +141,7 @@
- "non-persistent://tg/request/text-completion" - "non-persistent://tg/request/text-completion"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-response" - "non-persistent://tg/response/text-completion-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prompt-rag": "prompt-rag":
"command": "command":
@ -156,7 +156,7 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"pulsar": "pulsar":
"command": "bin/pulsar standalone" "command": "bin/pulsar standalone"
@ -183,7 +183,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"query-triples": "query-triples":
"command": "command":
@ -192,7 +192,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-graph-embeddings": "store-graph-embeddings":
"command": "command":
@ -201,7 +201,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-triples": "store-triples":
"command": "command":
@ -210,7 +210,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion": "text-completion":
"command": "command":
@ -219,7 +219,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-k" - "-k"
- "${CLAUDE_KEY}" - "${CLAUDE_KEY}"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion-rag": "text-completion-rag":
"command": "command":
@ -232,14 +232,14 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "-o" - "-o"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"vectorize": "vectorize":
"command": "command":
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"volumes": "volumes":
"cassandra": {} "cassandra": {}

30
docker-compose-cohere.yaml
View file

@ -15,14 +15,14 @@
- "1000" - "1000"
- "--chunk-overlap" - "--chunk-overlap"
- "50" - "50"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"embeddings": "embeddings":
"command": "command":
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"etcd": "etcd":
"command": "command":
@ -64,7 +64,7 @@
- "non-persistent://tg/request/prompt-rag" - "non-persistent://tg/request/prompt-rag"
- "--prompt-response-queue" - "--prompt-response-queue"
- "non-persistent://tg/response/prompt-rag-response" - "non-persistent://tg/response/prompt-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"init-pulsar": "init-pulsar":
"command": "command":
@ -81,14 +81,14 @@
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"kg-extract-relationships": "kg-extract-relationships":
"command": "command":
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"milvus": "milvus":
"command": "command":
@ -126,7 +126,7 @@
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prometheus": "prometheus":
"image": "docker.io/prom/prometheus:v2.53.1" "image": "docker.io/prom/prometheus:v2.53.1"
@ -145,7 +145,7 @@
- "non-persistent://tg/request/text-completion" - "non-persistent://tg/request/text-completion"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-response" - "non-persistent://tg/response/text-completion-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prompt-rag": "prompt-rag":
"command": "command":
@ -160,7 +160,7 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"pulsar": "pulsar":
"command": "bin/pulsar standalone" "command": "bin/pulsar standalone"
@ -187,7 +187,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"query-triples": "query-triples":
"command": "command":
@ -196,7 +196,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-graph-embeddings": "store-graph-embeddings":
"command": "command":
@ -205,7 +205,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-triples": "store-triples":
"command": "command":
@ -214,7 +214,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion": "text-completion":
"command": "command":
@ -223,7 +223,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-k" - "-k"
- "${COHERE_KEY}" - "${COHERE_KEY}"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion-rag": "text-completion-rag":
"command": "command":
@ -236,14 +236,14 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "-o" - "-o"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"vectorize": "vectorize":
"command": "command":
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"volumes": "volumes":
"cassandra": {} "cassandra": {}

30
docker-compose-mix.yaml
View file

@ -15,14 +15,14 @@
- "4000" - "4000"
- "--chunk-overlap" - "--chunk-overlap"
- "120" - "120"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"embeddings": "embeddings":
"command": "command":
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"etcd": "etcd":
"command": "command":
@ -64,7 +64,7 @@
- "non-persistent://tg/request/prompt-rag" - "non-persistent://tg/request/prompt-rag"
- "--prompt-response-queue" - "--prompt-response-queue"
- "non-persistent://tg/response/prompt-rag-response" - "non-persistent://tg/response/prompt-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"init-pulsar": "init-pulsar":
"command": "command":
@ -81,14 +81,14 @@
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"kg-extract-relationships": "kg-extract-relationships":
"command": "command":
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"milvus": "milvus":
"command": "command":
@ -126,7 +126,7 @@
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prometheus": "prometheus":
"image": "docker.io/prom/prometheus:v2.53.1" "image": "docker.io/prom/prometheus:v2.53.1"
@ -145,7 +145,7 @@
- "non-persistent://tg/request/text-completion" - "non-persistent://tg/request/text-completion"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-response" - "non-persistent://tg/response/text-completion-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prompt-rag": "prompt-rag":
"command": "command":
@ -160,7 +160,7 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"pulsar": "pulsar":
"command": "bin/pulsar standalone" "command": "bin/pulsar standalone"
@ -187,7 +187,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"query-triples": "query-triples":
"command": "command":
@ -196,7 +196,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-graph-embeddings": "store-graph-embeddings":
"command": "command":
@ -205,7 +205,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-triples": "store-triples":
"command": "command":
@ -214,7 +214,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion": "text-completion":
"command": "command":
@ -225,7 +225,7 @@
- "${COHERE_KEY}" - "${COHERE_KEY}"
- "-m" - "-m"
- "c4ai-aya-23-35b" - "c4ai-aya-23-35b"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion-rag": "text-completion-rag":
"command": "command":
@ -240,14 +240,14 @@
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
- "-m" - "-m"
- "c4ai-aya-23-8b" - "c4ai-aya-23-8b"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"vectorize": "vectorize":
"command": "command":
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"volumes": "volumes":
"cassandra": {} "cassandra": {}

30
docker-compose-ollama.yaml
View file

@ -11,14 +11,14 @@
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"embeddings": "embeddings":
"command": "command":
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"etcd": "etcd":
"command": "command":
@ -60,7 +60,7 @@
- "non-persistent://tg/request/prompt-rag" - "non-persistent://tg/request/prompt-rag"
- "--prompt-response-queue" - "--prompt-response-queue"
- "non-persistent://tg/response/prompt-rag-response" - "non-persistent://tg/response/prompt-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"init-pulsar": "init-pulsar":
"command": "command":
@ -77,14 +77,14 @@
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"kg-extract-relationships": "kg-extract-relationships":
"command": "command":
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"milvus": "milvus":
"command": "command":
@ -122,7 +122,7 @@
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prometheus": "prometheus":
"image": "docker.io/prom/prometheus:v2.53.1" "image": "docker.io/prom/prometheus:v2.53.1"
@ -141,7 +141,7 @@
- "non-persistent://tg/request/text-completion" - "non-persistent://tg/request/text-completion"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-response" - "non-persistent://tg/response/text-completion-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prompt-rag": "prompt-rag":
"command": "command":
@ -156,7 +156,7 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"pulsar": "pulsar":
"command": "bin/pulsar standalone" "command": "bin/pulsar standalone"
@ -183,7 +183,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"query-triples": "query-triples":
"command": "command":
@ -192,7 +192,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-graph-embeddings": "store-graph-embeddings":
"command": "command":
@ -201,7 +201,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-triples": "store-triples":
"command": "command":
@ -210,7 +210,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion": "text-completion":
"command": "command":
@ -219,7 +219,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-r" - "-r"
- "${OLLAMA_HOST}" - "${OLLAMA_HOST}"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion-rag": "text-completion-rag":
"command": "command":
@ -232,14 +232,14 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "-o" - "-o"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"vectorize": "vectorize":
"command": "command":
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"volumes": "volumes":
"cassandra": {} "cassandra": {}

30
docker-compose-openai-neo4j.yaml
View file

@ -4,14 +4,14 @@
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"embeddings": "embeddings":
"command": "command":
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"etcd": "etcd":
"command": "command":
@ -53,7 +53,7 @@
- "non-persistent://tg/request/prompt-rag" - "non-persistent://tg/request/prompt-rag"
- "--prompt-response-queue" - "--prompt-response-queue"
- "non-persistent://tg/response/prompt-rag-response" - "non-persistent://tg/response/prompt-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"init-pulsar": "init-pulsar":
"command": "command":
@ -70,14 +70,14 @@
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"kg-extract-relationships": "kg-extract-relationships":
"command": "command":
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"milvus": "milvus":
"command": "command":
@ -125,7 +125,7 @@
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prometheus": "prometheus":
"image": "docker.io/prom/prometheus:v2.53.1" "image": "docker.io/prom/prometheus:v2.53.1"
@ -144,7 +144,7 @@
- "non-persistent://tg/request/text-completion" - "non-persistent://tg/request/text-completion"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-response" - "non-persistent://tg/response/text-completion-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prompt-rag": "prompt-rag":
"command": "command":
@ -159,7 +159,7 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"pulsar": "pulsar":
"command": "bin/pulsar standalone" "command": "bin/pulsar standalone"
@ -186,7 +186,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"query-triples": "query-triples":
"command": "command":
@ -195,7 +195,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "bolt://neo4j:7687" - "bolt://neo4j:7687"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-graph-embeddings": "store-graph-embeddings":
"command": "command":
@ -204,7 +204,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-triples": "store-triples":
"command": "command":
@ -213,7 +213,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "bolt://neo4j:7687" - "bolt://neo4j:7687"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion": "text-completion":
"command": "command":
@ -222,7 +222,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-k" - "-k"
- "${OPENAI_KEY}" - "${OPENAI_KEY}"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion-rag": "text-completion-rag":
"command": "command":
@ -235,14 +235,14 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "-o" - "-o"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"vectorize": "vectorize":
"command": "command":
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"volumes": "volumes":
"etcd": {} "etcd": {}

30
docker-compose-openai.yaml
View file

@ -11,14 +11,14 @@
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"embeddings": "embeddings":
"command": "command":
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"etcd": "etcd":
"command": "command":
@ -60,7 +60,7 @@
- "non-persistent://tg/request/prompt-rag" - "non-persistent://tg/request/prompt-rag"
- "--prompt-response-queue" - "--prompt-response-queue"
- "non-persistent://tg/response/prompt-rag-response" - "non-persistent://tg/response/prompt-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"init-pulsar": "init-pulsar":
"command": "command":
@ -77,14 +77,14 @@
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"kg-extract-relationships": "kg-extract-relationships":
"command": "command":
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"milvus": "milvus":
"command": "command":
@ -122,7 +122,7 @@
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prometheus": "prometheus":
"image": "docker.io/prom/prometheus:v2.53.1" "image": "docker.io/prom/prometheus:v2.53.1"
@ -141,7 +141,7 @@
- "non-persistent://tg/request/text-completion" - "non-persistent://tg/request/text-completion"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-response" - "non-persistent://tg/response/text-completion-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prompt-rag": "prompt-rag":
"command": "command":
@ -156,7 +156,7 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"pulsar": "pulsar":
"command": "bin/pulsar standalone" "command": "bin/pulsar standalone"
@ -183,7 +183,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"query-triples": "query-triples":
"command": "command":
@ -192,7 +192,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-graph-embeddings": "store-graph-embeddings":
"command": "command":
@ -201,7 +201,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-triples": "store-triples":
"command": "command":
@ -210,7 +210,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion": "text-completion":
"command": "command":
@ -219,7 +219,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-k" - "-k"
- "${OPENAI_KEY}" - "${OPENAI_KEY}"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion-rag": "text-completion-rag":
"command": "command":
@ -232,14 +232,14 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "-o" - "-o"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"vectorize": "vectorize":
"command": "command":
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"volumes": "volumes":
"cassandra": {} "cassandra": {}

30
docker-compose-vertexai.yaml
View file

@ -11,14 +11,14 @@
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"embeddings": "embeddings":
"command": "command":
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"etcd": "etcd":
"command": "command":
@ -60,7 +60,7 @@
- "non-persistent://tg/request/prompt-rag" - "non-persistent://tg/request/prompt-rag"
- "--prompt-response-queue" - "--prompt-response-queue"
- "non-persistent://tg/response/prompt-rag-response" - "non-persistent://tg/response/prompt-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"init-pulsar": "init-pulsar":
"command": "command":
@ -77,14 +77,14 @@
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"kg-extract-relationships": "kg-extract-relationships":
"command": "command":
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"milvus": "milvus":
"command": "command":
@ -122,7 +122,7 @@
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prometheus": "prometheus":
"image": "docker.io/prom/prometheus:v2.53.1" "image": "docker.io/prom/prometheus:v2.53.1"
@ -141,7 +141,7 @@
- "non-persistent://tg/request/text-completion" - "non-persistent://tg/request/text-completion"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-response" - "non-persistent://tg/response/text-completion-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"prompt-rag": "prompt-rag":
"command": "command":
@ -156,7 +156,7 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "--text-completion-response-queue" - "--text-completion-response-queue"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"pulsar": "pulsar":
"command": "bin/pulsar standalone" "command": "bin/pulsar standalone"
@ -183,7 +183,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"query-triples": "query-triples":
"command": "command":
@ -192,7 +192,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-graph-embeddings": "store-graph-embeddings":
"command": "command":
@ -201,7 +201,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
- "http://milvus:19530" - "http://milvus:19530"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"store-triples": "store-triples":
"command": "command":
@ -210,7 +210,7 @@
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
- "cassandra" - "cassandra"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"text-completion": "text-completion":
"command": "command":
@ -221,7 +221,7 @@
- "/vertexai/private.json" - "/vertexai/private.json"
- "-r" - "-r"
- "us-west1" - "us-west1"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"volumes": "volumes":
- "./vertexai:/vertexai" - "./vertexai:/vertexai"
@ -238,7 +238,7 @@
- "non-persistent://tg/request/text-completion-rag" - "non-persistent://tg/request/text-completion-rag"
- "-o" - "-o"
- "non-persistent://tg/response/text-completion-rag-response" - "non-persistent://tg/response/text-completion-rag-response"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"volumes": "volumes":
- "./vertexai:/vertexai" - "./vertexai:/vertexai"
@ -247,7 +247,7 @@
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
"image": "docker.io/trustgraph/trustgraph-flow:0.6.6" "image": "docker.io/trustgraph/trustgraph-flow:0.6.8"
"restart": "on-failure:100" "restart": "on-failure:100"
"volumes": "volumes":
"cassandra": {} "cassandra": {}

6
scripts/chunker-token Executable file
View file

@ -0,0 +1,6 @@
#!/usr/bin/env python3
from trustgraph.chunking.token import run
run()

4
setup.py
View file

@ -4,7 +4,7 @@ import os
with open("README.md", "r") as fh: with open("README.md", "r") as fh:
long_description = fh.read() long_description = fh.read()
version = "0.6.7" version = "0.6.8"
setuptools.setup( setuptools.setup(
name="trustgraph", name="trustgraph",
@ -49,9 +49,11 @@ setuptools.setup(
"boto3", "boto3",
"openai", "openai",
"neo4j", "neo4j",
"tiktoken",
], ],
scripts=[ scripts=[
"scripts/chunker-recursive", "scripts/chunker-recursive",
"scripts/chunker-token",
"scripts/concat-parquet", "scripts/concat-parquet",
"scripts/dump-parquet", "scripts/dump-parquet",
"scripts/embeddings-hf", "scripts/embeddings-hf",

2
trustgraph/chunking/recursive/chunker.py
View file

@ -69,7 +69,7 @@ class Processor(ConsumerProducer):
self.send(r) self.send(r)
print("Done.", flush=True) print("Done.", flush=True)
@staticmethod @staticmethod
def add_args(parser): def add_args(parser):

3
trustgraph/chunking/token/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from . chunker import *

7
trustgraph/chunking/token/__main__.py Normal file
View file

@ -0,0 +1,7 @@
#!/usr/bin/env python3
from . chunker import run
if __name__ == '__main__':
run()

97
trustgraph/chunking/token/chunker.py Executable file
View file

@ -0,0 +1,97 @@
"""
Simple decoder, accepts text documents on input, outputs chunks from the
as text as separate output objects.
"""
from langchain_text_splitters import TokenTextSplitter
from ... schema import TextDocument, Chunk, Source
from ... schema import text_ingest_queue, chunk_ingest_queue
from ... log_level import LogLevel
from ... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
default_input_queue = text_ingest_queue
default_output_queue = chunk_ingest_queue
default_subscriber = module
class Processor(ConsumerProducer):
def __init__(self, **params):
input_queue = params.get("input_queue", default_input_queue)
output_queue = params.get("output_queue", default_output_queue)
subscriber = params.get("subscriber", default_subscriber)
chunk_size = params.get("chunk_size", 250)
chunk_overlap = params.get("chunk_overlap", 15)
super(Processor, self).__init__(
**params | {
"input_queue": input_queue,
"output_queue": output_queue,
"subscriber": subscriber,
"input_schema": TextDocument,
"output_schema": Chunk,
}
)
self.text_splitter = TokenTextSplitter(
encoding_name="cl100k_base",
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
)
def handle(self, msg):
v = msg.value()
print(f"Chunking {v.source.id}...", flush=True)
texts = self.text_splitter.create_documents(
[v.text.decode("utf-8")]
)
for ix, chunk in enumerate(texts):
id = v.source.id + "-c" + str(ix)
r = Chunk(
source=Source(
source=v.source.source,
id=id,
title=v.source.title
),
chunk=chunk.page_content.encode("utf-8"),
)
self.send(r)
print("Done.", flush=True)
@staticmethod
def add_args(parser):
ConsumerProducer.add_args(
parser, default_input_queue, default_subscriber,
default_output_queue,
)
parser.add_argument(
'-z', '--chunk-size',
type=int,
default=250,
help=f'Chunk size (default: 250)'
)
parser.add_argument(
'-v', '--chunk-overlap',
type=int,
default=15,
help=f'Chunk overlap (default: 15)'
)
def run():
Processor.start(module, __doc__)