Template generation of docker compose files

Makefile

@@ -1,6 +1,6 @@
 
 # VERSION=$(shell git describe | sed 's/^v//')
-VERSION=0.5.9
+VERSION=0.6.0
 
 all: container
 
@@ -28,4 +28,14 @@ clean:
 	rm -rf wheels/
 
 set-version:
-	sed -i 's/trustgraph-flow:[0-9]*\.[0-9]*\.[0-9]*/trustgraph-flow:'${VERSION}'/' docker-compose*.yaml
+# 	sed -i 's/trustgraph-flow:[0-9]*\.[0-9]*\.[0-9]*/trustgraph-flow:'${VERSION}'/' docker-compose*.yaml
+	echo '"${VERSION}"' > templates/version.jsonnet
+
+TEMPLATES=azure bedrock claude cohere mix ollama openai vertexai
+DCS=$(foreach template,${TEMPLATES},${template:%=docker-compose-%.yaml})
+
+update-templates: set-version ${DCS}
+
+docker-compose-%.yaml: templates/docker-compose-%.jsonnet templates/version.jsonnet
+	jsonnet -S ${@:docker-compose-%.yaml=templates/docker-compose-%.jsonnet} > $@
+

docker-compose-azure.yaml

@@ -1,241 +1,230 @@
-
-volumes:
-  cassandra:
-  pulsar-conf:
-  pulsar-data:
-  etcd:
-  minio-data:
-  milvus:
-  prometheus-data:
-  grafana-storage:
-
-services:
-
-  cassandra:
-    image: docker.io/cassandra:4.1.5
-    ports:
-      - "9042:9042"
-    volumes:
-      - "cassandra:/var/lib/cassandra"
-    restart: on-failure:100
-
-  pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command: bin/pulsar standalone
-    ports:
-      - "6650:6650"
-      - "8080:8080"
-    volumes:
-      - "pulsar-conf:/pulsar/conf"
-      - "pulsar-data:/pulsar/data"
-    restart: on-failure:100
-
-  init-pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command:
-      - "sh"
-      - "-c"
-      - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
-    depends_on:
-      pulsar:
-        condition: service_started
-    restart: on-failure:100
-
-  pulsar-manager:
-    image: docker.io/apachepulsar/pulsar-manager:v0.3.0
-    ports:
-      - "9527:9527"
-      - "7750:7750"
-    environment:
-      SPRING_CONFIGURATION_FILE: /pulsar-manager/pulsar-manager/application.properties
-    restart: on-failure:100
-
-  etcd:
-    image: quay.io/coreos/etcd:v3.5.5
-    command:
-      - "etcd"
-      - "-advertise-client-urls=http://127.0.0.1:2379"
-      - "-listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--data-dir"
-      - "/etcd"
-    environment:
-      ETCD_AUTO_COMPACTION_MODE: revision
-      ETCD_AUTO_COMPACTION_RETENTION: "1000"
-      ETCD_QUOTA_BACKEND_BYTES: "4294967296"
-      ETCD_SNAPSHOT_COUNT: "50000"
-    ports:
-      - "2379:2379"
-    volumes:
-      - "etcd:/etcd"
-    restart: on-failure:100
-    
-  minio:
-    image: docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z
-    command:
-      - "minio"
-      - "server"
-      - "/minio_data"
-      - "--console-address"
-      - ":9001"
-    environment:
-      MINIO_ROOT_USER: minioadmin
-      MINIO_ROOT_PASSWORD: minioadmin
-    ports:
-      - "9001:9001"
-    volumes:
-      - "minio-data:/minio_data"
-    restart: on-failure:100
-
-  milvus:
-    image: docker.io/milvusdb/milvus:v2.4.5
-    command:
-      - "milvus"
-      - "run"
-      - "standalone"
-    environment:
-      ETCD_ENDPOINTS: etcd:2379
-      MINIO_ADDRESS: minio:9000
-    ports:
-      - "9091:9091"
-      - "19530:19530"
-    volumes:
-      - "milvus:/var/lib/milvus"
-    restart: on-failure:100
-
-  prometheus:
-    image: docker.io/prom/prometheus:v2.53.1
-    ports:
-      - "9090:9090"
-    volumes:
-      - "./prometheus:/etc/prometheus"
-      - "prometheus-data:/prometheus"
-    restart: on-failure:100
-
-  grafana:
-    image: docker.io/grafana/grafana:10.0.0
-    ports:
-      - "3000:3000"
-    volumes:
-      - "grafana-storage:/var/lib/grafana"
-      - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
-      - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
-      - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
-    environment:
-#       GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
-#       GF_AUTH_ANONYMOUS_ENABLED: true
-#       GF_ORG_ROLE: Admin
-      GF_ORG_NAME: trustgraph.ai
-#     GF_SERVER_ROOT_URL: https://example.com
-    restart: on-failure:100
-
-  pdf-decoder:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "pdf-decoder"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  chunker:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "chunker-recursive"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  vectorize:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-vectorize"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-hf"
-      - "-p"
-      - "pulsar://pulsar:6650"
-#       - "-m"
-#       - "mixedbread-ai/mxbai-embed-large-v1"
-    restart: on-failure:100
-    
-  kg-extract-definitions:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-definitions"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  kg-extract-relationships:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-relationships"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  store-graph-embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "ge-write-milvus"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-t"
-      - "http://milvus:19530"
-    restart: on-failure:100
-    
-  store-triples:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "triples-write-cassandra"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-g"
-      - "cassandra"
-    restart: on-failure:100
-    
-  text-completion:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-azure"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-k"
-      - ${AZURE_TOKEN}
-      - "-e"
-      - ${AZURE_ENDPOINT}
-    restart: on-failure:100
-
-  text-completion-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-azure"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-k"
-      - ${AZURE_TOKEN}
-      - "-e"
-      - ${AZURE_ENDPOINT}
-      - "-i"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "-o"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
-
-  graph-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "graph-rag"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "--text-completion-request-queue"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "--text-completion-response-queue"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
-
+"services":
+  "cassandra":
+    "image": "docker.io/cassandra:4.1.5"
+    "ports":
+    - "9042:9042"
+    "restart": "on-failure:100"
+    "volumes":
+    - "cassandra:/var/lib/cassandra"
+  "chunker":
+    "command":
+    - "chunker-recursive"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "embeddings":
+    "command":
+    - "embeddings-hf"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "etcd":
+    "command":
+    - "etcd"
+    - "-advertise-client-urls=http://127.0.0.1:2379"
+    - "-listen-client-urls"
+    - "http://0.0.0.0:2379"
+    - "--data-dir"
+    - "/etcd"
+    "environment":
+      "ETCD_AUTO_COMPACTION_MODE": "revision"
+      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
+      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
+      "ETCD_SNAPSHOT_COUNT": "50000"
+    "image": "quay.io/coreos/etcd:v3.5.5"
+    "ports":
+    - "2379:2379"
+    "restart": "on-failure:100"
+    "volumes":
+    - "etcd:/etcd"
+  "grafana":
+    "environment":
+      "GF_ORG_NAME": "trustgraph.ai"
+    "image": "docker.io/grafana/grafana:10.0.0"
+    "ports":
+    - "3000:3000"
+    "restart": "on-failure:100"
+    "volumes":
+    - "grafana-storage:/var/lib/grafana"
+    - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
+    - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
+    - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
+  "graph-rag":
+    "command":
+    - "graph-rag"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "--text-completion-request-queue"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "--text-completion-response-queue"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "init-pulsar":
+    "command":
+    - "sh"
+    - "-c"
+    - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
+    "depends_on":
+      "pulsar":
+        "condition": "service_started"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "restart": "on-failure:100"
+  "kg-extract-definitions":
+    "command":
+    - "kg-extract-definitions"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "kg-extract-relationships":
+    "command":
+    - "kg-extract-relationships"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "milvus":
+    "command":
+    - "milvus"
+    - "run"
+    - "standalone"
+    "environment":
+      "ETCD_ENDPOINTS": "etcd:2379"
+      "MINIO_ADDRESS": "minio:9000"
+    "image": "docker.io/milvusdb/milvus:v2.4.5"
+    "ports":
+    - "9091:9091"
+    - "19530:19530"
+    "restart": "on-failure:100"
+    "volumes":
+    - "milvus:/var/lib/milvus"
+  "minio":
+    "command":
+    - "minio"
+    - "server"
+    - "/minio_data"
+    - "--console-address"
+    - ":9001"
+    "environment":
+      "MINIO_ROOT_PASSWORD": "minioadmin"
+      "MINIO_ROOT_USER": "minioadmin"
+    "image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z"
+    "ports":
+    - "9001:9001"
+    "restart": "on-failure:100"
+    "volumes":
+    - "minio-data:/minio_data"
+  "pdf-decoder":
+    "command":
+    - "pdf-decoder"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "prometheus":
+    "image": "docker.io/prom/prometheus:v2.53.1"
+    "ports":
+    - "9090:9090"
+    "restart": "on-failure:100"
+    "volumes":
+    - "./prometheus:/etc/prometheus"
+    - "prometheus-data:/prometheus"
+  "pulsar":
+    "command": "bin/pulsar standalone"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "ports":
+    - "6650:6650"
+    - "8080:8080"
+    "restart": "on-failure:100"
+    "volumes":
+    - "pulsar-conf:/pulsar/conf"
+    - "pulsar-data:/pulsar/data"
+  "pulsar-manager":
+    "environment":
+      "SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties"
+    "image": "docker.io/apachepulsar/pulsar-manager:v0.3.0"
+    "ports":
+    - "9527:9527"
+    - "7750:7750"
+    "restart": "on-failure:100"
+  "query-graph-embeddings":
+    "command":
+    - "ge-query-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "query-triples":
+    "command":
+    - "triples-query-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-graph-embeddings":
+    "command":
+    - "ge-write-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-triples":
+    "command":
+    - "triples-write-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion":
+    "command":
+    - "text-completion-azure"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-k"
+    - "${AZURE_TOKEN}"
+    - "-e"
+    - "${AZURE_ENDPOINT}"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion-rag":
+    "command":
+    - "text-completion-azure"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-k"
+    - "${AZURE_TOKEN}"
+    - "-e"
+    - "${AZURE_ENDPOINT}"
+    - "-i"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "-o"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "vectorize":
+    "command":
+    - "embeddings-vectorize"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+"volumes":
+  "cassandra": {}
+  "etcd": {}
+  "grafana-storage": {}
+  "milvus": {}
+  "minio-data": {}
+  "prometheus-data": {}
+  "pulsar-conf": {}
+  "pulsar-data": {}

docker-compose-bedrock.yaml

@@ -1,267 +1,238 @@
-
-volumes:
-  cassandra:
-  pulsar-conf:
-  pulsar-data:
-  etcd:
-  minio-data:
-  milvus:
-  prometheus-data:
-  grafana-storage:
-
-services:
-
-  cassandra:
-    image: docker.io/cassandra:4.1.5
-    ports:
-      - "9042:9042"
-    volumes:
-      - "cassandra:/var/lib/cassandra"
-    restart: on-failure:100
-
-  pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command: bin/pulsar standalone
-    ports:
-      - "6650:6650"
-      - "8080:8080"
-    volumes:
-      - "pulsar-conf:/pulsar/conf"
-      - "pulsar-data:/pulsar/data"
-    restart: on-failure:100
-
-  init-pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command:
-      - "sh"
-      - "-c"
-      - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
-    depends_on:
-      pulsar:
-        condition: service_started
-    restart: on-failure:100
-
-  pulsar-manager:
-    image: docker.io/apachepulsar/pulsar-manager:v0.3.0
-    ports:
-      - "9527:9527"
-      - "7750:7750"
-    environment:
-      SPRING_CONFIGURATION_FILE: /pulsar-manager/pulsar-manager/application.properties
-    restart: on-failure:100
-
-  etcd:
-    image: quay.io/coreos/etcd:v3.5.5
-    command:
-      - "etcd"
-      - "-advertise-client-urls=http://127.0.0.1:2379"
-      - "-listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--data-dir"
-      - "/etcd"
-    environment:
-      ETCD_AUTO_COMPACTION_MODE: revision
-      ETCD_AUTO_COMPACTION_RETENTION: "1000"
-      ETCD_QUOTA_BACKEND_BYTES: "4294967296"
-      ETCD_SNAPSHOT_COUNT: "50000"
-    ports:
-      - "2379:2379"
-    volumes:
-      - "etcd:/etcd"
-    restart: on-failure:100
-    
-  minio:
-    image: docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z
-    command:
-      - "minio"
-      - "server"
-      - "/minio_data"
-      - "--console-address"
-      - ":9001"
-    environment:
-      MINIO_ROOT_USER: minioadmin
-      MINIO_ROOT_PASSWORD: minioadmin
-    ports:
-      - "9001:9001"
-    volumes:
-      - "minio-data:/minio_data"
-    restart: on-failure:100
-
-  milvus:
-    image: docker.io/milvusdb/milvus:v2.4.5
-    command:
-      - "milvus"
-      - "run"
-      - "standalone"
-    environment:
-      ETCD_ENDPOINTS: etcd:2379
-      MINIO_ADDRESS: minio:9000
-    ports:
-      - "9091:9091"
-      - "19530:19530"
-    volumes:
-      - "milvus:/var/lib/milvus"
-    restart: on-failure:100
-
-  prometheus:
-    image: docker.io/prom/prometheus:v2.53.1
-    ports:
-      - "9090:9090"
-    volumes:
-      - "./prometheus:/etc/prometheus"
-      - "prometheus-data:/prometheus"
-    restart: on-failure:100
-
-  grafana:
-    image: docker.io/grafana/grafana:10.0.0
-    ports:
-      - "3000:3000"
-    volumes:
-      - "grafana-storage:/var/lib/grafana"
-      - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
-      - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
-      - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
-    environment:
-#       GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
-#       GF_AUTH_ANONYMOUS_ENABLED: true
-#       GF_ORG_ROLE: Admin
-      GF_ORG_NAME: trustgraph.ai
-#     GF_SERVER_ROOT_URL: https://example.com
-    restart: on-failure:100
-
-  pdf-decoder:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "pdf-decoder"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  chunker:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "chunker-recursive"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "--chunk-size"
-      - "2000"
-      - "--chunk-overlap"
-      - "100"
-    restart: on-failure:100
-
-  vectorize:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-vectorize"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-hf"
-      - "-p"
-      - "pulsar://pulsar:6650"
-#       - "-m"
-#       - "mixedbread-ai/mxbai-embed-large-v1"
-    restart: on-failure:100
-    
-  kg-extract-definitions:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-definitions"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  kg-extract-relationships:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-relationships"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  store-graph-embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "ge-write-milvus"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-t"
-      - "http://milvus:19530"
-    restart: on-failure:100
-    
-  store-triples:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "triples-write-cassandra"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-g"
-      - "cassandra"
-    restart: on-failure:100
-    
-  text-completion:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-bedrock"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-z"
-      - "${AWS_ID_KEY}"
-      - "-k"
-      - "${AWS_SECRET_KEY}"
-      - "-r"
-      - "us-west-2"
-    restart: on-failure:100
-    
-  text-completion-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-bedrock"
-      - "-p"
-      - "pulsar://pulsar:6650"
-  #    - "-m"
-  #    - "mistral.mistral-large-2407-v1:0"
-      - "-z"
-      - "${AWS_ID_KEY}"
-      - "-k"
-      - "${AWS_SECRET_KEY}"
-      - "-r"
-      - "us-west-2" 
-      - "-i"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "-o"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
-  
-  #text-completion-rag:
-  #  image: docker.io/trustgraph/trustgraph-flow:0.5.9
-  #  command:
-  #    - "text-completion-ollama"
-  #    - "-p"
-  #    - "pulsar://pulsar:6650"
-  #    - "-r"
-  #    - "http://${OLLAMA_HOST}:11434/"
-  #    - "-i"
-  #    - "non-persistent://tg/request/text-completion-rag"
-  #    - "-o"
-  #    - "non-persistent://tg/response/text-completion-rag-response"
-  #    - "-m"
-  #    - "gemma2:2b"
-  #  restart: on-failure:100
-
-  graph-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "graph-rag"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "--text-completion-request-queue"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "--text-completion-response-queue"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
-
+"services":
+  "cassandra":
+    "image": "docker.io/cassandra:4.1.5"
+    "ports":
+    - "9042:9042"
+    "restart": "on-failure:100"
+    "volumes":
+    - "cassandra:/var/lib/cassandra"
+  "chunker":
+    "command":
+    - "chunker-recursive"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "--chunk-size"
+    - "2000"
+    - "--chunk-overlap"
+    - "100"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "embeddings":
+    "command":
+    - "embeddings-hf"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "etcd":
+    "command":
+    - "etcd"
+    - "-advertise-client-urls=http://127.0.0.1:2379"
+    - "-listen-client-urls"
+    - "http://0.0.0.0:2379"
+    - "--data-dir"
+    - "/etcd"
+    "environment":
+      "ETCD_AUTO_COMPACTION_MODE": "revision"
+      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
+      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
+      "ETCD_SNAPSHOT_COUNT": "50000"
+    "image": "quay.io/coreos/etcd:v3.5.5"
+    "ports":
+    - "2379:2379"
+    "restart": "on-failure:100"
+    "volumes":
+    - "etcd:/etcd"
+  "grafana":
+    "environment":
+      "GF_ORG_NAME": "trustgraph.ai"
+    "image": "docker.io/grafana/grafana:10.0.0"
+    "ports":
+    - "3000:3000"
+    "restart": "on-failure:100"
+    "volumes":
+    - "grafana-storage:/var/lib/grafana"
+    - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
+    - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
+    - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
+  "graph-rag":
+    "command":
+    - "graph-rag"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "--text-completion-request-queue"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "--text-completion-response-queue"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "init-pulsar":
+    "command":
+    - "sh"
+    - "-c"
+    - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
+    "depends_on":
+      "pulsar":
+        "condition": "service_started"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "restart": "on-failure:100"
+  "kg-extract-definitions":
+    "command":
+    - "kg-extract-definitions"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "kg-extract-relationships":
+    "command":
+    - "kg-extract-relationships"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "milvus":
+    "command":
+    - "milvus"
+    - "run"
+    - "standalone"
+    "environment":
+      "ETCD_ENDPOINTS": "etcd:2379"
+      "MINIO_ADDRESS": "minio:9000"
+    "image": "docker.io/milvusdb/milvus:v2.4.5"
+    "ports":
+    - "9091:9091"
+    - "19530:19530"
+    "restart": "on-failure:100"
+    "volumes":
+    - "milvus:/var/lib/milvus"
+  "minio":
+    "command":
+    - "minio"
+    - "server"
+    - "/minio_data"
+    - "--console-address"
+    - ":9001"
+    "environment":
+      "MINIO_ROOT_PASSWORD": "minioadmin"
+      "MINIO_ROOT_USER": "minioadmin"
+    "image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z"
+    "ports":
+    - "9001:9001"
+    "restart": "on-failure:100"
+    "volumes":
+    - "minio-data:/minio_data"
+  "pdf-decoder":
+    "command":
+    - "pdf-decoder"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "prometheus":
+    "image": "docker.io/prom/prometheus:v2.53.1"
+    "ports":
+    - "9090:9090"
+    "restart": "on-failure:100"
+    "volumes":
+    - "./prometheus:/etc/prometheus"
+    - "prometheus-data:/prometheus"
+  "pulsar":
+    "command": "bin/pulsar standalone"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "ports":
+    - "6650:6650"
+    - "8080:8080"
+    "restart": "on-failure:100"
+    "volumes":
+    - "pulsar-conf:/pulsar/conf"
+    - "pulsar-data:/pulsar/data"
+  "pulsar-manager":
+    "environment":
+      "SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties"
+    "image": "docker.io/apachepulsar/pulsar-manager:v0.3.0"
+    "ports":
+    - "9527:9527"
+    - "7750:7750"
+    "restart": "on-failure:100"
+  "query-graph-embeddings":
+    "command":
+    - "ge-query-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "query-triples":
+    "command":
+    - "triples-query-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-graph-embeddings":
+    "command":
+    - "ge-write-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-triples":
+    "command":
+    - "triples-write-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion":
+    "command":
+    - "text-completion-bedrock"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-z"
+    - "${AWS_ID_KEY}"
+    - "-k"
+    - "${AWS_SECRET_KEY}"
+    - "-r"
+    - "us-west-2"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion-rag":
+    "command":
+    - "text-completion-bedrock"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-z"
+    - "${AWS_ID_KEY}"
+    - "-k"
+    - "${AWS_SECRET_KEY}"
+    - "-r"
+    - "us-west-2"
+    - "-i"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "-o"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "vectorize":
+    "command":
+    - "embeddings-vectorize"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+"volumes":
+  "cassandra": {}
+  "etcd": {}
+  "grafana-storage": {}
+  "milvus": {}
+  "minio-data": {}
+  "prometheus-data": {}
+  "pulsar-conf": {}
+  "pulsar-data": {}

docker-compose-claude.yaml

@@ -1,236 +1,226 @@
-
-volumes:
-  cassandra:
-  pulsar-conf:
-  pulsar-data:
-  etcd:
-  minio-data:
-  milvus:
-  prometheus-data:
-  grafana-storage:
-
-services:
-
-  cassandra:
-    image: docker.io/cassandra:4.1.5
-    ports:
-      - "9042:9042"
-    volumes:
-      - "cassandra:/var/lib/cassandra"
-    restart: on-failure:100
-
-  pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command: bin/pulsar standalone
-    ports:
-      - "6650:6650"
-      - "8080:8080"
-    volumes:
-      - "pulsar-conf:/pulsar/conf"
-      - "pulsar-data:/pulsar/data"
-    restart: on-failure:100
-
-  init-pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command:
-      - "sh"
-      - "-c"
-      - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
-    depends_on:
-      pulsar:
-        condition: service_started
-    restart: on-failure:100
-
-  pulsar-manager:
-    image: docker.io/apachepulsar/pulsar-manager:v0.3.0
-    ports:
-      - "9527:9527"
-      - "7750:7750"
-    environment:
-      SPRING_CONFIGURATION_FILE: /pulsar-manager/pulsar-manager/application.properties
-    restart: on-failure:100
-
-  etcd:
-    image: quay.io/coreos/etcd:v3.5.5
-    command:
-      - "etcd"
-      - "-advertise-client-urls=http://127.0.0.1:2379"
-      - "-listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--data-dir"
-      - "/etcd"
-    environment:
-      ETCD_AUTO_COMPACTION_MODE: revision
-      ETCD_AUTO_COMPACTION_RETENTION: "1000"
-      ETCD_QUOTA_BACKEND_BYTES: "4294967296"
-      ETCD_SNAPSHOT_COUNT: "50000"
-    ports:
-      - "2379:2379"
-    volumes:
-      - "etcd:/etcd"
-    restart: on-failure:100
-    
-  minio:
-    image: docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z
-    command:
-      - "minio"
-      - "server"
-      - "/minio_data"
-      - "--console-address"
-      - ":9001"
-    environment:
-      MINIO_ROOT_USER: minioadmin
-      MINIO_ROOT_PASSWORD: minioadmin
-    ports:
-      - "9001:9001"
-    volumes:
-      - "minio-data:/minio_data"
-    restart: on-failure:100
-
-  milvus:
-    image: docker.io/milvusdb/milvus:v2.4.5
-    command:
-      - "milvus"
-      - "run"
-      - "standalone"
-    environment:
-      ETCD_ENDPOINTS: etcd:2379
-      MINIO_ADDRESS: minio:9000
-    ports:
-      - "9091:9091"
-      - "19530:19530"
-    volumes:
-      - "milvus:/var/lib/milvus"
-    restart: on-failure:100
-
-  prometheus:
-    image: docker.io/prom/prometheus:v2.53.1
-    ports:
-      - "9090:9090"
-    volumes:
-      - "./prometheus:/etc/prometheus"
-      - "prometheus-data:/prometheus"
-    restart: on-failure:100
-
-  grafana:
-    image: docker.io/grafana/grafana:10.0.0
-    ports:
-      - "3000:3000"
-    volumes:
-      - "grafana-storage:/var/lib/grafana"
-      - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
-      - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
-      - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
-    environment:
-#       GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
-#       GF_AUTH_ANONYMOUS_ENABLED: true
-#       GF_ORG_ROLE: Admin
-      GF_ORG_NAME: trustgraph.ai
-#     GF_SERVER_ROOT_URL: https://example.com
-    restart: on-failure:100
-
-  pdf-decoder:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "pdf-decoder"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  chunker:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "chunker-recursive"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  vectorize:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-vectorize"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-hf"
-      - "-p"
-      - "pulsar://pulsar:6650"
-#       - "-m"
-#       - "mixedbread-ai/mxbai-embed-large-v1"
-    restart: on-failure:100
-    
-  kg-extract-definitions:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-definitions"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  kg-extract-relationships:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-relationships"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  store-graph-embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "ge-write-milvus"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-t"
-      - "http://milvus:19530"
-    restart: on-failure:100
-    
-  store-triples:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "triples-write-cassandra"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-g"
-      - "cassandra"
-    restart: on-failure:100
-    
-  text-completion:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-claude"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-k"
-      - ${CLAUDE_KEY}
-    restart: on-failure:100
-
-  text-completion-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-ollama"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-k"
-      - ${CLAUDE_KEY}
-      - "-i"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "-o"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
-
-  graph-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "graph-rag"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "--text-completion-request-queue"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "--text-completion-response-queue"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
+"services":
+  "cassandra":
+    "image": "docker.io/cassandra:4.1.5"
+    "ports":
+    - "9042:9042"
+    "restart": "on-failure:100"
+    "volumes":
+    - "cassandra:/var/lib/cassandra"
+  "chunker":
+    "command":
+    - "chunker-recursive"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "embeddings":
+    "command":
+    - "embeddings-hf"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "etcd":
+    "command":
+    - "etcd"
+    - "-advertise-client-urls=http://127.0.0.1:2379"
+    - "-listen-client-urls"
+    - "http://0.0.0.0:2379"
+    - "--data-dir"
+    - "/etcd"
+    "environment":
+      "ETCD_AUTO_COMPACTION_MODE": "revision"
+      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
+      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
+      "ETCD_SNAPSHOT_COUNT": "50000"
+    "image": "quay.io/coreos/etcd:v3.5.5"
+    "ports":
+    - "2379:2379"
+    "restart": "on-failure:100"
+    "volumes":
+    - "etcd:/etcd"
+  "grafana":
+    "environment":
+      "GF_ORG_NAME": "trustgraph.ai"
+    "image": "docker.io/grafana/grafana:10.0.0"
+    "ports":
+    - "3000:3000"
+    "restart": "on-failure:100"
+    "volumes":
+    - "grafana-storage:/var/lib/grafana"
+    - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
+    - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
+    - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
+  "graph-rag":
+    "command":
+    - "graph-rag"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "--text-completion-request-queue"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "--text-completion-response-queue"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "init-pulsar":
+    "command":
+    - "sh"
+    - "-c"
+    - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
+    "depends_on":
+      "pulsar":
+        "condition": "service_started"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "restart": "on-failure:100"
+  "kg-extract-definitions":
+    "command":
+    - "kg-extract-definitions"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "kg-extract-relationships":
+    "command":
+    - "kg-extract-relationships"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "milvus":
+    "command":
+    - "milvus"
+    - "run"
+    - "standalone"
+    "environment":
+      "ETCD_ENDPOINTS": "etcd:2379"
+      "MINIO_ADDRESS": "minio:9000"
+    "image": "docker.io/milvusdb/milvus:v2.4.5"
+    "ports":
+    - "9091:9091"
+    - "19530:19530"
+    "restart": "on-failure:100"
+    "volumes":
+    - "milvus:/var/lib/milvus"
+  "minio":
+    "command":
+    - "minio"
+    - "server"
+    - "/minio_data"
+    - "--console-address"
+    - ":9001"
+    "environment":
+      "MINIO_ROOT_PASSWORD": "minioadmin"
+      "MINIO_ROOT_USER": "minioadmin"
+    "image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z"
+    "ports":
+    - "9001:9001"
+    "restart": "on-failure:100"
+    "volumes":
+    - "minio-data:/minio_data"
+  "pdf-decoder":
+    "command":
+    - "pdf-decoder"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "prometheus":
+    "image": "docker.io/prom/prometheus:v2.53.1"
+    "ports":
+    - "9090:9090"
+    "restart": "on-failure:100"
+    "volumes":
+    - "./prometheus:/etc/prometheus"
+    - "prometheus-data:/prometheus"
+  "pulsar":
+    "command": "bin/pulsar standalone"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "ports":
+    - "6650:6650"
+    - "8080:8080"
+    "restart": "on-failure:100"
+    "volumes":
+    - "pulsar-conf:/pulsar/conf"
+    - "pulsar-data:/pulsar/data"
+  "pulsar-manager":
+    "environment":
+      "SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties"
+    "image": "docker.io/apachepulsar/pulsar-manager:v0.3.0"
+    "ports":
+    - "9527:9527"
+    - "7750:7750"
+    "restart": "on-failure:100"
+  "query-graph-embeddings":
+    "command":
+    - "ge-query-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "query-triples":
+    "command":
+    - "triples-query-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-graph-embeddings":
+    "command":
+    - "ge-write-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-triples":
+    "command":
+    - "triples-write-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion":
+    "command":
+    - "text-completion-claude"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-k"
+    - "${CLAUDE_KEY}"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion-rag":
+    "command":
+    - "text-completion-claude"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-k"
+    - "${CLAUDE_KEY}"
+    - "-i"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "-o"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "vectorize":
+    "command":
+    - "embeddings-vectorize"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+"volumes":
+  "cassandra": {}
+  "etcd": {}
+  "grafana-storage": {}
+  "milvus": {}
+  "minio-data": {}
+  "prometheus-data": {}
+  "pulsar-conf": {}
+  "pulsar-data": {}

docker-compose-cohere.yaml

@@ -1,223 +1,230 @@
-
-volumes:
-  cassandra:
-  pulsar-conf:
-  pulsar-data:
-  etcd:
-  minio-data:
-  milvus:
-  prometheus-data:
-  grafana-storage:
-
-services:
-
-  cassandra:
-    image: docker.io/cassandra:4.1.5
-    ports:
-      - "9042:9042"
-    volumes:
-      - "cassandra:/var/lib/cassandra"
-    restart: on-failure:100
-
-  pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command: bin/pulsar standalone
-    ports:
-      - "6650:6650"
-      - "8080:8080"
-    volumes:
-      - "pulsar-conf:/pulsar/conf"
-      - "pulsar-data:/pulsar/data"
-    restart: on-failure:100
-
-  init-pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command:
-      - "sh"
-      - "-c"
-      - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
-    depends_on:
-      pulsar:
-        condition: service_started
-    restart: on-failure:100
-
-  pulsar-manager:
-    image: docker.io/apachepulsar/pulsar-manager:v0.3.0
-    ports:
-      - "9527:9527"
-      - "7750:7750"
-    environment:
-      SPRING_CONFIGURATION_FILE: /pulsar-manager/pulsar-manager/application.properties
-    restart: on-failure:100
-
-  etcd:
-    image: quay.io/coreos/etcd:v3.5.5
-    command:
-      - "etcd"
-      - "-advertise-client-urls=http://127.0.0.1:2379"
-      - "-listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--data-dir"
-      - "/etcd"
-    environment:
-      ETCD_AUTO_COMPACTION_MODE: revision
-      ETCD_AUTO_COMPACTION_RETENTION: "1000"
-      ETCD_QUOTA_BACKEND_BYTES: "4294967296"
-      ETCD_SNAPSHOT_COUNT: "50000"
-    ports:
-      - "2379:2379"
-    volumes:
-      - "etcd:/etcd"
-    restart: on-failure:100
-    
-  minio:
-    image: docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z
-    command:
-      - "minio"
-      - "server"
-      - "/minio_data"
-      - "--console-address"
-      - ":9001"
-    environment:
-      MINIO_ROOT_USER: minioadmin
-      MINIO_ROOT_PASSWORD: minioadmin
-    ports:
-      - "9001:9001"
-    volumes:
-      - "minio-data:/minio_data"
-    restart: on-failure:100
-
-  milvus:
-    image: docker.io/milvusdb/milvus:v2.4.5
-    command:
-      - "milvus"
-      - "run"
-      - "standalone"
-    environment:
-      ETCD_ENDPOINTS: etcd:2379
-      MINIO_ADDRESS: minio:9000
-    ports:
-      - "9091:9091"
-      - "19530:19530"
-    volumes:
-      - "milvus:/var/lib/milvus"
-    restart: on-failure:100
-
-  prometheus:
-    image: docker.io/prom/prometheus:v2.53.1
-    ports:
-      - "9090:9090"
-    volumes:
-      - "./prometheus:/etc/prometheus"
-      - "prometheus-data:/prometheus"
-    restart: on-failure:100
-
-  grafana:
-    image: docker.io/grafana/grafana:10.0.0
-    ports:
-      - "3000:3000"
-    volumes:
-      - "grafana-storage:/var/lib/grafana"
-      - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
-      - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
-      - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
-    environment:
-#       GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
-#       GF_AUTH_ANONYMOUS_ENABLED: true
-#       GF_ORG_ROLE: Admin
-      GF_ORG_NAME: trustgraph.ai
-#     GF_SERVER_ROOT_URL: https://example.com
-    restart: on-failure:100
-
-  pdf-decoder:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "pdf-decoder"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  chunker:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "chunker-recursive"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "--chunk-size"
-      - "1000"
-      - "--chunk-overlap"
-      - "50"
-    restart: on-failure:100
-
-  vectorize:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-vectorize"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-hf"
-      - "-p"
-      - "pulsar://pulsar:6650"
-#       - "-m"
-#       - "mixedbread-ai/mxbai-embed-large-v1"
-    restart: on-failure:100
-    
-  kg-extract-definitions:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-definitions"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  kg-extract-relationships:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-relationships"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  store-graph-embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "ge-write-milvus"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-t"
-      - "http://milvus:19530"
-    restart: on-failure:100
-    
-  store-triples:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "triples-write-cassandra"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-g"
-      - "cassandra"
-    restart: on-failure:100
-    
-  text-completion:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-cohere"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-k"
-      - ${COHERE_KEY}
-    restart: on-failure:100
-
-  graph-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "graph-rag"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
+"services":
+  "cassandra":
+    "image": "docker.io/cassandra:4.1.5"
+    "ports":
+    - "9042:9042"
+    "restart": "on-failure:100"
+    "volumes":
+    - "cassandra:/var/lib/cassandra"
+  "chunker":
+    "command":
+    - "chunker-recursive"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "--chunk-size"
+    - "1000"
+    - "--chunk-overlap"
+    - "50"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "embeddings":
+    "command":
+    - "embeddings-hf"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "etcd":
+    "command":
+    - "etcd"
+    - "-advertise-client-urls=http://127.0.0.1:2379"
+    - "-listen-client-urls"
+    - "http://0.0.0.0:2379"
+    - "--data-dir"
+    - "/etcd"
+    "environment":
+      "ETCD_AUTO_COMPACTION_MODE": "revision"
+      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
+      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
+      "ETCD_SNAPSHOT_COUNT": "50000"
+    "image": "quay.io/coreos/etcd:v3.5.5"
+    "ports":
+    - "2379:2379"
+    "restart": "on-failure:100"
+    "volumes":
+    - "etcd:/etcd"
+  "grafana":
+    "environment":
+      "GF_ORG_NAME": "trustgraph.ai"
+    "image": "docker.io/grafana/grafana:10.0.0"
+    "ports":
+    - "3000:3000"
+    "restart": "on-failure:100"
+    "volumes":
+    - "grafana-storage:/var/lib/grafana"
+    - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
+    - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
+    - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
+  "graph-rag":
+    "command":
+    - "graph-rag"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "--text-completion-request-queue"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "--text-completion-response-queue"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "init-pulsar":
+    "command":
+    - "sh"
+    - "-c"
+    - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
+    "depends_on":
+      "pulsar":
+        "condition": "service_started"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "restart": "on-failure:100"
+  "kg-extract-definitions":
+    "command":
+    - "kg-extract-definitions"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "kg-extract-relationships":
+    "command":
+    - "kg-extract-relationships"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "milvus":
+    "command":
+    - "milvus"
+    - "run"
+    - "standalone"
+    "environment":
+      "ETCD_ENDPOINTS": "etcd:2379"
+      "MINIO_ADDRESS": "minio:9000"
+    "image": "docker.io/milvusdb/milvus:v2.4.5"
+    "ports":
+    - "9091:9091"
+    - "19530:19530"
+    "restart": "on-failure:100"
+    "volumes":
+    - "milvus:/var/lib/milvus"
+  "minio":
+    "command":
+    - "minio"
+    - "server"
+    - "/minio_data"
+    - "--console-address"
+    - ":9001"
+    "environment":
+      "MINIO_ROOT_PASSWORD": "minioadmin"
+      "MINIO_ROOT_USER": "minioadmin"
+    "image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z"
+    "ports":
+    - "9001:9001"
+    "restart": "on-failure:100"
+    "volumes":
+    - "minio-data:/minio_data"
+  "pdf-decoder":
+    "command":
+    - "pdf-decoder"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "prometheus":
+    "image": "docker.io/prom/prometheus:v2.53.1"
+    "ports":
+    - "9090:9090"
+    "restart": "on-failure:100"
+    "volumes":
+    - "./prometheus:/etc/prometheus"
+    - "prometheus-data:/prometheus"
+  "pulsar":
+    "command": "bin/pulsar standalone"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "ports":
+    - "6650:6650"
+    - "8080:8080"
+    "restart": "on-failure:100"
+    "volumes":
+    - "pulsar-conf:/pulsar/conf"
+    - "pulsar-data:/pulsar/data"
+  "pulsar-manager":
+    "environment":
+      "SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties"
+    "image": "docker.io/apachepulsar/pulsar-manager:v0.3.0"
+    "ports":
+    - "9527:9527"
+    - "7750:7750"
+    "restart": "on-failure:100"
+  "query-graph-embeddings":
+    "command":
+    - "ge-query-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "query-triples":
+    "command":
+    - "triples-query-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-graph-embeddings":
+    "command":
+    - "ge-write-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-triples":
+    "command":
+    - "triples-write-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion":
+    "command":
+    - "text-completion-cohere"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-k"
+    - "${COHERE_KEY}"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion-rag":
+    "command":
+    - "text-completion-cohere"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-k"
+    - "${COHERE_KEY}"
+    - "-i"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "-o"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "vectorize":
+    "command":
+    - "embeddings-vectorize"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+"volumes":
+  "cassandra": {}
+  "etcd": {}
+  "grafana-storage": {}
+  "milvus": {}
+  "minio-data": {}
+  "prometheus-data": {}
+  "pulsar-conf": {}
+  "pulsar-data": {}

docker-compose-mix.yaml

@@ -1,261 +1,234 @@
-
-volumes:
-  cassandra:
-  pulsar-conf:
-  pulsar-data:
-  etcd:
-  minio-data:
-  milvus:
-  prometheus-data:
-  grafana-storage:
-
-services:
-
-  cassandra:
-    image: docker.io/cassandra:4.1.5
-    ports:
-      - "9042:9042"
-    volumes:
-      - "cassandra:/var/lib/cassandra"
-    restart: on-failure:100
-
-  pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command: bin/pulsar standalone
-    ports:
-      - "6650:6650"
-      - "8080:8080"
-    volumes:
-      - "pulsar-conf:/pulsar/conf"
-      - "pulsar-data:/pulsar/data"
-    restart: on-failure:100
-
-  init-pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command:
-      - "sh"
-      - "-c"
-      - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
-    depends_on:
-      pulsar:
-        condition: service_started
-    restart: on-failure:100
-
-  pulsar-manager:
-    image: docker.io/apachepulsar/pulsar-manager:v0.3.0
-    ports:
-      - "9527:9527"
-      - "7750:7750"
-    environment:
-      SPRING_CONFIGURATION_FILE: /pulsar-manager/pulsar-manager/application.properties
-    restart: on-failure:100
-
-  etcd:
-    image: quay.io/coreos/etcd:v3.5.5
-    command:
-      - "etcd"
-      - "-advertise-client-urls=http://127.0.0.1:2379"
-      - "-listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--data-dir"
-      - "/etcd"
-    environment:
-      ETCD_AUTO_COMPACTION_MODE: revision
-      ETCD_AUTO_COMPACTION_RETENTION: "1000"
-      ETCD_QUOTA_BACKEND_BYTES: "4294967296"
-      ETCD_SNAPSHOT_COUNT: "50000"
-    ports:
-      - "2379:2379"
-    volumes:
-      - "etcd:/etcd"
-    restart: on-failure:100
-    
-  minio:
-    image: docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z
-    command:
-      - "minio"
-      - "server"
-      - "/minio_data"
-      - "--console-address"
-      - ":9001"
-    environment:
-      MINIO_ROOT_USER: minioadmin
-      MINIO_ROOT_PASSWORD: minioadmin
-    ports:
-      - "9001:9001"
-    volumes:
-      - "minio-data:/minio_data"
-    restart: on-failure:100
-
-  milvus:
-    image: docker.io/milvusdb/milvus:v2.4.5
-    command:
-      - "milvus"
-      - "run"
-      - "standalone"
-    environment:
-      ETCD_ENDPOINTS: etcd:2379
-      MINIO_ADDRESS: minio:9000
-    ports:
-      - "9091:9091"
-      - "19530:19530"
-    volumes:
-      - "milvus:/var/lib/milvus"
-    restart: on-failure:100
-
-  prometheus:
-    image: docker.io/prom/prometheus:v2.53.1
-    ports:
-      - "9090:9090"
-    volumes:
-      - "./prometheus:/etc/prometheus"
-      - "prometheus-data:/prometheus"
-    restart: on-failure:100
-
-  grafana:
-    image: docker.io/grafana/grafana:10.0.0
-    ports:
-      - "3000:3000"
-    volumes:
-      - "grafana-storage:/var/lib/grafana"
-      - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
-      - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
-      - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
-    environment:
-#       GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
-#       GF_AUTH_ANONYMOUS_ENABLED: true
-#       GF_ORG_ROLE: Admin
-      GF_ORG_NAME: trustgraph.ai
-#     GF_SERVER_ROOT_URL: https://example.com
-    restart: on-failure:100
-
-  pdf-decoder:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "pdf-decoder"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  chunker:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "chunker-recursive"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "--chunk-size"
-      - "4000"
-      - "--chunk-overlap"
-      - "120"
-    restart: on-failure:100
-
-  vectorize:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-vectorize"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-hf"
-      - "-p"
-      - "pulsar://pulsar:6650"
-#       - "-m"
-#       - "mixedbread-ai/mxbai-embed-large-v1"
-    restart: on-failure:100
-    
-  kg-extract-definitions:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-definitions"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  kg-extract-relationships:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-relationships"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  store-graph-embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "ge-write-milvus"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-t"
-      - "http://milvus:19530"
-    restart: on-failure:100
-    
-  store-triples:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "triples-write-cassandra"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-g"
-      - "cassandra"
-    restart: on-failure:100
-    
-  text-completion:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-cohere"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-k"
-      - ${COHERE_KEY}
-      - "-m"
-      - "c4ai-aya-23-35b"
-    restart: on-failure:100
-    
-  text-completion-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-cohere"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-k"
-      - ${COHERE_KEY}
-      - "-m"
-      - "c4ai-aya-23-8b"
-      - "-i"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "-o"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
-  
-  #text-completion-rag:
-  #  image: docker.io/trustgraph/trustgraph-flow:0.5.9
-  #  command:
-  #    - "text-completion-ollama"
-  #    - "-p"
-  #    - "pulsar://pulsar:6650"
-  #    - "-r"
-  #    - "http://${OLLAMA_HOST}:11434/"
-  #    - "-i"
-  #    - "non-persistent://tg/request/text-completion-rag"
-  #    - "-o"
-  #    - "non-persistent://tg/response/text-completion-rag-response"
-  #    - "-m"
-  #    - "gemma2:2b"
-  #  restart: on-failure:100
-
-  graph-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "graph-rag"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "--text-completion-request-queue"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "--text-completion-response-queue"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
-
+"services":
+  "cassandra":
+    "image": "docker.io/cassandra:4.1.5"
+    "ports":
+    - "9042:9042"
+    "restart": "on-failure:100"
+    "volumes":
+    - "cassandra:/var/lib/cassandra"
+  "chunker":
+    "command":
+    - "chunker-recursive"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "--chunk-size"
+    - "4000"
+    - "--chunk-overlap"
+    - "120"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "embeddings":
+    "command":
+    - "embeddings-hf"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "etcd":
+    "command":
+    - "etcd"
+    - "-advertise-client-urls=http://127.0.0.1:2379"
+    - "-listen-client-urls"
+    - "http://0.0.0.0:2379"
+    - "--data-dir"
+    - "/etcd"
+    "environment":
+      "ETCD_AUTO_COMPACTION_MODE": "revision"
+      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
+      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
+      "ETCD_SNAPSHOT_COUNT": "50000"
+    "image": "quay.io/coreos/etcd:v3.5.5"
+    "ports":
+    - "2379:2379"
+    "restart": "on-failure:100"
+    "volumes":
+    - "etcd:/etcd"
+  "grafana":
+    "environment":
+      "GF_ORG_NAME": "trustgraph.ai"
+    "image": "docker.io/grafana/grafana:10.0.0"
+    "ports":
+    - "3000:3000"
+    "restart": "on-failure:100"
+    "volumes":
+    - "grafana-storage:/var/lib/grafana"
+    - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
+    - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
+    - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
+  "graph-rag":
+    "command":
+    - "graph-rag"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "--text-completion-request-queue"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "--text-completion-response-queue"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "init-pulsar":
+    "command":
+    - "sh"
+    - "-c"
+    - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
+    "depends_on":
+      "pulsar":
+        "condition": "service_started"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "restart": "on-failure:100"
+  "kg-extract-definitions":
+    "command":
+    - "kg-extract-definitions"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "kg-extract-relationships":
+    "command":
+    - "kg-extract-relationships"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "milvus":
+    "command":
+    - "milvus"
+    - "run"
+    - "standalone"
+    "environment":
+      "ETCD_ENDPOINTS": "etcd:2379"
+      "MINIO_ADDRESS": "minio:9000"
+    "image": "docker.io/milvusdb/milvus:v2.4.5"
+    "ports":
+    - "9091:9091"
+    - "19530:19530"
+    "restart": "on-failure:100"
+    "volumes":
+    - "milvus:/var/lib/milvus"
+  "minio":
+    "command":
+    - "minio"
+    - "server"
+    - "/minio_data"
+    - "--console-address"
+    - ":9001"
+    "environment":
+      "MINIO_ROOT_PASSWORD": "minioadmin"
+      "MINIO_ROOT_USER": "minioadmin"
+    "image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z"
+    "ports":
+    - "9001:9001"
+    "restart": "on-failure:100"
+    "volumes":
+    - "minio-data:/minio_data"
+  "pdf-decoder":
+    "command":
+    - "pdf-decoder"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "prometheus":
+    "image": "docker.io/prom/prometheus:v2.53.1"
+    "ports":
+    - "9090:9090"
+    "restart": "on-failure:100"
+    "volumes":
+    - "./prometheus:/etc/prometheus"
+    - "prometheus-data:/prometheus"
+  "pulsar":
+    "command": "bin/pulsar standalone"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "ports":
+    - "6650:6650"
+    - "8080:8080"
+    "restart": "on-failure:100"
+    "volumes":
+    - "pulsar-conf:/pulsar/conf"
+    - "pulsar-data:/pulsar/data"
+  "pulsar-manager":
+    "environment":
+      "SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties"
+    "image": "docker.io/apachepulsar/pulsar-manager:v0.3.0"
+    "ports":
+    - "9527:9527"
+    - "7750:7750"
+    "restart": "on-failure:100"
+  "query-graph-embeddings":
+    "command":
+    - "ge-query-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "query-triples":
+    "command":
+    - "triples-query-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-graph-embeddings":
+    "command":
+    - "ge-write-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-triples":
+    "command":
+    - "triples-write-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion":
+    "command":
+    - "text-completion-cohere"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-k"
+    - "${COHERE_KEY}"
+    - "-m"
+    - "c4ai-aya-23-35b"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion-rag":
+    "command":
+    - "text-completion-cohere"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-k"
+    - "${COHERE_KEY}"
+    - "-i"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "-o"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    - "-m"
+    - "c4ai-aya-23-8b"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "vectorize":
+    "command":
+    - "embeddings-vectorize"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+"volumes":
+  "cassandra": {}
+  "etcd": {}
+  "grafana-storage": {}
+  "milvus": {}
+  "minio-data": {}
+  "prometheus-data": {}
+  "pulsar-conf": {}
+  "pulsar-data": {}

docker-compose-ollama.yaml

@@ -1,241 +1,226 @@
-
-volumes:
-  cassandra:
-  pulsar-conf:
-  pulsar-data:
-  etcd:
-  minio-data:
-  milvus:
-  prometheus-data:
-  grafana-storage:
-
-services:
-
-  cassandra:
-    image: docker.io/cassandra:4.1.5
-    ports:
-      - "9042:9042"
-    volumes:
-      - "cassandra:/var/lib/cassandra"
-    restart: on-failure:100
-
-  pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command: bin/pulsar standalone
-    ports:
-      - "6650:6650"
-      - "8080:8080"
-    volumes:
-      - "pulsar-conf:/pulsar/conf"
-      - "pulsar-data:/pulsar/data"
-    restart: on-failure:100
-
-  init-pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command:
-      - "sh"
-      - "-c"
-      - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
-    depends_on:
-      pulsar:
-        condition: service_started
-    restart: on-failure:100
-
-  pulsar-manager:
-    image: docker.io/apachepulsar/pulsar-manager:v0.3.0
-    ports:
-      - "9527:9527"
-      - "7750:7750"
-    environment:
-      SPRING_CONFIGURATION_FILE: /pulsar-manager/pulsar-manager/application.properties
-    restart: on-failure:100
-
-  etcd:
-    image: quay.io/coreos/etcd:v3.5.5
-    command:
-      - "etcd"
-      - "-advertise-client-urls=http://127.0.0.1:2379"
-      - "-listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--data-dir"
-      - "/etcd"
-    environment:
-      ETCD_AUTO_COMPACTION_MODE: revision
-      ETCD_AUTO_COMPACTION_RETENTION: "1000"
-      ETCD_QUOTA_BACKEND_BYTES: "4294967296"
-      ETCD_SNAPSHOT_COUNT: "50000"
-    ports:
-      - "2379:2379"
-    volumes:
-      - "etcd:/etcd"
-    restart: on-failure:100
-    
-  minio:
-    image: docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z
-    command:
-      - "minio"
-      - "server"
-      - "/minio_data"
-      - "--console-address"
-      - ":9001"
-    environment:
-      MINIO_ROOT_USER: minioadmin
-      MINIO_ROOT_PASSWORD: minioadmin
-    ports:
-      - "9001:9001"
-    volumes:
-      - "minio-data:/minio_data"
-    restart: on-failure:100
-
-  milvus:
-    image: docker.io/milvusdb/milvus:v2.4.5
-    command:
-      - "milvus"
-      - "run"
-      - "standalone"
-    environment:
-      ETCD_ENDPOINTS: etcd:2379
-      MINIO_ADDRESS: minio:9000
-    ports:
-      - "9091:9091"
-      - "19530:19530"
-    volumes:
-      - "milvus:/var/lib/milvus"
-    restart: on-failure:100
-
-  prometheus:
-    image: docker.io/prom/prometheus:v2.53.1
-    ports:
-      - "9090:9090"
-    volumes:
-      - "./prometheus:/etc/prometheus"
-      - "prometheus-data:/prometheus"
-    restart: on-failure:100
-
-  grafana:
-    image: docker.io/grafana/grafana:10.0.0
-    ports:
-      - "3000:3000"
-    volumes:
-      - "grafana-storage:/var/lib/grafana"
-      - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
-      - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
-      - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
-    environment:
-#       GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
-#       GF_AUTH_ANONYMOUS_ENABLED: true
-#       GF_ORG_ROLE: Admin
-      GF_ORG_NAME: trustgraph.ai
-#     GF_SERVER_ROOT_URL: https://example.com
-    restart: on-failure:100
-
-  pdf-decoder:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "pdf-decoder"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  chunker:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "chunker-recursive"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  vectorize:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-vectorize"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-hf"
-      - "-p"
-      - "pulsar://pulsar:6650"
-#       - "-m"
-#       - "mixedbread-ai/mxbai-embed-large-v1"
-    restart: on-failure:100
-    
-  kg-extract-definitions:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-definitions"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  kg-extract-relationships:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-relationships"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  store-graph-embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "ge-write-milvus"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-t"
-      - "http://milvus:19530"
-    restart: on-failure:100
-    
-  store-triples:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "triples-write-cassandra"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-g"
-      - "cassandra"
-    restart: on-failure:100
-    
-  text-completion:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-ollama"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-r"
-      - "http://${OLLAMA_HOST}:11434/"
-#       - "-m"
-#       - "llama3.1:8b"
-    restart: on-failure:100
-    
-  text-completion-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-ollama"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-r"
-      - "http://${OLLAMA_HOST}:11434/"
-      - "-i"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "-o"
-      - "non-persistent://tg/response/text-completion-rag-response"
-      - "-m"
-      - "llama3.1:8b"
-    restart: on-failure:100
-
-  graph-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "graph-rag"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "--text-completion-request-queue"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "--text-completion-response-queue"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
-
+"services":
+  "cassandra":
+    "image": "docker.io/cassandra:4.1.5"
+    "ports":
+    - "9042:9042"
+    "restart": "on-failure:100"
+    "volumes":
+    - "cassandra:/var/lib/cassandra"
+  "chunker":
+    "command":
+    - "chunker-recursive"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "embeddings":
+    "command":
+    - "embeddings-hf"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "etcd":
+    "command":
+    - "etcd"
+    - "-advertise-client-urls=http://127.0.0.1:2379"
+    - "-listen-client-urls"
+    - "http://0.0.0.0:2379"
+    - "--data-dir"
+    - "/etcd"
+    "environment":
+      "ETCD_AUTO_COMPACTION_MODE": "revision"
+      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
+      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
+      "ETCD_SNAPSHOT_COUNT": "50000"
+    "image": "quay.io/coreos/etcd:v3.5.5"
+    "ports":
+    - "2379:2379"
+    "restart": "on-failure:100"
+    "volumes":
+    - "etcd:/etcd"
+  "grafana":
+    "environment":
+      "GF_ORG_NAME": "trustgraph.ai"
+    "image": "docker.io/grafana/grafana:10.0.0"
+    "ports":
+    - "3000:3000"
+    "restart": "on-failure:100"
+    "volumes":
+    - "grafana-storage:/var/lib/grafana"
+    - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
+    - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
+    - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
+  "graph-rag":
+    "command":
+    - "graph-rag"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "--text-completion-request-queue"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "--text-completion-response-queue"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "init-pulsar":
+    "command":
+    - "sh"
+    - "-c"
+    - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
+    "depends_on":
+      "pulsar":
+        "condition": "service_started"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "restart": "on-failure:100"
+  "kg-extract-definitions":
+    "command":
+    - "kg-extract-definitions"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "kg-extract-relationships":
+    "command":
+    - "kg-extract-relationships"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "milvus":
+    "command":
+    - "milvus"
+    - "run"
+    - "standalone"
+    "environment":
+      "ETCD_ENDPOINTS": "etcd:2379"
+      "MINIO_ADDRESS": "minio:9000"
+    "image": "docker.io/milvusdb/milvus:v2.4.5"
+    "ports":
+    - "9091:9091"
+    - "19530:19530"
+    "restart": "on-failure:100"
+    "volumes":
+    - "milvus:/var/lib/milvus"
+  "minio":
+    "command":
+    - "minio"
+    - "server"
+    - "/minio_data"
+    - "--console-address"
+    - ":9001"
+    "environment":
+      "MINIO_ROOT_PASSWORD": "minioadmin"
+      "MINIO_ROOT_USER": "minioadmin"
+    "image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z"
+    "ports":
+    - "9001:9001"
+    "restart": "on-failure:100"
+    "volumes":
+    - "minio-data:/minio_data"
+  "pdf-decoder":
+    "command":
+    - "pdf-decoder"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "prometheus":
+    "image": "docker.io/prom/prometheus:v2.53.1"
+    "ports":
+    - "9090:9090"
+    "restart": "on-failure:100"
+    "volumes":
+    - "./prometheus:/etc/prometheus"
+    - "prometheus-data:/prometheus"
+  "pulsar":
+    "command": "bin/pulsar standalone"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "ports":
+    - "6650:6650"
+    - "8080:8080"
+    "restart": "on-failure:100"
+    "volumes":
+    - "pulsar-conf:/pulsar/conf"
+    - "pulsar-data:/pulsar/data"
+  "pulsar-manager":
+    "environment":
+      "SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties"
+    "image": "docker.io/apachepulsar/pulsar-manager:v0.3.0"
+    "ports":
+    - "9527:9527"
+    - "7750:7750"
+    "restart": "on-failure:100"
+  "query-graph-embeddings":
+    "command":
+    - "ge-query-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "query-triples":
+    "command":
+    - "triples-query-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-graph-embeddings":
+    "command":
+    - "ge-write-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-triples":
+    "command":
+    - "triples-write-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion":
+    "command":
+    - "text-completion-ollama"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-r"
+    - "${OLLAMA_HOST}"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion-rag":
+    "command":
+    - "text-completion-ollama"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-r"
+    - "${OLLAMA_HOST}"
+    - "-i"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "-o"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "vectorize":
+    "command":
+    - "embeddings-vectorize"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+"volumes":
+  "cassandra": {}
+  "etcd": {}
+  "grafana-storage": {}
+  "milvus": {}
+  "minio-data": {}
+  "prometheus-data": {}
+  "pulsar-conf": {}
+  "pulsar-data": {}

docker-compose-openai.yaml

@@ -1,236 +1,226 @@
-
-volumes:
-  cassandra:
-  pulsar-conf:
-  pulsar-data:
-  etcd:
-  minio-data:
-  milvus:
-  prometheus-data:
-  grafana-storage:
-
-services:
-
-  cassandra:
-    image: docker.io/cassandra:4.1.5
-    ports:
-      - "9042:9042"
-    volumes:
-      - "cassandra:/var/lib/cassandra"
-    restart: on-failure:100
-
-  pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command: bin/pulsar standalone
-    ports:
-      - "6650:6650"
-      - "8080:8080"
-    volumes:
-      - "pulsar-conf:/pulsar/conf"
-      - "pulsar-data:/pulsar/data"
-    restart: on-failure:100
-
-  init-pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command:
-      - "sh"
-      - "-c"
-      - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
-    depends_on:
-      pulsar:
-        condition: service_started
-    restart: on-failure:100
-
-  pulsar-manager:
-    image: docker.io/apachepulsar/pulsar-manager:v0.3.0
-    ports:
-      - "9527:9527"
-      - "7750:7750"
-    environment:
-      SPRING_CONFIGURATION_FILE: /pulsar-manager/pulsar-manager/application.properties
-    restart: on-failure:100
-
-  etcd:
-    image: quay.io/coreos/etcd:v3.5.5
-    command:
-      - "etcd"
-      - "-advertise-client-urls=http://127.0.0.1:2379"
-      - "-listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--data-dir"
-      - "/etcd"
-    environment:
-      ETCD_AUTO_COMPACTION_MODE: revision
-      ETCD_AUTO_COMPACTION_RETENTION: "1000"
-      ETCD_QUOTA_BACKEND_BYTES: "4294967296"
-      ETCD_SNAPSHOT_COUNT: "50000"
-    ports:
-      - "2379:2379"
-    volumes:
-      - "etcd:/etcd"
-    restart: on-failure:100
-    
-  minio:
-    image: docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z
-    command:
-      - "minio"
-      - "server"
-      - "/minio_data"
-      - "--console-address"
-      - ":9001"
-    environment:
-      MINIO_ROOT_USER: minioadmin
-      MINIO_ROOT_PASSWORD: minioadmin
-    ports:
-      - "9001:9001"
-    volumes:
-      - "minio-data:/minio_data"
-    restart: on-failure:100
-
-  milvus:
-    image: docker.io/milvusdb/milvus:v2.4.5
-    command:
-      - "milvus"
-      - "run"
-      - "standalone"
-    environment:
-      ETCD_ENDPOINTS: etcd:2379
-      MINIO_ADDRESS: minio:9000
-    ports:
-      - "9091:9091"
-      - "19530:19530"
-    volumes:
-      - "milvus:/var/lib/milvus"
-    restart: on-failure:100
-
-  prometheus:
-    image: docker.io/prom/prometheus:v2.53.1
-    ports:
-      - "9090:9090"
-    volumes:
-      - "./prometheus:/etc/prometheus"
-      - "prometheus-data:/prometheus"
-    restart: on-failure:100
-
-  grafana:
-    image: docker.io/grafana/grafana:10.0.0
-    ports:
-      - "3000:3000"
-    volumes:
-      - "grafana-storage:/var/lib/grafana"
-      - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
-      - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
-      - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
-    environment:
-#       GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
-#       GF_AUTH_ANONYMOUS_ENABLED: true
-#       GF_ORG_ROLE: Admin
-      GF_ORG_NAME: trustgraph.ai
-#     GF_SERVER_ROOT_URL: https://example.com
-    restart: on-failure:100
-
-  pdf-decoder:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "pdf-decoder"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  chunker:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "chunker-recursive"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  vectorize:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-vectorize"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-hf"
-      - "-p"
-      - "pulsar://pulsar:6650"
-#       - "-m"
-#       - "mixedbread-ai/mxbai-embed-large-v1"
-    restart: on-failure:100
-    
-  kg-extract-definitions:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-definitions"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  kg-extract-relationships:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-relationships"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  store-graph-embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "ge-write-milvus"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-t"
-      - "http://milvus:19530"
-    restart: on-failure:100
-    
-  store-triples:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "triples-write-cassandra"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-g"
-      - "cassandra"
-    restart: on-failure:100
-    
-  text-completion:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-openai"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-k"
-      - ${OPENAI_KEY}
-    restart: on-failure:100
-
-  text-completion-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-openai"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-k"
-      - ${OPENAI_KEY}
-      - "-i"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "-o"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
-
-  graph-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "graph-rag"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "--text-completion-request-queue"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "--text-completion-response-queue"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
+"services":
+  "cassandra":
+    "image": "docker.io/cassandra:4.1.5"
+    "ports":
+    - "9042:9042"
+    "restart": "on-failure:100"
+    "volumes":
+    - "cassandra:/var/lib/cassandra"
+  "chunker":
+    "command":
+    - "chunker-recursive"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "embeddings":
+    "command":
+    - "embeddings-hf"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "etcd":
+    "command":
+    - "etcd"
+    - "-advertise-client-urls=http://127.0.0.1:2379"
+    - "-listen-client-urls"
+    - "http://0.0.0.0:2379"
+    - "--data-dir"
+    - "/etcd"
+    "environment":
+      "ETCD_AUTO_COMPACTION_MODE": "revision"
+      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
+      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
+      "ETCD_SNAPSHOT_COUNT": "50000"
+    "image": "quay.io/coreos/etcd:v3.5.5"
+    "ports":
+    - "2379:2379"
+    "restart": "on-failure:100"
+    "volumes":
+    - "etcd:/etcd"
+  "grafana":
+    "environment":
+      "GF_ORG_NAME": "trustgraph.ai"
+    "image": "docker.io/grafana/grafana:10.0.0"
+    "ports":
+    - "3000:3000"
+    "restart": "on-failure:100"
+    "volumes":
+    - "grafana-storage:/var/lib/grafana"
+    - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
+    - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
+    - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
+  "graph-rag":
+    "command":
+    - "graph-rag"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "--text-completion-request-queue"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "--text-completion-response-queue"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "init-pulsar":
+    "command":
+    - "sh"
+    - "-c"
+    - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
+    "depends_on":
+      "pulsar":
+        "condition": "service_started"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "restart": "on-failure:100"
+  "kg-extract-definitions":
+    "command":
+    - "kg-extract-definitions"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "kg-extract-relationships":
+    "command":
+    - "kg-extract-relationships"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "milvus":
+    "command":
+    - "milvus"
+    - "run"
+    - "standalone"
+    "environment":
+      "ETCD_ENDPOINTS": "etcd:2379"
+      "MINIO_ADDRESS": "minio:9000"
+    "image": "docker.io/milvusdb/milvus:v2.4.5"
+    "ports":
+    - "9091:9091"
+    - "19530:19530"
+    "restart": "on-failure:100"
+    "volumes":
+    - "milvus:/var/lib/milvus"
+  "minio":
+    "command":
+    - "minio"
+    - "server"
+    - "/minio_data"
+    - "--console-address"
+    - ":9001"
+    "environment":
+      "MINIO_ROOT_PASSWORD": "minioadmin"
+      "MINIO_ROOT_USER": "minioadmin"
+    "image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z"
+    "ports":
+    - "9001:9001"
+    "restart": "on-failure:100"
+    "volumes":
+    - "minio-data:/minio_data"
+  "pdf-decoder":
+    "command":
+    - "pdf-decoder"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "prometheus":
+    "image": "docker.io/prom/prometheus:v2.53.1"
+    "ports":
+    - "9090:9090"
+    "restart": "on-failure:100"
+    "volumes":
+    - "./prometheus:/etc/prometheus"
+    - "prometheus-data:/prometheus"
+  "pulsar":
+    "command": "bin/pulsar standalone"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "ports":
+    - "6650:6650"
+    - "8080:8080"
+    "restart": "on-failure:100"
+    "volumes":
+    - "pulsar-conf:/pulsar/conf"
+    - "pulsar-data:/pulsar/data"
+  "pulsar-manager":
+    "environment":
+      "SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties"
+    "image": "docker.io/apachepulsar/pulsar-manager:v0.3.0"
+    "ports":
+    - "9527:9527"
+    - "7750:7750"
+    "restart": "on-failure:100"
+  "query-graph-embeddings":
+    "command":
+    - "ge-query-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "query-triples":
+    "command":
+    - "triples-query-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-graph-embeddings":
+    "command":
+    - "ge-write-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-triples":
+    "command":
+    - "triples-write-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion":
+    "command":
+    - "text-completion-openai"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-k"
+    - "${OPENAI_KEY}"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion-rag":
+    "command":
+    - "text-completion-openai"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-k"
+    - "${OPENAI_KEY}"
+    - "-i"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "-o"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "vectorize":
+    "command":
+    - "embeddings-vectorize"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+"volumes":
+  "cassandra": {}
+  "etcd": {}
+  "grafana-storage": {}
+  "milvus": {}
+  "minio-data": {}
+  "prometheus-data": {}
+  "pulsar-conf": {}
+  "pulsar-data": {}

docker-compose-stores.yaml

@@ -1,92 +0,0 @@
-
-volumes:
-  cassandra:
-  pulsar-conf:
-  pulsar-data:
-  etcd:
-  minio-data:
-  milvus:
-
-services:
-
-  cassandra:
-    image: docker.io/cassandra:4.1.5
-    ports:
-      - "9042:9042"
-    volumes:
-      - "cassandra:/var/lib/cassandra"
-    restart: on-failure:100
-
-  pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command: bin/pulsar standalone
-    ports:
-      - "6650:6650"
-      - "8080:8080"
-    volumes:
-      - "pulsar-conf:/pulsar/conf"
-      - "pulsar-data:/pulsar/data"
-    restart: on-failure:100
-
-  pulsar-manager:
-    image: docker.io/apachepulsar/pulsar-manager:v0.3.0
-    ports:
-      - "9527:9527"
-      - "7750:7750"
-    environment:
-      SPRING_CONFIGURATION_FILE: /pulsar-manager/pulsar-manager/application.properties
-    restart: on-failure:100
-
-  etcd:
-    image: quay.io/coreos/etcd:v3.5.5
-    command:
-      - "etcd"
-      - "-advertise-client-urls=http://127.0.0.1:2379"
-      - "-listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--data-dir"
-      - "/etcd"
-    environment:
-      ETCD_AUTO_COMPACTION_MODE: revision
-      ETCD_AUTO_COMPACTION_RETENTION: "1000"
-      ETCD_QUOTA_BACKEND_BYTES: "4294967296"
-      ETCD_SNAPSHOT_COUNT: "50000"
-    ports:
-      - "2379:2379"
-    volumes:
-      - "etcd:/etcd"
-    restart: on-failure:100
-    
-  minio:
-    image: docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z
-    command:
-      - "minio"
-      - "server"
-      - "/minio_data"
-      - "--console-address"
-      - ":9001"
-    environment:
-      MINIO_ROOT_USER: minioadmin
-      MINIO_ROOT_PASSWORD: minioadmin
-    ports:
-      - "9001:9001"
-    volumes:
-      - "minio-data:/minio_data"
-    restart: on-failure:100
-
-  milvus:
-    image: docker.io/milvusdb/milvus:v2.4.5
-    command:
-      - "milvus"
-      - "run"
-      - "standalone"
-    environment:
-      ETCD_ENDPOINTS: etcd:2379
-      MINIO_ADDRESS: minio:9000
-    ports:
-      - "9091:9091"
-      - "19530:19530"
-    volumes:
-      - "milvus:/var/lib/milvus"
-    restart: on-failure:100
-

docker-compose-vertexai.yaml

@@ -1,243 +1,234 @@
-
-volumes:
-  cassandra:
-  pulsar-conf:
-  pulsar-data:
-  etcd:
-  minio-data:
-  milvus:
-  prometheus-data:
-  grafana-storage:
-
-services:
-
-  cassandra:
-    image: docker.io/cassandra:4.1.5
-    ports:
-      - "9042:9042"
-    volumes:
-      - "cassandra:/var/lib/cassandra"
-    restart: on-failure:100
-
-  pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command: bin/pulsar standalone
-    ports:
-      - "6650:6650"
-      - "8080:8080"
-    volumes:
-      - "pulsar-conf:/pulsar/conf"
-      - "pulsar-data:/pulsar/data"
-    restart: on-failure:100
-
-  init-pulsar:
-    image: docker.io/apachepulsar/pulsar:3.3.0
-    command:
-      - "sh"
-      - "-c"
-      - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
-    depends_on:
-      pulsar:
-        condition: service_started
-    restart: on-failure:100
-
-  pulsar-manager:
-    image: docker.io/apachepulsar/pulsar-manager:v0.3.0
-    ports:
-      - "9527:9527"
-      - "7750:7750"
-    environment:
-      SPRING_CONFIGURATION_FILE: /pulsar-manager/pulsar-manager/application.properties
-    restart: on-failure:100
-
-  etcd:
-    image: quay.io/coreos/etcd:v3.5.5
-    command:
-      - "etcd"
-      - "-advertise-client-urls=http://127.0.0.1:2379"
-      - "-listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--data-dir"
-      - "/etcd"
-    environment:
-      ETCD_AUTO_COMPACTION_MODE: revision
-      ETCD_AUTO_COMPACTION_RETENTION: "1000"
-      ETCD_QUOTA_BACKEND_BYTES: "4294967296"
-      ETCD_SNAPSHOT_COUNT: "50000"
-    ports:
-      - "2379:2379"
-    volumes:
-      - "etcd:/etcd"
-    restart: on-failure:100
-    
-  minio:
-    image: docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z
-    command:
-      - "minio"
-      - "server"
-      - "/minio_data"
-      - "--console-address"
-      - ":9001"
-    environment:
-      MINIO_ROOT_USER: minioadmin
-      MINIO_ROOT_PASSWORD: minioadmin
-    ports:
-      - "9001:9001"
-    volumes:
-      - "minio-data:/minio_data"
-    restart: on-failure:100
-
-  milvus:
-    image: docker.io/milvusdb/milvus:v2.4.5
-    command:
-      - "milvus"
-      - "run"
-      - "standalone"
-    environment:
-      ETCD_ENDPOINTS: etcd:2379
-      MINIO_ADDRESS: minio:9000
-    ports:
-      - "9091:9091"
-      - "19530:19530"
-    volumes:
-      - "milvus:/var/lib/milvus"
-    restart: on-failure:100
-
-  prometheus:
-    image: docker.io/prom/prometheus:v2.53.1
-    ports:
-      - "9090:9090"
-    volumes:
-      - "./prometheus:/etc/prometheus"
-      - "prometheus-data:/prometheus"
-    restart: on-failure:100
-
-  grafana:
-    image: docker.io/grafana/grafana:10.0.0
-    ports:
-      - "3000:3000"
-    volumes:
-      - "grafana-storage:/var/lib/grafana"
-      - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
-      - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
-      - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
-    environment:
-#       GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
-#       GF_AUTH_ANONYMOUS_ENABLED: true
-#       GF_ORG_ROLE: Admin
-      GF_ORG_NAME: trustgraph.ai
-#     GF_SERVER_ROOT_URL: https://example.com
-    restart: on-failure:100
-
-  pdf-decoder:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "pdf-decoder"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  chunker:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "chunker-recursive"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-
-  vectorize:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-vectorize"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "embeddings-hf"
-      - "-p"
-      - "pulsar://pulsar:6650"
-#       - "-m"
-#       - "mixedbread-ai/mxbai-embed-large-v1"
-    restart: on-failure:100
-    
-  kg-extract-definitions:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-definitions"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  kg-extract-relationships:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "kg-extract-relationships"
-      - "-p"
-      - "pulsar://pulsar:6650"
-    restart: on-failure:100
-    
-  store-graph-embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "ge-write-milvus"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-t"
-      - "http://milvus:19530"
-    restart: on-failure:100
-    
-  store-triples:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "triples-write-cassandra"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-g"
-      - "cassandra"
-    restart: on-failure:100
-    
-  text-completion:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-vertexai"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-k"
-      - "/vertexai/private.json"
-      - "-r"
-      - "us-west1"
-    volumes:
-      - "./vertexai:/vertexai"
-    restart: on-failure:100
-    
-  text-completion-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "text-completion-vertexai"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "-k"
-      - "/vertexai/private.json"
-      - "-r"
-      - "us-west1"
-      - "-i"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "-o"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
-
-  graph-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.5.9
-    command:
-      - "graph-rag"
-      - "-p"
-      - "pulsar://pulsar:6650"
-      - "--text-completion-request-queue"
-      - "non-persistent://tg/request/text-completion-rag"
-      - "--text-completion-response-queue"
-      - "non-persistent://tg/response/text-completion-rag-response"
-    restart: on-failure:100
-
+"services":
+  "cassandra":
+    "image": "docker.io/cassandra:4.1.5"
+    "ports":
+    - "9042:9042"
+    "restart": "on-failure:100"
+    "volumes":
+    - "cassandra:/var/lib/cassandra"
+  "chunker":
+    "command":
+    - "chunker-recursive"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "embeddings":
+    "command":
+    - "embeddings-hf"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "etcd":
+    "command":
+    - "etcd"
+    - "-advertise-client-urls=http://127.0.0.1:2379"
+    - "-listen-client-urls"
+    - "http://0.0.0.0:2379"
+    - "--data-dir"
+    - "/etcd"
+    "environment":
+      "ETCD_AUTO_COMPACTION_MODE": "revision"
+      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
+      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
+      "ETCD_SNAPSHOT_COUNT": "50000"
+    "image": "quay.io/coreos/etcd:v3.5.5"
+    "ports":
+    - "2379:2379"
+    "restart": "on-failure:100"
+    "volumes":
+    - "etcd:/etcd"
+  "grafana":
+    "environment":
+      "GF_ORG_NAME": "trustgraph.ai"
+    "image": "docker.io/grafana/grafana:10.0.0"
+    "ports":
+    - "3000:3000"
+    "restart": "on-failure:100"
+    "volumes":
+    - "grafana-storage:/var/lib/grafana"
+    - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
+    - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
+    - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
+  "graph-rag":
+    "command":
+    - "graph-rag"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "--text-completion-request-queue"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "--text-completion-response-queue"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "init-pulsar":
+    "command":
+    - "sh"
+    - "-c"
+    - "pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response"
+    "depends_on":
+      "pulsar":
+        "condition": "service_started"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "restart": "on-failure:100"
+  "kg-extract-definitions":
+    "command":
+    - "kg-extract-definitions"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "kg-extract-relationships":
+    "command":
+    - "kg-extract-relationships"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "milvus":
+    "command":
+    - "milvus"
+    - "run"
+    - "standalone"
+    "environment":
+      "ETCD_ENDPOINTS": "etcd:2379"
+      "MINIO_ADDRESS": "minio:9000"
+    "image": "docker.io/milvusdb/milvus:v2.4.5"
+    "ports":
+    - "9091:9091"
+    - "19530:19530"
+    "restart": "on-failure:100"
+    "volumes":
+    - "milvus:/var/lib/milvus"
+  "minio":
+    "command":
+    - "minio"
+    - "server"
+    - "/minio_data"
+    - "--console-address"
+    - ":9001"
+    "environment":
+      "MINIO_ROOT_PASSWORD": "minioadmin"
+      "MINIO_ROOT_USER": "minioadmin"
+    "image": "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z"
+    "ports":
+    - "9001:9001"
+    "restart": "on-failure:100"
+    "volumes":
+    - "minio-data:/minio_data"
+  "pdf-decoder":
+    "command":
+    - "pdf-decoder"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "prometheus":
+    "image": "docker.io/prom/prometheus:v2.53.1"
+    "ports":
+    - "9090:9090"
+    "restart": "on-failure:100"
+    "volumes":
+    - "./prometheus:/etc/prometheus"
+    - "prometheus-data:/prometheus"
+  "pulsar":
+    "command": "bin/pulsar standalone"
+    "image": "docker.io/apachepulsar/pulsar:3.3.0"
+    "ports":
+    - "6650:6650"
+    - "8080:8080"
+    "restart": "on-failure:100"
+    "volumes":
+    - "pulsar-conf:/pulsar/conf"
+    - "pulsar-data:/pulsar/data"
+  "pulsar-manager":
+    "environment":
+      "SPRING_CONFIGURATION_FILE": "/pulsar-manager/pulsar-manager/application.properties"
+    "image": "docker.io/apachepulsar/pulsar-manager:v0.3.0"
+    "ports":
+    - "9527:9527"
+    - "7750:7750"
+    "restart": "on-failure:100"
+  "query-graph-embeddings":
+    "command":
+    - "ge-query-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "query-triples":
+    "command":
+    - "triples-query-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-graph-embeddings":
+    "command":
+    - "ge-write-milvus"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-t"
+    - "http://milvus:19530"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "store-triples":
+    "command":
+    - "triples-write-cassandra"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-g"
+    - "cassandra"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+  "text-completion":
+    "command":
+    - "text-completion-vertexai"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-k"
+    - "/vertexai/private.json"
+    - "-r"
+    - "us-west1"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+    "volumes":
+    - "./vertexai:/vertexai"
+  "text-completion-rag":
+    "command":
+    - "text-completion-vertexai"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    - "-k"
+    - "/vertexai/private.json"
+    - "-r"
+    - "us-west1"
+    - "-i"
+    - "non-persistent://tg/request/text-completion-rag"
+    - "-o"
+    - "non-persistent://tg/response/text-completion-rag-response"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+    "volumes":
+    - "./vertexai:/vertexai"
+  "vectorize":
+    "command":
+    - "embeddings-vectorize"
+    - "-p"
+    - "pulsar://pulsar:6650"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.6.0"
+    "restart": "on-failure:100"
+"volumes":
+  "cassandra": {}
+  "etcd": {}
+  "grafana-storage": {}
+  "milvus": {}
+  "minio-data": {}
+  "prometheus-data": {}
+  "pulsar-conf": {}
+  "pulsar-data": {}

setup.py

@@ -4,7 +4,7 @@ import os
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
-version = "0.5.9"
+version = "0.6.0"
 
 setuptools.setup(
     name="trustgraph",

templates/base.jsonnet

@@ -0,0 +1,3 @@
+{
+    restart: "on-failure:100",
+}

templates/cassandra.jsonnet

@@ -0,0 +1,18 @@
+local base = import "base.jsonnet";
+local images = import "images.jsonnet";
+{
+    volumes +: {
+	cassandra: {},
+    },
+    services +: {
+	cassandra: base + {
+	    image: images.cassandra,
+	    ports: [
+		"9042:9042"
+	    ],
+	    volumes: [
+		"cassandra:/var/lib/cassandra"
+	    ],
+	}
+    },
+}

templates/docker-compose-azure.jsonnet

@@ -0,0 +1,49 @@
+
+local base = import "base.jsonnet";
+local url = import "url.jsonnet";
+local images = import "images.jsonnet";
+
+local cassandra = import "cassandra.jsonnet";
+local pulsar = import "pulsar.jsonnet";
+local milvus = import "milvus.jsonnet";
+local grafana = import "grafana.jsonnet";
+local trustgraph = import "trustgraph.jsonnet";
+
+local config = cassandra + pulsar + milvus + grafana + trustgraph + {
+    services +: {
+
+	"text-completion": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-azure",
+		"-p",
+		url.pulsar,
+		"-k",
+		"${AZURE_TOKEN}",
+		"-e",
+		"${AZURE_ENDPOINT}",
+	    ],
+	},
+
+	"text-completion-rag": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-azure",
+		"-p",
+		url.pulsar,
+		"-k",
+		"${AZURE_TOKEN}",
+		"-e",
+		"${AZURE_ENDPOINT}",
+		"-i",
+		"non-persistent://tg/request/text-completion-rag",
+		"-o",
+		"non-persistent://tg/response/text-completion-rag-response",
+	    ],
+	},
+	
+    }
+};
+
+std.manifestYamlDoc(config)
+

templates/docker-compose-bedrock.jsonnet

@@ -0,0 +1,68 @@
+
+local base = import "base.jsonnet";
+local url = import "url.jsonnet";
+local images = import "images.jsonnet";
+
+local cassandra = import "cassandra.jsonnet";
+local pulsar = import "pulsar.jsonnet";
+local milvus = import "milvus.jsonnet";
+local grafana = import "grafana.jsonnet";
+local trustgraph = import "trustgraph.jsonnet";
+
+local config = cassandra + pulsar + milvus + grafana + trustgraph + {
+    services +: {
+
+	chunker: base + {
+	    image: images.trustgraph,
+	    command: [
+		"chunker-recursive",
+		"-p",
+		url.pulsar,
+		"--chunk-size",
+		"2000",
+		"--chunk-overlap",
+		"100",
+	    ],
+	},
+
+	"text-completion": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-bedrock",
+		"-p",
+		url.pulsar,
+		"-z",
+		"${AWS_ID_KEY}",
+		"-k",
+		"${AWS_SECRET_KEY}",
+		"-r",
+		"us-west-2",
+	    ],
+	},
+
+	"text-completion-rag": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-bedrock",
+		"-p",
+		url.pulsar,
+		// "-m",
+		// "mistral.mistral-large-2407-v1:0",
+		"-z",
+		"${AWS_ID_KEY}",
+		"-k",
+		"${AWS_SECRET_KEY}",
+		"-r",
+		"us-west-2",
+		"-i",
+		"non-persistent://tg/request/text-completion-rag",
+		"-o",
+		"non-persistent://tg/response/text-completion-rag-response",
+	    ],
+	},
+	
+    }
+};
+
+std.manifestYamlDoc(config)
+

templates/docker-compose-claude.jsonnet

@@ -0,0 +1,45 @@
+
+local base = import "base.jsonnet";
+local url = import "url.jsonnet";
+local images = import "images.jsonnet";
+
+local cassandra = import "cassandra.jsonnet";
+local pulsar = import "pulsar.jsonnet";
+local milvus = import "milvus.jsonnet";
+local grafana = import "grafana.jsonnet";
+local trustgraph = import "trustgraph.jsonnet";
+
+local config = cassandra + pulsar + milvus + grafana + trustgraph + {
+    services +: {
+
+	"text-completion": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-claude",
+		"-p",
+		url.pulsar,
+		"-k",
+		"${CLAUDE_KEY}",
+	    ],
+	},
+
+	"text-completion-rag": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-claude",
+		"-p",
+		url.pulsar,
+		"-k",
+		"${CLAUDE_KEY}",
+		"-i",
+		"non-persistent://tg/request/text-completion-rag",
+		"-o",
+		"non-persistent://tg/response/text-completion-rag-response",
+	    ],
+	},
+	
+    }
+};
+
+std.manifestYamlDoc(config)
+

templates/docker-compose-cohere.jsonnet

@@ -0,0 +1,58 @@
+
+local base = import "base.jsonnet";
+local url = import "url.jsonnet";
+local images = import "images.jsonnet";
+
+local cassandra = import "cassandra.jsonnet";
+local pulsar = import "pulsar.jsonnet";
+local milvus = import "milvus.jsonnet";
+local grafana = import "grafana.jsonnet";
+local trustgraph = import "trustgraph.jsonnet";
+
+local config = cassandra + pulsar + milvus + grafana + trustgraph + {
+    services +: {
+
+	chunker: base + {
+	    image: images.trustgraph,
+	    command: [
+		"chunker-recursive",
+		"-p",
+		url.pulsar,
+		"--chunk-size",
+		"1000",
+		"--chunk-overlap",
+		"50",
+	    ],
+	},
+
+	"text-completion": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-cohere",
+		"-p",
+		url.pulsar,
+		"-k",
+		"${COHERE_KEY}",
+	    ],
+	},
+
+	"text-completion-rag": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-cohere",
+		"-p",
+		url.pulsar,
+		"-k",
+		"${COHERE_KEY}",
+		"-i",
+		"non-persistent://tg/request/text-completion-rag",
+		"-o",
+		"non-persistent://tg/response/text-completion-rag-response",
+	    ],
+	},
+	
+    }
+};
+
+std.manifestYamlDoc(config)
+

templates/docker-compose-mix.jsonnet

@@ -0,0 +1,62 @@
+
+local base = import "base.jsonnet";
+local url = import "url.jsonnet";
+local images = import "images.jsonnet";
+
+local cassandra = import "cassandra.jsonnet";
+local pulsar = import "pulsar.jsonnet";
+local milvus = import "milvus.jsonnet";
+local grafana = import "grafana.jsonnet";
+local trustgraph = import "trustgraph.jsonnet";
+
+local config = cassandra + pulsar + milvus + grafana + trustgraph + {
+    services +: {
+
+	chunker: base + {
+	    image: images.trustgraph,
+	    command: [
+		"chunker-recursive",
+		"-p",
+		url.pulsar,
+		"--chunk-size",
+		"4000",
+		"--chunk-overlap",
+		"120",
+	    ],
+	},
+
+	"text-completion": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-cohere",
+		"-p",
+		url.pulsar,
+		"-k",
+		"${COHERE_KEY}",
+		"-m",
+		"c4ai-aya-23-35b",
+	    ],
+	},
+
+	"text-completion-rag": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-cohere",
+		"-p",
+		url.pulsar,
+		"-k",
+		"${COHERE_KEY}",
+		"-i",
+		"non-persistent://tg/request/text-completion-rag",
+		"-o",
+		"non-persistent://tg/response/text-completion-rag-response",
+		"-m",
+		"c4ai-aya-23-8b",
+	    ],
+	},
+	
+    }
+};
+
+std.manifestYamlDoc(config)
+

templates/docker-compose-ollama.jsonnet

@@ -0,0 +1,49 @@
+
+local base = import "base.jsonnet";
+local url = import "url.jsonnet";
+local images = import "images.jsonnet";
+
+local cassandra = import "cassandra.jsonnet";
+local pulsar = import "pulsar.jsonnet";
+local milvus = import "milvus.jsonnet";
+local grafana = import "grafana.jsonnet";
+local trustgraph = import "trustgraph.jsonnet";
+
+local config = cassandra + pulsar + milvus + grafana + trustgraph + {
+    services +: {
+
+	"text-completion": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-ollama",
+		"-p",
+		url.pulsar,
+                // "-m",
+                // "llama3.1:8b",
+		"-r",
+		"${OLLAMA_HOST}",
+	    ],
+	},
+
+	"text-completion-rag": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-ollama",
+		"-p",
+		url.pulsar,
+                // "-m",
+                // "llama3.1:8b",
+		"-r",
+		"${OLLAMA_HOST}",
+		"-i",
+		"non-persistent://tg/request/text-completion-rag",
+		"-o",
+		"non-persistent://tg/response/text-completion-rag-response",
+	    ],
+	},
+	
+    }
+};
+
+std.manifestYamlDoc(config)
+

templates/docker-compose-openai.jsonnet

@@ -0,0 +1,45 @@
+
+local base = import "base.jsonnet";
+local url = import "url.jsonnet";
+local images = import "images.jsonnet";
+
+local cassandra = import "cassandra.jsonnet";
+local pulsar = import "pulsar.jsonnet";
+local milvus = import "milvus.jsonnet";
+local grafana = import "grafana.jsonnet";
+local trustgraph = import "trustgraph.jsonnet";
+
+local config = cassandra + pulsar + milvus + grafana + trustgraph + {
+    services +: {
+
+	"text-completion": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-openai",
+		"-p",
+		url.pulsar,
+		"-k",
+		"${OPENAI_KEY}",
+	    ],
+	},
+
+	"text-completion-rag": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-openai",
+		"-p",
+		url.pulsar,
+		"-k",
+		"${OPENAI_KEY}",
+		"-i",
+		"non-persistent://tg/request/text-completion-rag",
+		"-o",
+		"non-persistent://tg/response/text-completion-rag-response",
+	    ],
+	},
+	
+    }
+};
+
+std.manifestYamlDoc(config)
+

templates/docker-compose-vertexai.jsonnet

@@ -0,0 +1,55 @@
+
+local base = import "base.jsonnet";
+local url = import "url.jsonnet";
+local images = import "images.jsonnet";
+
+local cassandra = import "cassandra.jsonnet";
+local pulsar = import "pulsar.jsonnet";
+local milvus = import "milvus.jsonnet";
+local grafana = import "grafana.jsonnet";
+local trustgraph = import "trustgraph.jsonnet";
+
+local config = cassandra + pulsar + milvus + grafana + trustgraph + {
+    services +: {
+
+	"text-completion": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-vertexai",
+		"-p",
+		url.pulsar,
+		"-k",
+		"/vertexai/private.json",
+		"-r",
+		"us-west1",
+	    ],
+	    volumes: [
+		"./vertexai:/vertexai"
+	    ],
+	},
+
+	"text-completion-rag": base + {
+	    image: images.trustgraph,
+	    command: [
+		"text-completion-vertexai",
+		"-p",
+		url.pulsar,
+		"-k",
+		"/vertexai/private.json",
+		"-r",
+		"us-west1",
+		"-i",
+		"non-persistent://tg/request/text-completion-rag",
+		"-o",
+		"non-persistent://tg/response/text-completion-rag-response",
+	    ],
+	    volumes: [
+		"./vertexai:/vertexai"
+	    ],
+	},
+	
+    }
+};
+
+std.manifestYamlDoc(config)
+

templates/docker-compose.jsonnet

@@ -0,0 +1,11 @@
+
+local cassandra = import "cassandra.jsonnet";
+local pulsar = import "pulsar.jsonnet";
+local milvus = import "milvus.jsonnet";
+local grafana = import "grafana.jsonnet";
+local trustgraph = import "trustgraph.jsonnet";
+
+local config = cassandra + pulsar + milvus + grafana + trustgraph;
+
+std.manifestYamlDoc(config)
+

templates/grafana.jsonnet

@@ -0,0 +1,41 @@
+local base = import "base.jsonnet";
+local images = import "images.jsonnet";
+{
+    volumes +: {
+        "prometheus-data": {},
+        "grafana-storage": {},
+    },
+    services +: {
+	prometheus: base + {
+	    image: images.prometheus,
+	    ports: [
+		"9090:9090",
+	    ],
+	    volumes: [
+		"./prometheus:/etc/prometheus",
+		"prometheus-data:/prometheus",
+	    ],
+	},
+	grafana: base + {
+	    image: images.grafana,
+	    ports: [
+		"3000:3000",
+	    ],
+	    volumes: [
+		"grafana-storage:/var/lib/grafana",
+		"./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml",
+		"./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml",
+		"./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json",
+	    ], 
+	    environment: {
+		// GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin",
+		// GF_AUTH_ANONYMOUS_ENABLED: "true",
+		// GF_ORG_ROLE: "Admin",
+		GF_ORG_NAME: "trustgraph.ai",
+		// GF_SERVER_ROOT_URL: "https://example.com",
+	    },
+	},
+    },
+}
+
+    

templates/images.jsonnet

@@ -0,0 +1,12 @@
+local version = import "version.jsonnet";
+{
+    cassandra: "docker.io/cassandra:4.1.5",
+    pulsar: "docker.io/apachepulsar/pulsar:3.3.0",
+    pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.3.0",
+    etcd: "quay.io/coreos/etcd:v3.5.5",
+    minio: "docker.io/minio/minio:RELEASE.2024-07-04T14-25-45Z",
+    milvus: "docker.io/milvusdb/milvus:v2.4.5",
+    prometheus: "docker.io/prom/prometheus:v2.53.1",
+    grafana: "docker.io/grafana/grafana:10.0.0",
+    trustgraph: "docker.io/trustgraph/trustgraph-flow:" + version,
+}

templates/milvus.jsonnet

@@ -0,0 +1,71 @@
+local base = import "base.jsonnet";
+local images = import "images.jsonnet";
+{
+    volumes +: {
+	etcd: {},
+	"minio-data": {},
+	milvus: {},
+    },
+    services +: {
+	etcd: base + {
+	    image: images.etcd,
+	    command: [
+		"etcd",
+		"-advertise-client-urls=http://127.0.0.1:2379",
+		"-listen-client-urls",
+		"http://0.0.0.0:2379",
+		"--data-dir",
+		"/etcd",
+	    ],
+	    environment: {
+		ETCD_AUTO_COMPACTION_MODE: "revision",
+		ETCD_AUTO_COMPACTION_RETENTION: "1000",
+		ETCD_QUOTA_BACKEND_BYTES: "4294967296",
+		ETCD_SNAPSHOT_COUNT: "50000"
+	    },
+	    ports: [
+		"2379:2379",
+	    ],
+	    volumes: [
+		"etcd:/etcd"
+	    ],
+	},
+	minio: base + {
+	    image: images.minio,
+	    command: [
+		"minio",
+		"server",
+		"/minio_data",
+		"--console-address",
+		":9001",
+	    ],
+	    environment: {
+		MINIO_ROOT_USER: "minioadmin",
+		MINIO_ROOT_PASSWORD: "minioadmin",
+	    },
+	    ports: [
+		"9001:9001",
+	    ],
+	    volumes: [
+		"minio-data:/minio_data",
+	    ],	
+	},
+	milvus: base + {
+	    image: images.milvus,
+	    command: [
+		"milvus", "run", "standalone"
+	    ],
+	    environment: {
+		ETCD_ENDPOINTS: "etcd:2379",
+		MINIO_ADDRESS: "minio:9000",
+	    },
+	    ports: [
+		"9091:9091",
+		"19530:19530",
+	    ],
+	    volumes: [
+		"milvus:/var/lib/milvus"
+	    ],
+	},
+    },
+}

templates/pulsar.jsonnet

@@ -0,0 +1,45 @@
+local base = import "base.jsonnet";
+local images = import "images.jsonnet";
+{
+    volumes +: {
+        "pulsar-conf": {},
+        "pulsar-data": {},
+    },
+    services +: {
+	pulsar: base + {
+	    image: images.pulsar,
+	    command: "bin/pulsar standalone",
+	    ports: [
+		"6650:6650",
+		"8080:8080",
+	    ],
+	    volumes: [
+		"pulsar-conf:/pulsar/conf",
+		"pulsar-data:/pulsar/data",
+	    ]
+	},
+	"init-pulsar": base + {
+	    image: images.pulsar,
+	    command: [
+		"sh",
+		"-c",
+		"pulsar-admin --admin-url http://pulsar:8080 tenants create tg && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request && pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response && pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response",
+	    ],
+	    depends_on: {
+		pulsar: {
+		    condition: "service_started",
+		}
+	    },	
+	},
+	"pulsar-manager": base + {
+	    image: images.pulsar_manager,
+	    ports: [
+		"9527:9527",
+		"7750:7750",
+	    ],
+	    environment: {
+		SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties",
+	    },	
+	},
+    }
+}

templates/trustgraph.jsonnet

@@ -0,0 +1,128 @@
+
+local base = import "base.jsonnet";
+
+local images = import "images.jsonnet";
+
+local url = import "url.jsonnet";
+
+local cassandra_hosts = "cassandra";
+
+{
+    services +: {
+
+	"pdf-decoder": base + {
+	    image: images.trustgraph,
+	    command: [
+		"pdf-decoder",
+		"-p",
+		url.pulsar,
+	    ],
+	},
+
+	chunker: base + {
+	    image: images.trustgraph,
+	    command: [
+		"chunker-recursive",
+		"-p",
+		url.pulsar,
+	    ],
+	},
+
+	vectorize: base + {
+	    image: images.trustgraph,
+	    command: [
+		"embeddings-vectorize",
+		"-p",
+		url.pulsar,
+	    ],
+	},
+
+	embeddings: base + {
+	    image: images.trustgraph,
+	    command: [
+		"embeddings-hf",
+		"-p",
+		url.pulsar,
+		// "-m",
+		// "mixedbread-ai/mxbai-embed-large-v1",
+	    ],
+	},
+
+	"kg-extract-definitions": base + {
+	    image: images.trustgraph,
+	    command: [
+		"kg-extract-definitions",
+		"-p",
+		url.pulsar,
+	    ],
+	},
+
+	"kg-extract-relationships": base + {
+	    image: images.trustgraph,
+	    command: [
+		"kg-extract-relationships",
+		"-p",
+		url.pulsar,
+	    ],
+	},
+
+	"store-graph-embeddings": base + {
+	    image: images.trustgraph,
+	    command: [
+		"ge-write-milvus",
+		"-p",
+		url.pulsar,
+		"-t",
+		url.milvus,
+	    ],
+	},
+
+	"query-graph-embeddings": base + {
+	    image: images.trustgraph,
+	    command: [
+		"ge-query-milvus",
+		"-p",
+		url.pulsar,
+		"-t",
+		url.milvus,
+	    ],
+	},
+
+	"store-triples": base + {
+	    image: images.trustgraph,
+	    command: [
+		"triples-write-cassandra",
+		"-p",
+		url.pulsar,
+		"-g",
+		cassandra_hosts,
+	    ],
+	},
+
+	"query-triples": base + {
+	    image: images.trustgraph,
+	    command: [
+		"triples-query-cassandra",
+		"-p",
+		url.pulsar,
+		"-g",
+		cassandra_hosts,
+	    ],
+	},
+
+	"graph-rag": base + {
+	    image: images.trustgraph,
+	    command: [
+		"graph-rag",
+		"-p",
+		url.pulsar,
+		"--text-completion-request-queue",
+		"non-persistent://tg/request/text-completion-rag",
+		"--text-completion-response-queue",
+		"non-persistent://tg/response/text-completion-rag-response",
+	    ],
+	},
+
+    },
+
+}

templates/url.jsonnet

@@ -0,0 +1,4 @@
+{
+    pulsar: "pulsar://pulsar:6650",
+    milvus: "http://milvus:19530",
+}