Switch Milvus for Qdrant in YAMLs (#43)

* Qdrant working

* - Fix missing prompt templates
- Bump version
- Add Qdrant to packages

* Switch Milvus for Qdrant in config files

Makefile

@@ -1,6 +1,6 @@
 
 # VERSION=$(shell git describe | sed 's/^v//')
-VERSION=0.7.19
+VERSION=0.7.20
 
 DOCKER=podman
 
@@ -43,9 +43,12 @@ GRAPHS=cassandra neo4j
 tg-launch-%.yaml: templates/%.jsonnet templates/components/version.jsonnet
 	jsonnet -S ${@:tg-launch-%.yaml=templates/%.jsonnet} > $@
 
+# VECTORDB=milvus
+VECTORDB=qdrant
+
 update-templates: set-version
 	for graph in ${GRAPHS}; do \
-	    cm=$${graph},pulsar,milvus,grafana; \
+	    cm=$${graph},pulsar,${VECTORDB},grafana; \
 	    input=templates/main.jsonnet; \
 	    output=tg-storage-$${graph}.yaml; \
 	    echo $${graph} '->' $${output}; \
@@ -53,7 +56,7 @@ update-templates: set-version
 	done
 	for model in ${MODELS}; do \
 	  for graph in ${GRAPHS}; do \
-	    cm=$${graph},pulsar,milvus,grafana,trustgraph,$${model}; \
+	    cm=$${graph},pulsar,${VECTORDB},grafana,trustgraph,$${model}; \
 	    input=templates/main.jsonnet; \
 	    output=tg-launch-$${model}-$${graph}.yaml; \
 	    echo $${model} + $${graph} '->' $${output}; \

scripts/ge-query-qdrant

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.query.graph_embeddings.qdrant import run
+
+run()
+

scripts/ge-write-qdrant

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.graph_embeddings.qdrant import run
+
+run()
+

setup.py

@@ -4,7 +4,7 @@ import os
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
-version = "0.7.19"
+version = "0.7.20"
 
 setuptools.setup(
     name="trustgraph",
@@ -40,6 +40,7 @@ setuptools.setup(
         "cassandra-driver",
         "pulsar-client",
         "pypdf",
+        "qdrant-client",
         "anthropic",
         "google-cloud-aiplatform",
         "pyyaml",
@@ -65,6 +66,8 @@ setuptools.setup(
         "scripts/ge-dump-parquet",
         "scripts/ge-query-milvus",
         "scripts/ge-write-milvus",
+        "scripts/ge-query-qdrant",
+        "scripts/ge-write-qdrant",
         "scripts/graph-rag",
         "scripts/graph-show",
         "scripts/graph-to-turtle",

templates/components/images.jsonnet

@@ -10,4 +10,5 @@ local version = import "version.jsonnet";
     prometheus: "docker.io/prom/prometheus:v2.53.2",
     grafana: "docker.io/grafana/grafana:11.1.4",
     trustgraph: "docker.io/trustgraph/trustgraph-flow:" + version,
+    qdrant: "docker.io/qdrant/qdrant:v1.11.1"
 }

templates/components/qdrant.jsonnet

@@ -0,0 +1,111 @@
+local base = import "base.jsonnet";
+local images = import "images.jsonnet";
+local url = import "url.jsonnet";
+local qdrant = import "stores/qdrant.jsonnet";
+
+qdrant + {
+
+    services +: {
+
+	"store-graph-embeddings": base + {
+	    image: images.trustgraph,
+	    command: [
+		"ge-write-qdrant",
+		"-p",
+		url.pulsar,
+		"-t",
+		url.qdrant,
+	    ],
+            deploy: {
+		resources: {
+		    limits: {
+			cpus: '0.5',
+			memory: '128M'
+		    },
+		    reservations: {
+			cpus: '0.1',
+			memory: '128M'
+		    }
+		}
+	    },
+	},
+
+	"query-graph-embeddings": base + {
+	    image: images.trustgraph,
+	    command: [
+		"ge-query-qdrant",
+		"-p",
+		url.pulsar,
+		"-t",
+		url.qdrant,
+	    ],
+            deploy: {
+		resources: {
+		    limits: {
+			cpus: '0.5',
+			memory: '128M'
+		    },
+		    reservations: {
+			cpus: '0.1',
+			memory: '128M'
+		    }
+		}
+	    },
+	},
+
+/*
+
+// Document embeddings writer & query service.  Not currently enabled.
+
+	"store-doc-embeddings": base + {
+	    image: images.trustgraph,
+	    command: [
+		"de-write-qdrant",
+		"-p",
+		url.pulsar,
+		"-t",
+		url.qdrant,
+	    ],
+            deploy: {
+		resources: {
+		    limits: {
+			cpus: '0.5',
+			memory: '128M'
+		    },
+		    reservations: {
+			cpus: '0.1',
+			memory: '128M'
+		    }
+		}
+	    },
+	},
+
+	"query-doc-embeddings": base + {
+	    image: images.trustgraph,
+	    command: [
+		"de-query-qdrant",
+		"-p",
+		url.pulsar,
+		"-t",
+		url.qdrant,
+	    ],
+            deploy: {
+		resources: {
+		    limits: {
+			cpus: '0.5',
+			memory: '128M'
+		    },
+		    reservations: {
+			cpus: '0.1',
+			memory: '128M'
+		    }
+		}
+	    },
+	},
+
+*/
+
+    }
+
+}
+

templates/components/stores/qdrant.jsonnet

@@ -0,0 +1,37 @@
+local base = import "../base.jsonnet";
+local images = import "../images.jsonnet";
+
+{
+
+    volumes +: {
+	qdrant: {},
+    },
+
+    services +: {
+
+	qdrant: base + {
+	    image: images.qdrant,
+	    ports: [
+		"6333:6333",
+		"6334:6334",
+	    ],
+	    volumes: [
+		"qdrant:/qdrant/storage"
+	    ],
+            deploy: {
+		resources: {
+		    limits: {
+			cpus: '1.0',
+			memory: '256M'
+		    },
+		    reservations: {
+			cpus: '0.5',
+			memory: '256M'
+		    }
+		}
+            },
+	},
+
+    },
+
+}

templates/components/url.jsonnet

@@ -1,4 +1,5 @@
 {
     pulsar: "pulsar://pulsar:6650",
     milvus: "http://milvus:19530",
+    qdrant: "http://qdrant:6333",
 }

templates/main.jsonnet

@@ -3,6 +3,7 @@ local components = {
     cassandra: import "components/cassandra.jsonnet",
     pulsar: import "components/pulsar.jsonnet",
     milvus: import "components/milvus.jsonnet",
+    qdrant: import "components/qdrant.jsonnet",
     grafana: import "components/grafana.jsonnet",
     trustgraph: import "components/trustgraph.jsonnet",
     azure: import "components/azure.jsonnet",

templates/prompts/gemini.jsonnet

@@ -25,6 +25,10 @@ local url = import "../components/url.jsonnet";
                 "<instructions>\nStudy the following text and derive entity relationships.  For each\nrelationship, derive the subject, predicate and object of the relationship.\nOutput relationships in JSON format as an arary of objects with fields:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: false if the object is a simple data type: name, value or date.  true if it is an entity.\n</instructions>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not use special characters in the abstract text. The\nabstract must be written as plain text.  Do not add markdown formatting\nor headers or prefixes.\n</requirements>",
                 "--knowledge-query-template",
                 "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{graph}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n",
+                "--document-query-template",
+                "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{documents}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n",
+                "--rows-template",
+                "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>",
 	    ],
             deploy: {
 		resources: {

templates/prompts/mixtral.jsonnet

@@ -25,6 +25,10 @@ local url = import "../components/url.jsonnet";
                 "<instructions>\nStudy the following text and derive entity relationships.  For each\nrelationship, derive the subject, predicate and object of the relationship.\nOutput relationships in JSON format as an arary of objects with fields:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: false if the object is a simple data type: name, value or date.  true if it is an entity.\n</instructions>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not use special characters in the abstract text. The\nabstract must be written as plain text.  Do not add markdown formatting\nor headers or prefixes.\n</requirements>",
                 "--knowledge-query-template",
                 "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{graph}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n",
+                "--document-query-template",
+                "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{documents}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n",
+                "--rows-template",
+                "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>",
 	    ],
             deploy: {
 		resources: {

templates/prompts/openai.jsonnet

@@ -25,6 +25,10 @@ local url = import "../components/url.jsonnet";
                 "<instructions>\nStudy the following text and derive entity relationships.  For each\nrelationship, derive the subject, predicate and object of the relationship.\nOutput relationships in JSON format as an arary of objects with fields:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: false if the object is a simple data type: name, value or date.  true if it is an entity.\n</instructions>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not use special characters in the abstract text. The\nabstract must be written as plain text.  Do not add markdown formatting\nor headers or prefixes.\n</requirements>",
                 "--knowledge-query-template",
                 "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{graph}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n",
+                "--document-query-template",
+                "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{documents}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n",
+                "--rows-template",
+                "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>",
 	    ],
             deploy: {
 		resources: {

templates/test.jsonnet

@@ -0,0 +1 @@
+std.extVar("asd")

tg-launch-azure-cassandra.yaml

@@ -33,7 +33,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -50,35 +50,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -122,7 +95,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -155,7 +128,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -170,55 +143,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "pdf-decoder":
     "command":
     - "pdf-decoder"
@@ -232,7 +158,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -272,6 +198,17 @@
       
       Use only the provided knowledge statements to respond to the following:
       {query}
+    - "--document-query-template"
+    - |
+      Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.
+      
+      Here is the context:
+      {documents}
+      
+      Use only the provided knowledge statements to respond to the following:
+      {query}
+    - "--rows-template"
+    - "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>"
     "deploy":
       "resources":
         "limits":
@@ -280,7 +217,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -327,7 +264,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -365,13 +302,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -380,7 +333,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -397,15 +350,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -414,7 +367,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -431,7 +384,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -454,7 +407,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion-rag":
     "command":
@@ -481,7 +434,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "vectorize":
     "command":
@@ -496,14 +449,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
   "cassandra": {}
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-azure-neo4j.yaml

@@ -16,7 +16,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -33,35 +33,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -105,7 +78,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -138,7 +111,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -153,55 +126,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "neo4j":
     "deploy":
       "resources":
@@ -233,7 +159,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -273,6 +199,17 @@
       
       Use only the provided knowledge statements to respond to the following:
       {query}
+    - "--document-query-template"
+    - |
+      Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.
+      
+      Here is the context:
+      {documents}
+      
+      Use only the provided knowledge statements to respond to the following:
+      {query}
+    - "--rows-template"
+    - "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>"
     "deploy":
       "resources":
         "limits":
@@ -281,7 +218,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -328,7 +265,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -366,13 +303,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -381,7 +334,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -398,15 +351,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -415,7 +368,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -432,7 +385,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -455,7 +408,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion-rag":
     "command":
@@ -482,7 +435,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "vectorize":
     "command":
@@ -497,14 +450,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "neo4j": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-bedrock-cassandra.yaml

@@ -33,7 +33,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -50,35 +50,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -122,7 +95,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -155,7 +128,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -170,55 +143,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "pdf-decoder":
     "command":
     - "pdf-decoder"
@@ -232,7 +158,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -272,6 +198,17 @@
       
       Use only the provided knowledge statements to respond to the following:
       {query}
+    - "--document-query-template"
+    - |
+      Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.
+      
+      Here is the context:
+      {documents}
+      
+      Use only the provided knowledge statements to respond to the following:
+      {query}
+    - "--rows-template"
+    - "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>"
     "deploy":
       "resources":
         "limits":
@@ -280,7 +217,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -327,7 +264,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -365,13 +302,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -380,7 +333,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -397,15 +350,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -414,7 +367,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -431,7 +384,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -458,7 +411,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion-rag":
     "command":
@@ -489,7 +442,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "vectorize":
     "command":
@@ -504,14 +457,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
   "cassandra": {}
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-bedrock-neo4j.yaml

@@ -16,7 +16,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -33,35 +33,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -105,7 +78,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -138,7 +111,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -153,55 +126,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "neo4j":
     "deploy":
       "resources":
@@ -233,7 +159,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -273,6 +199,17 @@
       
       Use only the provided knowledge statements to respond to the following:
       {query}
+    - "--document-query-template"
+    - |
+      Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.
+      
+      Here is the context:
+      {documents}
+      
+      Use only the provided knowledge statements to respond to the following:
+      {query}
+    - "--rows-template"
+    - "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>"
     "deploy":
       "resources":
         "limits":
@@ -281,7 +218,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -328,7 +265,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -366,13 +303,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -381,7 +334,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -398,15 +351,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -415,7 +368,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -432,7 +385,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -459,7 +412,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion-rag":
     "command":
@@ -490,7 +443,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "vectorize":
     "command":
@@ -505,14 +458,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "neo4j": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-claude-cassandra.yaml

@@ -33,7 +33,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -50,35 +50,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -122,7 +95,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -155,7 +128,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -170,55 +143,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "pdf-decoder":
     "command":
     - "pdf-decoder"
@@ -232,7 +158,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -272,6 +198,17 @@
       
       Use only the provided knowledge statements to respond to the following:
       {query}
+    - "--document-query-template"
+    - |
+      Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.
+      
+      Here is the context:
+      {documents}
+      
+      Use only the provided knowledge statements to respond to the following:
+      {query}
+    - "--rows-template"
+    - "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>"
     "deploy":
       "resources":
         "limits":
@@ -280,7 +217,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -327,7 +264,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -365,13 +302,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -380,7 +333,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -397,15 +350,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -414,7 +367,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -431,7 +384,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -452,7 +405,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion-rag":
     "command":
@@ -477,7 +430,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "vectorize":
     "command":
@@ -492,14 +445,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
   "cassandra": {}
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-claude-neo4j.yaml

@@ -16,7 +16,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -33,35 +33,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -105,7 +78,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -138,7 +111,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -153,55 +126,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "neo4j":
     "deploy":
       "resources":
@@ -233,7 +159,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -273,6 +199,17 @@
       
       Use only the provided knowledge statements to respond to the following:
       {query}
+    - "--document-query-template"
+    - |
+      Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.
+      
+      Here is the context:
+      {documents}
+      
+      Use only the provided knowledge statements to respond to the following:
+      {query}
+    - "--rows-template"
+    - "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>"
     "deploy":
       "resources":
         "limits":
@@ -281,7 +218,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -328,7 +265,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -366,13 +303,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -381,7 +334,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -398,15 +351,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -415,7 +368,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -432,7 +385,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -453,7 +406,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion-rag":
     "command":
@@ -478,7 +431,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "vectorize":
     "command":
@@ -493,14 +446,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "neo4j": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-cohere-cassandra.yaml

@@ -33,7 +33,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -50,35 +50,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -122,7 +95,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -155,7 +128,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -170,55 +143,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "pdf-decoder":
     "command":
     - "pdf-decoder"
@@ -232,7 +158,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -272,6 +198,17 @@
       
       Use only the provided knowledge statements to respond to the following:
       {query}
+    - "--document-query-template"
+    - |
+      Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.
+      
+      Here is the context:
+      {documents}
+      
+      Use only the provided knowledge statements to respond to the following:
+      {query}
+    - "--rows-template"
+    - "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>"
     "deploy":
       "resources":
         "limits":
@@ -280,7 +217,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -327,7 +264,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -365,13 +302,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -380,7 +333,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -397,15 +350,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -414,7 +367,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -431,7 +384,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -450,7 +403,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion-rag":
     "command":
@@ -473,7 +426,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "vectorize":
     "command":
@@ -488,14 +441,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
   "cassandra": {}
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-cohere-neo4j.yaml

@@ -16,7 +16,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -33,35 +33,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -105,7 +78,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -138,7 +111,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -153,55 +126,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "neo4j":
     "deploy":
       "resources":
@@ -233,7 +159,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -273,6 +199,17 @@
       
       Use only the provided knowledge statements to respond to the following:
       {query}
+    - "--document-query-template"
+    - |
+      Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.
+      
+      Here is the context:
+      {documents}
+      
+      Use only the provided knowledge statements to respond to the following:
+      {query}
+    - "--rows-template"
+    - "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>"
     "deploy":
       "resources":
         "limits":
@@ -281,7 +218,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -328,7 +265,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -366,13 +303,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -381,7 +334,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -398,15 +351,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -415,7 +368,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -432,7 +385,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -451,7 +404,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion-rag":
     "command":
@@ -474,7 +427,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "vectorize":
     "command":
@@ -489,14 +442,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "neo4j": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-ollama-cassandra.yaml

@@ -33,7 +33,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -50,35 +50,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -122,7 +95,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -155,7 +128,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -170,55 +143,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "pdf-decoder":
     "command":
     - "pdf-decoder"
@@ -232,7 +158,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -267,7 +193,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -290,7 +216,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -328,13 +254,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -343,7 +285,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -360,15 +302,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -377,7 +319,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -394,7 +336,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -413,7 +355,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion-rag":
     "command":
@@ -436,7 +378,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "vectorize":
     "command":
@@ -451,14 +393,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
   "cassandra": {}
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-ollama-neo4j.yaml

@@ -16,7 +16,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -33,35 +33,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -105,7 +78,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -138,7 +111,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -153,55 +126,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "neo4j":
     "deploy":
       "resources":
@@ -233,7 +159,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -268,7 +194,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -291,7 +217,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -329,13 +255,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -344,7 +286,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -361,15 +303,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -378,7 +320,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -395,7 +337,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -414,7 +356,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion-rag":
     "command":
@@ -437,7 +379,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "vectorize":
     "command":
@@ -452,14 +394,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "neo4j": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-openai-cassandra.yaml

@@ -33,7 +33,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -50,35 +50,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -122,7 +95,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -155,7 +128,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -170,55 +143,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "pdf-decoder":
     "command":
     - "pdf-decoder"
@@ -232,7 +158,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -272,6 +198,17 @@
       
       Use only the provided knowledge statements to respond to the following:
       {query}
+    - "--document-query-template"
+    - |
+      Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.
+      
+      Here is the context:
+      {documents}
+      
+      Use only the provided knowledge statements to respond to the following:
+      {query}
+    - "--rows-template"
+    - "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>"
     "deploy":
       "resources":
         "limits":
@@ -280,7 +217,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -327,7 +264,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -365,13 +302,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -380,7 +333,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -397,15 +350,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -414,7 +367,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -431,7 +384,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -452,7 +405,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion-rag":
     "command":
@@ -477,7 +430,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "vectorize":
     "command":
@@ -492,14 +445,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
   "cassandra": {}
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-openai-neo4j.yaml

@@ -16,7 +16,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -33,35 +33,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -105,7 +78,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -138,7 +111,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -153,55 +126,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "neo4j":
     "deploy":
       "resources":
@@ -233,7 +159,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -273,6 +199,17 @@
       
       Use only the provided knowledge statements to respond to the following:
       {query}
+    - "--document-query-template"
+    - |
+      Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.
+      
+      Here is the context:
+      {documents}
+      
+      Use only the provided knowledge statements to respond to the following:
+      {query}
+    - "--rows-template"
+    - "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>"
     "deploy":
       "resources":
         "limits":
@@ -281,7 +218,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -328,7 +265,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -366,13 +303,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -381,7 +334,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -398,15 +351,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -415,7 +368,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -432,7 +385,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -453,7 +406,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion-rag":
     "command":
@@ -478,7 +431,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "vectorize":
     "command":
@@ -493,14 +446,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "neo4j": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-vertexai-cassandra.yaml

@@ -33,7 +33,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -50,35 +50,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -122,7 +95,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -155,7 +128,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -170,55 +143,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "pdf-decoder":
     "command":
     - "pdf-decoder"
@@ -232,7 +158,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -272,6 +198,17 @@
       
       Use only the provided knowledge statements to respond to the following:
       {query}
+    - "--document-query-template"
+    - |
+      Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.
+      
+      Here is the context:
+      {documents}
+      
+      Use only the provided knowledge statements to respond to the following:
+      {query}
+    - "--rows-template"
+    - "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>"
     "deploy":
       "resources":
         "limits":
@@ -280,7 +217,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -327,7 +264,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -365,13 +302,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -380,7 +333,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -397,15 +350,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -414,7 +367,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -431,7 +384,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -454,7 +407,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
     "volumes":
     - "./vertexai:/vertexai"
@@ -483,7 +436,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
     "volumes":
     - "./vertexai:/vertexai"
@@ -500,14 +453,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
   "cassandra": {}
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-launch-vertexai-neo4j.yaml

@@ -16,7 +16,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "embeddings":
     "command":
@@ -33,35 +33,8 @@
         "reservations":
           "cpus": "0.5"
           "memory": "256M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -105,7 +78,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "init-pulsar":
     "command":
@@ -138,7 +111,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "kg-extract-relationships":
     "command":
@@ -153,55 +126,8 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "neo4j":
     "deploy":
       "resources":
@@ -233,7 +159,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prometheus":
     "deploy":
@@ -273,6 +199,17 @@
       
       Use only the provided knowledge statements to respond to the following:
       {query}
+    - "--document-query-template"
+    - |
+      Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.
+      
+      Here is the context:
+      {documents}
+      
+      Use only the provided knowledge statements to respond to the following:
+      {query}
+    - "--rows-template"
+    - "<instructions>\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema.  For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n</instructions>\n\n<schema>\n{schema}\n</schema>\n\n<text>\n{text}\n</text>\n\n<requirements>\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n</requirements>"
     "deploy":
       "resources":
         "limits":
@@ -281,7 +218,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "prompt-rag":
     "command":
@@ -328,7 +265,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "pulsar":
     "command": "bin/pulsar standalone"
@@ -366,13 +303,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -381,7 +334,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -398,15 +351,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -415,7 +368,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -432,7 +385,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "text-completion":
     "command":
@@ -455,7 +408,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
     "volumes":
     - "./vertexai:/vertexai"
@@ -484,7 +437,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
     "volumes":
     - "./vertexai:/vertexai"
@@ -501,14 +454,12 @@
         "reservations":
           "cpus": "0.5"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "neo4j": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-storage-cassandra.yaml

@@ -16,33 +16,6 @@
     "restart": "on-failure:100"
     "volumes":
     - "cassandra:/var/lib/cassandra"
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -81,53 +54,6 @@
           "memory": "128M"
     "image": "docker.io/apachepulsar/pulsar:3.3.1"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "prometheus":
     "deploy":
       "resources":
@@ -180,13 +106,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -195,7 +137,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -212,15 +154,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "512M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -229,7 +171,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -246,14 +188,12 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
   "cassandra": {}
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

tg-storage-neo4j.yaml

@@ -1,31 +1,4 @@
 "services":
-  "etcd":
-    "command":
-    - "etcd"
-    - "-advertise-client-urls=http://127.0.0.1:2379"
-    - "-listen-client-urls"
-    - "http://0.0.0.0:2379"
-    - "--data-dir"
-    - "/etcd"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "ETCD_AUTO_COMPACTION_MODE": "revision"
-      "ETCD_AUTO_COMPACTION_RETENTION": "1000"
-      "ETCD_QUOTA_BACKEND_BYTES": "4294967296"
-      "ETCD_SNAPSHOT_COUNT": "50000"
-    "image": "quay.io/coreos/etcd:v3.5.15"
-    "ports":
-    - "2379:2379"
-    "restart": "on-failure:100"
-    "volumes":
-    - "etcd:/etcd"
   "grafana":
     "deploy":
       "resources":
@@ -64,53 +37,6 @@
           "memory": "128M"
     "image": "docker.io/apachepulsar/pulsar:3.3.1"
     "restart": "on-failure:100"
-  "milvus":
-    "command":
-    - "milvus"
-    - "run"
-    - "standalone"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "1.0"
-          "memory": "256M"
-        "reservations":
-          "cpus": "0.5"
-          "memory": "256M"
-    "environment":
-      "ETCD_ENDPOINTS": "etcd:2379"
-      "MINIO_ADDRESS": "minio:9000"
-    "image": "docker.io/milvusdb/milvus:v2.4.9"
-    "ports":
-    - "9091:9091"
-    - "19530:19530"
-    "restart": "on-failure:100"
-    "volumes":
-    - "milvus:/var/lib/milvus"
-  "minio":
-    "command":
-    - "minio"
-    - "server"
-    - "/minio_data"
-    - "--console-address"
-    - ":9001"
-    "deploy":
-      "resources":
-        "limits":
-          "cpus": "0.5"
-          "memory": "128M"
-        "reservations":
-          "cpus": "0.25"
-          "memory": "128M"
-    "environment":
-      "MINIO_ROOT_PASSWORD": "minioadmin"
-      "MINIO_ROOT_USER": "minioadmin"
-    "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z"
-    "ports":
-    - "9001:9001"
-    "restart": "on-failure:100"
-    "volumes":
-    - "minio-data:/minio_data"
   "neo4j":
     "deploy":
       "resources":
@@ -181,13 +107,29 @@
     - "9527:9527"
     - "7750:7750"
     "restart": "on-failure:100"
+  "qdrant":
+    "deploy":
+      "resources":
+        "limits":
+          "cpus": "1.0"
+          "memory": "256M"
+        "reservations":
+          "cpus": "0.5"
+          "memory": "256M"
+    "image": "docker.io/qdrant/qdrant:v1.11.1"
+    "ports":
+    - "6333:6333"
+    - "6334:6334"
+    "restart": "on-failure:100"
+    "volumes":
+    - "qdrant:/qdrant/storage"
   "query-graph-embeddings":
     "command":
-    - "ge-query-milvus"
+    - "ge-query-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -196,7 +138,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "query-triples":
     "command":
@@ -213,15 +155,15 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-graph-embeddings":
     "command":
-    - "ge-write-milvus"
+    - "ge-write-qdrant"
     - "-p"
     - "pulsar://pulsar:6650"
     - "-t"
-    - "http://milvus:19530"
+    - "http://qdrant:6333"
     "deploy":
       "resources":
         "limits":
@@ -230,7 +172,7 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
   "store-triples":
     "command":
@@ -247,14 +189,12 @@
         "reservations":
           "cpus": "0.1"
           "memory": "128M"
-    "image": "docker.io/trustgraph/trustgraph-flow:0.7.19"
+    "image": "docker.io/trustgraph/trustgraph-flow:0.7.20"
     "restart": "on-failure:100"
 "volumes":
-  "etcd": {}
   "grafana-storage": {}
-  "milvus": {}
-  "minio-data": {}
   "neo4j": {}
   "prometheus-data": {}
   "pulsar-conf": {}
   "pulsar-data": {}
+  "qdrant": {}

trustgraph/base/consumer.py

@@ -10,7 +10,6 @@ class Consumer(BaseProcessor):
 
     def __init__(self, **params):
 
-        print("HERE2")
         super(Consumer, self).__init__(**params)
 
         input_queue = params.get("input_queue")
@@ -30,7 +29,6 @@ class Consumer(BaseProcessor):
                 'pubsub', 'Pub/sub configuration'
             )
 
-        print("HERE")
         if not hasattr(__class__, "processing_metric"):
             __class__.processing_metric = Counter(
                 'processing_count', 'Processing count', ["status"]

trustgraph/query/graph_embeddings/qdrant/__init__.py

@@ -0,0 +1,3 @@
+
+from . service import *
+

trustgraph/query/graph_embeddings/qdrant/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+from . hf import run
+
+if __name__ == '__main__':
+    run()
+

trustgraph/query/graph_embeddings/qdrant/service.py

@@ -0,0 +1,133 @@
+
+"""
+Graph embeddings query service.  Input is vector, output is list of
+entities
+"""
+
+from qdrant_client import QdrantClient
+from qdrant_client.models import PointStruct
+from qdrant_client.models import Distance, VectorParams
+import uuid
+
+from .... schema import GraphEmbeddingsRequest, GraphEmbeddingsResponse
+from .... schema import Error, Value
+from .... schema import graph_embeddings_request_queue
+from .... schema import graph_embeddings_response_queue
+from .... base import ConsumerProducer
+
+module = ".".join(__name__.split(".")[1:-1])
+
+default_input_queue = graph_embeddings_request_queue
+default_output_queue = graph_embeddings_response_queue
+default_subscriber = module
+default_store_uri = 'http://localhost:6333'
+
+class Processor(ConsumerProducer):
+
+    def __init__(self, **params):
+
+        input_queue = params.get("input_queue", default_input_queue)
+        output_queue = params.get("output_queue", default_output_queue)
+        subscriber = params.get("subscriber", default_subscriber)
+        store_uri = params.get("store_uri", default_store_uri)
+
+        super(Processor, self).__init__(
+            **params | {
+                "input_queue": input_queue,
+                "output_queue": output_queue,
+                "subscriber": subscriber,
+                "input_schema": GraphEmbeddingsRequest,
+                "output_schema": GraphEmbeddingsResponse,
+                "store_uri": store_uri,
+            }
+        )
+
+        self.client = QdrantClient(url=store_uri)
+
+    def create_value(self, ent):
+        if ent.startswith("http://") or ent.startswith("https://"):
+            return Value(value=ent, is_uri=True)
+        else:
+            return Value(value=ent, is_uri=False)
+        
+    def handle(self, msg):
+
+        try:
+
+            v = msg.value()
+
+            # Sender-produced ID
+            id = msg.properties()["id"]
+
+            print(f"Handling input {id}...", flush=True)
+
+            entities = set()
+
+            for vec in v.vectors:
+
+                dim = len(vec)
+                collection = "triples_" + str(dim)
+
+                search_result = self.client.query_points(
+                    collection_name=collection,
+                    query=vec,
+                    limit=v.limit,
+                    with_payload=True,
+                ).points
+
+                for r in search_result:
+                    ent = r.payload["entity"]
+                    entities.add(ent)
+
+            # Convert set to list
+            entities = list(entities)
+
+            ents2 = []
+
+            for ent in entities:
+                ents2.append(self.create_value(ent))
+
+            entities = ents2
+
+            print("Send response...", flush=True)
+            r = GraphEmbeddingsResponse(entities=entities, error=None)
+            self.producer.send(r, properties={"id": id})
+
+            print("Done.", flush=True)
+
+        except Exception as e:
+
+            print(f"Exception: {e}")
+
+            print("Send error response...", flush=True)
+
+            r = GraphEmbeddingsResponse(
+                error=Error(
+                    type = "llm-error",
+                    message = str(e),
+                ),
+                entities=None,
+            )
+
+            self.producer.send(r, properties={"id": id})
+
+            self.consumer.acknowledge(msg)
+
+    @staticmethod
+    def add_args(parser):
+
+        ConsumerProducer.add_args(
+            parser, default_input_queue, default_subscriber,
+            default_output_queue,
+        )
+
+        parser.add_argument(
+            '-t', '--store-uri',
+            default=default_store_uri,
+            help=f'Milvus store URI (default: {default_store_uri})'
+        )
+
+def run():
+
+    Processor.start(module, __doc__)
+

trustgraph/storage/graph_embeddings/qdrant/__init__.py

@@ -0,0 +1,3 @@
+
+from . write import *
+

trustgraph/storage/graph_embeddings/qdrant/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+from . write import run
+
+if __name__ == '__main__':
+    run()
+

trustgraph/storage/graph_embeddings/qdrant/write.py

@@ -0,0 +1,102 @@
+
+"""
+Accepts entity/vector pairs and writes them to a Qdrant store.
+"""
+
+from qdrant_client import QdrantClient
+from qdrant_client.models import PointStruct
+from qdrant_client.models import Distance, VectorParams
+import uuid
+
+from .... schema import GraphEmbeddings
+from .... schema import graph_embeddings_store_queue
+from .... log_level import LogLevel
+from .... base import Consumer
+
+module = ".".join(__name__.split(".")[1:-1])
+
+default_input_queue = graph_embeddings_store_queue
+default_subscriber = module
+default_store_uri = 'http://localhost:6333'
+
+class Processor(Consumer):
+
+    def __init__(self, **params):
+
+        input_queue = params.get("input_queue", default_input_queue)
+        subscriber = params.get("subscriber", default_subscriber)
+        store_uri = params.get("store_uri", default_store_uri)
+
+        super(Processor, self).__init__(
+            **params | {
+                "input_queue": input_queue,
+                "subscriber": subscriber,
+                "input_schema": GraphEmbeddings,
+                "store_uri": store_uri,
+            }
+        )
+
+        self.last_collection = None
+        self.last_dim = None
+
+        self.client = QdrantClient(url=store_uri)
+
+    def handle(self, msg):
+
+        v = msg.value()
+
+        if v.entity.value == "" or v.entity.value is None: return
+
+        for vec in v.vectors:
+
+            dim = len(vec)
+            collection = "triples_" + str(dim)
+
+            if dim != self.last_dim:
+
+                if not self.client.collection_exists(collection):
+
+                    try:
+                        self.client.create_collection(
+                            collection_name=collection,
+                            vectors_config=VectorParams(
+                                size=dim, distance=Distance.DOT
+                            ),
+                        )
+                    except Exception as e:
+                        print("Qdrant collection creation failed")
+                        raise e
+
+                self.last_collection = collection
+                self.last_dim = dim
+
+            self.client.upsert(
+                collection_name=collection,
+                points=[
+                    PointStruct(
+                        id=str(uuid.uuid4()),
+                        vector=vec,
+                        payload={
+                            "entity": v.entity.value,
+                        }
+                    )
+                ]
+            )
+
+    @staticmethod
+    def add_args(parser):
+
+        Consumer.add_args(
+            parser, default_input_queue, default_subscriber,
+        )
+
+        parser.add_argument(
+            '-t', '--store-uri',
+            default=default_store_uri,
+            help=f'Qdrant store URI (default: {default_store_uri})'
+        )
+
+def run():
+
+    Processor.start(module, __doc__)
+