diff --git a/Makefile b/Makefile index 66ba658d..b6a2e49d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # VERSION=$(shell git describe | sed 's/^v//') -VERSION=0.7.19 +VERSION=0.7.20 DOCKER=podman @@ -43,9 +43,12 @@ GRAPHS=cassandra neo4j tg-launch-%.yaml: templates/%.jsonnet templates/components/version.jsonnet jsonnet -S ${@:tg-launch-%.yaml=templates/%.jsonnet} > $@ +# VECTORDB=milvus +VECTORDB=qdrant + update-templates: set-version for graph in ${GRAPHS}; do \ - cm=$${graph},pulsar,milvus,grafana; \ + cm=$${graph},pulsar,${VECTORDB},grafana; \ input=templates/main.jsonnet; \ output=tg-storage-$${graph}.yaml; \ echo $${graph} '->' $${output}; \ @@ -53,7 +56,7 @@ update-templates: set-version done for model in ${MODELS}; do \ for graph in ${GRAPHS}; do \ - cm=$${graph},pulsar,milvus,grafana,trustgraph,$${model}; \ + cm=$${graph},pulsar,${VECTORDB},grafana,trustgraph,$${model}; \ input=templates/main.jsonnet; \ output=tg-launch-$${model}-$${graph}.yaml; \ echo $${model} + $${graph} '->' $${output}; \ diff --git a/scripts/ge-query-qdrant b/scripts/ge-query-qdrant new file mode 100755 index 00000000..7039d17a --- /dev/null +++ b/scripts/ge-query-qdrant @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from trustgraph.query.graph_embeddings.qdrant import run + +run() + diff --git a/scripts/ge-write-qdrant b/scripts/ge-write-qdrant new file mode 100755 index 00000000..4276fd2b --- /dev/null +++ b/scripts/ge-write-qdrant @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from trustgraph.storage.graph_embeddings.qdrant import run + +run() + diff --git a/setup.py b/setup.py index 2d35975f..8243f3ba 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import os with open("README.md", "r") as fh: long_description = fh.read() -version = "0.7.19" +version = "0.7.20" setuptools.setup( name="trustgraph", @@ -40,6 +40,7 @@ setuptools.setup( "cassandra-driver", "pulsar-client", "pypdf", + "qdrant-client", "anthropic", "google-cloud-aiplatform", "pyyaml", @@ -65,6 +66,8 @@ setuptools.setup( "scripts/ge-dump-parquet", "scripts/ge-query-milvus", "scripts/ge-write-milvus", + "scripts/ge-query-qdrant", + "scripts/ge-write-qdrant", "scripts/graph-rag", "scripts/graph-show", "scripts/graph-to-turtle", diff --git a/templates/components/images.jsonnet b/templates/components/images.jsonnet index 76fbd0bd..01ecee4d 100644 --- a/templates/components/images.jsonnet +++ b/templates/components/images.jsonnet @@ -10,4 +10,5 @@ local version = import "version.jsonnet"; prometheus: "docker.io/prom/prometheus:v2.53.2", grafana: "docker.io/grafana/grafana:11.1.4", trustgraph: "docker.io/trustgraph/trustgraph-flow:" + version, + qdrant: "docker.io/qdrant/qdrant:v1.11.1" } diff --git a/templates/components/qdrant.jsonnet b/templates/components/qdrant.jsonnet new file mode 100644 index 00000000..e2d69f39 --- /dev/null +++ b/templates/components/qdrant.jsonnet @@ -0,0 +1,111 @@ +local base = import "base.jsonnet"; +local images = import "images.jsonnet"; +local url = import "url.jsonnet"; +local qdrant = import "stores/qdrant.jsonnet"; + +qdrant + { + + services +: { + + "store-graph-embeddings": base + { + image: images.trustgraph, + command: [ + "ge-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + ], + deploy: { + resources: { + limits: { + cpus: '0.5', + memory: '128M' + }, + reservations: { + cpus: '0.1', + memory: '128M' + } + } + }, + }, + + "query-graph-embeddings": base + { + image: images.trustgraph, + command: [ + "ge-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + ], + deploy: { + resources: { + limits: { + cpus: '0.5', + memory: '128M' + }, + reservations: { + cpus: '0.1', + memory: '128M' + } + } + }, + }, + +/* + +// Document embeddings writer & query service. Not currently enabled. + + "store-doc-embeddings": base + { + image: images.trustgraph, + command: [ + "de-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + ], + deploy: { + resources: { + limits: { + cpus: '0.5', + memory: '128M' + }, + reservations: { + cpus: '0.1', + memory: '128M' + } + } + }, + }, + + "query-doc-embeddings": base + { + image: images.trustgraph, + command: [ + "de-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + ], + deploy: { + resources: { + limits: { + cpus: '0.5', + memory: '128M' + }, + reservations: { + cpus: '0.1', + memory: '128M' + } + } + }, + }, + +*/ + + } + +} + diff --git a/templates/components/stores/qdrant.jsonnet b/templates/components/stores/qdrant.jsonnet new file mode 100644 index 00000000..63c41325 --- /dev/null +++ b/templates/components/stores/qdrant.jsonnet @@ -0,0 +1,37 @@ +local base = import "../base.jsonnet"; +local images = import "../images.jsonnet"; + +{ + + volumes +: { + qdrant: {}, + }, + + services +: { + + qdrant: base + { + image: images.qdrant, + ports: [ + "6333:6333", + "6334:6334", + ], + volumes: [ + "qdrant:/qdrant/storage" + ], + deploy: { + resources: { + limits: { + cpus: '1.0', + memory: '256M' + }, + reservations: { + cpus: '0.5', + memory: '256M' + } + } + }, + }, + + }, + +} diff --git a/templates/components/url.jsonnet b/templates/components/url.jsonnet index 9987ae63..36c7ca41 100644 --- a/templates/components/url.jsonnet +++ b/templates/components/url.jsonnet @@ -1,4 +1,5 @@ { pulsar: "pulsar://pulsar:6650", milvus: "http://milvus:19530", + qdrant: "http://qdrant:6333", } diff --git a/templates/main.jsonnet b/templates/main.jsonnet index 26e6ecda..24773618 100644 --- a/templates/main.jsonnet +++ b/templates/main.jsonnet @@ -3,6 +3,7 @@ local components = { cassandra: import "components/cassandra.jsonnet", pulsar: import "components/pulsar.jsonnet", milvus: import "components/milvus.jsonnet", + qdrant: import "components/qdrant.jsonnet", grafana: import "components/grafana.jsonnet", trustgraph: import "components/trustgraph.jsonnet", azure: import "components/azure.jsonnet", diff --git a/templates/prompts/gemini.jsonnet b/templates/prompts/gemini.jsonnet index 4c91a75d..03dadfd1 100644 --- a/templates/prompts/gemini.jsonnet +++ b/templates/prompts/gemini.jsonnet @@ -25,6 +25,10 @@ local url = import "../components/url.jsonnet"; "\nStudy the following text and derive entity relationships. For each\nrelationship, derive the subject, predicate and object of the relationship.\nOutput relationships in JSON format as an arary of objects with fields:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: false if the object is a simple data type: name, value or date. true if it is an entity.\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not use special characters in the abstract text. The\nabstract must be written as plain text. Do not add markdown formatting\nor headers or prefixes.\n", "--knowledge-query-template", "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{graph}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n", + "--document-query-template", + "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{documents}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n", + "--rows-template", + "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", ], deploy: { resources: { diff --git a/templates/prompts/mixtral.jsonnet b/templates/prompts/mixtral.jsonnet index 8b1f1e94..74f9ff09 100644 --- a/templates/prompts/mixtral.jsonnet +++ b/templates/prompts/mixtral.jsonnet @@ -25,6 +25,10 @@ local url = import "../components/url.jsonnet"; "\nStudy the following text and derive entity relationships. For each\nrelationship, derive the subject, predicate and object of the relationship.\nOutput relationships in JSON format as an arary of objects with fields:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: false if the object is a simple data type: name, value or date. true if it is an entity.\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not use special characters in the abstract text. The\nabstract must be written as plain text. Do not add markdown formatting\nor headers or prefixes.\n", "--knowledge-query-template", "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{graph}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n", + "--document-query-template", + "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{documents}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n", + "--rows-template", + "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", ], deploy: { resources: { diff --git a/templates/prompts/openai.jsonnet b/templates/prompts/openai.jsonnet index ccd83705..ffbb78d5 100644 --- a/templates/prompts/openai.jsonnet +++ b/templates/prompts/openai.jsonnet @@ -25,6 +25,10 @@ local url = import "../components/url.jsonnet"; "\nStudy the following text and derive entity relationships. For each\nrelationship, derive the subject, predicate and object of the relationship.\nOutput relationships in JSON format as an arary of objects with fields:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: false if the object is a simple data type: name, value or date. true if it is an entity.\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not use special characters in the abstract text. The\nabstract must be written as plain text. Do not add markdown formatting\nor headers or prefixes.\n", "--knowledge-query-template", "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{graph}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n", + "--document-query-template", + "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{documents}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n", + "--rows-template", + "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", ], deploy: { resources: { diff --git a/templates/test.jsonnet b/templates/test.jsonnet new file mode 100644 index 00000000..d80b056c --- /dev/null +++ b/templates/test.jsonnet @@ -0,0 +1 @@ +std.extVar("asd") diff --git a/tg-launch-azure-cassandra.yaml b/tg-launch-azure-cassandra.yaml index c91bd030..61da8ab0 100644 --- a/tg-launch-azure-cassandra.yaml +++ b/tg-launch-azure-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -50,35 +50,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -122,7 +95,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +128,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,55 +143,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "pdf-decoder": "command": - "pdf-decoder" @@ -232,7 +158,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -272,6 +198,17 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--document-query-template" + - | + Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + + Here is the context: + {documents} + + Use only the provided knowledge statements to respond to the following: + {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -280,7 +217,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -327,7 +264,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -365,13 +302,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -380,7 +333,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -397,15 +350,15 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -414,7 +367,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -431,7 +384,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -454,7 +407,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion-rag": "command": @@ -481,7 +434,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "vectorize": "command": @@ -496,14 +449,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": "cassandra": {} - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-azure-neo4j.yaml b/tg-launch-azure-neo4j.yaml index cc3b7484..4f339732 100644 --- a/tg-launch-azure-neo4j.yaml +++ b/tg-launch-azure-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -33,35 +33,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -105,7 +78,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +111,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,55 +126,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "neo4j": "deploy": "resources": @@ -233,7 +159,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -273,6 +199,17 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--document-query-template" + - | + Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + + Here is the context: + {documents} + + Use only the provided knowledge statements to respond to the following: + {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -281,7 +218,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -328,7 +265,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -366,13 +303,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -381,7 +334,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -398,15 +351,15 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -415,7 +368,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -432,7 +385,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -455,7 +408,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion-rag": "command": @@ -482,7 +435,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "vectorize": "command": @@ -497,14 +450,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "neo4j": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-bedrock-cassandra.yaml b/tg-launch-bedrock-cassandra.yaml index b1c6dc43..46660d69 100644 --- a/tg-launch-bedrock-cassandra.yaml +++ b/tg-launch-bedrock-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -50,35 +50,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -122,7 +95,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +128,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,55 +143,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "pdf-decoder": "command": - "pdf-decoder" @@ -232,7 +158,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -272,6 +198,17 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--document-query-template" + - | + Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + + Here is the context: + {documents} + + Use only the provided knowledge statements to respond to the following: + {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -280,7 +217,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -327,7 +264,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -365,13 +302,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -380,7 +333,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -397,15 +350,15 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -414,7 +367,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -431,7 +384,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -458,7 +411,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion-rag": "command": @@ -489,7 +442,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "vectorize": "command": @@ -504,14 +457,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": "cassandra": {} - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-bedrock-neo4j.yaml b/tg-launch-bedrock-neo4j.yaml index 12c0ade5..8d4f8c57 100644 --- a/tg-launch-bedrock-neo4j.yaml +++ b/tg-launch-bedrock-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -33,35 +33,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -105,7 +78,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +111,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,55 +126,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "neo4j": "deploy": "resources": @@ -233,7 +159,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -273,6 +199,17 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--document-query-template" + - | + Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + + Here is the context: + {documents} + + Use only the provided knowledge statements to respond to the following: + {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -281,7 +218,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -328,7 +265,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -366,13 +303,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -381,7 +334,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -398,15 +351,15 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -415,7 +368,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -432,7 +385,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -459,7 +412,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion-rag": "command": @@ -490,7 +443,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "vectorize": "command": @@ -505,14 +458,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "neo4j": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-claude-cassandra.yaml b/tg-launch-claude-cassandra.yaml index 834e8649..1dc098c1 100644 --- a/tg-launch-claude-cassandra.yaml +++ b/tg-launch-claude-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -50,35 +50,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -122,7 +95,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +128,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,55 +143,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "pdf-decoder": "command": - "pdf-decoder" @@ -232,7 +158,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -272,6 +198,17 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--document-query-template" + - | + Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + + Here is the context: + {documents} + + Use only the provided knowledge statements to respond to the following: + {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -280,7 +217,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -327,7 +264,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -365,13 +302,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -380,7 +333,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -397,15 +350,15 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -414,7 +367,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -431,7 +384,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -452,7 +405,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion-rag": "command": @@ -477,7 +430,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "vectorize": "command": @@ -492,14 +445,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": "cassandra": {} - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-claude-neo4j.yaml b/tg-launch-claude-neo4j.yaml index ab726b43..3b6f32fb 100644 --- a/tg-launch-claude-neo4j.yaml +++ b/tg-launch-claude-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -33,35 +33,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -105,7 +78,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +111,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,55 +126,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "neo4j": "deploy": "resources": @@ -233,7 +159,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -273,6 +199,17 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--document-query-template" + - | + Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + + Here is the context: + {documents} + + Use only the provided knowledge statements to respond to the following: + {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -281,7 +218,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -328,7 +265,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -366,13 +303,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -381,7 +334,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -398,15 +351,15 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -415,7 +368,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -432,7 +385,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -453,7 +406,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion-rag": "command": @@ -478,7 +431,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "vectorize": "command": @@ -493,14 +446,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "neo4j": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-cohere-cassandra.yaml b/tg-launch-cohere-cassandra.yaml index a1807392..cffa2806 100644 --- a/tg-launch-cohere-cassandra.yaml +++ b/tg-launch-cohere-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -50,35 +50,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -122,7 +95,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +128,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,55 +143,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "pdf-decoder": "command": - "pdf-decoder" @@ -232,7 +158,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -272,6 +198,17 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--document-query-template" + - | + Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + + Here is the context: + {documents} + + Use only the provided knowledge statements to respond to the following: + {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -280,7 +217,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -327,7 +264,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -365,13 +302,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -380,7 +333,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -397,15 +350,15 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -414,7 +367,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -431,7 +384,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -450,7 +403,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion-rag": "command": @@ -473,7 +426,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "vectorize": "command": @@ -488,14 +441,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": "cassandra": {} - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-cohere-neo4j.yaml b/tg-launch-cohere-neo4j.yaml index eb6a1204..3adcd792 100644 --- a/tg-launch-cohere-neo4j.yaml +++ b/tg-launch-cohere-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -33,35 +33,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -105,7 +78,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +111,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,55 +126,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "neo4j": "deploy": "resources": @@ -233,7 +159,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -273,6 +199,17 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--document-query-template" + - | + Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + + Here is the context: + {documents} + + Use only the provided knowledge statements to respond to the following: + {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -281,7 +218,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -328,7 +265,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -366,13 +303,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -381,7 +334,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -398,15 +351,15 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -415,7 +368,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -432,7 +385,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -451,7 +404,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion-rag": "command": @@ -474,7 +427,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "vectorize": "command": @@ -489,14 +442,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "neo4j": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-ollama-cassandra.yaml b/tg-launch-ollama-cassandra.yaml index b776ef5e..e21e34d3 100644 --- a/tg-launch-ollama-cassandra.yaml +++ b/tg-launch-ollama-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -50,35 +50,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -122,7 +95,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +128,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,55 +143,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "pdf-decoder": "command": - "pdf-decoder" @@ -232,7 +158,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -267,7 +193,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -290,7 +216,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -328,13 +254,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -343,7 +285,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -360,15 +302,15 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -377,7 +319,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -394,7 +336,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -413,7 +355,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion-rag": "command": @@ -436,7 +378,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "vectorize": "command": @@ -451,14 +393,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": "cassandra": {} - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-ollama-neo4j.yaml b/tg-launch-ollama-neo4j.yaml index 87eccca0..5739e5f4 100644 --- a/tg-launch-ollama-neo4j.yaml +++ b/tg-launch-ollama-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -33,35 +33,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -105,7 +78,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +111,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,55 +126,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "neo4j": "deploy": "resources": @@ -233,7 +159,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -268,7 +194,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -291,7 +217,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -329,13 +255,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -344,7 +286,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -361,15 +303,15 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -378,7 +320,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -395,7 +337,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -414,7 +356,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion-rag": "command": @@ -437,7 +379,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "vectorize": "command": @@ -452,14 +394,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "neo4j": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-openai-cassandra.yaml b/tg-launch-openai-cassandra.yaml index a95ebede..90675766 100644 --- a/tg-launch-openai-cassandra.yaml +++ b/tg-launch-openai-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -50,35 +50,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -122,7 +95,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +128,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,55 +143,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "pdf-decoder": "command": - "pdf-decoder" @@ -232,7 +158,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -272,6 +198,17 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--document-query-template" + - | + Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + + Here is the context: + {documents} + + Use only the provided knowledge statements to respond to the following: + {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -280,7 +217,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -327,7 +264,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -365,13 +302,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -380,7 +333,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -397,15 +350,15 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -414,7 +367,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -431,7 +384,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -452,7 +405,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion-rag": "command": @@ -477,7 +430,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "vectorize": "command": @@ -492,14 +445,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": "cassandra": {} - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-openai-neo4j.yaml b/tg-launch-openai-neo4j.yaml index 73aa8eb5..f80ce5cf 100644 --- a/tg-launch-openai-neo4j.yaml +++ b/tg-launch-openai-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -33,35 +33,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -105,7 +78,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +111,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,55 +126,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "neo4j": "deploy": "resources": @@ -233,7 +159,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -273,6 +199,17 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--document-query-template" + - | + Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + + Here is the context: + {documents} + + Use only the provided knowledge statements to respond to the following: + {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -281,7 +218,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -328,7 +265,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -366,13 +303,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -381,7 +334,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -398,15 +351,15 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -415,7 +368,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -432,7 +385,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -453,7 +406,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion-rag": "command": @@ -478,7 +431,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "vectorize": "command": @@ -493,14 +446,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "neo4j": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-vertexai-cassandra.yaml b/tg-launch-vertexai-cassandra.yaml index 39bd0ac7..18938ada 100644 --- a/tg-launch-vertexai-cassandra.yaml +++ b/tg-launch-vertexai-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -50,35 +50,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -122,7 +95,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +128,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,55 +143,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "pdf-decoder": "command": - "pdf-decoder" @@ -232,7 +158,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -272,6 +198,17 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--document-query-template" + - | + Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + + Here is the context: + {documents} + + Use only the provided knowledge statements to respond to the following: + {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -280,7 +217,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -327,7 +264,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -365,13 +302,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -380,7 +333,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -397,15 +350,15 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -414,7 +367,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -431,7 +384,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -454,7 +407,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": - "./vertexai:/vertexai" @@ -483,7 +436,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": - "./vertexai:/vertexai" @@ -500,14 +453,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": "cassandra": {} - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-launch-vertexai-neo4j.yaml b/tg-launch-vertexai-neo4j.yaml index dc4dc8e3..cc480e04 100644 --- a/tg-launch-vertexai-neo4j.yaml +++ b/tg-launch-vertexai-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "embeddings": "command": @@ -33,35 +33,8 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -105,7 +78,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +111,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,55 +126,8 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "neo4j": "deploy": "resources": @@ -233,7 +159,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prometheus": "deploy": @@ -273,6 +199,17 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--document-query-template" + - | + Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + + Here is the context: + {documents} + + Use only the provided knowledge statements to respond to the following: + {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -281,7 +218,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "prompt-rag": "command": @@ -328,7 +265,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -366,13 +303,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -381,7 +334,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -398,15 +351,15 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -415,7 +368,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -432,7 +385,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "text-completion": "command": @@ -455,7 +408,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": - "./vertexai:/vertexai" @@ -484,7 +437,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": - "./vertexai:/vertexai" @@ -501,14 +454,12 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "neo4j": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-storage-cassandra.yaml b/tg-storage-cassandra.yaml index e1d6935c..4bf4c27d 100644 --- a/tg-storage-cassandra.yaml +++ b/tg-storage-cassandra.yaml @@ -16,33 +16,6 @@ "restart": "on-failure:100" "volumes": - "cassandra:/var/lib/cassandra" - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -81,53 +54,6 @@ "memory": "128M" "image": "docker.io/apachepulsar/pulsar:3.3.1" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "prometheus": "deploy": "resources": @@ -180,13 +106,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -195,7 +137,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -212,15 +154,15 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -229,7 +171,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -246,14 +188,12 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": "cassandra": {} - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/tg-storage-neo4j.yaml b/tg-storage-neo4j.yaml index 51e969f9..b6f0c49d 100644 --- a/tg-storage-neo4j.yaml +++ b/tg-storage-neo4j.yaml @@ -1,31 +1,4 @@ "services": - "etcd": - "command": - - "etcd" - - "-advertise-client-urls=http://127.0.0.1:2379" - - "-listen-client-urls" - - "http://0.0.0.0:2379" - - "--data-dir" - - "/etcd" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "ETCD_AUTO_COMPACTION_MODE": "revision" - "ETCD_AUTO_COMPACTION_RETENTION": "1000" - "ETCD_QUOTA_BACKEND_BYTES": "4294967296" - "ETCD_SNAPSHOT_COUNT": "50000" - "image": "quay.io/coreos/etcd:v3.5.15" - "ports": - - "2379:2379" - "restart": "on-failure:100" - "volumes": - - "etcd:/etcd" "grafana": "deploy": "resources": @@ -64,53 +37,6 @@ "memory": "128M" "image": "docker.io/apachepulsar/pulsar:3.3.1" "restart": "on-failure:100" - "milvus": - "command": - - "milvus" - - "run" - - "standalone" - "deploy": - "resources": - "limits": - "cpus": "1.0" - "memory": "256M" - "reservations": - "cpus": "0.5" - "memory": "256M" - "environment": - "ETCD_ENDPOINTS": "etcd:2379" - "MINIO_ADDRESS": "minio:9000" - "image": "docker.io/milvusdb/milvus:v2.4.9" - "ports": - - "9091:9091" - - "19530:19530" - "restart": "on-failure:100" - "volumes": - - "milvus:/var/lib/milvus" - "minio": - "command": - - "minio" - - "server" - - "/minio_data" - - "--console-address" - - ":9001" - "deploy": - "resources": - "limits": - "cpus": "0.5" - "memory": "128M" - "reservations": - "cpus": "0.25" - "memory": "128M" - "environment": - "MINIO_ROOT_PASSWORD": "minioadmin" - "MINIO_ROOT_USER": "minioadmin" - "image": "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z" - "ports": - - "9001:9001" - "restart": "on-failure:100" - "volumes": - - "minio-data:/minio_data" "neo4j": "deploy": "resources": @@ -181,13 +107,29 @@ - "9527:9527" - "7750:7750" "restart": "on-failure:100" + "qdrant": + "deploy": + "resources": + "limits": + "cpus": "1.0" + "memory": "256M" + "reservations": + "cpus": "0.5" + "memory": "256M" + "image": "docker.io/qdrant/qdrant:v1.11.1" + "ports": + - "6333:6333" + - "6334:6334" + "restart": "on-failure:100" + "volumes": + - "qdrant:/qdrant/storage" "query-graph-embeddings": "command": - - "ge-query-milvus" + - "ge-query-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -196,7 +138,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "query-triples": "command": @@ -213,15 +155,15 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-graph-embeddings": "command": - - "ge-write-milvus" + - "ge-write-qdrant" - "-p" - "pulsar://pulsar:6650" - "-t" - - "http://milvus:19530" + - "http://qdrant:6333" "deploy": "resources": "limits": @@ -230,7 +172,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "store-triples": "command": @@ -247,14 +189,12 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.20" "restart": "on-failure:100" "volumes": - "etcd": {} "grafana-storage": {} - "milvus": {} - "minio-data": {} "neo4j": {} "prometheus-data": {} "pulsar-conf": {} "pulsar-data": {} + "qdrant": {} diff --git a/trustgraph/base/consumer.py b/trustgraph/base/consumer.py index 5d79db5e..0c975cf5 100644 --- a/trustgraph/base/consumer.py +++ b/trustgraph/base/consumer.py @@ -10,7 +10,6 @@ class Consumer(BaseProcessor): def __init__(self, **params): - print("HERE2") super(Consumer, self).__init__(**params) input_queue = params.get("input_queue") @@ -30,7 +29,6 @@ class Consumer(BaseProcessor): 'pubsub', 'Pub/sub configuration' ) - print("HERE") if not hasattr(__class__, "processing_metric"): __class__.processing_metric = Counter( 'processing_count', 'Processing count', ["status"] diff --git a/trustgraph/query/graph_embeddings/qdrant/__init__.py b/trustgraph/query/graph_embeddings/qdrant/__init__.py new file mode 100644 index 00000000..ba844705 --- /dev/null +++ b/trustgraph/query/graph_embeddings/qdrant/__init__.py @@ -0,0 +1,3 @@ + +from . service import * + diff --git a/trustgraph/query/graph_embeddings/qdrant/__main__.py b/trustgraph/query/graph_embeddings/qdrant/__main__.py new file mode 100755 index 00000000..89684e3e --- /dev/null +++ b/trustgraph/query/graph_embeddings/qdrant/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from . hf import run + +if __name__ == '__main__': + run() + diff --git a/trustgraph/query/graph_embeddings/qdrant/service.py b/trustgraph/query/graph_embeddings/qdrant/service.py new file mode 100755 index 00000000..e61a00a7 --- /dev/null +++ b/trustgraph/query/graph_embeddings/qdrant/service.py @@ -0,0 +1,133 @@ + +""" +Graph embeddings query service. Input is vector, output is list of +entities +""" + +from qdrant_client import QdrantClient +from qdrant_client.models import PointStruct +from qdrant_client.models import Distance, VectorParams +import uuid + +from .... schema import GraphEmbeddingsRequest, GraphEmbeddingsResponse +from .... schema import Error, Value +from .... schema import graph_embeddings_request_queue +from .... schema import graph_embeddings_response_queue +from .... base import ConsumerProducer + +module = ".".join(__name__.split(".")[1:-1]) + +default_input_queue = graph_embeddings_request_queue +default_output_queue = graph_embeddings_response_queue +default_subscriber = module +default_store_uri = 'http://localhost:6333' + +class Processor(ConsumerProducer): + + def __init__(self, **params): + + input_queue = params.get("input_queue", default_input_queue) + output_queue = params.get("output_queue", default_output_queue) + subscriber = params.get("subscriber", default_subscriber) + store_uri = params.get("store_uri", default_store_uri) + + super(Processor, self).__init__( + **params | { + "input_queue": input_queue, + "output_queue": output_queue, + "subscriber": subscriber, + "input_schema": GraphEmbeddingsRequest, + "output_schema": GraphEmbeddingsResponse, + "store_uri": store_uri, + } + ) + + self.client = QdrantClient(url=store_uri) + + def create_value(self, ent): + if ent.startswith("http://") or ent.startswith("https://"): + return Value(value=ent, is_uri=True) + else: + return Value(value=ent, is_uri=False) + + def handle(self, msg): + + try: + + v = msg.value() + + # Sender-produced ID + id = msg.properties()["id"] + + print(f"Handling input {id}...", flush=True) + + entities = set() + + for vec in v.vectors: + + dim = len(vec) + collection = "triples_" + str(dim) + + search_result = self.client.query_points( + collection_name=collection, + query=vec, + limit=v.limit, + with_payload=True, + ).points + + for r in search_result: + ent = r.payload["entity"] + entities.add(ent) + + # Convert set to list + entities = list(entities) + + ents2 = [] + + for ent in entities: + ents2.append(self.create_value(ent)) + + entities = ents2 + + print("Send response...", flush=True) + r = GraphEmbeddingsResponse(entities=entities, error=None) + self.producer.send(r, properties={"id": id}) + + print("Done.", flush=True) + + except Exception as e: + + print(f"Exception: {e}") + + print("Send error response...", flush=True) + + r = GraphEmbeddingsResponse( + error=Error( + type = "llm-error", + message = str(e), + ), + entities=None, + ) + + self.producer.send(r, properties={"id": id}) + + self.consumer.acknowledge(msg) + + @staticmethod + def add_args(parser): + + ConsumerProducer.add_args( + parser, default_input_queue, default_subscriber, + default_output_queue, + ) + + parser.add_argument( + '-t', '--store-uri', + default=default_store_uri, + help=f'Milvus store URI (default: {default_store_uri})' + ) + +def run(): + + Processor.start(module, __doc__) + diff --git a/trustgraph/storage/graph_embeddings/qdrant/__init__.py b/trustgraph/storage/graph_embeddings/qdrant/__init__.py new file mode 100644 index 00000000..d891d55f --- /dev/null +++ b/trustgraph/storage/graph_embeddings/qdrant/__init__.py @@ -0,0 +1,3 @@ + +from . write import * + diff --git a/trustgraph/storage/graph_embeddings/qdrant/__main__.py b/trustgraph/storage/graph_embeddings/qdrant/__main__.py new file mode 100755 index 00000000..c05d8c6d --- /dev/null +++ b/trustgraph/storage/graph_embeddings/qdrant/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from . write import run + +if __name__ == '__main__': + run() + diff --git a/trustgraph/storage/graph_embeddings/qdrant/write.py b/trustgraph/storage/graph_embeddings/qdrant/write.py new file mode 100755 index 00000000..95448750 --- /dev/null +++ b/trustgraph/storage/graph_embeddings/qdrant/write.py @@ -0,0 +1,102 @@ + +""" +Accepts entity/vector pairs and writes them to a Qdrant store. +""" + +from qdrant_client import QdrantClient +from qdrant_client.models import PointStruct +from qdrant_client.models import Distance, VectorParams +import uuid + +from .... schema import GraphEmbeddings +from .... schema import graph_embeddings_store_queue +from .... log_level import LogLevel +from .... base import Consumer + +module = ".".join(__name__.split(".")[1:-1]) + +default_input_queue = graph_embeddings_store_queue +default_subscriber = module +default_store_uri = 'http://localhost:6333' + +class Processor(Consumer): + + def __init__(self, **params): + + input_queue = params.get("input_queue", default_input_queue) + subscriber = params.get("subscriber", default_subscriber) + store_uri = params.get("store_uri", default_store_uri) + + super(Processor, self).__init__( + **params | { + "input_queue": input_queue, + "subscriber": subscriber, + "input_schema": GraphEmbeddings, + "store_uri": store_uri, + } + ) + + self.last_collection = None + self.last_dim = None + + self.client = QdrantClient(url=store_uri) + + def handle(self, msg): + + v = msg.value() + + if v.entity.value == "" or v.entity.value is None: return + + for vec in v.vectors: + + dim = len(vec) + collection = "triples_" + str(dim) + + if dim != self.last_dim: + + if not self.client.collection_exists(collection): + + try: + self.client.create_collection( + collection_name=collection, + vectors_config=VectorParams( + size=dim, distance=Distance.DOT + ), + ) + except Exception as e: + print("Qdrant collection creation failed") + raise e + + self.last_collection = collection + self.last_dim = dim + + self.client.upsert( + collection_name=collection, + points=[ + PointStruct( + id=str(uuid.uuid4()), + vector=vec, + payload={ + "entity": v.entity.value, + } + ) + ] + ) + + @staticmethod + def add_args(parser): + + Consumer.add_args( + parser, default_input_queue, default_subscriber, + ) + + parser.add_argument( + '-t', '--store-uri', + default=default_store_uri, + help=f'Qdrant store URI (default: {default_store_uri})' + ) + +def run(): + + Processor.start(module, __doc__) +