diff --git a/Makefile b/Makefile index b07e3dc8..66ba658d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # VERSION=$(shell git describe | sed 's/^v//') -VERSION=0.7.17 +VERSION=0.7.19 DOCKER=podman diff --git a/scripts/kg-extract-definitions b/scripts/kg-extract-definitions index 327ec06f..7f20225b 100755 --- a/scripts/kg-extract-definitions +++ b/scripts/kg-extract-definitions @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from trustgraph.kg.extract_definitions import run +from trustgraph.extract.kg.definitions import run run() diff --git a/scripts/kg-extract-relationships b/scripts/kg-extract-relationships index 91040589..f57d7c89 100755 --- a/scripts/kg-extract-relationships +++ b/scripts/kg-extract-relationships @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from trustgraph.kg.extract_relationships import run +from trustgraph.extract.kg.relationships import run run() diff --git a/scripts/object-extract-row b/scripts/object-extract-row new file mode 100755 index 00000000..04cbcfef --- /dev/null +++ b/scripts/object-extract-row @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from trustgraph.extract.object.row import run + +run() + diff --git a/scripts/oe-write-milvus b/scripts/oe-write-milvus new file mode 100755 index 00000000..c78f2000 --- /dev/null +++ b/scripts/oe-write-milvus @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from trustgraph.storage.object_embeddings.milvus import run + +run() + diff --git a/scripts/rows-write-cassandra b/scripts/rows-write-cassandra new file mode 100755 index 00000000..a1358f5e --- /dev/null +++ b/scripts/rows-write-cassandra @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from trustgraph.storage.rows.cassandra import run + +run() + diff --git a/setup.py b/setup.py index 2f6a11f0..2d35975f 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import os with open("README.md", "r") as fh: long_description = fh.read() -version = "0.7.17" +version = "0.7.19" setuptools.setup( name="trustgraph", @@ -74,10 +74,13 @@ setuptools.setup( "scripts/load-graph-embeddings", "scripts/load-triples", "scripts/loader", + "scripts/object-extract-row", + "scripts/oe-write-milvus", "scripts/pdf-decoder", "scripts/prompt-generic", "scripts/prompt-template", "scripts/query", + "scripts/rows-write-cassandra", "scripts/run-processing", "scripts/text-completion-azure", "scripts/text-completion-bedrock", @@ -88,8 +91,8 @@ setuptools.setup( "scripts/text-completion-vertexai", "scripts/triples-dump-parquet", "scripts/triples-query-cassandra", - "scripts/triples-write-cassandra", "scripts/triples-query-neo4j", + "scripts/triples-write-cassandra", "scripts/triples-write-neo4j", ] ) diff --git a/templates/prompts/gemini.jsonnet b/templates/prompts/gemini.jsonnet index 5fb19a1c..4c91a75d 100644 --- a/templates/prompts/gemini.jsonnet +++ b/templates/prompts/gemini.jsonnet @@ -62,6 +62,8 @@ local url = import "../components/url.jsonnet"; "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{graph}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n", "--document-query-template", "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{documents}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n", + "--rows-template", + "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", ], deploy: { resources: { diff --git a/templates/prompts/mixtral.jsonnet b/templates/prompts/mixtral.jsonnet index 33984033..8b1f1e94 100644 --- a/templates/prompts/mixtral.jsonnet +++ b/templates/prompts/mixtral.jsonnet @@ -62,6 +62,8 @@ local url = import "../components/url.jsonnet"; "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{graph}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n", "--document-query-template", "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{documents}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n", + "--rows-template", + "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", ], deploy: { resources: { diff --git a/templates/prompts/openai.jsonnet b/templates/prompts/openai.jsonnet index edc41a37..ccd83705 100644 --- a/templates/prompts/openai.jsonnet +++ b/templates/prompts/openai.jsonnet @@ -62,6 +62,8 @@ local url = import "../components/url.jsonnet"; "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{graph}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n", "--document-query-template", "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{documents}\n\nUse only the provided knowledge statements to respond to the following:\n{query}\n", + "--rows-template", + "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", ], deploy: { resources: { diff --git a/tests/test-rows-prompt b/tests/test-rows-prompt new file mode 100755 index 00000000..450d7f34 --- /dev/null +++ b/tests/test-rows-prompt @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import pulsar +from trustgraph.clients.prompt_client import PromptClient +from trustgraph.objects.object import Schema +from trustgraph.objects.field import Field, FieldType + +schema = Schema( + name="actors", + description="actors in this story", + fields=[ + Field( + name="name", type=FieldType.STRING, + description="Name of the animal or person in the story" + ), + Field( + name="legs", type=FieldType.INT, + description="Number of legs of the animal or person" + ), + Field( + name="notes", type=FieldType.STRING, + description="Additional notes or observations about this animal or person" + ), + ] +) + +chunk = """I noticed a cat in my garden. It is a four-legged animal +which is a mammal and can be tame or wild. I wonder if it will be friends +with me? I think the cat's name is Fred and it has 4 legs. +There is also a dog barking outside. The dog has 4 legs also. +The dog comes to my call when I shout "Come here, Bernard". + +I am also standing in the garden, my name is Steve and I have 2 legs. + +My friend Clifford is coming to visit shortly, he has 3 legs due to +a freak accident at birth. +""" + +p = PromptClient(pulsar_host="pulsar://localhost:6650") + +resp = p.request_rows( + schema=schema, + chunk=chunk, +) + +for d in resp: + print(f"Name: {d['name']}") + print(f" No. of legs: {d['legs']}") + print(f" Notes: {d['notes']}") + print() + diff --git a/tests/test-run-extract-row b/tests/test-run-extract-row new file mode 100644 index 00000000..5018e07f --- /dev/null +++ b/tests/test-run-extract-row @@ -0,0 +1,13 @@ + +scripts/object-extract-row \ + -p pulsar://localhost:6650 \ + --field 'name:string:100:pri:Name of the person in the story' \ + --field 'job:string:100::Job title or role' \ + --field 'date:string:20::Date entered into role if known' \ + --field 'supervisor:string:100::Supervisor or manager of this person, if known' \ + --field 'location:string:100::Main base or location of work, if known' \ + --field 'notes:string:1000::Additional notes or observations about this animal or person' \ + --no-metrics \ + --name actors \ + --description 'Relevant people' + diff --git a/tg-launch-azure-cassandra.yaml b/tg-launch-azure-cassandra.yaml index eedd1878..c91bd030 100644 --- a/tg-launch-azure-cassandra.yaml +++ b/tg-launch-azure-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -50,7 +50,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -122,7 +122,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +155,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,7 +170,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -232,7 +232,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -280,7 +280,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -317,6 +317,8 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -325,7 +327,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -378,7 +380,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -395,7 +397,7 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -412,7 +414,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -429,7 +431,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -452,7 +454,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion-rag": "command": @@ -479,7 +481,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "vectorize": "command": @@ -494,7 +496,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/tg-launch-azure-neo4j.yaml b/tg-launch-azure-neo4j.yaml index 9b82a29b..cc3b7484 100644 --- a/tg-launch-azure-neo4j.yaml +++ b/tg-launch-azure-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -33,7 +33,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -105,7 +105,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +138,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,7 +153,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -233,7 +233,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -281,7 +281,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -318,6 +318,8 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -326,7 +328,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -379,7 +381,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -396,7 +398,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -413,7 +415,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -430,7 +432,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -453,7 +455,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion-rag": "command": @@ -480,7 +482,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "vectorize": "command": @@ -495,7 +497,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "etcd": {} diff --git a/tg-launch-bedrock-cassandra.yaml b/tg-launch-bedrock-cassandra.yaml index d569bb04..b1c6dc43 100644 --- a/tg-launch-bedrock-cassandra.yaml +++ b/tg-launch-bedrock-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -50,7 +50,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -122,7 +122,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +155,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,7 +170,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -232,7 +232,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -280,7 +280,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -317,6 +317,8 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -325,7 +327,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -378,7 +380,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -395,7 +397,7 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -412,7 +414,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -429,7 +431,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -456,7 +458,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion-rag": "command": @@ -487,7 +489,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "vectorize": "command": @@ -502,7 +504,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/tg-launch-bedrock-neo4j.yaml b/tg-launch-bedrock-neo4j.yaml index 8c14f007..12c0ade5 100644 --- a/tg-launch-bedrock-neo4j.yaml +++ b/tg-launch-bedrock-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -33,7 +33,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -105,7 +105,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +138,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,7 +153,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -233,7 +233,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -281,7 +281,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -318,6 +318,8 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -326,7 +328,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -379,7 +381,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -396,7 +398,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -413,7 +415,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -430,7 +432,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -457,7 +459,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion-rag": "command": @@ -488,7 +490,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "vectorize": "command": @@ -503,7 +505,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "etcd": {} diff --git a/tg-launch-claude-cassandra.yaml b/tg-launch-claude-cassandra.yaml index bab31894..834e8649 100644 --- a/tg-launch-claude-cassandra.yaml +++ b/tg-launch-claude-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -50,7 +50,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -122,7 +122,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +155,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,7 +170,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -232,7 +232,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -280,7 +280,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -317,6 +317,8 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -325,7 +327,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -378,7 +380,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -395,7 +397,7 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -412,7 +414,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -429,7 +431,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -450,7 +452,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion-rag": "command": @@ -475,7 +477,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "vectorize": "command": @@ -490,7 +492,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/tg-launch-claude-neo4j.yaml b/tg-launch-claude-neo4j.yaml index 7c151513..ab726b43 100644 --- a/tg-launch-claude-neo4j.yaml +++ b/tg-launch-claude-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -33,7 +33,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -105,7 +105,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +138,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,7 +153,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -233,7 +233,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -281,7 +281,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -318,6 +318,8 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -326,7 +328,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -379,7 +381,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -396,7 +398,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -413,7 +415,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -430,7 +432,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -451,7 +453,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion-rag": "command": @@ -476,7 +478,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "vectorize": "command": @@ -491,7 +493,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "etcd": {} diff --git a/tg-launch-cohere-cassandra.yaml b/tg-launch-cohere-cassandra.yaml index 65bced52..a1807392 100644 --- a/tg-launch-cohere-cassandra.yaml +++ b/tg-launch-cohere-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -50,7 +50,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -122,7 +122,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +155,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,7 +170,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -232,7 +232,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -280,7 +280,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -317,6 +317,8 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -325,7 +327,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -378,7 +380,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -395,7 +397,7 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -412,7 +414,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -429,7 +431,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -448,7 +450,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion-rag": "command": @@ -471,7 +473,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "vectorize": "command": @@ -486,7 +488,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/tg-launch-cohere-neo4j.yaml b/tg-launch-cohere-neo4j.yaml index 8e8c394f..eb6a1204 100644 --- a/tg-launch-cohere-neo4j.yaml +++ b/tg-launch-cohere-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -33,7 +33,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -105,7 +105,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +138,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,7 +153,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -233,7 +233,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -281,7 +281,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -318,6 +318,8 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -326,7 +328,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -379,7 +381,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -396,7 +398,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -413,7 +415,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -430,7 +432,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -449,7 +451,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion-rag": "command": @@ -472,7 +474,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "vectorize": "command": @@ -487,7 +489,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "etcd": {} diff --git a/tg-launch-ollama-cassandra.yaml b/tg-launch-ollama-cassandra.yaml index c475527b..b776ef5e 100644 --- a/tg-launch-ollama-cassandra.yaml +++ b/tg-launch-ollama-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -50,7 +50,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -122,7 +122,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +155,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,7 +170,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -232,7 +232,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -267,7 +267,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -290,7 +290,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -343,7 +343,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -360,7 +360,7 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -377,7 +377,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -394,7 +394,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -413,7 +413,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion-rag": "command": @@ -436,7 +436,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "vectorize": "command": @@ -451,7 +451,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/tg-launch-ollama-neo4j.yaml b/tg-launch-ollama-neo4j.yaml index f5a76f57..87eccca0 100644 --- a/tg-launch-ollama-neo4j.yaml +++ b/tg-launch-ollama-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -33,7 +33,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -105,7 +105,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +138,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,7 +153,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -233,7 +233,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -268,7 +268,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -291,7 +291,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -344,7 +344,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -361,7 +361,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -378,7 +378,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -395,7 +395,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -414,7 +414,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion-rag": "command": @@ -437,7 +437,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "vectorize": "command": @@ -452,7 +452,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "etcd": {} diff --git a/tg-launch-openai-cassandra.yaml b/tg-launch-openai-cassandra.yaml index 906c87ae..a95ebede 100644 --- a/tg-launch-openai-cassandra.yaml +++ b/tg-launch-openai-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -50,7 +50,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -122,7 +122,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +155,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,7 +170,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -232,7 +232,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -280,7 +280,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -317,6 +317,8 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -325,7 +327,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -378,7 +380,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -395,7 +397,7 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -412,7 +414,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -429,7 +431,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -450,7 +452,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion-rag": "command": @@ -475,7 +477,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "vectorize": "command": @@ -490,7 +492,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/tg-launch-openai-neo4j.yaml b/tg-launch-openai-neo4j.yaml index 97156f11..73aa8eb5 100644 --- a/tg-launch-openai-neo4j.yaml +++ b/tg-launch-openai-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -33,7 +33,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -105,7 +105,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +138,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,7 +153,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -233,7 +233,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -281,7 +281,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -318,6 +318,8 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -326,7 +328,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -379,7 +381,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -396,7 +398,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -413,7 +415,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -430,7 +432,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -451,7 +453,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion-rag": "command": @@ -476,7 +478,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "vectorize": "command": @@ -491,7 +493,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "etcd": {} diff --git a/tg-launch-vertexai-cassandra.yaml b/tg-launch-vertexai-cassandra.yaml index a8ad2a20..39bd0ac7 100644 --- a/tg-launch-vertexai-cassandra.yaml +++ b/tg-launch-vertexai-cassandra.yaml @@ -33,7 +33,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -50,7 +50,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -122,7 +122,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -155,7 +155,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -170,7 +170,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -232,7 +232,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -280,7 +280,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -317,6 +317,8 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -325,7 +327,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -378,7 +380,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -395,7 +397,7 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -412,7 +414,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -429,7 +431,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -452,7 +454,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": - "./vertexai:/vertexai" @@ -481,7 +483,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": - "./vertexai:/vertexai" @@ -498,7 +500,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/tg-launch-vertexai-neo4j.yaml b/tg-launch-vertexai-neo4j.yaml index a604befa..dc4dc8e3 100644 --- a/tg-launch-vertexai-neo4j.yaml +++ b/tg-launch-vertexai-neo4j.yaml @@ -16,7 +16,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "embeddings": "command": @@ -33,7 +33,7 @@ "reservations": "cpus": "0.5" "memory": "256M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "etcd": "command": @@ -105,7 +105,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "init-pulsar": "command": @@ -138,7 +138,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "kg-extract-relationships": "command": @@ -153,7 +153,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "milvus": "command": @@ -233,7 +233,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prometheus": "deploy": @@ -281,7 +281,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "prompt-rag": "command": @@ -318,6 +318,8 @@ Use only the provided knowledge statements to respond to the following: {query} + - "--rows-template" + - "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{schema}\n\n\n\n{text}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n" "deploy": "resources": "limits": @@ -326,7 +328,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "pulsar": "command": "bin/pulsar standalone" @@ -379,7 +381,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -396,7 +398,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -413,7 +415,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -430,7 +432,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "text-completion": "command": @@ -453,7 +455,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": - "./vertexai:/vertexai" @@ -482,7 +484,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": - "./vertexai:/vertexai" @@ -499,7 +501,7 @@ "reservations": "cpus": "0.5" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "etcd": {} diff --git a/tg-storage-cassandra.yaml b/tg-storage-cassandra.yaml index 08997ca1..e1d6935c 100644 --- a/tg-storage-cassandra.yaml +++ b/tg-storage-cassandra.yaml @@ -195,7 +195,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -212,7 +212,7 @@ "reservations": "cpus": "0.1" "memory": "512M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -229,7 +229,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -246,7 +246,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "cassandra": {} diff --git a/tg-storage-neo4j.yaml b/tg-storage-neo4j.yaml index ad188310..51e969f9 100644 --- a/tg-storage-neo4j.yaml +++ b/tg-storage-neo4j.yaml @@ -196,7 +196,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "query-triples": "command": @@ -213,7 +213,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-graph-embeddings": "command": @@ -230,7 +230,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "store-triples": "command": @@ -247,7 +247,7 @@ "reservations": "cpus": "0.1" "memory": "128M" - "image": "docker.io/trustgraph/trustgraph-flow:0.7.17" + "image": "docker.io/trustgraph/trustgraph-flow:0.7.19" "restart": "on-failure:100" "volumes": "etcd": {} diff --git a/trustgraph/base/base_processor.py b/trustgraph/base/base_processor.py index a71d724a..9b2e29ac 100644 --- a/trustgraph/base/base_processor.py +++ b/trustgraph/base/base_processor.py @@ -116,4 +116,4 @@ class BaseProcessor: print("Exception:", e, flush=True) print("Will retry...", flush=True) - time.sleep(10) + time.sleep(4) diff --git a/trustgraph/base/consumer.py b/trustgraph/base/consumer.py index a7e5e041..5d79db5e 100644 --- a/trustgraph/base/consumer.py +++ b/trustgraph/base/consumer.py @@ -10,6 +10,7 @@ class Consumer(BaseProcessor): def __init__(self, **params): + print("HERE2") super(Consumer, self).__init__(**params) input_queue = params.get("input_queue") @@ -29,6 +30,7 @@ class Consumer(BaseProcessor): 'pubsub', 'Pub/sub configuration' ) + print("HERE") if not hasattr(__class__, "processing_metric"): __class__.processing_metric = Counter( 'processing_count', 'Processing count', ["status"] @@ -65,7 +67,7 @@ class Consumer(BaseProcessor): self.consumer.negative_acknowledge(msg) print("TooManyRequests: will retry") __class__.processing_metric.labels(status="rate-limit").inc() - time.sleep(2) + time.sleep(5) continue except Exception as e: diff --git a/trustgraph/base/consumer_producer.py b/trustgraph/base/consumer_producer.py index 35bf6be2..3f9d0f4f 100644 --- a/trustgraph/base/consumer_producer.py +++ b/trustgraph/base/consumer_producer.py @@ -84,7 +84,7 @@ class ConsumerProducer(BaseProcessor): self.consumer.negative_acknowledge(msg) print("TooManyRequests: will retry") __class__.processing_metric.labels(status="rate-limit").inc() - time.sleep(2) + time.sleep(5) continue except Exception as e: diff --git a/trustgraph/clients/prompt_client.py b/trustgraph/clients/prompt_client.py index 13f36c67..dd1f4e5d 100644 --- a/trustgraph/clients/prompt_client.py +++ b/trustgraph/clients/prompt_client.py @@ -1,7 +1,7 @@ import _pulsar -from .. schema import PromptRequest, PromptResponse, Fact +from .. schema import PromptRequest, PromptResponse, Fact, RowSchema, Field from .. schema import prompt_request_queue from .. schema import prompt_response_queue from . base import BaseClient @@ -52,6 +52,24 @@ class PromptClient(BaseClient): timeout=timeout ).relationships + def request_rows(self, schema, chunk, timeout=300): + + return self.call( + kind="extract-rows", chunk=chunk, + row_schema=RowSchema( + name=schema.name, + description=schema.description, + fields=[ + Field( + name=f.name, type=str(f.type), size=f.size, + primary=f.primary, description=f.description, + ) + for f in schema.fields + ] + ), + timeout=timeout + ).rows + def request_kg_prompt(self, query, kg, timeout=300): return self.call( diff --git a/trustgraph/direct/milvus_object_embeddings.py b/trustgraph/direct/milvus_object_embeddings.py new file mode 100644 index 00000000..92cacfc7 --- /dev/null +++ b/trustgraph/direct/milvus_object_embeddings.py @@ -0,0 +1,154 @@ + +from pymilvus import MilvusClient, CollectionSchema, FieldSchema, DataType +import time + +class ObjectVectors: + + def __init__(self, uri="http://localhost:19530", prefix='obj'): + + self.client = MilvusClient(uri=uri) + + # Strategy is to create collections per dimension. Probably only + # going to be using 1 anyway, but that means we don't need to + # hard-code the dimension anywhere, and no big deal if more than + # one are created. + self.collections = {} + + self.prefix = prefix + + # Time between reloads + self.reload_time = 90 + + # Next time to reload - this forces a reload at next window + self.next_reload = time.time() + self.reload_time + print("Reload at", self.next_reload) + + def init_collection(self, dimension, name): + + collection_name = self.prefix + "_" + name + "_" + str(dimension) + + pkey_field = FieldSchema( + name="id", + dtype=DataType.INT64, + is_primary=True, + auto_id=True, + ) + + vec_field = FieldSchema( + name="vector", + dtype=DataType.FLOAT_VECTOR, + dim=dimension, + ) + + name_field = FieldSchema( + name="name", + dtype=DataType.VARCHAR, + max_length=65535, + ) + + key_name_field = FieldSchema( + name="key_name", + dtype=DataType.VARCHAR, + max_length=65535, + ) + + key_field = FieldSchema( + name="key", + dtype=DataType.VARCHAR, + max_length=65535, + ) + + schema = CollectionSchema( + fields = [ + pkey_field, vec_field, name_field, key_name_field, key_field + ], + description = "Object embedding schema", + ) + + self.client.create_collection( + collection_name=collection_name, + schema=schema, + metric_type="COSINE", + ) + + index_params = MilvusClient.prepare_index_params() + + index_params.add_index( + field_name="vector", + metric_type="COSINE", + index_type="IVF_SQ8", + index_name="vector_index", + params={ "nlist": 128 } + ) + + self.client.create_index( + collection_name=collection_name, + index_params=index_params + ) + + self.collections[(dimension, name)] = collection_name + + def insert(self, embeds, name, key_name, key): + + dim = len(embeds) + + if (dim, name) not in self.collections: + self.init_collection(dim, name) + + data = [ + { + "vector": embeds, + "name": name, + "key_name": key_name, + "key": key, + } + ] + + self.client.insert( + collection_name=self.collections[(dim, name)], + data=data + ) + + def search(self, embeds, name, fields=["key_name", "name"], limit=10): + + dim = len(embeds) + + if dim not in self.collections: + self.init_collection(dim, name) + + coll = self.collections[(dim, name)] + + search_params = { + "metric_type": "COSINE", + "params": { + "radius": 0.1, + "range_filter": 0.8 + } + } + + print("Loading...") + self.client.load_collection( + collection_name=coll, + ) + + print("Searching...") + + res = self.client.search( + collection_name=coll, + data=[embeds], + limit=limit, + output_fields=fields, + search_params=search_params, + )[0] + + + # If reload time has passed, unload collection + if time.time() > self.next_reload: + print("Unloading, reload at", self.next_reload) + self.client.release_collection( + collection_name=coll, + ) + self.next_reload = time.time() + self.reload_time + + return res + diff --git a/trustgraph/kg/__init__.py b/trustgraph/extract/__init__.py similarity index 100% rename from trustgraph/kg/__init__.py rename to trustgraph/extract/__init__.py diff --git a/trustgraph/extract/kg/__init__.py b/trustgraph/extract/kg/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/trustgraph/kg/extract_definitions/__init__.py b/trustgraph/extract/kg/definitions/__init__.py similarity index 100% rename from trustgraph/kg/extract_definitions/__init__.py rename to trustgraph/extract/kg/definitions/__init__.py diff --git a/trustgraph/kg/extract_definitions/__main__.py b/trustgraph/extract/kg/definitions/__main__.py similarity index 100% rename from trustgraph/kg/extract_definitions/__main__.py rename to trustgraph/extract/kg/definitions/__main__.py diff --git a/trustgraph/kg/extract_definitions/extract.py b/trustgraph/extract/kg/definitions/extract.py similarity index 89% rename from trustgraph/kg/extract_definitions/extract.py rename to trustgraph/extract/kg/definitions/extract.py index 81976dcb..fee54736 100755 --- a/trustgraph/kg/extract_definitions/extract.py +++ b/trustgraph/extract/kg/definitions/extract.py @@ -7,14 +7,14 @@ get entity definitions which are output as graph edges. import urllib.parse import json -from ... schema import ChunkEmbeddings, Triple, Source, Value -from ... schema import chunk_embeddings_ingest_queue, triples_store_queue -from ... schema import prompt_request_queue -from ... schema import prompt_response_queue -from ... log_level import LogLevel -from ... clients.prompt_client import PromptClient -from ... rdf import TRUSTGRAPH_ENTITIES, DEFINITION -from ... base import ConsumerProducer +from .... schema import ChunkEmbeddings, Triple, Source, Value +from .... schema import chunk_embeddings_ingest_queue, triples_store_queue +from .... schema import prompt_request_queue +from .... schema import prompt_response_queue +from .... log_level import LogLevel +from .... clients.prompt_client import PromptClient +from .... rdf import TRUSTGRAPH_ENTITIES, DEFINITION +from .... base import ConsumerProducer DEFINITION_VALUE = Value(value=DEFINITION, is_uri=True) diff --git a/trustgraph/kg/extract_relationships/__init__.py b/trustgraph/extract/kg/relationships/__init__.py similarity index 100% rename from trustgraph/kg/extract_relationships/__init__.py rename to trustgraph/extract/kg/relationships/__init__.py diff --git a/trustgraph/kg/extract_relationships/__main__.py b/trustgraph/extract/kg/relationships/__main__.py similarity index 100% rename from trustgraph/kg/extract_relationships/__main__.py rename to trustgraph/extract/kg/relationships/__main__.py diff --git a/trustgraph/kg/extract_relationships/extract.py b/trustgraph/extract/kg/relationships/extract.py similarity index 92% rename from trustgraph/kg/extract_relationships/extract.py rename to trustgraph/extract/kg/relationships/extract.py index 3c99a428..49ef9072 100755 --- a/trustgraph/kg/extract_relationships/extract.py +++ b/trustgraph/extract/kg/relationships/extract.py @@ -9,15 +9,15 @@ import urllib.parse import os from pulsar.schema import JsonSchema -from ... schema import ChunkEmbeddings, Triple, GraphEmbeddings, Source, Value -from ... schema import chunk_embeddings_ingest_queue, triples_store_queue -from ... schema import graph_embeddings_store_queue -from ... schema import prompt_request_queue -from ... schema import prompt_response_queue -from ... log_level import LogLevel -from ... clients.prompt_client import PromptClient -from ... rdf import RDF_LABEL, TRUSTGRAPH_ENTITIES -from ... base import ConsumerProducer +from .... schema import ChunkEmbeddings, Triple, GraphEmbeddings, Source, Value +from .... schema import chunk_embeddings_ingest_queue, triples_store_queue +from .... schema import graph_embeddings_store_queue +from .... schema import prompt_request_queue +from .... schema import prompt_response_queue +from .... log_level import LogLevel +from .... clients.prompt_client import PromptClient +from .... rdf import RDF_LABEL, TRUSTGRAPH_ENTITIES +from .... base import ConsumerProducer RDF_LABEL_VALUE = Value(value=RDF_LABEL, is_uri=True) diff --git a/trustgraph/extract/object/__init__.py b/trustgraph/extract/object/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/trustgraph/extract/object/row/__init__.py b/trustgraph/extract/object/row/__init__.py new file mode 100644 index 00000000..81287a3c --- /dev/null +++ b/trustgraph/extract/object/row/__init__.py @@ -0,0 +1,3 @@ + +from . extract import * + diff --git a/trustgraph/extract/object/row/__main__.py b/trustgraph/extract/object/row/__main__.py new file mode 100755 index 00000000..403fe672 --- /dev/null +++ b/trustgraph/extract/object/row/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from . extract import run + +if __name__ == '__main__': + run() + diff --git a/trustgraph/extract/object/row/extract.py b/trustgraph/extract/object/row/extract.py new file mode 100755 index 00000000..aa53f2a6 --- /dev/null +++ b/trustgraph/extract/object/row/extract.py @@ -0,0 +1,220 @@ + +""" +Simple decoder, accepts vector+text chunks input, applies analysis to pull +out a row of fields. Output as a vector plus object. +""" + +import urllib.parse +import os +from pulsar.schema import JsonSchema + +from .... schema import ChunkEmbeddings, Rows, ObjectEmbeddings, Source +from .... schema import RowSchema, Field +from .... schema import chunk_embeddings_ingest_queue, rows_store_queue +from .... schema import object_embeddings_store_queue +from .... schema import prompt_request_queue +from .... schema import prompt_response_queue +from .... log_level import LogLevel +from .... clients.prompt_client import PromptClient +from .... base import ConsumerProducer + +from .... objects.field import Field as FieldParser +from .... objects.object import Schema + +module = ".".join(__name__.split(".")[1:-1]) + +default_input_queue = chunk_embeddings_ingest_queue +default_output_queue = rows_store_queue +default_vector_queue = object_embeddings_store_queue +default_subscriber = module + +class Processor(ConsumerProducer): + + def __init__(self, **params): + + input_queue = params.get("input_queue", default_input_queue) + output_queue = params.get("output_queue", default_output_queue) + vector_queue = params.get("vector_queue", default_vector_queue) + subscriber = params.get("subscriber", default_subscriber) + pr_request_queue = params.get( + "prompt_request_queue", prompt_request_queue + ) + pr_response_queue = params.get( + "prompt_response_queue", prompt_response_queue + ) + + super(Processor, self).__init__( + **params | { + "input_queue": input_queue, + "output_queue": output_queue, + "subscriber": subscriber, + "input_schema": ChunkEmbeddings, + "output_schema": Rows, + "prompt_request_queue": pr_request_queue, + "prompt_response_queue": pr_response_queue, + } + ) + + self.vec_prod = self.client.create_producer( + topic=vector_queue, + schema=JsonSchema(ObjectEmbeddings), + ) + + __class__.pubsub_metric.info({ + "input_queue": input_queue, + "output_queue": output_queue, + "vector_queue": vector_queue, + "prompt_request_queue": pr_request_queue, + "prompt_response_queue": pr_response_queue, + "subscriber": subscriber, + "input_schema": ChunkEmbeddings.__name__, + "output_schema": Rows.__name__, + "vector_schema": ObjectEmbeddings.__name__, + }) + + flds = __class__.parse_fields(params["field"]) + + for fld in flds: + print(fld) + + self.primary = None + + for f in flds: + if f.primary: + if self.primary: + raise RuntimeError( + "Only one primary key field is supported" + ) + self.primary = f + + if self.primary == None: + raise RuntimeError( + "Must have exactly one primary key field" + ) + + self.schema = Schema( + name = params["name"], + description = params["description"], + fields = flds + ) + + self.row_schema=RowSchema( + name=self.schema.name, + description=self.schema.description, + fields=[ + Field( + name=f.name, type=str(f.type), size=f.size, + primary=f.primary, description=f.description, + ) + for f in self.schema.fields + ] + ) + + self.prompt = PromptClient( + pulsar_host=self.pulsar_host, + input_queue=pr_request_queue, + output_queue=pr_response_queue, + subscriber = module + "-prompt", + ) + + @staticmethod + def parse_fields(fields): + return [ FieldParser.parse(f) for f in fields ] + + def get_rows(self, chunk): + return self.prompt.request_rows(self.schema, chunk) + + def emit_rows(self, source, rows): + + t = Rows( + source=source, row_schema=self.row_schema, rows=rows + ) + self.producer.send(t) + + def emit_vec(self, source, name, vec, key_name, key): + + r = ObjectEmbeddings( + source=source, vectors=vec, name=name, key_name=key_name, id=key + ) + self.vec_prod.send(r) + + def handle(self, msg): + + v = msg.value() + print(f"Indexing {v.source.id}...", flush=True) + + chunk = v.chunk.decode("utf-8") + + try: + + rows = self.get_rows(chunk) + + self.emit_rows( + source=v.source, + rows=rows + ) + + for row in rows: + self.emit_vec( + source=v.source, vec=v.vectors, + name=self.schema.name, key_name=self.primary.name, + key=row[self.primary.name] + ) + + for row in rows: + print(row) + + except Exception as e: + print("Exception: ", e, flush=True) + + print("Done.", flush=True) + + @staticmethod + def add_args(parser): + + ConsumerProducer.add_args( + parser, default_input_queue, default_subscriber, + default_output_queue, + ) + + parser.add_argument( + '-c', '--vector-queue', + default=default_vector_queue, + help=f'Vector output queue (default: {default_vector_queue})' + ) + + parser.add_argument( + '--prompt-request-queue', + default=prompt_request_queue, + help=f'Prompt request queue (default: {prompt_request_queue})', + ) + + parser.add_argument( + '--prompt-response-queue', + default=prompt_response_queue, + help=f'Prompt response queue (default: {prompt_response_queue})', + ) + + parser.add_argument( + '-f', '--field', + required=True, + action='append', + help=f'Field definition, format name:type:size:pri:descriptionn', + ) + + parser.add_argument( + '-n', '--name', + required=True, + help=f'Name of row object', + ) + + parser.add_argument( + '-d', '--description', + required=True, + help=f'Description of object', + ) + +def run(): + + Processor.start(module, __doc__) + diff --git a/trustgraph/model/prompt/generic/prompts.py b/trustgraph/model/prompt/generic/prompts.py index 3d21bf79..e9254dd9 100644 --- a/trustgraph/model/prompt/generic/prompts.py +++ b/trustgraph/model/prompt/generic/prompts.py @@ -48,6 +48,44 @@ or headers or prefixes. Do not include null or unknown definitions. return prompt +def to_rows(schema, text): + + field_schema = [ + f"- Name: {f.name}\n Type: {f.type}\n Definition: {f.description}" + for f in schema.fields + ] + + field_schema = "\n".join(field_schema) + + schema = f"""Object name: {schema.name} +Description: {schema.description} + +Fields: +{field_schema}""" + + prompt = f""" +Study the following text and derive objects which match the schema provided. + +You must output an array of JSON objects for each object you discover +which matches the schema. For each object, output a JSON object whose fields +carry the name field specified in the schema. + + + +{schema} + + + +{text} + + + +You will respond only with raw JSON format data. Do not provide +explanations. Do not add markdown formatting or headers or prefixes. +""" + + return prompt + def get_cypher(kg): sg2 = [] diff --git a/trustgraph/model/prompt/generic/service.py b/trustgraph/model/prompt/generic/service.py index 7f52201d..0c6f8095 100755 --- a/trustgraph/model/prompt/generic/service.py +++ b/trustgraph/model/prompt/generic/service.py @@ -14,7 +14,7 @@ from .... base import ConsumerProducer from .... clients.llm_client import LlmClient from . prompts import to_definitions, to_relationships -from . prompts import to_kg_query, to_document_query +from . prompts import to_kg_query, to_document_query, to_rows module = ".".join(__name__.split(".")[1:-1]) @@ -77,6 +77,11 @@ class Processor(ConsumerProducer): self.handle_extract_relationships(id, v) return + elif kind == "extract-rows": + + self.handle_extract_rows(id, v) + return + elif kind == "kg-prompt": self.handle_kg_prompt(id, v) @@ -222,6 +227,77 @@ class Processor(ConsumerProducer): ) self.producer.send(r, properties={"id": id}) + + def handle_extract_rows(self, id, v): + + try: + + fields = v.row_schema.fields + + prompt = to_rows(v.row_schema, v.chunk) + + print(prompt) + + ans = self.llm.request(prompt) + + print(ans) + + # Silently ignore JSON parse error + try: + objs = json.loads(ans) + except: + print("JSON parse error, ignored", flush=True) + objs = [] + + output = [] + + for obj in objs: + + try: + + row = {} + + for f in fields: + + if f.name not in obj: + print(f"Object ignored, missing field {f.name}") + row = {} + break + + row[f.name] = obj[f.name] + + if row == {}: + continue + + output.append(row) + + except Exception as e: + print("row fields missing, ignored", flush=True) + + for row in output: + print(row) + + print("Send response...", flush=True) + r = PromptResponse(rows=output, error=None) + self.producer.send(r, properties={"id": id}) + + print("Done.", flush=True) + + except Exception as e: + + print(f"Exception: {e}") + + print("Send error response...", flush=True) + + r = PromptResponse( + error=Error( + type = "llm-error", + message = str(e), + ), + response=None, + ) + + self.producer.send(r, properties={"id": id}) def handle_kg_prompt(self, id, v): diff --git a/trustgraph/model/prompt/template/prompts.py b/trustgraph/model/prompt/template/prompts.py index 232bbacc..8b25c621 100644 --- a/trustgraph/model/prompt/template/prompts.py +++ b/trustgraph/model/prompt/template/prompts.py @@ -5,6 +5,27 @@ def to_relationships(template, text): def to_definitions(template, text): return template.format(text=text) +def to_rows(template, schema, text): + + field_schema = [ + f"- Name: {f.name}\n Type: {f.type}\n Definition: {f.description}" + for f in schema.fields + ] + + field_schema = "\n".join(field_schema) + + return template.format(schema=schema, text=text) + + schema = f"""Object name: {schema.name} +Description: {schema.description} + +Fields: +{schema}""" + + prompt = f"""""" + + return prompt + def get_cypher(kg): sg2 = [] for f in kg: diff --git a/trustgraph/model/prompt/template/service.py b/trustgraph/model/prompt/template/service.py index febaab85..f54b5071 100755 --- a/trustgraph/model/prompt/template/service.py +++ b/trustgraph/model/prompt/template/service.py @@ -14,7 +14,7 @@ from .... schema import prompt_request_queue, prompt_response_queue from .... base import ConsumerProducer from .... clients.llm_client import LlmClient -from . prompts import to_definitions, to_relationships +from . prompts import to_definitions, to_relationships, to_rows from . prompts import to_kg_query, to_document_query module = ".".join(__name__.split(".")[1:-1]) @@ -38,6 +38,7 @@ class Processor(ConsumerProducer): ) definition_template = params.get("definition_template") relationship_template = params.get("relationship_template") + rows_template = params.get("rows_template") knowledge_query_template = params.get("knowledge_query_template") document_query_template = params.get("document_query_template") @@ -62,6 +63,7 @@ class Processor(ConsumerProducer): self.definition_template = definition_template self.relationship_template = relationship_template + self.rows_template = rows_template self.knowledge_query_template = knowledge_query_template self.document_query_template = document_query_template @@ -87,6 +89,11 @@ class Processor(ConsumerProducer): self.handle_extract_relationships(id, v) return + elif kind == "extract-rows": + + self.handle_extract_rows(id, v) + return + elif kind == "kg-prompt": self.handle_kg_prompt(id, v) @@ -232,6 +239,77 @@ class Processor(ConsumerProducer): ) self.producer.send(r, properties={"id": id}) + + def handle_extract_rows(self, id, v): + + try: + + fields = v.row_schema.fields + + prompt = to_rows(self.rows_template, v.row_schema, v.chunk) + + print(prompt) + + ans = self.llm.request(prompt) + + print(ans) + + # Silently ignore JSON parse error + try: + objs = json.loads(ans) + except: + print("JSON parse error, ignored", flush=True) + objs = [] + + output = [] + + for obj in objs: + + try: + + row = {} + + for f in fields: + + if f.name not in obj: + print(f"Object ignored, missing field {f.name}") + row = {} + break + + row[f.name] = obj[f.name] + + if row == {}: + continue + + output.append(row) + + except Exception as e: + print("row fields missing, ignored", flush=True) + + for row in output: + print(row) + + print("Send response...", flush=True) + r = PromptResponse(rows=output, error=None) + self.producer.send(r, properties={"id": id}) + + print("Done.", flush=True) + + except Exception as e: + + print(f"Exception: {e}") + + print("Send error response...", flush=True) + + r = PromptResponse( + error=Error( + type = "llm-error", + message = str(e), + ), + response=None, + ) + + self.producer.send(r, properties={"id": id}) def handle_kg_prompt(self, id, v): @@ -329,6 +407,12 @@ class Processor(ConsumerProducer): help=f'Definition extraction template', ) + parser.add_argument( + '--rows-template', + required=True, + help=f'Rows extraction template', + ) + parser.add_argument( '--relationship-template', required=True, diff --git a/trustgraph/objects/__init__.py b/trustgraph/objects/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/trustgraph/objects/field.py b/trustgraph/objects/field.py new file mode 100644 index 00000000..e4d84991 --- /dev/null +++ b/trustgraph/objects/field.py @@ -0,0 +1,72 @@ + +from dataclasses import dataclass +from enum import Enum + +class FieldType(Enum): + STRING = 0 + INT = 1 + LONG = 2 + BOOL = 3 + FLOAT = 4 + DOUBLE = 5 + + def __str__(self): + return self.name.lower() + +@dataclass +class Field: + name: str + size: int = -1 + primary: bool = False + type: str = "undefined" + description: str = "" + + @staticmethod + def parse(defn): + + if defn == "" or defn is None: + raise RuntimeError("Field definition cannot be empty") + + parts = defn.split(":") + + if len(parts) == 0: + raise RuntimeError("Field definition cannot be empty") + + if len(parts) == 1: parts.append("string") + if len(parts) == 2: parts.append("0") + if len(parts) == 3: parts.append("") + if len(parts) == 4: parts.append("") + + name, type, size, pri, description = parts + + size = int(size) + + try: + type = FieldType[type.upper()] + except: + raise RuntimeError(f"Field type {type} is not known") + + pri = True if pri == "pri" else False + + return Field( + name=name, type=type, size=size, primary=pri, + description=description + ) + + def __repr__(self): + name = self.name + type = self.type + size = self.size + pri = "pri" if self.primary else "" + description = self.description + + return f"{name}:{type}:{size}:{pri}:{description}" + + def __str__(self): + name = self.name + type = self.type + size = self.size + pri = "pri" if self.primary else "" + description = self.description + + return f"{name}:{type}:{size}:{pri}:{description}" diff --git a/trustgraph/objects/object.py b/trustgraph/objects/object.py new file mode 100644 index 00000000..bb41ecda --- /dev/null +++ b/trustgraph/objects/object.py @@ -0,0 +1,8 @@ + +class Schema: + def __init__(self, name, description, fields): + self.name = name + self.description = description + self.fields = fields + + diff --git a/trustgraph/schema.py b/trustgraph/schema.py deleted file mode 100644 index ddc4c67d..00000000 --- a/trustgraph/schema.py +++ /dev/null @@ -1,274 +0,0 @@ - -from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double - -from enum import Enum - -def topic(topic, kind='persistent', tenant='tg', namespace='flow'): - return f"{kind}://{tenant}/{namespace}/{topic}" - -############################################################################ - -class Error(Record): - type = String() - message = String() - -############################################################################ - -class Value(Record): - value = String() - is_uri = Boolean() - type = String() - -class Source(Record): - source = String() - id = String() - title = String() - -############################################################################ - -# PDF docs etc. -class Document(Record): - source = Source() - data = Bytes() - -document_ingest_queue = topic('document-load') - -############################################################################ - -# Text documents / text from PDF - -class TextDocument(Record): - source = Source() - text = Bytes() - -text_ingest_queue = topic('text-document-load') - -############################################################################ - -# Chunks of text - -class Chunk(Record): - source = Source() - chunk = Bytes() - -chunk_ingest_queue = topic('chunk-load') - -############################################################################ - -# Chunk embeddings are an embeddings associated with a text chunk - -class ChunkEmbeddings(Record): - source = Source() - vectors = Array(Array(Double())) - chunk = Bytes() - -chunk_embeddings_ingest_queue = topic('chunk-embeddings-load') - -############################################################################ - -# Graph embeddings are embeddings associated with a graph entity - -class GraphEmbeddings(Record): - source = Source() - vectors = Array(Array(Double())) - entity = Value() - -graph_embeddings_store_queue = topic('graph-embeddings-store') - -############################################################################ - -# Graph embeddings query - -class GraphEmbeddingsRequest(Record): - vectors = Array(Array(Double())) - limit = Integer() - -class GraphEmbeddingsResponse(Record): - error = Error() - entities = Array(Value()) - -graph_embeddings_request_queue = topic( - 'graph-embeddings', kind='non-persistent', namespace='request' -) -graph_embeddings_response_queue = topic( - 'graph-embeddings-response', kind='non-persistent', namespace='response', -) - -############################################################################ - -# Doc embeddings query - -class DocumentEmbeddingsRequest(Record): - vectors = Array(Array(Double())) - limit = Integer() - -class DocumentEmbeddingsResponse(Record): - error = Error() - documents = Array(Bytes()) - -document_embeddings_request_queue = topic( - 'doc-embeddings', kind='non-persistent', namespace='request' -) -document_embeddings_response_queue = topic( - 'doc-embeddings-response', kind='non-persistent', namespace='response', -) - -############################################################################ - -# Graph triples - -class Triple(Record): - source = Source() - s = Value() - p = Value() - o = Value() - -triples_store_queue = topic('triples-store') - -############################################################################ - -# Triples query - -class TriplesQueryRequest(Record): - s = Value() - p = Value() - o = Value() - limit = Integer() - -class TriplesQueryResponse(Record): - error = Error() - triples = Array(Triple()) - -triples_request_queue = topic( - 'triples', kind='non-persistent', namespace='request' -) -triples_response_queue = topic( - 'triples-response', kind='non-persistent', namespace='response', -) - -############################################################################ - -# chunk_embeddings_store_queue = topic('chunk-embeddings-store') - -############################################################################ - -# LLM text completion - -class TextCompletionRequest(Record): - prompt = String() - -class TextCompletionResponse(Record): - error = Error() - response = String() - -text_completion_request_queue = topic( - 'text-completion', kind='non-persistent', namespace='request' -) -text_completion_response_queue = topic( - 'text-completion-response', kind='non-persistent', namespace='response', -) - -############################################################################ - -# Embeddings - -class EmbeddingsRequest(Record): - text = String() - -class EmbeddingsResponse(Record): - error = Error() - vectors = Array(Array(Double())) - -embeddings_request_queue = topic( - 'embeddings', kind='non-persistent', namespace='request' -) -embeddings_response_queue = topic( - 'embeddings-response', kind='non-persistent', namespace='response' -) - -############################################################################ - -# Graph RAG text retrieval - -class GraphRagQuery(Record): - query = String() - -class GraphRagResponse(Record): - error = Error() - response = String() - -graph_rag_request_queue = topic( - 'graph-rag', kind='non-persistent', namespace='request' -) -graph_rag_response_queue = topic( - 'graph-rag-response', kind='non-persistent', namespace='response' -) - -############################################################################ - -# Document RAG text retrieval - -class DocumentRagQuery(Record): - query = String() - -class DocumentRagResponse(Record): - error = Error() - response = String() - -document_rag_request_queue = topic( - 'doc-rag', kind='non-persistent', namespace='request' -) -document_rag_response_queue = topic( - 'doc-rag-response', kind='non-persistent', namespace='response' -) - -############################################################################ - -# Prompt services, abstract the prompt generation - -class Definition(Record): - name = String() - definition = String() - -class Relationship(Record): - s = String() - p = String() - o = String() - o_entity = Boolean() - -class Fact(Record): - s = String() - p = String() - o = String() - -# extract-definitions: -# chunk -> definitions -# extract-relationships: -# chunk -> relationships -# kg-prompt: -# query, triples -> answer -# document-prompt: -# query, documents -> answer - -class PromptRequest(Record): - kind = String() - chunk = String() - query = String() - kg = Array(Fact()) - documents = Array(Bytes()) - -class PromptResponse(Record): - error = Error() - answer = String() - definitions = Array(Definition()) - relationships = Array(Relationship()) - -prompt_request_queue = topic( - 'prompt', kind='non-persistent', namespace='request' -) -prompt_response_queue = topic( - 'prompt-response', kind='non-persistent', namespace='response' -) - -############################################################################ - diff --git a/trustgraph/schema/__init__.py b/trustgraph/schema/__init__.py new file mode 100644 index 00000000..0cd5a370 --- /dev/null +++ b/trustgraph/schema/__init__.py @@ -0,0 +1,12 @@ + +from . types import * +from . prompt import * +from . documents import * +from . models import * +from . object import * +from . topic import * +from . graph import * +from . retrieval import * + + + diff --git a/trustgraph/schema/documents.py b/trustgraph/schema/documents.py new file mode 100644 index 00000000..d80ff38f --- /dev/null +++ b/trustgraph/schema/documents.py @@ -0,0 +1,68 @@ + +from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double +from . topic import topic +from . types import Error + +class Source(Record): + source = String() + id = String() + title = String() + +############################################################################ + +# PDF docs etc. +class Document(Record): + source = Source() + data = Bytes() + +document_ingest_queue = topic('document-load') + +############################################################################ + +# Text documents / text from PDF + +class TextDocument(Record): + source = Source() + text = Bytes() + +text_ingest_queue = topic('text-document-load') + +############################################################################ + +# Chunks of text + +class Chunk(Record): + source = Source() + chunk = Bytes() + +chunk_ingest_queue = topic('chunk-load') + +############################################################################ + +# Chunk embeddings are an embeddings associated with a text chunk + +class ChunkEmbeddings(Record): + source = Source() + vectors = Array(Array(Double())) + chunk = Bytes() + +chunk_embeddings_ingest_queue = topic('chunk-embeddings-load') + +############################################################################ + +# Doc embeddings query + +class DocumentEmbeddingsRequest(Record): + vectors = Array(Array(Double())) + limit = Integer() + +class DocumentEmbeddingsResponse(Record): + error = Error() + documents = Array(Bytes()) + +document_embeddings_request_queue = topic( + 'doc-embeddings', kind='non-persistent', namespace='request' +) +document_embeddings_response_queue = topic( + 'doc-embeddings-response', kind='non-persistent', namespace='response', +) diff --git a/trustgraph/schema/graph.py b/trustgraph/schema/graph.py new file mode 100644 index 00000000..234a0bed --- /dev/null +++ b/trustgraph/schema/graph.py @@ -0,0 +1,69 @@ + +from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double + +from . documents import Source +from . types import Error, Value +from . topic import topic + +############################################################################ + +# Graph embeddings are embeddings associated with a graph entity + +class GraphEmbeddings(Record): + source = Source() + vectors = Array(Array(Double())) + entity = Value() + +graph_embeddings_store_queue = topic('graph-embeddings-store') + +############################################################################ + +# Graph embeddings query + +class GraphEmbeddingsRequest(Record): + vectors = Array(Array(Double())) + limit = Integer() + +class GraphEmbeddingsResponse(Record): + error = Error() + entities = Array(Value()) + +graph_embeddings_request_queue = topic( + 'graph-embeddings', kind='non-persistent', namespace='request' +) +graph_embeddings_response_queue = topic( + 'graph-embeddings-response', kind='non-persistent', namespace='response', +) + +############################################################################ + +# Graph triples + +class Triple(Record): + source = Source() + s = Value() + p = Value() + o = Value() + +triples_store_queue = topic('triples-store') + +############################################################################ + +# Triples query + +class TriplesQueryRequest(Record): + s = Value() + p = Value() + o = Value() + limit = Integer() + +class TriplesQueryResponse(Record): + error = Error() + triples = Array(Triple()) + +triples_request_queue = topic( + 'triples', kind='non-persistent', namespace='request' +) +triples_response_queue = topic( + 'triples-response', kind='non-persistent', namespace='response', +) diff --git a/trustgraph/schema/models.py b/trustgraph/schema/models.py new file mode 100644 index 00000000..cf73a203 --- /dev/null +++ b/trustgraph/schema/models.py @@ -0,0 +1,41 @@ + +from pulsar.schema import Record, String, Array, Double + +from . topic import topic +from . types import Error + +############################################################################ + +# LLM text completion + +class TextCompletionRequest(Record): + prompt = String() + +class TextCompletionResponse(Record): + error = Error() + response = String() + +text_completion_request_queue = topic( + 'text-completion', kind='non-persistent', namespace='request' +) +text_completion_response_queue = topic( + 'text-completion-response', kind='non-persistent', namespace='response', +) + +############################################################################ + +# Embeddings + +class EmbeddingsRequest(Record): + text = String() + +class EmbeddingsResponse(Record): + error = Error() + vectors = Array(Array(Double())) + +embeddings_request_queue = topic( + 'embeddings', kind='non-persistent', namespace='request' +) +embeddings_response_queue = topic( + 'embeddings-response', kind='non-persistent', namespace='response' +) diff --git a/trustgraph/schema/object.py b/trustgraph/schema/object.py new file mode 100644 index 00000000..3377e2df --- /dev/null +++ b/trustgraph/schema/object.py @@ -0,0 +1,33 @@ + +from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array +from pulsar.schema import Double, Map + +from . documents import Source +from . types import Value, RowSchema +from . topic import topic + +############################################################################ + +# Object embeddings are embeddings associated with the primary key of an +# object + +class ObjectEmbeddings(Record): + source = Source() + vectors = Array(Array(Double())) + name = String() + key_name = String() + id = String() + +object_embeddings_store_queue = topic('object-embeddings-store') + +############################################################################ + +# Stores rows of information + +class Rows(Record): + source = Source() + row_schema = RowSchema() + rows = Array(Map(String())) + +rows_store_queue = topic('rows-store') + diff --git a/trustgraph/schema/prompt.py b/trustgraph/schema/prompt.py new file mode 100644 index 00000000..69f81ff3 --- /dev/null +++ b/trustgraph/schema/prompt.py @@ -0,0 +1,60 @@ + +from pulsar.schema import Record, Bytes, String, Boolean, Array, Map, Integer + +from . topic import topic +from . types import Error, RowSchema + +############################################################################ + +# Prompt services, abstract the prompt generation + +class Definition(Record): + name = String() + definition = String() + +class Relationship(Record): + s = String() + p = String() + o = String() + o_entity = Boolean() + +class Fact(Record): + s = String() + p = String() + o = String() + +# extract-definitions: +# chunk -> definitions +# extract-relationships: +# chunk -> relationships +# kg-prompt: +# query, triples -> answer +# document-prompt: +# query, documents -> answer +# extract-rows +# schema, chunk -> rows + +class PromptRequest(Record): + kind = String() + chunk = String() + query = String() + kg = Array(Fact()) + documents = Array(Bytes()) + row_schema = RowSchema() + +class PromptResponse(Record): + error = Error() + answer = String() + definitions = Array(Definition()) + relationships = Array(Relationship()) + rows = Array(Map(String())) + +prompt_request_queue = topic( + 'prompt', kind='non-persistent', namespace='request' +) +prompt_response_queue = topic( + 'prompt-response', kind='non-persistent', namespace='response' +) + +############################################################################ + diff --git a/trustgraph/schema/retrieval.py b/trustgraph/schema/retrieval.py new file mode 100644 index 00000000..fa0288fc --- /dev/null +++ b/trustgraph/schema/retrieval.py @@ -0,0 +1,40 @@ + +from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double +from . topic import topic +from . types import Error, Value + +############################################################################ + +# Graph RAG text retrieval + +class GraphRagQuery(Record): + query = String() + +class GraphRagResponse(Record): + error = Error() + response = String() + +graph_rag_request_queue = topic( + 'graph-rag', kind='non-persistent', namespace='request' +) +graph_rag_response_queue = topic( + 'graph-rag-response', kind='non-persistent', namespace='response' +) + +############################################################################ + +# Document RAG text retrieval + +class DocumentRagQuery(Record): + query = String() + +class DocumentRagResponse(Record): + error = Error() + response = String() + +document_rag_request_queue = topic( + 'doc-rag', kind='non-persistent', namespace='request' +) +document_rag_response_queue = topic( + 'doc-rag-response', kind='non-persistent', namespace='response' +) diff --git a/trustgraph/schema/topic.py b/trustgraph/schema/topic.py new file mode 100644 index 00000000..cdd643b7 --- /dev/null +++ b/trustgraph/schema/topic.py @@ -0,0 +1,4 @@ + +def topic(topic, kind='persistent', tenant='tg', namespace='flow'): + return f"{kind}://{tenant}/{namespace}/{topic}" + diff --git a/trustgraph/schema/types.py b/trustgraph/schema/types.py new file mode 100644 index 00000000..4cad70ac --- /dev/null +++ b/trustgraph/schema/types.py @@ -0,0 +1,25 @@ + +from pulsar.schema import Record, String, Boolean, Array, Integer + +class Error(Record): + type = String() + message = String() + +class Value(Record): + value = String() + is_uri = Boolean() + type = String() + +class Field(Record): + name = String() + # int, string, long, bool, float, double + type = String() + size = Integer() + primary = Boolean() + description = String() + +class RowSchema(Record): + name = String() + description = String() + fields = Array(Field()) + diff --git a/trustgraph/storage/object_embeddings/__init__.py b/trustgraph/storage/object_embeddings/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/trustgraph/storage/object_embeddings/milvus/__init__.py b/trustgraph/storage/object_embeddings/milvus/__init__.py new file mode 100644 index 00000000..d891d55f --- /dev/null +++ b/trustgraph/storage/object_embeddings/milvus/__init__.py @@ -0,0 +1,3 @@ + +from . write import * + diff --git a/trustgraph/storage/object_embeddings/milvus/__main__.py b/trustgraph/storage/object_embeddings/milvus/__main__.py new file mode 100755 index 00000000..c05d8c6d --- /dev/null +++ b/trustgraph/storage/object_embeddings/milvus/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from . write import run + +if __name__ == '__main__': + run() + diff --git a/trustgraph/storage/object_embeddings/milvus/write.py b/trustgraph/storage/object_embeddings/milvus/write.py new file mode 100755 index 00000000..468b357a --- /dev/null +++ b/trustgraph/storage/object_embeddings/milvus/write.py @@ -0,0 +1,61 @@ + +""" +Accepts entity/vector pairs and writes them to a Milvus store. +""" + +from .... schema import ObjectEmbeddings +from .... schema import object_embeddings_store_queue +from .... log_level import LogLevel +from .... direct.milvus_object_embeddings import ObjectVectors +from .... base import Consumer + +module = ".".join(__name__.split(".")[1:-1]) + +default_input_queue = object_embeddings_store_queue +default_subscriber = module +default_store_uri = 'http://localhost:19530' + +class Processor(Consumer): + + def __init__(self, **params): + + input_queue = params.get("input_queue", default_input_queue) + subscriber = params.get("subscriber", default_subscriber) + store_uri = params.get("store_uri", default_store_uri) + + super(Processor, self).__init__( + **params | { + "input_queue": input_queue, + "subscriber": subscriber, + "input_schema": ObjectEmbeddings, + "store_uri": store_uri, + } + ) + + self.vecstore = ObjectVectors(store_uri) + + def handle(self, msg): + + v = msg.value() + + if v.id != "" and v.id is not None: + for vec in v.vectors: + self.vecstore.insert(vec, v.name, v.key_name, v.id) + + @staticmethod + def add_args(parser): + + Consumer.add_args( + parser, default_input_queue, default_subscriber, + ) + + parser.add_argument( + '-t', '--store-uri', + default=default_store_uri, + help=f'Milvus store URI (default: {default_store_uri})' + ) + +def run(): + + Processor.start(module, __doc__) + diff --git a/trustgraph/storage/rows/__init__.py b/trustgraph/storage/rows/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/trustgraph/storage/rows/cassandra/__init__.py b/trustgraph/storage/rows/cassandra/__init__.py new file mode 100644 index 00000000..d891d55f --- /dev/null +++ b/trustgraph/storage/rows/cassandra/__init__.py @@ -0,0 +1,3 @@ + +from . write import * + diff --git a/trustgraph/storage/rows/cassandra/__main__.py b/trustgraph/storage/rows/cassandra/__main__.py new file mode 100755 index 00000000..c05d8c6d --- /dev/null +++ b/trustgraph/storage/rows/cassandra/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from . write import run + +if __name__ == '__main__': + run() + diff --git a/trustgraph/storage/rows/cassandra/write.py b/trustgraph/storage/rows/cassandra/write.py new file mode 100755 index 00000000..d44864fe --- /dev/null +++ b/trustgraph/storage/rows/cassandra/write.py @@ -0,0 +1,127 @@ + +""" +Graph writer. Input is graph edge. Writes edges to Cassandra graph. +""" + +import pulsar +import base64 +import os +import argparse +import time +from cassandra.cluster import Cluster +from cassandra.auth import PlainTextAuthProvider + +from .... schema import Rows +from .... schema import rows_store_queue +from .... log_level import LogLevel +from .... base import Consumer + +module = ".".join(__name__.split(".")[1:-1]) + +default_input_queue = rows_store_queue +default_subscriber = module +default_graph_host='localhost' + +class Processor(Consumer): + + def __init__(self, **params): + + input_queue = params.get("input_queue", default_input_queue) + subscriber = params.get("subscriber", default_subscriber) + graph_host = params.get("graph_host", default_graph_host) + + super(Processor, self).__init__( + **params | { + "input_queue": input_queue, + "subscriber": subscriber, + "input_schema": Rows, + "graph_host": graph_host, + } + ) + + self.cluster = Cluster(graph_host.split(",")) + self.session = self.cluster.connect() + + self.tables = set() + + self.session.execute(""" + create keyspace if not exists trustgraph + with replication = { + 'class' : 'SimpleStrategy', + 'replication_factor' : 1 + }; + """); + + self.session.execute("use trustgraph"); + + def handle(self, msg): + + try: + + v = msg.value() + name = v.row_schema.name + + if name not in self.tables: + + # FIXME: SQL injection? + + pkey = [] + + stmt = "create table if not exists " + name + " ( " + + for field in v.row_schema.fields: + + stmt += field.name + " text, " + + if field.primary: + pkey.append(field.name) + + stmt += "PRIMARY KEY (" + ", ".join(pkey) + "));" + + self.session.execute(stmt) + + self.tables.add(name); + + for row in v.rows: + + field_names = [] + values = [] + + for field in v.row_schema.fields: + field_names.append(field.name) + values.append(row[field.name]) + + # FIXME: SQL injection? + stmt = ( + "insert into " + name + " (" + ", ".join(field_names) + + ") values (" + ",".join(["%s"] * len(values)) + ")" + ) + + self.session.execute(stmt, values) + + except Exception as e: + + print("Exception:", str(e), flush=True) + + # If there's an error make sure to do table creation etc. + self.tables.remove(name) + + raise e + + @staticmethod + def add_args(parser): + + Consumer.add_args( + parser, default_input_queue, default_subscriber, + ) + + parser.add_argument( + '-g', '--graph-host', + default="localhost", + help=f'Graph host (default: localhost)' + ) + +def run(): + + Processor.start(module, __doc__) +