diff --git a/Makefile b/Makefile index 6951087b..503a32e7 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,6 @@ DCS=$(foreach template,${TEMPLATES},${template:%=docker-compose-%.yaml}) update-templates: set-version ${DCS} -docker-compose-%.yaml: templates/docker-compose-%.jsonnet templates/version.jsonnet +docker-compose-%.yaml: templates/docker-compose-%.jsonnet templates/components/version.jsonnet jsonnet -S ${@:docker-compose-%.yaml=templates/docker-compose-%.jsonnet} > $@ diff --git a/templates/components/azure.jsonnet b/templates/components/azure.jsonnet new file mode 100644 index 00000000..cbf5a3d1 --- /dev/null +++ b/templates/components/azure.jsonnet @@ -0,0 +1,38 @@ +local base = import "base.jsonnet"; +local images = import "images.jsonnet"; +local url = import "url.jsonnet"; +{ + services +: { + + "text-completion": base + { + image: images.trustgraph, + command: [ + "text-completion-azure", + "-p", + url.pulsar, + "-k", + "${AZURE_TOKEN}", + "-e", + "${AZURE_ENDPOINT}", + ], + }, + + "text-completion-rag": base + { + image: images.trustgraph, + command: [ + "text-completion-azure", + "-p", + url.pulsar, + "-k", + "${AZURE_TOKEN}", + "-e", + "${AZURE_ENDPOINT}", + "-i", + "non-persistent://tg/request/text-completion-rag", + "-o", + "non-persistent://tg/response/text-completion-rag-response", + ], + }, + + }, +} diff --git a/templates/components/bedrock.jsonnet b/templates/components/bedrock.jsonnet new file mode 100644 index 00000000..454e593a --- /dev/null +++ b/templates/components/bedrock.jsonnet @@ -0,0 +1,57 @@ +local base = import "base.jsonnet"; +local images = import "images.jsonnet"; +local url = import "url.jsonnet"; +{ + services +: { + + chunker: base + { + image: images.trustgraph, + command: [ + "chunker-recursive", + "-p", + url.pulsar, + "--chunk-size", + "2000", + "--chunk-overlap", + "100", + ], + }, + + "text-completion": base + { + image: images.trustgraph, + command: [ + "text-completion-bedrock", + "-p", + url.pulsar, + "-z", + "${AWS_ID_KEY}", + "-k", + "${AWS_SECRET_KEY}", + "-r", + "us-west-2", + ], + }, + + "text-completion-rag": base + { + image: images.trustgraph, + command: [ + "text-completion-bedrock", + "-p", + url.pulsar, + // "-m", + // "mistral.mistral-large-2407-v1:0", + "-z", + "${AWS_ID_KEY}", + "-k", + "${AWS_SECRET_KEY}", + "-r", + "us-west-2", + "-i", + "non-persistent://tg/request/text-completion-rag", + "-o", + "non-persistent://tg/response/text-completion-rag-response", + ], + }, + + }, +} diff --git a/templates/components/cassandra.jsonnet b/templates/components/cassandra.jsonnet index eb0cd466..af9dd5f1 100644 --- a/templates/components/cassandra.jsonnet +++ b/templates/components/cassandra.jsonnet @@ -1,10 +1,13 @@ local base = import "base.jsonnet"; local images = import "images.jsonnet"; +local url = import "url.jsonnet"; +local cassandra_hosts = "cassandra"; { volumes +: { cassandra: {}, }, services +: { + cassandra: base + { image: images.cassandra, ports: [ @@ -13,6 +16,29 @@ local images = import "images.jsonnet"; volumes: [ "cassandra:/var/lib/cassandra" ], - } + }, + + "store-triples": base + { + image: images.trustgraph, + command: [ + "triples-write-cassandra", + "-p", + url.pulsar, + "-g", + cassandra_hosts, + ], + }, + + "query-triples": base + { + image: images.trustgraph, + command: [ + "triples-query-cassandra", + "-p", + url.pulsar, + "-g", + cassandra_hosts, + ], + }, + }, } diff --git a/templates/components/claude.jsonnet b/templates/components/claude.jsonnet new file mode 100644 index 00000000..d446b748 --- /dev/null +++ b/templates/components/claude.jsonnet @@ -0,0 +1,34 @@ +local base = import "base.jsonnet"; +local images = import "images.jsonnet"; +local url = import "url.jsonnet"; +{ + services +: { + + "text-completion": base + { + image: images.trustgraph, + command: [ + "text-completion-claude", + "-p", + url.pulsar, + "-k", + "${CLAUDE_KEY}", + ], + }, + + "text-completion-rag": base + { + image: images.trustgraph, + command: [ + "text-completion-claude", + "-p", + url.pulsar, + "-k", + "${CLAUDE_KEY}", + "-i", + "non-persistent://tg/request/text-completion-rag", + "-o", + "non-persistent://tg/response/text-completion-rag-response", + ], + }, + + }, +} diff --git a/templates/components/cohere.jsonnet b/templates/components/cohere.jsonnet new file mode 100644 index 00000000..391ae054 --- /dev/null +++ b/templates/components/cohere.jsonnet @@ -0,0 +1,47 @@ +local base = import "base.jsonnet"; +local images = import "images.jsonnet"; +local url = import "url.jsonnet"; +{ + services +: { + + chunker: base + { + image: images.trustgraph, + command: [ + "chunker-recursive", + "-p", + url.pulsar, + "--chunk-size", + "1000", + "--chunk-overlap", + "50", + ], + }, + + "text-completion": base + { + image: images.trustgraph, + command: [ + "text-completion-cohere", + "-p", + url.pulsar, + "-k", + "${COHERE_KEY}", + ], + }, + + "text-completion-rag": base + { + image: images.trustgraph, + command: [ + "text-completion-cohere", + "-p", + url.pulsar, + "-k", + "${COHERE_KEY}", + "-i", + "non-persistent://tg/request/text-completion-rag", + "-o", + "non-persistent://tg/response/text-completion-rag-response", + ], + }, + + }, +} diff --git a/templates/components/neo4j.jsonnet b/templates/components/neo4j.jsonnet index 47f42394..dd6670d2 100644 --- a/templates/components/neo4j.jsonnet +++ b/templates/components/neo4j.jsonnet @@ -1,10 +1,14 @@ local base = import "base.jsonnet"; local images = import "images.jsonnet"; +local url = import "url.jsonnet"; { + volumes +: { neo4j: {}, }, + services +: { + neo4j: base + { image: images.neo4j, ports: [ @@ -20,6 +24,30 @@ local images = import "images.jsonnet"; volumes: [ "neo4j:/data" ], + }, + + "query-triples": base + { + image: images.trustgraph, + command: [ + "triples-query-neo4j", + "-p", + url.pulsar, + "-g", + "bolt://neo4j:7687", + ], + }, + + "store-triples": base + { + image: images.trustgraph, + command: [ + "triples-write-neo4j", + "-p", + url.pulsar, + "-g", + "bolt://neo4j:7687", + ], } + }, + } diff --git a/templates/components/ollama.jsonnet b/templates/components/ollama.jsonnet new file mode 100644 index 00000000..af9922c1 --- /dev/null +++ b/templates/components/ollama.jsonnet @@ -0,0 +1,38 @@ +local base = import "base.jsonnet"; +local images = import "images.jsonnet"; +local url = import "url.jsonnet"; +{ + services +: { + + "text-completion": base + { + image: images.trustgraph, + command: [ + "text-completion-ollama", + "-p", + url.pulsar, + // "-m", + // "llama3.1:8b", + "-r", + "${OLLAMA_HOST}", + ], + }, + + "text-completion-rag": base + { + image: images.trustgraph, + command: [ + "text-completion-ollama", + "-p", + url.pulsar, + // "-m", + // "llama3.1:8b", + "-r", + "${OLLAMA_HOST}", + "-i", + "non-persistent://tg/request/text-completion-rag", + "-o", + "non-persistent://tg/response/text-completion-rag-response", + ], + }, + + }, +} diff --git a/templates/components/openai.jsonnet b/templates/components/openai.jsonnet new file mode 100644 index 00000000..7bd21e55 --- /dev/null +++ b/templates/components/openai.jsonnet @@ -0,0 +1,34 @@ +local base = import "base.jsonnet"; +local images = import "images.jsonnet"; +local url = import "url.jsonnet"; +{ + services +: { + + "text-completion": base + { + image: images.trustgraph, + command: [ + "text-completion-openai", + "-p", + url.pulsar, + "-k", + "${OPENAI_KEY}", + ], + }, + + "text-completion-rag": base + { + image: images.trustgraph, + command: [ + "text-completion-openai", + "-p", + url.pulsar, + "-k", + "${OPENAI_KEY}", + "-i", + "non-persistent://tg/request/text-completion-rag", + "-o", + "non-persistent://tg/response/text-completion-rag-response", + ], + }, + + }, +} diff --git a/templates/components/trustgraph.jsonnet b/templates/components/trustgraph.jsonnet index 7281d906..0962c98f 100644 --- a/templates/components/trustgraph.jsonnet +++ b/templates/components/trustgraph.jsonnet @@ -5,8 +5,6 @@ local images = import "images.jsonnet"; local url = import "url.jsonnet"; -local cassandra_hosts = "cassandra"; - { services +: { @@ -88,28 +86,6 @@ local cassandra_hosts = "cassandra"; ], }, - "store-triples": base + { - image: images.trustgraph, - command: [ - "triples-write-cassandra", - "-p", - url.pulsar, - "-g", - cassandra_hosts, - ], - }, - - "query-triples": base + { - image: images.trustgraph, - command: [ - "triples-query-cassandra", - "-p", - url.pulsar, - "-g", - cassandra_hosts, - ], - }, - "graph-rag": base + { image: images.trustgraph, command: [ diff --git a/templates/components/version.jsonnet b/templates/components/version.jsonnet new file mode 100644 index 00000000..3ee47627 --- /dev/null +++ b/templates/components/version.jsonnet @@ -0,0 +1 @@ +"0.6.1" diff --git a/templates/components/vertexai.jsonnet b/templates/components/vertexai.jsonnet new file mode 100644 index 00000000..a0780f9b --- /dev/null +++ b/templates/components/vertexai.jsonnet @@ -0,0 +1,44 @@ +local base = import "base.jsonnet"; +local images = import "images.jsonnet"; +local url = import "url.jsonnet"; +{ + services +: { + + "text-completion": base + { + image: images.trustgraph, + command: [ + "text-completion-vertexai", + "-p", + url.pulsar, + "-k", + "/vertexai/private.json", + "-r", + "us-west1", + ], + volumes: [ + "./vertexai:/vertexai" + ], + }, + + "text-completion-rag": base + { + image: images.trustgraph, + command: [ + "text-completion-vertexai", + "-p", + url.pulsar, + "-k", + "/vertexai/private.json", + "-r", + "us-west1", + "-i", + "non-persistent://tg/request/text-completion-rag", + "-o", + "non-persistent://tg/response/text-completion-rag-response", + ], + volumes: [ + "./vertexai:/vertexai" + ], + }, + + }, +} diff --git a/templates/docker-compose-azure.jsonnet b/templates/docker-compose-azure.jsonnet index 0ceba3a4..4dba5d1a 100644 --- a/templates/docker-compose-azure.jsonnet +++ b/templates/docker-compose-azure.jsonnet @@ -1,49 +1,12 @@ -local base = import "components/base.jsonnet"; -local url = import "components/url.jsonnet"; -local images = import "components/images.jsonnet"; - local cassandra = import "components/cassandra.jsonnet"; local pulsar = import "components/pulsar.jsonnet"; local milvus = import "components/milvus.jsonnet"; local grafana = import "components/grafana.jsonnet"; local trustgraph = import "components/trustgraph.jsonnet"; +local azure = import "components/azure.jsonnet"; -local config = cassandra + pulsar + milvus + grafana + trustgraph + { - services +: { - - "text-completion": base + { - image: images.trustgraph, - command: [ - "text-completion-azure", - "-p", - url.pulsar, - "-k", - "${AZURE_TOKEN}", - "-e", - "${AZURE_ENDPOINT}", - ], - }, - - "text-completion-rag": base + { - image: images.trustgraph, - command: [ - "text-completion-azure", - "-p", - url.pulsar, - "-k", - "${AZURE_TOKEN}", - "-e", - "${AZURE_ENDPOINT}", - "-i", - "non-persistent://tg/request/text-completion-rag", - "-o", - "non-persistent://tg/response/text-completion-rag-response", - ], - }, - - } -}; +local config = cassandra + pulsar + milvus + grafana + trustgraph + azure; std.manifestYamlDoc(config) diff --git a/templates/docker-compose-bedrock.jsonnet b/templates/docker-compose-bedrock.jsonnet index 1f100694..4b233ba9 100644 --- a/templates/docker-compose-bedrock.jsonnet +++ b/templates/docker-compose-bedrock.jsonnet @@ -1,68 +1,12 @@ -local base = import "components/base.jsonnet"; -local url = import "components/url.jsonnet"; -local images = import "components/images.jsonnet"; - local cassandra = import "components/cassandra.jsonnet"; local pulsar = import "components/pulsar.jsonnet"; local milvus = import "components/milvus.jsonnet"; local grafana = import "components/grafana.jsonnet"; local trustgraph = import "components/trustgraph.jsonnet"; +local bedrock = import "components/bedrock.jsonnet"; -local config = cassandra + pulsar + milvus + grafana + trustgraph + { - services +: { - - chunker: base + { - image: images.trustgraph, - command: [ - "chunker-recursive", - "-p", - url.pulsar, - "--chunk-size", - "2000", - "--chunk-overlap", - "100", - ], - }, - - "text-completion": base + { - image: images.trustgraph, - command: [ - "text-completion-bedrock", - "-p", - url.pulsar, - "-z", - "${AWS_ID_KEY}", - "-k", - "${AWS_SECRET_KEY}", - "-r", - "us-west-2", - ], - }, - - "text-completion-rag": base + { - image: images.trustgraph, - command: [ - "text-completion-bedrock", - "-p", - url.pulsar, - // "-m", - // "mistral.mistral-large-2407-v1:0", - "-z", - "${AWS_ID_KEY}", - "-k", - "${AWS_SECRET_KEY}", - "-r", - "us-west-2", - "-i", - "non-persistent://tg/request/text-completion-rag", - "-o", - "non-persistent://tg/response/text-completion-rag-response", - ], - }, - - } -}; +local config = cassandra + pulsar + milvus + grafana + trustgraph + bedrock; std.manifestYamlDoc(config) diff --git a/templates/docker-compose-claude.jsonnet b/templates/docker-compose-claude.jsonnet index 32de766f..d698a079 100644 --- a/templates/docker-compose-claude.jsonnet +++ b/templates/docker-compose-claude.jsonnet @@ -1,45 +1,12 @@ -local base = import "components/base.jsonnet"; -local url = import "components/url.jsonnet"; -local images = import "components/images.jsonnet"; - local cassandra = import "components/cassandra.jsonnet"; local pulsar = import "components/pulsar.jsonnet"; local milvus = import "components/milvus.jsonnet"; local grafana = import "components/grafana.jsonnet"; local trustgraph = import "components/trustgraph.jsonnet"; +local claude = import "components/claude.jsonnet"; -local config = cassandra + pulsar + milvus + grafana + trustgraph + { - services +: { - - "text-completion": base + { - image: images.trustgraph, - command: [ - "text-completion-claude", - "-p", - url.pulsar, - "-k", - "${CLAUDE_KEY}", - ], - }, - - "text-completion-rag": base + { - image: images.trustgraph, - command: [ - "text-completion-claude", - "-p", - url.pulsar, - "-k", - "${CLAUDE_KEY}", - "-i", - "non-persistent://tg/request/text-completion-rag", - "-o", - "non-persistent://tg/response/text-completion-rag-response", - ], - }, - - } -}; +local config = cassandra + pulsar + milvus + grafana + trustgraph + claude; std.manifestYamlDoc(config) diff --git a/templates/docker-compose-cohere.jsonnet b/templates/docker-compose-cohere.jsonnet index 86c2fb29..1da8bab5 100644 --- a/templates/docker-compose-cohere.jsonnet +++ b/templates/docker-compose-cohere.jsonnet @@ -1,58 +1,12 @@ -local base = import "components/base.jsonnet"; -local url = import "components/url.jsonnet"; -local images = import "components/images.jsonnet"; - local cassandra = import "components/cassandra.jsonnet"; local pulsar = import "components/pulsar.jsonnet"; local milvus = import "components/milvus.jsonnet"; local grafana = import "components/grafana.jsonnet"; local trustgraph = import "components/trustgraph.jsonnet"; +local cohere = import "components/cohere.jsonnet"; -local config = cassandra + pulsar + milvus + grafana + trustgraph + { - services +: { - - chunker: base + { - image: images.trustgraph, - command: [ - "chunker-recursive", - "-p", - url.pulsar, - "--chunk-size", - "1000", - "--chunk-overlap", - "50", - ], - }, - - "text-completion": base + { - image: images.trustgraph, - command: [ - "text-completion-cohere", - "-p", - url.pulsar, - "-k", - "${COHERE_KEY}", - ], - }, - - "text-completion-rag": base + { - image: images.trustgraph, - command: [ - "text-completion-cohere", - "-p", - url.pulsar, - "-k", - "${COHERE_KEY}", - "-i", - "non-persistent://tg/request/text-completion-rag", - "-o", - "non-persistent://tg/response/text-completion-rag-response", - ], - }, - - } -}; +local config = cassandra + pulsar + milvus + grafana + trustgraph + cohere; std.manifestYamlDoc(config) diff --git a/templates/docker-compose-ollama.jsonnet b/templates/docker-compose-ollama.jsonnet index ca8359a8..0bd1abb6 100644 --- a/templates/docker-compose-ollama.jsonnet +++ b/templates/docker-compose-ollama.jsonnet @@ -1,49 +1,12 @@ -local base = import "components/base.jsonnet"; -local url = import "components/url.jsonnet"; -local images = import "components/images.jsonnet"; - local cassandra = import "components/cassandra.jsonnet"; local pulsar = import "components/pulsar.jsonnet"; local milvus = import "components/milvus.jsonnet"; local grafana = import "components/grafana.jsonnet"; local trustgraph = import "components/trustgraph.jsonnet"; +local ollama = import "components/ollama.jsonnet"; -local config = cassandra + pulsar + milvus + grafana + trustgraph + { - services +: { - - "text-completion": base + { - image: images.trustgraph, - command: [ - "text-completion-ollama", - "-p", - url.pulsar, - // "-m", - // "llama3.1:8b", - "-r", - "${OLLAMA_HOST}", - ], - }, - - "text-completion-rag": base + { - image: images.trustgraph, - command: [ - "text-completion-ollama", - "-p", - url.pulsar, - // "-m", - // "llama3.1:8b", - "-r", - "${OLLAMA_HOST}", - "-i", - "non-persistent://tg/request/text-completion-rag", - "-o", - "non-persistent://tg/response/text-completion-rag-response", - ], - }, - - } -}; +local config = cassandra + pulsar + milvus + grafana + trustgraph + ollama; std.manifestYamlDoc(config) diff --git a/templates/docker-compose-openai-neo4j.jsonnet b/templates/docker-compose-openai-neo4j.jsonnet index 391dcbf9..c0bf05b9 100644 --- a/templates/docker-compose-openai-neo4j.jsonnet +++ b/templates/docker-compose-openai-neo4j.jsonnet @@ -1,68 +1,12 @@ -local base = import "components/base.jsonnet"; -local url = import "components/url.jsonnet"; -local images = import "components/images.jsonnet"; - local neo4j = import "components/neo4j.jsonnet"; local pulsar = import "components/pulsar.jsonnet"; local milvus = import "components/milvus.jsonnet"; local grafana = import "components/grafana.jsonnet"; local trustgraph = import "components/trustgraph.jsonnet"; +local openai = import "components/openai.jsonnet"; -local config = neo4j + pulsar + milvus + grafana + trustgraph + { - - services: std.mergePatch(super.services, { - - "text-completion": base + { - image: images.trustgraph, - command: [ - "text-completion-openai", - "-p", - url.pulsar, - "-k", - "${OPENAI_KEY}", - ], - }, - - "text-completion-rag": base + { - image: images.trustgraph, - command: [ - "text-completion-openai", - "-p", - url.pulsar, - "-k", - "${OPENAI_KEY}", - "-i", - "non-persistent://tg/request/text-completion-rag", - "-o", - "non-persistent://tg/response/text-completion-rag-response", - ], - }, - - "query-triples": { - image: images.trustgraph, - command: [ - "triples-query-neo4j", - "-p", - url.pulsar, - "-g", - "bolt://neo4j:7687", - ], - }, - - "store-triples": { - image: images.trustgraph, - command: [ - "triples-write-neo4j", - "-p", - url.pulsar, - "-g", - "bolt://neo4j:7687", - ], - } - - }) -}; +local config = neo4j + pulsar + milvus + grafana + trustgraph + openai; std.manifestYamlDoc(config) diff --git a/templates/docker-compose-openai.jsonnet b/templates/docker-compose-openai.jsonnet index 2a5b6438..28386925 100644 --- a/templates/docker-compose-openai.jsonnet +++ b/templates/docker-compose-openai.jsonnet @@ -1,45 +1,12 @@ -local base = import "components/base.jsonnet"; -local url = import "components/url.jsonnet"; -local images = import "components/images.jsonnet"; - local cassandra = import "components/cassandra.jsonnet"; local pulsar = import "components/pulsar.jsonnet"; local milvus = import "components/milvus.jsonnet"; local grafana = import "components/grafana.jsonnet"; local trustgraph = import "components/trustgraph.jsonnet"; +local openai = import "components/openai.jsonnet"; -local config = cassandra + pulsar + milvus + grafana + trustgraph + { - services +: { - - "text-completion": base + { - image: images.trustgraph, - command: [ - "text-completion-openai", - "-p", - url.pulsar, - "-k", - "${OPENAI_KEY}", - ], - }, - - "text-completion-rag": base + { - image: images.trustgraph, - command: [ - "text-completion-openai", - "-p", - url.pulsar, - "-k", - "${OPENAI_KEY}", - "-i", - "non-persistent://tg/request/text-completion-rag", - "-o", - "non-persistent://tg/response/text-completion-rag-response", - ], - }, - - } -}; +local config = cassandra + pulsar + milvus + grafana + trustgraph + openai; std.manifestYamlDoc(config) diff --git a/templates/docker-compose-vertexai.jsonnet b/templates/docker-compose-vertexai.jsonnet index 55029de6..b148701b 100644 --- a/templates/docker-compose-vertexai.jsonnet +++ b/templates/docker-compose-vertexai.jsonnet @@ -1,55 +1,12 @@ -local base = import "components/base.jsonnet"; -local url = import "components/url.jsonnet"; -local images = import "components/images.jsonnet"; - local cassandra = import "components/cassandra.jsonnet"; local pulsar = import "components/pulsar.jsonnet"; local milvus = import "components/milvus.jsonnet"; local grafana = import "components/grafana.jsonnet"; local trustgraph = import "components/trustgraph.jsonnet"; +local vertexai = import "components/vertexai.jsonnet"; -local config = cassandra + pulsar + milvus + grafana + trustgraph + { - services +: { - - "text-completion": base + { - image: images.trustgraph, - command: [ - "text-completion-vertexai", - "-p", - url.pulsar, - "-k", - "/vertexai/private.json", - "-r", - "us-west1", - ], - volumes: [ - "./vertexai:/vertexai" - ], - }, - - "text-completion-rag": base + { - image: images.trustgraph, - command: [ - "text-completion-vertexai", - "-p", - url.pulsar, - "-k", - "/vertexai/private.json", - "-r", - "us-west1", - "-i", - "non-persistent://tg/request/text-completion-rag", - "-o", - "non-persistent://tg/response/text-completion-rag-response", - ], - volumes: [ - "./vertexai:/vertexai" - ], - }, - - } -}; +local config = cassandra + pulsar + milvus + grafana + trustgraph + vertexai; std.manifestYamlDoc(config)