From aab6472e3c0fc7a9ee9988656922f2485c0bd957 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Mon, 14 Oct 2024 20:02:19 +0100 Subject: [PATCH] More LLMs ported to environment secrets --- templates/components/azure-openai.jsonnet | 49 ++++++++----------- templates/components/azure.jsonnet | 39 +++++---------- templates/components/bedrock.jsonnet | 59 ++++++++--------------- templates/components/claude.jsonnet | 47 ++++++++---------- 4 files changed, 71 insertions(+), 123 deletions(-) diff --git a/templates/components/azure-openai.jsonnet b/templates/components/azure-openai.jsonnet index 31e19720..aa759b51 100644 --- a/templates/components/azure-openai.jsonnet +++ b/templates/components/azure-openai.jsonnet @@ -5,7 +5,6 @@ local prompts = import "prompts/mixtral.jsonnet"; { - "azure-openai-token":: "${AZURE_OPENAI_TOKEN}", "azure-openai-model":: "GPT-3.5-Turbo", "azure-openai-max-output-tokens":: 4192, "azure-openai-temperature":: 0.0, @@ -14,6 +13,9 @@ local prompts = import "prompts/mixtral.jsonnet"; create:: function(engine) + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token"); + local container = engine.container("text-completion") .with_image(images.trustgraph) @@ -21,8 +23,6 @@ local prompts = import "prompts/mixtral.jsonnet"; "text-completion-azure-openai", "-p", url.pulsar, - "-k", - $["azure-openai-token"], "-m", $["azure-openai-model"], "-x", @@ -30,39 +30,17 @@ local prompts = import "prompts/mixtral.jsonnet"; "-t", std.toString($["azure-openai-temperature"]), ]) + .with_env_var_secrets(envSecrets) .with_limits("0.5", "128M") .with_reservations("0.1", "128M"); - local containerSet = engine.containers( - "text-completion", [ container ] - ); - - local service = - engine.internalService(containerSet) - .with_port(8000, 8000, "metrics"); - - engine.resources([ - containerSet, - service, - ]) - - }, - - "text-completion-rag" +: { - - create:: function(engine) - - local container = + local containerRag = engine.container("text-completion-rag") .with_image(images.trustgraph) .with_command([ "text-completion-azure", "-p", url.pulsar, - "-k", - $["azure-openai-token"], - "-e", - $["azure-openai-model"], "-x", std.toString($["azure-openai-max-output-tokens"]), "-t", @@ -72,24 +50,35 @@ local prompts = import "prompts/mixtral.jsonnet"; "-o", "non-persistent://tg/response/text-completion-rag-response", ]) + .with_env_var_secrets(envSecrets) .with_limits("0.5", "128M") .with_reservations("0.1", "128M"); local containerSet = engine.containers( - "text-completion-rag", [ container ] + "text-completion", [ container ] + ); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] ); local service = engine.internalService(containerSet) .with_port(8000, 8000, "metrics"); + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + engine.resources([ + envSecrets, containerSet, + containerSetRag, service, + serviceRag, ]) - - } + }, } + prompts diff --git a/templates/components/azure.jsonnet b/templates/components/azure.jsonnet index e4ad3795..aacbeac4 100644 --- a/templates/components/azure.jsonnet +++ b/templates/components/azure.jsonnet @@ -32,31 +32,7 @@ local prompts = import "prompts/mixtral.jsonnet"; .with_limits("0.5", "128M") .with_reservations("0.1", "128M"); - local containerSet = engine.containers( - "text-completion", [ container ] - ); - - local service = - engine.internalService(containerSet) - .with_port(8000, 8000, "metrics"); - - engine.resources([ - envSecrets, - containerSet, - service, - ]) - - }, - - "text-completion-rag" +: { - - create:: function(engine) - - local envSecrets = engine.envSecrets("azure-credentials-rag") - .with_env_var("AZURE_TOKEN", "azure-token") - .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); - - local container = + local containerRag = engine.container("text-completion-rag") .with_image(images.trustgraph) .with_command([ @@ -77,20 +53,29 @@ local prompts = import "prompts/mixtral.jsonnet"; .with_reservations("0.1", "128M"); local containerSet = engine.containers( - "text-completion-rag", [ container ] + "text-completion", [ container ] + ); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] ); local service = engine.internalService(containerSet) .with_port(8000, 8000, "metrics"); + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + engine.resources([ envSecrets, containerSet, + containerSetRag, service, + serviceRag, ]) - } } + prompts diff --git a/templates/components/bedrock.jsonnet b/templates/components/bedrock.jsonnet index 1c375621..d519363e 100644 --- a/templates/components/bedrock.jsonnet +++ b/templates/components/bedrock.jsonnet @@ -6,9 +6,6 @@ local chunker = import "chunker-recursive.jsonnet"; { - "aws-id-key":: "${AWS_ID_KEY}", - "aws-secret-key":: "${AWS_SECRET_KEY}", - "aws-region":: "us-west-2", "bedrock-max-output-tokens":: 4096, "bedrock-temperature":: 0.0, "bedrock-model":: "mistral.mixtral-8x7b-instruct-v0:1", @@ -17,6 +14,11 @@ local chunker = import "chunker-recursive.jsonnet"; create:: function(engine) + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ID_KEY", "aws-id-key") + .with_env_var("AWS_SECRET_KEY", "aws-secret-key") + .with_env_var("AWS_REGION", "aws-region"); + local container = engine.container("text-completion") .with_image(images.trustgraph) @@ -24,12 +26,6 @@ local chunker = import "chunker-recursive.jsonnet"; "text-completion-bedrock", "-p", url.pulsar, - "-z", - $["aws-id-key"], - "-k", - $["aws-secret-key"], - "-r", - $["aws-region"], "-x", std.toString($["bedrock-max-output-tokens"]), "-t", @@ -37,41 +33,17 @@ local chunker = import "chunker-recursive.jsonnet"; "-m", $["bedrock-model"], ]) + .with_env_var_secrets(envSecrets) .with_limits("0.5", "128M") .with_reservations("0.1", "128M"); - local containerSet = engine.containers( - "text-completion", [ container ] - ); - - local service = - engine.internalService(containerSet) - .with_port(8000, 8000, "metrics"); - - engine.resources([ - containerSet, - service, - ]) - - }, - - "text-completion-rag" +: { - - create:: function(engine) - - local container = + local containerRag = engine.container("text-completion-rag") .with_image(images.trustgraph) .with_command([ "text-completion-bedrock", "-p", url.pulsar, - "-z", - $["aws-id-key"], - "-k", - $["aws-secret-key"], - "-r", - $["aws-region"], "-x", std.toString($["bedrock-max-output-tokens"]), "-t", @@ -83,24 +55,35 @@ local chunker = import "chunker-recursive.jsonnet"; "-o", "non-persistent://tg/response/text-completion-rag-response", ]) + .with_env_var_secrets(envSecrets) .with_limits("0.5", "128M") .with_reservations("0.1", "128M"); local containerSet = engine.containers( - "text-completion-rag", [ container ] + "text-completion", [ container ] + ); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] ); local service = engine.internalService(containerSet) .with_port(8000, 8000, "metrics"); + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + engine.resources([ + envSecrets, containerSet, + containerSetRag, service, + serviceRag, ]) - - } + }, } + prompts + chunker diff --git a/templates/components/claude.jsonnet b/templates/components/claude.jsonnet index 0cd190d4..b723a16f 100644 --- a/templates/components/claude.jsonnet +++ b/templates/components/claude.jsonnet @@ -5,7 +5,6 @@ local prompts = import "prompts/mixtral.jsonnet"; { - "claude-key":: "${CLAUDE_KEY}", "claude-max-output-tokens":: 4096, "claude-temperature":: 0.0, @@ -13,6 +12,9 @@ local prompts = import "prompts/mixtral.jsonnet"; create:: function(engine) + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY_TOKEN", "claude-key"); + local container = engine.container("text-completion") .with_image(images.trustgraph) @@ -20,44 +22,22 @@ local prompts = import "prompts/mixtral.jsonnet"; "text-completion-claude", "-p", url.pulsar, - "-k", - $["claude-key"], "-x", std.toString($["claude-max-output-tokens"]), "-t", std.toString($["claude-temperature"]), ]) + .with_env_var_secrets(envSecrets) .with_limits("0.5", "128M") .with_reservations("0.1", "128M"); - local containerSet = engine.containers( - "text-completion", [ container ] - ); - - local service = - engine.internalService(containerSet) - .with_port(8000, 8000, "metrics"); - - engine.resources([ - containerSet, - service, - ]) - - }, - - "text-completion-rag" +: { - - create:: function(engine) - - local container = + local containerRag = engine.container("text-completion-rag") .with_image(images.trustgraph) .with_command([ "text-completion-claude", "-p", url.pulsar, - "-k", - $["claude-key"], "-x", std.toString($["claude-max-output-tokens"]), "-t", @@ -67,24 +47,35 @@ local prompts = import "prompts/mixtral.jsonnet"; "-o", "non-persistent://tg/response/text-completion-rag-response", ]) + .with_env_var_secrets(envSecrets) .with_limits("0.5", "128M") .with_reservations("0.1", "128M"); local containerSet = engine.containers( - "text-completion-rag", [ container ] + "text-completion", [ container ] + ); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] ); local service = engine.internalService(containerSet) .with_port(8000, 8000, "metrics"); + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + engine.resources([ + envSecrets, containerSet, + containerSetRag, service, + serviceRag, ]) - - } + }, } + prompts