diff --git a/Makefile b/Makefile index b82117a2..b4aef413 100644 --- a/Makefile +++ b/Makefile @@ -32,12 +32,12 @@ clean: set-version: echo '"${VERSION}"' > templates/values/version.jsonnet -TEMPLATES=azure bedrock claude cohere mix ollama openai vertexai \ +TEMPLATES=azure bedrock claude cohere mix llamafile ollama openai vertexai \ openai-neo4j storage DCS=$(foreach template,${TEMPLATES},${template:%=tg-launch-%.yaml}) -MODELS=azure bedrock claude cohere ollama openai vertexai +MODELS=azure bedrock claude cohere llamafile ollama openai vertexai GRAPHS=cassandra neo4j # tg-launch-%.yaml: templates/%.jsonnet templates/components/version.jsonnet diff --git a/templates/all-patterns.jsonnet b/templates/all-patterns.jsonnet index 8d588d62..29384d1e 100644 --- a/templates/all-patterns.jsonnet +++ b/templates/all-patterns.jsonnet @@ -10,6 +10,7 @@ import "patterns/llm-bedrock.jsonnet", import "patterns/llm-claude.jsonnet", import "patterns/llm-cohere.jsonnet", + import "patterns/llm-llamafile.jsonnet", import "patterns/llm-ollama.jsonnet", import "patterns/llm-openai.jsonnet", import "patterns/llm-vertexai.jsonnet", diff --git a/templates/components.jsonnet b/templates/components.jsonnet index afeed58b..8ed2da0e 100644 --- a/templates/components.jsonnet +++ b/templates/components.jsonnet @@ -10,6 +10,7 @@ "graph-rag": import "components/graph-rag.jsonnet", "triple-store-cassandra": import "components/cassandra.jsonnet", "triple-store-neo4j": import "components/neo4j.jsonnet", + "llamafile": import "components/llamafile.jsonnet", "ollama": import "components/ollama.jsonnet", "openai": import "components/openai.jsonnet", "override-recursive-chunker": import "components/chunker-recursive.jsonnet", diff --git a/templates/components/llamafile.jsonnet b/templates/components/llamafile.jsonnet new file mode 100644 index 00000000..93163a14 --- /dev/null +++ b/templates/components/llamafile.jsonnet @@ -0,0 +1,75 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/slm.jsonnet"; + +{ + + "llamafile-model":: "LLaMA_CPP", + "llamafile-url":: "${LLAMAFILE_URL}", + + "text-completion" +: { + + create:: function(engine) + + local container = + engine.container("text-completion") + .with_image(images.trustgraph) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "-m", + $["llamafile-model"], + "-r", + $["llamafile-url"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + engine.resources([ + containerSet, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local container = + engine.container("text-completion-rag") + .with_image(images.trustgraph) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "-m", + $["llamafile-model"], + "-r", + $["llamafile-url"], + "-i", + "non-persistent://tg/request/text-completion-rag", + "-o", + "non-persistent://tg/response/text-completion-rag-response", + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion-rag", [ container ] + ); + + engine.resources([ + containerSet, + ]) + + + } + +} + prompts + diff --git a/templates/patterns/llm-llamafile.jsonnet b/templates/patterns/llm-llamafile.jsonnet new file mode 100644 index 00000000..de2d95ef --- /dev/null +++ b/templates/patterns/llm-llamafile.jsonnet @@ -0,0 +1,41 @@ +{ + pattern: { + name: "llamafile", + icon: "🤖💬", + title: "Add Llamafile-invoked LLMs for text completion", + description: "This pattern integrates a Llamafile service for text completion operations. You need to have a running Llamafile implementation executing the necessary model in order to be able to use this service.", + requires: ["pulsar", "trustgraph"], + features: ["llm"], + args: [ + { + name: "llamafile-max-output-tokens", + label: "Maximum output tokens", + type: "integer", + description: "Limit on number tokens to generate", + default: 4096, + required: true, + }, + { + name: "llamafile-temperature", + label: "Temperature", + type: "slider", + description: "Controlling predictability / creativity balance", + min: 0, + max: 1, + step: 0.05, + default: 0.5, + }, + { + name: "llamafile-url", + label: "URL", + type: "text", + width: 120, + description: "URL of the Llamafile service", + default: "http://llamafile:8080", + required: true, + }, + ], + category: [ "llm" ], + }, + module: "components/llamafile.jsonnet", +}