From bb746e237a5f5bb18626a9eb63e0a22f86a113fb Mon Sep 17 00:00:00 2001
From: Adil Hafeez <adil@katanemo.com>
Date: Mon, 30 Sep 2024 09:54:58 -0700
Subject: [PATCH] add support for 3b model (#96)

---
 ...Arch-Function-Calling-3B-Q4_K_M.model_file | 24 +++++++++++++++++++
 demos/function_calling/docker-compose.yaml    |  1 +
 2 files changed, 25 insertions(+)
 create mode 100644 demos/function_calling/Arch-Function-Calling-3B-Q4_K_M.model_file

diff --git a/demos/function_calling/Arch-Function-Calling-3B-Q4_K_M.model_file b/demos/function_calling/Arch-Function-Calling-3B-Q4_K_M.model_file
new file mode 100644
index 00000000..2d8619a4
--- /dev/null
+++ b/demos/function_calling/Arch-Function-Calling-3B-Q4_K_M.model_file
@@ -0,0 +1,24 @@
+FROM Arch-Function-Calling-3B-Q4_K_M.gguf
+
+# Set parameters for response generation
+PARAMETER num_ctx 4096
+PARAMETER num_predict 1024
+PARAMETER temperature 0.001
+PARAMETER top_p 1.0
+PARAMETER top_k 50
+PARAMETER repeat_penalty 1.0
+PARAMETER stop "<|im_start|>"
+PARAMETER stop "<|im_end|>"
+
+# Set the random number seed to use for generation
+PARAMETER seed 42
+
+# Set the prompt template to be passed into the model
+TEMPLATE """
+{{- if .System }}<|im_start|>system
+{{ .System }}<|im_end|>
+{{ end }}{{ if .Prompt }}
+<|im_start|>user
+{{ .Prompt }}<|im_end|>
+{{ end }}<|im_start|>assistant
+{{ .Response }}<|im_end|>"""
diff --git a/demos/function_calling/docker-compose.yaml b/demos/function_calling/docker-compose.yaml
index 5fc05cd6..a2bccf9f 100644
--- a/demos/function_calling/docker-compose.yaml
+++ b/demos/function_calling/docker-compose.yaml
@@ -57,6 +57,7 @@ services:
     environment:
       # use ollama endpoint that is hosted by host machine (no virtualization)
       - OLLAMA_ENDPOINT=${OLLAMA_ENDPOINT:-host.docker.internal}
+      - OLLAMA_MODEL=Arch-Function-Calling-3B-Q4_K_M
       # uncomment following line to use ollama endpoint that is hosted by docker
       # - OLLAMA_ENDPOINT=ollama
       # - OLLAMA_MODEL=Arch-Function-Calling-1.5B:Q4_K_M