add hurl tests for currency exchange demo (#435)

.github/workflows/e2e_test_demos.yml

@@ -32,6 +32,11 @@ jobs:
         run: |
           python -m venv venv
 
+      - name: install hurl
+        run: |
+          curl --location --remote-name https://github.com/Orange-OpenSource/hurl/releases/download/4.0.0/hurl_4.0.0_amd64.deb
+          sudo dpkg -i hurl_4.0.0_amd64.deb
+
       - name: install model server, arch gateway and test dependencies
         run: |
           source venv/bin/activate

demos/samples_python/currency_exchange/hurl_tests/simple.hurl

@@ -0,0 +1,19 @@
+POST http://localhost:10000/v1/chat/completions
+Content-Type: application/json
+
+{
+  "messages": [
+    {
+      "role": "user",
+      "content": "convert 100 eur"
+    }
+  ]
+}
+HTTP 200
+[Asserts]
+header "content-type" == "application/json"
+jsonpath "$.model" matches /^gpt-4o/
+jsonpath "$.metadata.x-arch-state" != null
+jsonpath "$.usage" != null
+jsonpath "$.choices[0].message.content" != null
+jsonpath "$.choices[0].message.role" == "assistant"

demos/samples_python/currency_exchange/hurl_tests/simple_stream.hurl

@@ -0,0 +1,17 @@
+POST http://localhost:10000/v1/chat/completions
+Content-Type: application/json
+
+{
+  "messages": [
+    {
+      "role": "user",
+      "content": "convert 100 eur"
+    }
+  ],
+  "stream": true
+}
+HTTP 200
+[Asserts]
+header "content-type" matches /text\/event-stream/
+body matches /^data: .*?currency_exchange.*?\n/
+body matches /^data: .*?EUR.*?\n/

demos/shared/test_runner/run_demo_tests.sh

@@ -8,11 +8,13 @@ do
   echo "Running tests for $demo ..."
   echo "****************************************"
   cd ../../samples_python/$demo
+  echo "starting archgw"
   archgw up arch_config.yaml
-  docker compose up -d
-  cd ../../shared/test_runner
-  TEST_DATA=../../samples_python/$demo/test_data.yaml poetry run pytest
-  cd ../../samples_python/$demo
+  echo "starting docker containers"
+  docker compose up -d 2>&1 > /dev/null
+  echo "starting hurl tests"
+  hurl --test hurl_tests/*.hurl
+  echo "stopping docker containers and archgw"
   archgw down
   docker compose down -v
   cd ../../shared/test_runner

tests/model_tests/arch_fc.hurl

@@ -0,0 +1,22 @@
+POST https://archfc.katanemo.dev/v1/chat/completions
+Content-Type: application/json
+
+{
+  "model": "Arch-Intent",
+  "messages": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant.\n\nYou task is to check if there are any tools that can be used to help the last user message in conversations according to the available tools listed below.\n\n<tools>\n{\"index\": \"T0\", \"type\": \"function\", \"function\": {\"name\": \"weather_forecast\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"str\"}, \"days\": {\"type\": \"int\"}}, \"required\": [\"city\", \"days\"]}}}\n</tools>\n\nProvide your tool assessment for ONLY THE LAST USER MESSAGE in the above conversation:\n- First line must read 'Yes' or 'No'.\n- If yes, a second line must include a comma-separated list of tool indexes.\n"
+    },
+    { "role": "user", "content": "how is the weather in seattle? Are there any tools can help?" }
+  ],
+  "stream": false
+}
+
+HTTP 200
+[Asserts]
+header "content-type" == "application/json"
+jsonpath "$.model" matches /^Arch-Function/
+jsonpath "$.usage" != null
+jsonpath "$.choices[0].message.content" matches /Yes/
+jsonpath "$.choices[0].message.role" == "assistant"