diff --git a/.github/workflows/e2e_test_demos.yml b/.github/workflows/e2e_test_demos.yml index 0dfe3dd7..9033ca29 100644 --- a/.github/workflows/e2e_test_demos.yml +++ b/.github/workflows/e2e_test_demos.yml @@ -32,6 +32,11 @@ jobs: run: | python -m venv venv + - name: install hurl + run: | + curl --location --remote-name https://github.com/Orange-OpenSource/hurl/releases/download/4.0.0/hurl_4.0.0_amd64.deb + sudo dpkg -i hurl_4.0.0_amd64.deb + - name: install model server, arch gateway and test dependencies run: | source venv/bin/activate diff --git a/demos/samples_python/currency_exchange/hurl_tests/simple.hurl b/demos/samples_python/currency_exchange/hurl_tests/simple.hurl new file mode 100644 index 00000000..579de34a --- /dev/null +++ b/demos/samples_python/currency_exchange/hurl_tests/simple.hurl @@ -0,0 +1,19 @@ +POST http://localhost:10000/v1/chat/completions +Content-Type: application/json + +{ + "messages": [ + { + "role": "user", + "content": "convert 100 eur" + } + ] +} +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^gpt-4o/ +jsonpath "$.metadata.x-arch-state" != null +jsonpath "$.usage" != null +jsonpath "$.choices[0].message.content" != null +jsonpath "$.choices[0].message.role" == "assistant" diff --git a/demos/samples_python/currency_exchange/hurl_tests/simple_stream.hurl b/demos/samples_python/currency_exchange/hurl_tests/simple_stream.hurl new file mode 100644 index 00000000..8315236c --- /dev/null +++ b/demos/samples_python/currency_exchange/hurl_tests/simple_stream.hurl @@ -0,0 +1,17 @@ +POST http://localhost:10000/v1/chat/completions +Content-Type: application/json + +{ + "messages": [ + { + "role": "user", + "content": "convert 100 eur" + } + ], + "stream": true +} +HTTP 200 +[Asserts] +header "content-type" matches /text\/event-stream/ +body matches /^data: .*?currency_exchange.*?\n/ +body matches /^data: .*?EUR.*?\n/ diff --git a/demos/shared/test_runner/run_demo_tests.sh b/demos/shared/test_runner/run_demo_tests.sh index a029345f..5b930565 100644 --- a/demos/shared/test_runner/run_demo_tests.sh +++ b/demos/shared/test_runner/run_demo_tests.sh @@ -8,11 +8,13 @@ do echo "Running tests for $demo ..." echo "****************************************" cd ../../samples_python/$demo + echo "starting archgw" archgw up arch_config.yaml - docker compose up -d - cd ../../shared/test_runner - TEST_DATA=../../samples_python/$demo/test_data.yaml poetry run pytest - cd ../../samples_python/$demo + echo "starting docker containers" + docker compose up -d 2>&1 > /dev/null + echo "starting hurl tests" + hurl --test hurl_tests/*.hurl + echo "stopping docker containers and archgw" archgw down docker compose down -v cd ../../shared/test_runner diff --git a/tests/model_tests/arch_fc.hurl b/tests/model_tests/arch_fc.hurl new file mode 100644 index 00000000..a16cdb92 --- /dev/null +++ b/tests/model_tests/arch_fc.hurl @@ -0,0 +1,22 @@ +POST https://archfc.katanemo.dev/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Intent", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant.\n\nYou task is to check if there are any tools that can be used to help the last user message in conversations according to the available tools listed below.\n\n\n{\"index\": \"T0\", \"type\": \"function\", \"function\": {\"name\": \"weather_forecast\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"str\"}, \"days\": {\"type\": \"int\"}}, \"required\": [\"city\", \"days\"]}}}\n\n\nProvide your tool assessment for ONLY THE LAST USER MESSAGE in the above conversation:\n- First line must read 'Yes' or 'No'.\n- If yes, a second line must include a comma-separated list of tool indexes.\n" + }, + { "role": "user", "content": "how is the weather in seattle? Are there any tools can help?" } + ], + "stream": false +} + +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^Arch-Function/ +jsonpath "$.usage" != null +jsonpath "$.choices[0].message.content" matches /Yes/ +jsonpath "$.choices[0].message.role" == "assistant"