From 5526314b3cf77e6d72456cc85f993258d2e869cb Mon Sep 17 00:00:00 2001 From: Adil Hafeez Date: Mon, 17 Mar 2025 11:50:14 -0700 Subject: [PATCH 1/3] fix ollama demo - add egress endpoint for llm routing (#433) --- demos/use_cases/ollama/arch_config.yaml | 4 ++-- demos/use_cases/ollama/docker-compose.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/demos/use_cases/ollama/arch_config.yaml b/demos/use_cases/ollama/arch_config.yaml index 394d3d6c..c933841d 100644 --- a/demos/use_cases/ollama/arch_config.yaml +++ b/demos/use_cases/ollama/arch_config.yaml @@ -1,9 +1,9 @@ version: v0.1 listeners: - ingress_traffic: + egress_traffic: address: 0.0.0.0 - port: 10000 + port: 12000 message_format: openai timeout: 30s diff --git a/demos/use_cases/ollama/docker-compose.yaml b/demos/use_cases/ollama/docker-compose.yaml index ee5465a5..44fb20b7 100644 --- a/demos/use_cases/ollama/docker-compose.yaml +++ b/demos/use_cases/ollama/docker-compose.yaml @@ -6,7 +6,7 @@ services: - "18080:8080" environment: # this is only because we are running the sample app in the same docker container environemtn as archgw - - CHAT_COMPLETION_ENDPOINT=http://host.docker.internal:10000/v1 + - CHAT_COMPLETION_ENDPOINT=http://host.docker.internal:12000/v1 extra_hosts: - "host.docker.internal:host-gateway" volumes: From 6072d6ef3019391e7ff922a394bca0c73b6c3215 Mon Sep 17 00:00:00 2001 From: Salman Paracha Date: Mon, 17 Mar 2025 14:07:06 -0700 Subject: [PATCH 2/3] updating the docs to improve usage guide for prompt_targets and function calling (#434) Co-authored-by: Salman Paracha --- docs/source/concepts/prompt_target.rst | 27 ++++++++++++------------- docs/source/guides/function_calling.rst | 3 +++ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/source/concepts/prompt_target.rst b/docs/source/concepts/prompt_target.rst index b489b0c1..d8f8df91 100644 --- a/docs/source/concepts/prompt_target.rst +++ b/docs/source/concepts/prompt_target.rst @@ -53,29 +53,28 @@ A prompt target configuration includes the following elements: - ``endpoint``: The API endpoint or function that handles the prompt. - ``parameters`` (Optional): A list of parameters to extract from the prompt. +.. _defining_prompt_target_parameters: + Defining Parameters ~~~~~~~~~~~~~~~~~~~ Parameters are the pieces of information that Arch needs to extract from the user's prompt to perform the desired action. -Each parameter can be marked as required or optional. -Here is a full list of parameter attributes that Arch can support: +Each parameter can be marked as required or optional. Here is a full list of parameter attributes that Arch can support: .. table:: :width: 100% - ==================== ============================================================================ + ======================== ============================================================================ **Attribute** **Description** - ==================== ============================================================================ - ``name`` Specifies identifier of parameters - ``type`` Specifies the data type of the parameter. - ``description`` Provides a human-readable explanation of the parameter's purpose. - ``required`` Indicates whether the parameter is mandatory or optional + ======================== ============================================================================ + ``name (req.)`` Specifies name of the parameter. + ``description (req.)`` Provides a human-readable explanation of the parameter's purpose. + ``type (req.)`` Specifies the data type. Supported types include: **int**, **str**, **float**, **bool**, **list**, **set**, **dict**, **tuple** ``default`` Specifies a default value for the parameter if not provided by the user. - ``items`` Used in the context of arrays to define the schema of items within an array. - ``format`` Specifies a format for the parameter value, e.g., date and email - ``enum`` Lists the allowable values for the parameter. - ``minimum`` Defines the minimum acceptable value for numeric parameters. - ``maximum`` Specifies the maximum acceptable value for numeric parameters. - ==================== ============================================================================ + ``format`` Specifies a format for the parameter value. For example: `2019-12-31` for a date value. + ``enum`` Lists of allowable values for the parameter with data type matching the ``type`` attribute. **Usage Example**: ``enum: ["celsius`", "fahrenheit"]`` + ``items`` Specifies the attribute of the elements when type euqals **list**, **set**, **dict**, **tuple**. **Usage Example**: ``items: {"type": "str"}`` + ``required`` Indicates whether the parameter is mandatory or optional. Valid values: **true** or **false** + ======================== ============================================================================ Example Configuration ~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/guides/function_calling.rst b/docs/source/guides/function_calling.rst index 036d0e00..072571d2 100644 --- a/docs/source/guides/function_calling.rst +++ b/docs/source/guides/function_calling.rst @@ -118,6 +118,9 @@ Specify the parameters your function needs and how Arch should interpret these. name: api_server path: /weather +.. Note:: + For a complete refernce of attributes that you can configure in a prompt target, see :ref:`here `. + Step 3: Arch Takes Over ~~~~~~~~~~~~~~~~~~~~~~~ Once you have defined the functions and configured the prompt targets, Arch Gateway takes care of the remaining work. From d2cb1427fb5c7b68fbc49b2c000174bc57fce4aa Mon Sep 17 00:00:00 2001 From: Adil Hafeez Date: Mon, 17 Mar 2025 14:21:41 -0700 Subject: [PATCH 3/3] add hurl tests for currency exchange demo (#435) --- .github/workflows/e2e_test_demos.yml | 5 +++++ .../currency_exchange/hurl_tests/simple.hurl | 19 ++++++++++++++++ .../hurl_tests/simple_stream.hurl | 17 ++++++++++++++ demos/shared/test_runner/run_demo_tests.sh | 10 +++++---- tests/model_tests/arch_fc.hurl | 22 +++++++++++++++++++ 5 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 demos/samples_python/currency_exchange/hurl_tests/simple.hurl create mode 100644 demos/samples_python/currency_exchange/hurl_tests/simple_stream.hurl create mode 100644 tests/model_tests/arch_fc.hurl diff --git a/.github/workflows/e2e_test_demos.yml b/.github/workflows/e2e_test_demos.yml index 0dfe3dd7..9033ca29 100644 --- a/.github/workflows/e2e_test_demos.yml +++ b/.github/workflows/e2e_test_demos.yml @@ -32,6 +32,11 @@ jobs: run: | python -m venv venv + - name: install hurl + run: | + curl --location --remote-name https://github.com/Orange-OpenSource/hurl/releases/download/4.0.0/hurl_4.0.0_amd64.deb + sudo dpkg -i hurl_4.0.0_amd64.deb + - name: install model server, arch gateway and test dependencies run: | source venv/bin/activate diff --git a/demos/samples_python/currency_exchange/hurl_tests/simple.hurl b/demos/samples_python/currency_exchange/hurl_tests/simple.hurl new file mode 100644 index 00000000..579de34a --- /dev/null +++ b/demos/samples_python/currency_exchange/hurl_tests/simple.hurl @@ -0,0 +1,19 @@ +POST http://localhost:10000/v1/chat/completions +Content-Type: application/json + +{ + "messages": [ + { + "role": "user", + "content": "convert 100 eur" + } + ] +} +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^gpt-4o/ +jsonpath "$.metadata.x-arch-state" != null +jsonpath "$.usage" != null +jsonpath "$.choices[0].message.content" != null +jsonpath "$.choices[0].message.role" == "assistant" diff --git a/demos/samples_python/currency_exchange/hurl_tests/simple_stream.hurl b/demos/samples_python/currency_exchange/hurl_tests/simple_stream.hurl new file mode 100644 index 00000000..8315236c --- /dev/null +++ b/demos/samples_python/currency_exchange/hurl_tests/simple_stream.hurl @@ -0,0 +1,17 @@ +POST http://localhost:10000/v1/chat/completions +Content-Type: application/json + +{ + "messages": [ + { + "role": "user", + "content": "convert 100 eur" + } + ], + "stream": true +} +HTTP 200 +[Asserts] +header "content-type" matches /text\/event-stream/ +body matches /^data: .*?currency_exchange.*?\n/ +body matches /^data: .*?EUR.*?\n/ diff --git a/demos/shared/test_runner/run_demo_tests.sh b/demos/shared/test_runner/run_demo_tests.sh index a029345f..5b930565 100644 --- a/demos/shared/test_runner/run_demo_tests.sh +++ b/demos/shared/test_runner/run_demo_tests.sh @@ -8,11 +8,13 @@ do echo "Running tests for $demo ..." echo "****************************************" cd ../../samples_python/$demo + echo "starting archgw" archgw up arch_config.yaml - docker compose up -d - cd ../../shared/test_runner - TEST_DATA=../../samples_python/$demo/test_data.yaml poetry run pytest - cd ../../samples_python/$demo + echo "starting docker containers" + docker compose up -d 2>&1 > /dev/null + echo "starting hurl tests" + hurl --test hurl_tests/*.hurl + echo "stopping docker containers and archgw" archgw down docker compose down -v cd ../../shared/test_runner diff --git a/tests/model_tests/arch_fc.hurl b/tests/model_tests/arch_fc.hurl new file mode 100644 index 00000000..a16cdb92 --- /dev/null +++ b/tests/model_tests/arch_fc.hurl @@ -0,0 +1,22 @@ +POST https://archfc.katanemo.dev/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Intent", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant.\n\nYou task is to check if there are any tools that can be used to help the last user message in conversations according to the available tools listed below.\n\n\n{\"index\": \"T0\", \"type\": \"function\", \"function\": {\"name\": \"weather_forecast\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"str\"}, \"days\": {\"type\": \"int\"}}, \"required\": [\"city\", \"days\"]}}}\n\n\nProvide your tool assessment for ONLY THE LAST USER MESSAGE in the above conversation:\n- First line must read 'Yes' or 'No'.\n- If yes, a second line must include a comma-separated list of tool indexes.\n" + }, + { "role": "user", "content": "how is the weather in seattle? Are there any tools can help?" } + ], + "stream": false +} + +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^Arch-Function/ +jsonpath "$.usage" != null +jsonpath "$.choices[0].message.content" matches /Yes/ +jsonpath "$.choices[0].message.role" == "assistant"