From 8322eade4cac4ac08ba1435ca636d80ea1ca9ccd Mon Sep 17 00:00:00 2001 From: Adil Hafeez Date: Tue, 18 Mar 2025 10:07:00 -0700 Subject: [PATCH] push rest/hurl changes --- tests/rest/model_test_adil.hurl | 105 ++++++++++++++- tests/rest/model_test_adil.rest | 220 ++++++++++++++++++++++++++++++++ 2 files changed, 319 insertions(+), 6 deletions(-) create mode 100644 tests/rest/model_test_adil.rest diff --git a/tests/rest/model_test_adil.hurl b/tests/rest/model_test_adil.hurl index a16cdb92..101cf08d 100644 --- a/tests/rest/model_test_adil.hurl +++ b/tests/rest/model_test_adil.hurl @@ -1,16 +1,108 @@ -POST https://archfc.katanemo.dev/v1/chat/completions +# POST http://localhost:8282/v1/chat/completions +# Content-Type: application/json + +# { +# "model": "Arch-Function", +# "messages": [ +# { +# "role": "system", +# "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" +# }, +# { +# "role": "user", +# "content": "I need help in converting currency" +# } +# ], +# "temperature": 0.6, +# "top_p": 1.0, +# "top_k": 10 +# } + +# HTTP 200 +# [Asserts] +# header "content-type" == "application/json" +# jsonpath "$.model" matches /^Arch-Function/ +# jsonpath "$.usage" != null +# # {"required_functions": ["convert_currency"], "clarification": "From which currency to which currency?"} +# jsonpath "$.choices[0].message.content" matches /required_functions.*convert_currency.*clarification/ +# jsonpath "$.choices[0].message.role" == "assistant" + +# POST http://localhost:8282/v1/chat/completions +# Content-Type: application/json + +# { +# "model": "Arch-Function", +# "messages": [ +# { +# "role": "system", +# "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" +# }, +# { +# "role": "user", +# "content": "I need help in converting currency" +# }, +# { +# "role": "assistant", +# "content": "From which currency to which currency?" +# }, +# { +# "role": "user", +# "content": "USD to INR" +# } +# ], +# "temperature": 0.6, +# "top_p": 1.0, +# "top_k": 10 +# } + +# HTTP 200 +# [Asserts] +# header "content-type" == "application/json" +# jsonpath "$.model" matches /^Arch-Function/ +# jsonpath "$.usage" != null +# # {"tool_calls": [{"name": "convert_currency", "arguments": {"from_currency": "USD", "to_currency": "INR"}}]} +# jsonpath "$.choices[0].message.content" matches /tool_calls.*convert_currency.*USD.*INR/ +# jsonpath "$.choices[0].message.role" == "assistant" + + +POST http://localhost:8282/v1/chat/completions Content-Type: application/json { - "model": "Arch-Intent", + "model": "Arch-Function", "messages": [ { "role": "system", - "content": "You are a helpful assistant.\n\nYou task is to check if there are any tools that can be used to help the last user message in conversations according to the available tools listed below.\n\n\n{\"index\": \"T0\", \"type\": \"function\", \"function\": {\"name\": \"weather_forecast\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"str\"}, \"days\": {\"type\": \"int\"}}, \"required\": [\"city\", \"days\"]}}}\n\n\nProvide your tool assessment for ONLY THE LAST USER MESSAGE in the above conversation:\n- First line must read 'Yes' or 'No'.\n- If yes, a second line must include a comma-separated list of tool indexes.\n" + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" }, - { "role": "user", "content": "how is the weather in seattle? Are there any tools can help?" } + { + "role": "user", + "content": "I need help in converting currency" + }, + { + "role": "assistant", + "content": "From which currency to which currency?" + }, + { + "role": "user", + "content": "USD to JPY" + }, + { + "role": "observation", + "content": "\n{'name': 'convert_currency', 'result': '400 JPY'}\n" + }, + { + "role": "assistant", + "content": "Here is the result: 400 JPY" + }, + { + "role": "user", + "content": "what is the rate for gbp in usd?" + } ], - "stream": false + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 } HTTP 200 @@ -18,5 +110,6 @@ HTTP 200 header "content-type" == "application/json" jsonpath "$.model" matches /^Arch-Function/ jsonpath "$.usage" != null -jsonpath "$.choices[0].message.content" matches /Yes/ +# {"tool_calls": [{"name": "convert_currency", "arguments": {"from_currency": "USD", "to_currency": "JPY"}}]} +jsonpath "$.choices[0].message.content" matches /tool_calls.*convert_currency.*USD.*JPY/ jsonpath "$.choices[0].message.role" == "assistant" diff --git a/tests/rest/model_test_adil.rest b/tests/rest/model_test_adil.rest new file mode 100644 index 00000000..8c965d5e --- /dev/null +++ b/tests/rest/model_test_adil.rest @@ -0,0 +1,220 @@ +### Missing required parameters - simple 1 +POST http://localhost:8282/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in converting currency" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +### Missing required parameters - simple 2 +POST http://localhost:8282/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "what is the exchange rate for jpy from usd?" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + + +### Missing required parameters - simple - default value - not working +POST http://localhost:8282/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "what is the exchange rate for jpy?" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + + +### Missing required parameters - parameter gathering +POST http://localhost:8282/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in converting currency" + }, + { + "role": "assistant", + "content": "From which currency to which currency?" + }, + { + "role": "user", + "content": "USD to INR" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +### Missing required parameters - parameter gathering with observation +POST http://localhost:8282/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in converting currency" + }, + { + "role": "assistant", + "content": "From which currency to which currency?" + }, + { + "role": "user", + "content": "USD to JPY" + }, + { + "role": "observation", + "content": "\n{'name': 'convert_currency', 'result': '400 JPY'}\n" + }, + { + "role": "assistant", + "content": "Here is the result: 400 JPY" + }, + { + "role": "user", + "content": "what is the rate for gbp in usd?" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + + +# Domain-specific parameter omissions - schedule meetings (domain specific parameter gathering) +POST http://localhost:8282/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"participants\": {\"type\": \"list\", \"description\": \"List of participants\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}}, \"required\": [\"participants\", \"start_time\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in scheduling a meeting with Joe and Jane tomorrow" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +#resp: {\"required_functions\": [\"schedule_meeting\"], \"clarification\": \"At what time would you like to schedule the meeting and for how long?\"} + +### parameter gathering +POST http://localhost:8282/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"participants\": {\"type\": \"list\", \"description\": \"List of participants\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}}, \"required\": [\"participants\", \"start_time\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in scheduling a meeting" + }, + { + "role": "assistant", + "content": "Could you please provide me with the list of participants and the start time for the meeting?" + }, + { + "role": "user", + "content": "Joe and Jane and for tomorrow please" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +# resp: {\"tool_calls\": [{\"name\": \"schedule_meeting\", \"arguments\": {\"participants\": [\"Joe\", \"Jane\"], \"start_time\": \"tomorrow\"}}]} + +### override default param value +POST http://localhost:8282/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"participants\": {\"type\": \"list\", \"description\": \"List of participants\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}}, \"required\": [\"participants\", \"start_time\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in scheduling a meeting" + }, + { + "role": "assistant", + "content": "Could you please provide me with the list of participants and the start time for the meeting?" + }, + { + "role": "user", + "content": "Joe and Jane and for tomorrow please. And the duration should be 2 hours" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +#resp: {\"tool_calls\": [{\"name\": \"schedule_meeting\", \"arguments\": {\"participants\": [\"Joe\", \"Jane\"], \"start_time\": \"tomorrow\", \"duration\": \"2 hours\"}}]}