diff --git a/tests/rest/hurl/missing_required_parameters_currency_exchange_1_simple.hurl b/tests/rest/hurl/missing_required_parameters_currency_exchange_1_simple.hurl new file mode 100644 index 00000000..2e20dac2 --- /dev/null +++ b/tests/rest/hurl/missing_required_parameters_currency_exchange_1_simple.hurl @@ -0,0 +1,29 @@ +### Missing required parameters - simple 1 +POST http://localhost:8282/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in converting currency" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +# {\"required_functions\": [\"exchange_rate\"], \"clarification\": \"Which currency would you like to convert from and to?\"} +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^Arch-Function/ +jsonpath "$.usage" != null +jsonpath "$.choices[0].message.content" matches /exchange_rate.*clarification/ +jsonpath "$.choices[0].message.role" == "assistant" diff --git a/tests/rest/hurl/missing_required_parameters_currency_exchange_2_happy_case.hurl b/tests/rest/hurl/missing_required_parameters_currency_exchange_2_happy_case.hurl new file mode 100644 index 00000000..0b81b0fa --- /dev/null +++ b/tests/rest/hurl/missing_required_parameters_currency_exchange_2_happy_case.hurl @@ -0,0 +1,30 @@ +### Missing required parameters - simple 1 +POST http://localhost:8282/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "what is the exchange rate for jpy from usd?" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +# {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]} + +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^Arch-Function/ +jsonpath "$.usage" != null +jsonpath "$.choices[0].message.content" matches /tool_calls.*exchange_rate.*USD.*JPY/ +jsonpath "$.choices[0].message.role" == "assistant" diff --git a/tests/rest/hurl/missing_required_parameters_currency_exchange_3_default_value.hurl b/tests/rest/hurl/missing_required_parameters_currency_exchange_3_default_value.hurl new file mode 100644 index 00000000..3935c150 --- /dev/null +++ b/tests/rest/hurl/missing_required_parameters_currency_exchange_3_default_value.hurl @@ -0,0 +1,31 @@ +### Missing required parameters - simple 1 +POST http://localhost:8282/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "what is the exchange rate for jpy?" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +# {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]} + +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^Arch-Function/ +jsonpath "$.usage" != null +# ignore param value for now +jsonpath "$.choices[0].message.content" matches /tool_calls.*exchange_rate/ +jsonpath "$.choices[0].message.role" == "assistant" diff --git a/tests/rest/hurl/missing_required_parameters_currency_exchange_4_param_gathering.hurl b/tests/rest/hurl/missing_required_parameters_currency_exchange_4_param_gathering.hurl new file mode 100644 index 00000000..e101a3b8 --- /dev/null +++ b/tests/rest/hurl/missing_required_parameters_currency_exchange_4_param_gathering.hurl @@ -0,0 +1,40 @@ +### Missing required parameters - simple 1 +POST http://localhost:8282/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in converting currency" + }, + { + "role": "assistant", + "content": "From which currency to which currency?" + }, + { + "role": "user", + "content": "USD to JPY" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +# {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]} + +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^Arch-Function/ +jsonpath "$.usage" != null +# ignore param value for now +jsonpath "$.choices[0].message.content" matches /tool_calls.*exchange_rate/ +# jsonpath "$.choices[0].message.content" matches /tool_calls.*exchange_rate.*USD.*JPY/ +jsonpath "$.choices[0].message.role" == "assistant" diff --git a/tests/rest/hurl/missing_required_parameters_currency_exchange_5_observation.hurl b/tests/rest/hurl/missing_required_parameters_currency_exchange_5_observation.hurl new file mode 100644 index 00000000..2fdd3b97 --- /dev/null +++ b/tests/rest/hurl/missing_required_parameters_currency_exchange_5_observation.hurl @@ -0,0 +1,55 @@ +### Missing required parameters - simple 1 +POST http://localhost:8282/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in converting currency" + }, + { + "role": "assistant", + "content": "From which currency to which currency?" + }, + { + "role": "user", + "content": "USD to JPY" + }, + { + "role": "assistant", + "content": "{\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]}" + }, + { + "role": "user", + "content": "\n{'name': 'exchange_rate', 'result': '149 JPY'}\n" + }, + { + "role": "assistant", + "content": "price from usd to jpy is 149" + }, + { + "role": "user", + "content": "what about eur?" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +# {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]} + +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^Arch-Function/ +jsonpath "$.usage" != null +# ignore param value check for now +jsonpath "$.choices[0].message.content" matches /tool_calls.*exchange_rate/ +jsonpath "$.choices[0].message.role" == "assistant" diff --git a/tests/rest/hurl/missing_required_parameters_currency_exchange_tool.py b/tests/rest/hurl/missing_required_parameters_currency_exchange_tool.py new file mode 100644 index 00000000..0338f9d9 --- /dev/null +++ b/tests/rest/hurl/missing_required_parameters_currency_exchange_tool.py @@ -0,0 +1,21 @@ +tools = [ + { + "name": "exchange_rate", + "description": "Get the current exchange rate", + "parameters": { + "type": "object", + "properties": { + "from_currency": { + "type": "string", + "description": "The currency to convert from", + "default": "USD", + }, + "to_currency": { + "type": "string", + "description": "The currency to convert to", + }, + }, + "required": ["to_currency"], + }, + }, +] diff --git a/tests/rest/hurl/missing_required_parameters_schedule_meeting_1_simple.hurl b/tests/rest/hurl/missing_required_parameters_schedule_meeting_1_simple.hurl new file mode 100644 index 00000000..2a696730 --- /dev/null +++ b/tests/rest/hurl/missing_required_parameters_schedule_meeting_1_simple.hurl @@ -0,0 +1,28 @@ +POST http://localhost:8282/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"subject\": {\"type\": \"string\", \"description\": \"The subject of the meeting\"}, \"start_date\": {\"type\": \"string\", \"description\": \"The start date of the meeting\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"participants\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}, \"description\": \"Email addresses of participants\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}, \"location\": {\"type\": \"string\", \"description\": \"The location of the meeting\"}}, \"required\": [\"subject\", \"start_date\", \"start_time\", \"participants\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in scheduling a meeting with Joe and Jane tomorrow at 10am with title 'Project Kickoff'" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^Arch-Function/ +jsonpath "$.usage" != null +# ignore param value check for now +jsonpath "$.choices[0].message.content" matches /tool_calls.*schedule_meeting/ +jsonpath "$.choices[0].message.role" == "assistant" diff --git a/tests/rest/hurl/missing_required_parameters_schedule_meeting_2_param_gathering.hurl b/tests/rest/hurl/missing_required_parameters_schedule_meeting_2_param_gathering.hurl new file mode 100644 index 00000000..90dcc251 --- /dev/null +++ b/tests/rest/hurl/missing_required_parameters_schedule_meeting_2_param_gathering.hurl @@ -0,0 +1,36 @@ +POST http://localhost:8282/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"subject\": {\"type\": \"string\", \"description\": \"The subject of the meeting\"}, \"start_date\": {\"type\": \"string\", \"description\": \"The start date of the meeting\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"participants\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}, \"description\": \"Email addresses of participants\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}, \"location\": {\"type\": \"string\", \"description\": \"The location of the meeting\"}}, \"required\": [\"subject\", \"start_date\", \"start_time\", \"participants\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in scheduling a meeting" + }, + { + "role": "assistant", + "content": "Could you please provide me with the subject of the meeting, the start date and time, and the email addresses of the participants?" + }, + { + "role": "user", + "content": "subject is 'Project Kickoff', start date is tomorrow, start time is 10am, participants are joe@gmail.com and jane@gmail.com" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^Arch-Function/ +jsonpath "$.usage" != null +# ignore param value check for now +jsonpath "$.choices[0].message.content" matches /tool_calls.*schedule_meeting/ +jsonpath "$.choices[0].message.role" == "assistant" diff --git a/tests/rest/hurl/missing_required_parameters_schedule_meeting_2_param_gathering_observation.hurl b/tests/rest/hurl/missing_required_parameters_schedule_meeting_2_param_gathering_observation.hurl new file mode 100644 index 00000000..d52e6dbd --- /dev/null +++ b/tests/rest/hurl/missing_required_parameters_schedule_meeting_2_param_gathering_observation.hurl @@ -0,0 +1,48 @@ +POST http://localhost:8282/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"subject\": {\"type\": \"string\", \"description\": \"The subject of the meeting\"}, \"start_date\": {\"type\": \"string\", \"description\": \"The start date of the meeting\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"participants\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}, \"description\": \"Email addresses of participants\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}, \"location\": {\"type\": \"string\", \"description\": \"The location of the meeting\"}}, \"required\": [\"subject\", \"start_date\", \"start_time\", \"participants\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in scheduling a meeting" + }, + { + "role": "assistant", + "content": "Could you please provide me with the subject of the meeting, the start date and time, and the email addresses of the participants?" + }, + { + "role": "user", + "content": "subject is 'Project Kickoff', start date is tomorrow, start time is 10am, participants are joe@gmail.com and jane@gmail.com" + }, + { + "role": "user", + "content": "\n{'name': 'schedule_meeting', 'result': 'success'}\n" + }, + { + "role": "assistant", + "content": "The meeting was scheduled successfully." + }, + { + "role": "user", + "content": "Thank you" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^Arch-Function/ +jsonpath "$.usage" != null +# ignore param value check for now +jsonpath "$.choices[0].message.content" matches /help|assist|welcome|day/ +jsonpath "$.choices[0].message.role" == "assistant" diff --git a/tests/rest/hurl/missing_required_parameters_schedule_meeting_2_param_gathering_observation_continuation.hurl b/tests/rest/hurl/missing_required_parameters_schedule_meeting_2_param_gathering_observation_continuation.hurl new file mode 100644 index 00000000..e1c238ba --- /dev/null +++ b/tests/rest/hurl/missing_required_parameters_schedule_meeting_2_param_gathering_observation_continuation.hurl @@ -0,0 +1,80 @@ +POST http://localhost:8282/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"subject\": {\"type\": \"string\", \"description\": \"The subject of the meeting\"}, \"start_date\": {\"type\": \"string\", \"description\": \"The start date of the meeting\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"participants\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}, \"description\": \"Email addresses of participants\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}, \"location\": {\"type\": \"string\", \"description\": \"The location of the meeting\"}}, \"required\": [\"subject\", \"start_date\", \"start_time\", \"participants\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I need help in scheduling a meeting" + }, + { + "role": "assistant", + "content": "Could you please provide me with the subject of the meeting, the start date and time, and the email addresses of the participants?" + }, + { + "role": "user", + "content": "subject is 'Project Kickoff', start date is tomorrow, start time is 10am, participants are joe@gmail.com and jane@gmail.com" + }, + { + "role": "user", + "content": "\n{'name': 'schedule_meeting', 'result': 'success'}\n" + }, + { + "role": "assistant", + "content": "The meeting was scheduled successfully." + }, + { + "role": "user", + "content": "I want to schedule another meeting" + }, + { + "role": "assistant", + "content": "Could you please provide me with the subject, start date and time, and the email addresses of the participants for the new meeting?" + }, + { + "role": "user", + "content": "Sure, subject is 'Project Kickoff'" + }, + { + "role": "assistant", + "content": "Could you please specify the start date and time for the second meeting?" + }, + { + "role": "user", + "content": "start date is tomorrow, start time is 10am" + }, + { + "role": "assistant", + "content": "Could you please confirm the email addresses of the participants for the second meeting?" + }, + { + "role": "user", + "content": "participants are tim@gmail.com and dave@gmail.com" + }, + { + "role": "assistant", + "content": "Please confirm the details: scheduling a meeting titled 'Project Kickoff' for tomorrow at 10am with participants 'tim@gmail.com' and 'dave@gmail.com'." + }, + { + "role": "user", + "content": "Yes please go ahead" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +HTTP 200 +[Asserts] +header "content-type" == "application/json" +jsonpath "$.model" matches /^Arch-Function/ +jsonpath "$.usage" != null +# ignore param value check for now +jsonpath "$.choices[0].message.content" matches /tool_response.*schedule_meeting/ +jsonpath "$.choices[0].message.role" == "assistant" diff --git a/tests/rest/hurl/missing_required_parameters_schedule_meeting_tool.py b/tests/rest/hurl/missing_required_parameters_schedule_meeting_tool.py new file mode 100644 index 00000000..d9d6e683 --- /dev/null +++ b/tests/rest/hurl/missing_required_parameters_schedule_meeting_tool.py @@ -0,0 +1,38 @@ +tools = [ + { + "name": "schedule_meeting", + "description": "Schedule a meeting", + "parameters": { + "type": "object", + "properties": { + "subject": { + "type": "string", + "description": "The subject of the meeting", + }, + "start_date": { + "type": "string", + "description": "The start date of the meeting", + }, + "start_time": { + "type": "string", + "description": "The start time of the meeting", + }, + "participants": { + "type": "array", + "items": {"type": "string"}, + "description": "Email addresses of participants", + }, + "duration": { + "type": "string", + "description": "The duration of the meeting", + "default": "1 hour", + }, + "location": { + "type": "string", + "description": "The location of the meeting", + }, + }, + "required": ["subject", "start_date", "start_time", "participants"], + }, + } +] diff --git a/tests/rest/model_test_adil.rest b/tests/rest/model_test_adil.rest index 8c965d5e..c678a689 100644 --- a/tests/rest/model_test_adil.rest +++ b/tests/rest/model_test_adil.rest @@ -7,7 +7,7 @@ Content-Type: application/json "messages": [ { "role": "system", - "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" }, { "role": "user", @@ -19,6 +19,8 @@ Content-Type: application/json "top_k": 10 } +# resp: {\"required_functions\": [\"exchange_rate\"], \"clarification\": \"Which currency would you like to convert from and to?\"} + ### Missing required parameters - simple 2 POST http://localhost:8282/v1/chat/completions HTTP/1.1 Content-Type: application/json @@ -28,7 +30,7 @@ Content-Type: application/json "messages": [ { "role": "system", - "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" }, { "role": "user", @@ -40,8 +42,10 @@ Content-Type: application/json "top_k": 10 } +# resp: {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]} -### Missing required parameters - simple - default value - not working + +### Missing required parameters - simple - default value POST http://localhost:8282/v1/chat/completions HTTP/1.1 Content-Type: application/json @@ -50,7 +54,7 @@ Content-Type: application/json "messages": [ { "role": "system", - "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" }, { "role": "user", @@ -62,6 +66,7 @@ Content-Type: application/json "top_k": 10 } +# resp: "{\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]}" ### Missing required parameters - parameter gathering POST http://localhost:8282/v1/chat/completions HTTP/1.1 @@ -72,7 +77,7 @@ Content-Type: application/json "messages": [ { "role": "system", - "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" }, { "role": "user", @@ -92,6 +97,8 @@ Content-Type: application/json "top_k": 10 } +# resp: {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"INR\"}}]} + ### Missing required parameters - parameter gathering with observation POST http://localhost:8282/v1/chat/completions HTTP/1.1 Content-Type: application/json @@ -101,7 +108,7 @@ Content-Type: application/json "messages": [ { "role": "system", - "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" }, { "role": "user", @@ -115,17 +122,21 @@ Content-Type: application/json "role": "user", "content": "USD to JPY" }, - { - "role": "observation", - "content": "\n{'name': 'convert_currency', 'result': '400 JPY'}\n" - }, { "role": "assistant", - "content": "Here is the result: 400 JPY" + "content": "{\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]}" }, { "role": "user", - "content": "what is the rate for gbp in usd?" + "content": "\n{'name': 'exchange_rate', 'result': '149 JPY'}\n" + }, + { + "role": "assistant", + "content": "price from usd to jpy is 149" + }, + { + "role": "user", + "content": "what about eur?" } ], "temperature": 0.6, @@ -133,8 +144,10 @@ Content-Type: application/json "top_k": 10 } +# resp: {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}}]} -# Domain-specific parameter omissions - schedule meetings (domain specific parameter gathering) + +### Domain-specific parameter omissions - schedule meetings POST http://localhost:8282/v1/chat/completions HTTP/1.1 Content-Type: application/json @@ -155,7 +168,8 @@ Content-Type: application/json "top_k": 10 } -#resp: {\"required_functions\": [\"schedule_meeting\"], \"clarification\": \"At what time would you like to schedule the meeting and for how long?\"} +#resp: {\"tool_calls\": [{\"name\": \"schedule_meeting\", \"arguments\": {\"participants\": [\"Joe\", \"Jane\"], \"start_time\": \"tomorrow\"}}]} + ### parameter gathering POST http://localhost:8282/v1/chat/completions HTTP/1.1