diff --git a/tests/rest/model_test_adil.hurl b/tests/rest/model_test_adil.hurl
index a16cdb92..101cf08d 100644
--- a/tests/rest/model_test_adil.hurl
+++ b/tests/rest/model_test_adil.hurl
@@ -1,16 +1,108 @@
-POST https://archfc.katanemo.dev/v1/chat/completions
+# POST http://localhost:8282/v1/chat/completions
+# Content-Type: application/json
+
+# {
+# "model": "Arch-Function",
+# "messages": [
+# {
+# "role": "system",
+# "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
+# },
+# {
+# "role": "user",
+# "content": "I need help in converting currency"
+# }
+# ],
+# "temperature": 0.6,
+# "top_p": 1.0,
+# "top_k": 10
+# }
+
+# HTTP 200
+# [Asserts]
+# header "content-type" == "application/json"
+# jsonpath "$.model" matches /^Arch-Function/
+# jsonpath "$.usage" != null
+# # {"required_functions": ["convert_currency"], "clarification": "From which currency to which currency?"}
+# jsonpath "$.choices[0].message.content" matches /required_functions.*convert_currency.*clarification/
+# jsonpath "$.choices[0].message.role" == "assistant"
+
+# POST http://localhost:8282/v1/chat/completions
+# Content-Type: application/json
+
+# {
+# "model": "Arch-Function",
+# "messages": [
+# {
+# "role": "system",
+# "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
+# },
+# {
+# "role": "user",
+# "content": "I need help in converting currency"
+# },
+# {
+# "role": "assistant",
+# "content": "From which currency to which currency?"
+# },
+# {
+# "role": "user",
+# "content": "USD to INR"
+# }
+# ],
+# "temperature": 0.6,
+# "top_p": 1.0,
+# "top_k": 10
+# }
+
+# HTTP 200
+# [Asserts]
+# header "content-type" == "application/json"
+# jsonpath "$.model" matches /^Arch-Function/
+# jsonpath "$.usage" != null
+# # {"tool_calls": [{"name": "convert_currency", "arguments": {"from_currency": "USD", "to_currency": "INR"}}]}
+# jsonpath "$.choices[0].message.content" matches /tool_calls.*convert_currency.*USD.*INR/
+# jsonpath "$.choices[0].message.role" == "assistant"
+
+
+POST http://localhost:8282/v1/chat/completions
Content-Type: application/json
{
- "model": "Arch-Intent",
+ "model": "Arch-Function",
"messages": [
{
"role": "system",
- "content": "You are a helpful assistant.\n\nYou task is to check if there are any tools that can be used to help the last user message in conversations according to the available tools listed below.\n\n\n{\"index\": \"T0\", \"type\": \"function\", \"function\": {\"name\": \"weather_forecast\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"str\"}, \"days\": {\"type\": \"int\"}}, \"required\": [\"city\", \"days\"]}}}\n\n\nProvide your tool assessment for ONLY THE LAST USER MESSAGE in the above conversation:\n- First line must read 'Yes' or 'No'.\n- If yes, a second line must include a comma-separated list of tool indexes.\n"
+ "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
},
- { "role": "user", "content": "how is the weather in seattle? Are there any tools can help?" }
+ {
+ "role": "user",
+ "content": "I need help in converting currency"
+ },
+ {
+ "role": "assistant",
+ "content": "From which currency to which currency?"
+ },
+ {
+ "role": "user",
+ "content": "USD to JPY"
+ },
+ {
+ "role": "observation",
+ "content": "\n{'name': 'convert_currency', 'result': '400 JPY'}\n"
+ },
+ {
+ "role": "assistant",
+ "content": "Here is the result: 400 JPY"
+ },
+ {
+ "role": "user",
+ "content": "what is the rate for gbp in usd?"
+ }
],
- "stream": false
+ "temperature": 0.6,
+ "top_p": 1.0,
+ "top_k": 10
}
HTTP 200
@@ -18,5 +110,6 @@ HTTP 200
header "content-type" == "application/json"
jsonpath "$.model" matches /^Arch-Function/
jsonpath "$.usage" != null
-jsonpath "$.choices[0].message.content" matches /Yes/
+# {"tool_calls": [{"name": "convert_currency", "arguments": {"from_currency": "USD", "to_currency": "JPY"}}]}
+jsonpath "$.choices[0].message.content" matches /tool_calls.*convert_currency.*USD.*JPY/
jsonpath "$.choices[0].message.role" == "assistant"
diff --git a/tests/rest/model_test_adil.rest b/tests/rest/model_test_adil.rest
new file mode 100644
index 00000000..8c965d5e
--- /dev/null
+++ b/tests/rest/model_test_adil.rest
@@ -0,0 +1,220 @@
+### Missing required parameters - simple 1
+POST http://localhost:8282/v1/chat/completions HTTP/1.1
+Content-Type: application/json
+
+{
+ "model": "Arch-Function",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
+ },
+ {
+ "role": "user",
+ "content": "I need help in converting currency"
+ }
+ ],
+ "temperature": 0.6,
+ "top_p": 1.0,
+ "top_k": 10
+}
+
+### Missing required parameters - simple 2
+POST http://localhost:8282/v1/chat/completions HTTP/1.1
+Content-Type: application/json
+
+{
+ "model": "Arch-Function",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
+ },
+ {
+ "role": "user",
+ "content": "what is the exchange rate for jpy from usd?"
+ }
+ ],
+ "temperature": 0.6,
+ "top_p": 1.0,
+ "top_k": 10
+}
+
+
+### Missing required parameters - simple - default value - not working
+POST http://localhost:8282/v1/chat/completions HTTP/1.1
+Content-Type: application/json
+
+{
+ "model": "Arch-Function",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
+ },
+ {
+ "role": "user",
+ "content": "what is the exchange rate for jpy?"
+ }
+ ],
+ "temperature": 0.6,
+ "top_p": 1.0,
+ "top_k": 10
+}
+
+
+### Missing required parameters - parameter gathering
+POST http://localhost:8282/v1/chat/completions HTTP/1.1
+Content-Type: application/json
+
+{
+ "model": "Arch-Function",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
+ },
+ {
+ "role": "user",
+ "content": "I need help in converting currency"
+ },
+ {
+ "role": "assistant",
+ "content": "From which currency to which currency?"
+ },
+ {
+ "role": "user",
+ "content": "USD to INR"
+ }
+ ],
+ "temperature": 0.6,
+ "top_p": 1.0,
+ "top_k": 10
+}
+
+### Missing required parameters - parameter gathering with observation
+POST http://localhost:8282/v1/chat/completions HTTP/1.1
+Content-Type: application/json
+
+{
+ "model": "Arch-Function",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
+ },
+ {
+ "role": "user",
+ "content": "I need help in converting currency"
+ },
+ {
+ "role": "assistant",
+ "content": "From which currency to which currency?"
+ },
+ {
+ "role": "user",
+ "content": "USD to JPY"
+ },
+ {
+ "role": "observation",
+ "content": "\n{'name': 'convert_currency', 'result': '400 JPY'}\n"
+ },
+ {
+ "role": "assistant",
+ "content": "Here is the result: 400 JPY"
+ },
+ {
+ "role": "user",
+ "content": "what is the rate for gbp in usd?"
+ }
+ ],
+ "temperature": 0.6,
+ "top_p": 1.0,
+ "top_k": 10
+}
+
+
+# Domain-specific parameter omissions - schedule meetings (domain specific parameter gathering)
+POST http://localhost:8282/v1/chat/completions HTTP/1.1
+Content-Type: application/json
+
+{
+ "model": "Arch-Function",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"participants\": {\"type\": \"list\", \"description\": \"List of participants\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}}, \"required\": [\"participants\", \"start_time\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
+ },
+ {
+ "role": "user",
+ "content": "I need help in scheduling a meeting with Joe and Jane tomorrow"
+ }
+ ],
+ "temperature": 0.6,
+ "top_p": 1.0,
+ "top_k": 10
+}
+
+#resp: {\"required_functions\": [\"schedule_meeting\"], \"clarification\": \"At what time would you like to schedule the meeting and for how long?\"}
+
+### parameter gathering
+POST http://localhost:8282/v1/chat/completions HTTP/1.1
+Content-Type: application/json
+
+{
+ "model": "Arch-Function",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"participants\": {\"type\": \"list\", \"description\": \"List of participants\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}}, \"required\": [\"participants\", \"start_time\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
+ },
+ {
+ "role": "user",
+ "content": "I need help in scheduling a meeting"
+ },
+ {
+ "role": "assistant",
+ "content": "Could you please provide me with the list of participants and the start time for the meeting?"
+ },
+ {
+ "role": "user",
+ "content": "Joe and Jane and for tomorrow please"
+ }
+ ],
+ "temperature": 0.6,
+ "top_p": 1.0,
+ "top_k": 10
+}
+
+# resp: {\"tool_calls\": [{\"name\": \"schedule_meeting\", \"arguments\": {\"participants\": [\"Joe\", \"Jane\"], \"start_time\": \"tomorrow\"}}]}
+
+### override default param value
+POST http://localhost:8282/v1/chat/completions HTTP/1.1
+Content-Type: application/json
+
+{
+ "model": "Arch-Function",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"participants\": {\"type\": \"list\", \"description\": \"List of participants\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}}, \"required\": [\"participants\", \"start_time\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
+ },
+ {
+ "role": "user",
+ "content": "I need help in scheduling a meeting"
+ },
+ {
+ "role": "assistant",
+ "content": "Could you please provide me with the list of participants and the start time for the meeting?"
+ },
+ {
+ "role": "user",
+ "content": "Joe and Jane and for tomorrow please. And the duration should be 2 hours"
+ }
+ ],
+ "temperature": 0.6,
+ "top_p": 1.0,
+ "top_k": 10
+}
+
+#resp: {\"tool_calls\": [{\"name\": \"schedule_meeting\", \"arguments\": {\"participants\": [\"Joe\", \"Jane\"], \"start_time\": \"tomorrow\", \"duration\": \"2 hours\"}}]}