mirror of
https://github.com/katanemo/plano.git
synced 2026-06-17 15:25:17 +02:00
add more tests
This commit is contained in:
Adil Hafeez
parent
24f75f5d98
commit
5163019a1d
12 changed files with 464 additions and 14 deletions
|
|
@ -0,0 +1,29 @@
|
|||
### Missing required parameters - simple 1
|
||||
POST http://localhost:8282/v1/chat/completions
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"model": "Arch-Function",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I need help in converting currency"
|
||||
}
|
||||
],
|
||||
"temperature": 0.6,
|
||||
"top_p": 1.0,
|
||||
"top_k": 10
|
||||
}
|
||||
|
||||
# {\"required_functions\": [\"exchange_rate\"], \"clarification\": \"Which currency would you like to convert from and to?\"}
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
header "content-type" == "application/json"
|
||||
jsonpath "$.model" matches /^Arch-Function/
|
||||
jsonpath "$.usage" != null
|
||||
jsonpath "$.choices[0].message.content" matches /exchange_rate.*clarification/
|
||||
jsonpath "$.choices[0].message.role" == "assistant"
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
### Missing required parameters - simple 1
|
||||
POST http://localhost:8282/v1/chat/completions
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"model": "Arch-Function",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what is the exchange rate for jpy from usd?"
|
||||
}
|
||||
],
|
||||
"temperature": 0.6,
|
||||
"top_p": 1.0,
|
||||
"top_k": 10
|
||||
}
|
||||
|
||||
# {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
header "content-type" == "application/json"
|
||||
jsonpath "$.model" matches /^Arch-Function/
|
||||
jsonpath "$.usage" != null
|
||||
jsonpath "$.choices[0].message.content" matches /tool_calls.*exchange_rate.*USD.*JPY/
|
||||
jsonpath "$.choices[0].message.role" == "assistant"
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
### Missing required parameters - simple 1
|
||||
POST http://localhost:8282/v1/chat/completions
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"model": "Arch-Function",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what is the exchange rate for jpy?"
|
||||
}
|
||||
],
|
||||
"temperature": 0.6,
|
||||
"top_p": 1.0,
|
||||
"top_k": 10
|
||||
}
|
||||
|
||||
# {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
header "content-type" == "application/json"
|
||||
jsonpath "$.model" matches /^Arch-Function/
|
||||
jsonpath "$.usage" != null
|
||||
# ignore param value for now
|
||||
jsonpath "$.choices[0].message.content" matches /tool_calls.*exchange_rate/
|
||||
jsonpath "$.choices[0].message.role" == "assistant"
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
### Missing required parameters - simple 1
|
||||
POST http://localhost:8282/v1/chat/completions
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"model": "Arch-Function",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I need help in converting currency"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "From which currency to which currency?"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "USD to JPY"
|
||||
}
|
||||
],
|
||||
"temperature": 0.6,
|
||||
"top_p": 1.0,
|
||||
"top_k": 10
|
||||
}
|
||||
|
||||
# {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
header "content-type" == "application/json"
|
||||
jsonpath "$.model" matches /^Arch-Function/
|
||||
jsonpath "$.usage" != null
|
||||
# ignore param value for now
|
||||
jsonpath "$.choices[0].message.content" matches /tool_calls.*exchange_rate/
|
||||
# jsonpath "$.choices[0].message.content" matches /tool_calls.*exchange_rate.*USD.*JPY/
|
||||
jsonpath "$.choices[0].message.role" == "assistant"
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
### Missing required parameters - simple 1
|
||||
POST http://localhost:8282/v1/chat/completions
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"model": "Arch-Function",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I need help in converting currency"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "From which currency to which currency?"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "USD to JPY"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "{\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]}"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "<tool_response>\n{'name': 'exchange_rate', 'result': '149 JPY'}\n</tool_response>"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "price from usd to jpy is 149"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what about eur?"
|
||||
}
|
||||
],
|
||||
"temperature": 0.6,
|
||||
"top_p": 1.0,
|
||||
"top_k": 10
|
||||
}
|
||||
|
||||
# {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
header "content-type" == "application/json"
|
||||
jsonpath "$.model" matches /^Arch-Function/
|
||||
jsonpath "$.usage" != null
|
||||
# ignore param value check for now
|
||||
jsonpath "$.choices[0].message.content" matches /tool_calls.*exchange_rate/
|
||||
jsonpath "$.choices[0].message.role" == "assistant"
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
tools = [
|
||||
{
|
||||
"name": "exchange_rate",
|
||||
"description": "Get the current exchange rate",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"from_currency": {
|
||||
"type": "string",
|
||||
"description": "The currency to convert from",
|
||||
"default": "USD",
|
||||
},
|
||||
"to_currency": {
|
||||
"type": "string",
|
||||
"description": "The currency to convert to",
|
||||
},
|
||||
},
|
||||
"required": ["to_currency"],
|
||||
},
|
||||
},
|
||||
]
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
POST http://localhost:8282/v1/chat/completions
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"model": "Arch-Function",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"subject\": {\"type\": \"string\", \"description\": \"The subject of the meeting\"}, \"start_date\": {\"type\": \"string\", \"description\": \"The start date of the meeting\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"participants\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}, \"description\": \"Email addresses of participants\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}, \"location\": {\"type\": \"string\", \"description\": \"The location of the meeting\"}}, \"required\": [\"subject\", \"start_date\", \"start_time\", \"participants\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I need help in scheduling a meeting with Joe and Jane tomorrow at 10am with title 'Project Kickoff'"
|
||||
}
|
||||
],
|
||||
"temperature": 0.6,
|
||||
"top_p": 1.0,
|
||||
"top_k": 10
|
||||
}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
header "content-type" == "application/json"
|
||||
jsonpath "$.model" matches /^Arch-Function/
|
||||
jsonpath "$.usage" != null
|
||||
# ignore param value check for now
|
||||
jsonpath "$.choices[0].message.content" matches /tool_calls.*schedule_meeting/
|
||||
jsonpath "$.choices[0].message.role" == "assistant"
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
POST http://localhost:8282/v1/chat/completions
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"model": "Arch-Function",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"subject\": {\"type\": \"string\", \"description\": \"The subject of the meeting\"}, \"start_date\": {\"type\": \"string\", \"description\": \"The start date of the meeting\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"participants\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}, \"description\": \"Email addresses of participants\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}, \"location\": {\"type\": \"string\", \"description\": \"The location of the meeting\"}}, \"required\": [\"subject\", \"start_date\", \"start_time\", \"participants\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I need help in scheduling a meeting"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Could you please provide me with the subject of the meeting, the start date and time, and the email addresses of the participants?"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "subject is 'Project Kickoff', start date is tomorrow, start time is 10am, participants are joe@gmail.com and jane@gmail.com"
|
||||
}
|
||||
],
|
||||
"temperature": 0.6,
|
||||
"top_p": 1.0,
|
||||
"top_k": 10
|
||||
}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
header "content-type" == "application/json"
|
||||
jsonpath "$.model" matches /^Arch-Function/
|
||||
jsonpath "$.usage" != null
|
||||
# ignore param value check for now
|
||||
jsonpath "$.choices[0].message.content" matches /tool_calls.*schedule_meeting/
|
||||
jsonpath "$.choices[0].message.role" == "assistant"
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
POST http://localhost:8282/v1/chat/completions
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"model": "Arch-Function",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"subject\": {\"type\": \"string\", \"description\": \"The subject of the meeting\"}, \"start_date\": {\"type\": \"string\", \"description\": \"The start date of the meeting\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"participants\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}, \"description\": \"Email addresses of participants\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}, \"location\": {\"type\": \"string\", \"description\": \"The location of the meeting\"}}, \"required\": [\"subject\", \"start_date\", \"start_time\", \"participants\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I need help in scheduling a meeting"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Could you please provide me with the subject of the meeting, the start date and time, and the email addresses of the participants?"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "subject is 'Project Kickoff', start date is tomorrow, start time is 10am, participants are joe@gmail.com and jane@gmail.com"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "<tool_response>\n{'name': 'schedule_meeting', 'result': 'success'}\n</tool_response>"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "The meeting was scheduled successfully."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Thank you"
|
||||
}
|
||||
],
|
||||
"temperature": 0.6,
|
||||
"top_p": 1.0,
|
||||
"top_k": 10
|
||||
}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
header "content-type" == "application/json"
|
||||
jsonpath "$.model" matches /^Arch-Function/
|
||||
jsonpath "$.usage" != null
|
||||
# ignore param value check for now
|
||||
jsonpath "$.choices[0].message.content" matches /help|assist|welcome|day/
|
||||
jsonpath "$.choices[0].message.role" == "assistant"
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
POST http://localhost:8282/v1/chat/completions
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"model": "Arch-Function",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"schedule_meeting\", \"description\": \"Schedule a meeting\", \"parameters\": {\"type\": \"object\", \"properties\": {\"subject\": {\"type\": \"string\", \"description\": \"The subject of the meeting\"}, \"start_date\": {\"type\": \"string\", \"description\": \"The start date of the meeting\"}, \"start_time\": {\"type\": \"string\", \"description\": \"The start time of the meeting\"}, \"participants\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}, \"description\": \"Email addresses of participants\"}, \"duration\": {\"type\": \"string\", \"description\": \"The duration of the meeting\", \"default\": \"1 hour\"}, \"location\": {\"type\": \"string\", \"description\": \"The location of the meeting\"}}, \"required\": [\"subject\", \"start_date\", \"start_time\", \"participants\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I need help in scheduling a meeting"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Could you please provide me with the subject of the meeting, the start date and time, and the email addresses of the participants?"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "subject is 'Project Kickoff', start date is tomorrow, start time is 10am, participants are joe@gmail.com and jane@gmail.com"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "<tool_response>\n{'name': 'schedule_meeting', 'result': 'success'}\n</tool_response>"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "The meeting was scheduled successfully."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I want to schedule another meeting"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Could you please provide me with the subject, start date and time, and the email addresses of the participants for the new meeting?"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Sure, subject is 'Project Kickoff'"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Could you please specify the start date and time for the second meeting?"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "start date is tomorrow, start time is 10am"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Could you please confirm the email addresses of the participants for the second meeting?"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "participants are tim@gmail.com and dave@gmail.com"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Please confirm the details: scheduling a meeting titled 'Project Kickoff' for tomorrow at 10am with participants 'tim@gmail.com' and 'dave@gmail.com'."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Yes please go ahead"
|
||||
}
|
||||
],
|
||||
"temperature": 0.6,
|
||||
"top_p": 1.0,
|
||||
"top_k": 10
|
||||
}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
header "content-type" == "application/json"
|
||||
jsonpath "$.model" matches /^Arch-Function/
|
||||
jsonpath "$.usage" != null
|
||||
# ignore param value check for now
|
||||
jsonpath "$.choices[0].message.content" matches /tool_response.*schedule_meeting/
|
||||
jsonpath "$.choices[0].message.role" == "assistant"
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
tools = [
|
||||
{
|
||||
"name": "schedule_meeting",
|
||||
"description": "Schedule a meeting",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"subject": {
|
||||
"type": "string",
|
||||
"description": "The subject of the meeting",
|
||||
},
|
||||
"start_date": {
|
||||
"type": "string",
|
||||
"description": "The start date of the meeting",
|
||||
},
|
||||
"start_time": {
|
||||
"type": "string",
|
||||
"description": "The start time of the meeting",
|
||||
},
|
||||
"participants": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Email addresses of participants",
|
||||
},
|
||||
"duration": {
|
||||
"type": "string",
|
||||
"description": "The duration of the meeting",
|
||||
"default": "1 hour",
|
||||
},
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The location of the meeting",
|
||||
},
|
||||
},
|
||||
"required": ["subject", "start_date", "start_time", "participants"],
|
||||
},
|
||||
}
|
||||
]
|
||||
|
|
@ -7,7 +7,7 @@ Content-Type: application/json
|
|||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
|
|
@ -19,6 +19,8 @@ Content-Type: application/json
|
|||
"top_k": 10
|
||||
}
|
||||
|
||||
# resp: {\"required_functions\": [\"exchange_rate\"], \"clarification\": \"Which currency would you like to convert from and to?\"}
|
||||
|
||||
### Missing required parameters - simple 2
|
||||
POST http://localhost:8282/v1/chat/completions HTTP/1.1
|
||||
Content-Type: application/json
|
||||
|
|
@ -28,7 +30,7 @@ Content-Type: application/json
|
|||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
|
|
@ -40,8 +42,10 @@ Content-Type: application/json
|
|||
"top_k": 10
|
||||
}
|
||||
|
||||
# resp: {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]}
|
||||
|
||||
### Missing required parameters - simple - default value - not working
|
||||
|
||||
### Missing required parameters - simple - default value
|
||||
POST http://localhost:8282/v1/chat/completions HTTP/1.1
|
||||
Content-Type: application/json
|
||||
|
||||
|
|
@ -50,7 +54,7 @@ Content-Type: application/json
|
|||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
|
|
@ -62,6 +66,7 @@ Content-Type: application/json
|
|||
"top_k": 10
|
||||
}
|
||||
|
||||
# resp: "{\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]}"
|
||||
|
||||
### Missing required parameters - parameter gathering
|
||||
POST http://localhost:8282/v1/chat/completions HTTP/1.1
|
||||
|
|
@ -72,7 +77,7 @@ Content-Type: application/json
|
|||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
|
|
@ -92,6 +97,8 @@ Content-Type: application/json
|
|||
"top_k": 10
|
||||
}
|
||||
|
||||
# resp: {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"INR\"}}]}
|
||||
|
||||
### Missing required parameters - parameter gathering with observation
|
||||
POST http://localhost:8282/v1/chat/completions HTTP/1.1
|
||||
Content-Type: application/json
|
||||
|
|
@ -101,7 +108,7 @@ Content-Type: application/json
|
|||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"convert_currency\", \"description\": \"Convert currency from one to another\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
"content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"name\": \"exchange_rate\", \"description\": \"Get the current exchange rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_currency\": {\"type\": \"string\", \"description\": \"The currency to convert from\", \"default\": \"USD\"}, \"to_currency\": {\"type\": \"string\", \"description\": \"The currency to convert to\"}}, \"required\": [\"to_currency\"]}}\n</tools>\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
|
|
@ -115,17 +122,21 @@ Content-Type: application/json
|
|||
"role": "user",
|
||||
"content": "USD to JPY"
|
||||
},
|
||||
{
|
||||
"role": "observation",
|
||||
"content": "<tool_response>\n{'name': 'convert_currency', 'result': '400 JPY'}\n</tool_response>"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Here is the result: 400 JPY"
|
||||
"content": "{\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"JPY\"}}]}"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what is the rate for gbp in usd?"
|
||||
"content": "<tool_response>\n{'name': 'exchange_rate', 'result': '149 JPY'}\n</tool_response>"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "price from usd to jpy is 149"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what about eur?"
|
||||
}
|
||||
],
|
||||
"temperature": 0.6,
|
||||
|
|
@ -133,8 +144,10 @@ Content-Type: application/json
|
|||
"top_k": 10
|
||||
}
|
||||
|
||||
# resp: {\"tool_calls\": [{\"name\": \"exchange_rate\", \"arguments\": {\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}}]}
|
||||
|
||||
# Domain-specific parameter omissions - schedule meetings (domain specific parameter gathering)
|
||||
|
||||
### Domain-specific parameter omissions - schedule meetings
|
||||
POST http://localhost:8282/v1/chat/completions HTTP/1.1
|
||||
Content-Type: application/json
|
||||
|
||||
|
|
@ -155,7 +168,8 @@ Content-Type: application/json
|
|||
"top_k": 10
|
||||
}
|
||||
|
||||
#resp: {\"required_functions\": [\"schedule_meeting\"], \"clarification\": \"At what time would you like to schedule the meeting and for how long?\"}
|
||||
#resp: {\"tool_calls\": [{\"name\": \"schedule_meeting\", \"arguments\": {\"participants\": [\"Joe\", \"Jane\"], \"start_time\": \"tomorrow\"}}]}
|
||||
|
||||
|
||||
### parameter gathering
|
||||
POST http://localhost:8282/v1/chat/completions HTTP/1.1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue