From 92eba95e45207711069ef954f16991838705a716 Mon Sep 17 00:00:00 2001 From: Adil Hafeez Date: Thu, 20 Mar 2025 12:58:25 -0700 Subject: [PATCH] add additional tests --- tests/rest/model_test_adil.rest | 347 +++++++++++++++++++++++++++++++- 1 file changed, 346 insertions(+), 1 deletion(-) diff --git a/tests/rest/model_test_adil.rest b/tests/rest/model_test_adil.rest index e2cbd2da..704c3c3d 100644 --- a/tests/rest/model_test_adil.rest +++ b/tests/rest/model_test_adil.rest @@ -137,7 +137,7 @@ Content-Type: application/json "messages": [ { "role": "system", - "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"verify_address\", \"description\": \"Verify the address\", \"parameters\": {\"type\": \"object\", \"properties\": {\"address\": {\"type\": \"str\", \"description\": \"The address to verify\"}}, \"required\": [\"address\"]}}\n{\"name\": \"track_shipment\", \"description\": \"Track the shipment\", \"parameters\": {\"type\": \"object\", \"properties\": {\"tracking_number\": {\"type\": \"str\", \"description\": \"The tracking number\"}}, \"required\": [\"tracking_number\"]}}\n{\"name\": \"generate_label\", \"description\": \"Generate the shipping label\", \"parameters\": {\"type\": \"object\", \"properties\": {\"sender\": {\"type\": \"str\", \"description\": \"The address to ship from\"}, \"recipient\": {\"type\": \"str\", \"description\": \"The address to ship to\"}, \"weight\": {\"type\": \"int\", \"description\": \"The weight of the package\"}}, \"required\": [\"sender\", \"recipient\", \"weight\"]}}\n{\"name\": \"calculate_shipping_rate\", \"description\": \"Calculate the shipping rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"sender\": {\"type\": \"str\", \"description\": \"The address to ship from\"}, \"recipient\": {\"type\": \"str\", \"description\": \"The address to ship to\"}, \"weight\": {\"type\": \"int\", \"description\": \"The weight of the package\"}}, \"required\": [\"sender\", \"recipient\", \"weight\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"verify_address\", \"description\": \"Verify the address\", \"parameters\": {\"type\": \"object\", \"property_address\": {\"type\": \"str\", \"description\": \"Complete address of the property\", \"format\": \"Street address, City, State/Province, ZIP/Postal Code, Country\"}, \"required\": [\"property_address\"]}}\n{\"name\": \"track_shipment\", \"description\": \"Track the shipment\", \"parameters\": {\"type\": \"object\", \"properties\": {\"tracking_number\": {\"type\": \"str\", \"description\": \"The tracking number\"}}, \"required\": [\"tracking_number\"]}}\n{\"name\": \"generate_label\", \"description\": \"Generate the shipping label\", \"parameters\": {\"type\": \"object\", \"properties\": {\"sender\": {\"type\": \"str\", \"description\": \"The address to ship from\"}, \"recipient\": {\"type\": \"str\", \"description\": \"The address to ship to\"}, \"weight\": {\"type\": \"int\", \"description\": \"The weight of the package\"}}, \"required\": [\"sender\", \"recipient\", \"weight\"]}}\n{\"name\": \"calculate_shipping_rate\", \"description\": \"Calculate the shipping rate\", \"parameters\": {\"type\": \"object\", \"properties\": {\"sender\": {\"type\": \"str\", \"description\": \"The address to ship from\"}, \"recipient\": {\"type\": \"str\", \"description\": \"The address to ship to\"}, \"weight\": {\"type\": \"int\", \"description\": \"The weight of the package\"}}, \"required\": [\"sender\", \"recipient\", \"weight\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" }, { "role": "user", @@ -377,3 +377,348 @@ Content-Type: application/json "top_p": 1.0, "top_k": 10 } + + +# tools = [ +# { +# "name": "account_balance", +# "description": "Get the account balance", +# "parameters": { +# "type": "object", +# "properties": { +# "account_id": {"type": "str", "description": "The account ID"} +# }, +# "required": ["account_id"], +# }, +# }, +# { +# "name": "transfer_funds", +# "description": "Transfer funds between accounts", +# "parameters": { +# "type": "object", +# "properties": { +# "from_account": { +# "type": "str", +# "description": "The account to transfer from", +# }, +# "to_account": { +# "type": "str", +# "description": "The account to transfer to", +# }, +# "amount": {"type": "int", "description": "The amount to transfer"}, +# "note": { +# "type": "str", +# "description": "The note to attach to the transaction", +# }, +# "transaction_type": { +# "type": "str", +# "description": "The type of transaction (e.g. 'transfer', 'payment')", +# }, +# }, +# "required": ["from_account", "to_account", "amount"], +# }, +# }, +# { +# "name": "account_transactions", +# "description": "Get the account transactions", +# "parameters": { +# "type": "object", +# "properties": { +# "account_id": {"type": "str", "description": "The account ID"}, +# "start_date": { +# "type": "str", +# "description": "The start date of the transactions", +# }, +# "end_date": { +# "type": "str", +# "description": "The end date of the transactions", +# }, +# }, +# "required": ["account_id"], +# }, +# }, +# ] + +### scenario: multi-turn - 3 required and two optional params +# tools used: account_balance, transfer_funds, account_transactions +# expected: model should ask for required parameters +# Status: Not Approved +# Notes: model is not making tool call even though all parameters have been passed. 3 out of 10 times model asked for confirmation but other times it kept asking for parameters that have been provided already. + +# sample response, +# "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"Please confirm that you want to transfer 25 USD from account 921249 to account 921211.\"}" +# "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"How much money do you want to transfer?\"}" +# "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"How much money do you want to transfer and do you want to make a payment or transfer?\"}" +# "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"You want to transfer 25 USD from account 921249 to account 921211. Is this correct?\"}" +# "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"Please confirm the transfer of 25 USD from account 921249 to account 921211.\"}" + +POST {{archfc_endpoint}}/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"account_balance\", \"description\": \"Get the account balance\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"str\", \"description\": \"The account ID\"}}, \"required\": [\"account_id\"]}}\n{\"name\": \"transfer_funds\", \"description\": \"Transfer funds between accounts\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_account\": {\"type\": \"str\", \"description\": \"The account to transfer from\"}, \"to_account\": {\"type\": \"str\", \"description\": \"The account to transfer to\"}, \"amount\": {\"type\": \"int\", \"description\": \"The amount to transfer\"}, \"note\": {\"type\": \"str\", \"description\": \"The note to attach to the transaction\"}, \"transaction_type\": {\"type\": \"str\", \"description\": \"The type of transaction (e.g. 'transfer', 'payment')\"}}, \"required\": [\"from_account\", \"to_account\", \"amount\"]}}\n{\"name\": \"account_transactions\", \"description\": \"Get the account transactions\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"str\", \"description\": \"The account ID\"}, \"start_date\": {\"type\": \"str\", \"description\": \"The start date of the transactions\"}, \"end_date\": {\"type\": \"str\", \"description\": \"The end date of the transactions\"}}, \"required\": [\"account_id\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I want to make a transfer of 25 USD" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"From which account do you want to transfer the funds?\"}" + }, + { + "role": "user", + "content": "921249" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"To which account do you want to transfer the funds?\"}" + }, + { + "role": "user", + "content": "921211" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + + +tools = [ + { + "name": "account_balance", + "description": "Get the account balance", + "parameters": { + "type": "object", + "properties": { + "account_id": {"type": "str", "description": "The account ID"} + }, + "required": ["account_id"], + }, + }, + { + "name": "transfer_funds", + "description": "Transfer funds between accounts", + "parameters": { + "type": "object", + "properties": { + "from_account": { + "type": "str", + "description": "The account to transfer from", + }, + "to_account": { + "type": "str", + "description": "The account to transfer to", + }, + "amount": {"type": "int", "description": "The amount to transfer"}, + "note": { + "type": "str", + "description": "The note to attach to the transaction", + }, + "transaction_type": { + "type": "str", + "description": "The type of transaction (e.g. 'transfer', 'payment')", + }, + }, + "required": ["from_account", "to_account", "amount", "note"], + }, + }, + { + "name": "account_transactions", + "description": "Get the account transactions", + "parameters": { + "type": "object", + "properties": { + "account_id": {"type": "str", "description": "The account ID"}, + "start_date": { + "type": "str", + "description": "The start date of the transactions", + }, + "end_date": { + "type": "str", + "description": "The end date of the transactions", + }, + }, + "required": ["account_id"], + }, + }, +] + +### scenario: multi-turn - 4 required and one optional params +# tools used: account_balance, transfer_funds, account_transactions +# expected: model should ask for required parameters +# Status: Not Approved +# Notes: model is not asking for required parameters. It is making tool calls with hallucinated values. +# 3/10 times tool calls with hallucinated values +# 2/10 times asks for confirmation +# rest of the times it asked for required parameter values + +# sample response, + +# "{\"tool_calls\": [{\"name\": \"transfer_funds\", \"arguments\": {\"from_account\": \"account_id\", \"to_account\": \"contractor_account_id\", \"amount\": 25, \"note\": \"payment to contractor for home repairs\"}}]}", +# "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"Please confirm the following: Transfer 25 USD from your account to the contractor's account with the note 'payment to contractor for home repairs'.\"}", +# "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"Who would you like to transfer the funds from?\"}" +# "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"What's the account ID for the account you want to transfer the funds from?\"}", +# "{\"tool_calls\": [{\"name\": \"transfer_funds\", \"arguments\": {\"from_account\": \"my_account\", \"to_account\": \"contractor_account\", \"amount\": 25, \"note\": \"payment to contractor for home repairs\", \"transaction_type\": \"transfer\"}}]}", + +POST {{archfc_endpoint}}/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"account_balance\", \"description\": \"Get the account balance\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"str\", \"description\": \"The account ID\"}}, \"required\": [\"account_id\"]}}\n{\"name\": \"transfer_funds\", \"description\": \"Transfer funds between accounts\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_account\": {\"type\": \"str\", \"description\": \"The account to transfer from\"}, \"to_account\": {\"type\": \"str\", \"description\": \"The account to transfer to\"}, \"amount\": {\"type\": \"int\", \"description\": \"The amount to transfer\"}, \"note\": {\"type\": \"str\", \"description\": \"The note to attach to the transaction\"}, \"transaction_type\": {\"type\": \"str\", \"description\": \"The type of transaction (e.g. 'transfer', 'payment')\"}}, \"required\": [\"from_account\", \"to_account\", \"amount\", \"note\"]}}\n{\"name\": \"account_transactions\", \"description\": \"Get the account transactions\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"str\", \"description\": \"The account ID\"}, \"start_date\": {\"type\": \"str\", \"description\": \"The start date of the transactions\"}, \"end_date\": {\"type\": \"str\", \"description\": \"The end date of the transactions\"}}, \"required\": [\"account_id\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I want to make a transfer of 25 USD" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"What's the note you want to attach to the transaction?\"}" + }, + { + "role": "user", + "content": "payment to contractor for home repairs" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +# tools = [ +# { +# "name": "account_balance", +# "description": "Get the account balance", +# "parameters": { +# "type": "object", +# "properties": { +# "account_id": {"type": "str", "description": "The account ID"} +# }, +# "required": ["account_id"], +# }, +# }, +# { +# "name": "transfer_funds", +# "description": "Transfer funds between accounts", +# "parameters": { +# "type": "object", +# "properties": { +# "from_account": { +# "type": "str", +# "description": "The account to transfer from", +# }, +# "to_account": { +# "type": "str", +# "description": "The account to transfer to", +# }, +# "amount": {"type": "int", "description": "The amount to transfer"}, +# "note": { +# "type": "str", +# "description": "The note to attach to the transaction", +# }, +# "transaction_type": { +# "type": "str", +# "description": "The type of transaction (e.g. 'transfer', 'payment')", +# }, +# }, +# "required": [ +# "from_account", +# "to_account", +# "amount", +# "note", +# "transaction_type", +# ], +# }, +# }, +# { +# "name": "account_transactions", +# "description": "Get the account transactions", +# "parameters": { +# "type": "object", +# "properties": { +# "account_id": {"type": "str", "description": "The account ID"}, +# "start_date": { +# "type": "str", +# "description": "The start date of the transactions", +# }, +# "end_date": { +# "type": "str", +# "description": "The end date of the transactions", +# }, +# }, +# "required": ["account_id"], +# }, +# }, +# ] + +### scenario: multi-turn - 5 required parameters +# tools used: account_balance, transfer_funds, account_transactions +# expected: model should ask for required parameters +# Status: Not Approved +# Notes: 7/10 times it made tool call without asking for additional required parameter. It hallucinated value for transaction type +# 3/10 times it asked for confirmation + +# sample response, +# "{\"tool_calls\": [{\"name\": \"transfer_funds\", \"arguments\": {\"from_account\": \"9221124\", \"to_account\": \"921211\", \"amount\": 25, \"note\": \"payment to contractor for home repairs\", \"transaction_type\": \"transfer\"}}]}" +# "{\"tool_calls\": [{\"name\": \"transfer_funds\", \"arguments\": {\"from_account\": \"9221124\", \"to_account\": \"921211\", \"amount\": 25, \"note\": \"payment to contractor for home repairs\", \"transaction_type\": \"transfer\"}}]}", +# "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"Please confirm: transfer 25 USD from account 9221124 to account 921211 with note 'payment to contractor for home repairs' and transaction type 'transfer'.\"}", +# "{\"tool_calls\": [{\"name\": \"transfer_funds\", \"arguments\": {\"from_account\": \"9221124\", \"to_account\": \"921211\", \"amount\": 25, \"note\": \"payment to contractor for home repairs\", \"transaction_type\": \"transfer\"}}]}", +# "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"Please confirm the following: Transfer of 25 USD from account 9221124 to account 921211 for the payment to contractor for home repairs.\"}", +# "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"Please confirm the transfer of 25 USD from account 9221124 to account 921211 with the note 'payment to contractor for home repairs'.\"}", +# "{\"tool_calls\": [{\"name\": \"transfer_funds\", \"arguments\": {\"from_account\": \"9221124\", \"to_account\": \"921211\", \"amount\": 25, \"note\": \"payment to contractor for home repairs\", \"transaction_type\": \"transfer\"}}]}", +# "{\"tool_calls\": [{\"name\": \"transfer_funds\", \"arguments\": {\"from_account\": \"9221124\", \"to_account\": \"921211\", \"amount\": 25, \"note\": \"payment to contractor for home repairs\", \"transaction_type\": \"transfer\"}}]}", +# "{\"tool_calls\": [{\"name\": \"transfer_funds\", \"arguments\": {\"from_account\": \"9221124\", \"to_account\": \"921211\", \"amount\": 25, \"note\": \"payment to contractor for home repairs\", \"transaction_type\": \"transfer\"}}]}", +# "{\"tool_calls\": [{\"name\": \"transfer_funds\", \"arguments\": {\"from_account\": \"9221124\", \"to_account\": \"921211\", \"amount\": 25, \"note\": \"payment to contractor for home repairs\", \"transaction_type\": \"transfer\"}}]}", + +POST {{archfc_endpoint}}/v1/chat/completions +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"account_balance\", \"description\": \"Get the account balance\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"str\", \"description\": \"The account ID\"}}, \"required\": [\"account_id\"]}}\n{\"name\": \"transfer_funds\", \"description\": \"Transfer funds between accounts\", \"parameters\": {\"type\": \"object\", \"properties\": {\"from_account\": {\"type\": \"str\", \"description\": \"The account to transfer from\"}, \"to_account\": {\"type\": \"str\", \"description\": \"The account to transfer to\"}, \"amount\": {\"type\": \"int\", \"description\": \"The amount to transfer\"}, \"note\": {\"type\": \"str\", \"description\": \"The note to attach to the transaction\"}, \"transaction_type\": {\"type\": \"str\", \"description\": \"The type of transaction (e.g. 'transfer', 'payment')\"}}, \"required\": [\"from_account\", \"to_account\", \"amount\", \"note\", \"transaction_type\"]}}\n{\"name\": \"account_transactions\", \"description\": \"Get the account transactions\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"str\", \"description\": \"The account ID\"}, \"start_date\": {\"type\": \"str\", \"description\": \"The start date of the transactions\"}, \"end_date\": {\"type\": \"str\", \"description\": \"The end date of the transactions\"}}, \"required\": [\"account_id\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I want to make a transfer of 25 USD" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"What's the note you want to attach to the transaction?\"}" + }, + { + "role": "user", + "content": "payment to contractor for home repairs" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"Which account is the transfer coming from?\"}" + }, + { + "role": "user", + "content": "9221124" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"transfer_funds\"], \"clarification\": \"And which account is the transfer going to?\"}" + }, + { + "role": "user", + "content": "921211" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +}