diff --git a/tests/rest/model_test_salman.rest b/tests/rest/model_test_salman.rest index b832e126..d0e76ada 100644 --- a/tests/rest/model_test_salman.rest +++ b/tests/rest/model_test_salman.rest @@ -1,8 +1,12 @@ @model_server_endpoint = http://localhost:51000 @archfc_endpoint = http://localhost:5670 - -### talk to Arch-Function directly for completion +### Scenario: Single-Turn. Check for single required parameter +### Tools Used: apply_loan and check_account_balance +### Expected output(s): "content": "{\"required_functions\": [\"check_account_balance\"], \"clarification\": \"Could you please provide me with your account identifier?\"}", +### Notes: Tested 10 times. 9/10 times language is consistent and precise. The assistant asks for the missing parameter in a single-turn in the right format +### Status: Approved +### Tested By: Salman Paracha POST {{archfc_endpoint}}/v1/chat/completions HTTP/1.1 Content-Type: application/json @@ -11,19 +15,411 @@ Content-Type: application/json "messages": [ { "role": "system", - "content": "You are a helpful assistant designed to analyze user queries and determine if functions are needed to respond.\n\nYou have access to the following tools:\n\n{\"name\": \"get_weather\", \"description\": \"Determine weather in my location\", \"parameters\": {\"type\": \"object\", \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The city and state e.g. San Francisco, CA\"}, \"unit\": {\"type\": \"string\", \"enum\": [\"c\", \"f\"]}}, \"required\": [\"location\", \"unit\"]}}\n{\"name\": \"get_stock_price\", \"description\": \"Get the current stock price\", \"parameters\": {\"type\": \"object\", \"properties\": {\"symbol\": {\"type\": \"string\", \"description\": \"The stock symbol\"}}, \"required\": [\"symbol\"]}}\n{\"name\": \"get_networking_device_sumary\", \"description\": \"retrieve networking device statistics\", \"parameters\": {\"type\": \"object\", \"properties\": {\"device_ids\": {\"type\": \"list\", \"description\": \"A list of device identifier to retrieve statistics for\", \"items\": {\"type\": \"int\"}}, \"days\": {\"type\": \"int\", \"description\": \"the number of days for which to get a device\"}}, \"required\": [\"device_ids\"]}}\n\n\nYour task is to decide which functions are required and and collect missing parameters, if any.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"function1\", \"function2\", ...], \"clarification\": \"Text asking for missing information\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"apply_loan\", \"description\": \"Apply for a personal loan\", \"parameters\": {\"type\": \"object\", \"properties\": {\"amount\": {\"type\": \"number\", \"description\": \"Loan amount in USD\"}, \"term_months\": {\"type\": \"integer\", \"description\": \"Term duration in months\"}, \"credit_score\": {\"type\": \"integer\", \"description\": \"Applicant's credit score\"}}, \"required\": [\"amount\", \"term_months\", \"credit_score\"]}}\n{\"name\": \"check_account_balance\", \"description\": \"Check balance of a financial account\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"string\", \"description\": \"User's account number or identifier\"}}, \"required\": [\"account_id\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" }, { "role": "user", - "content": "give me device network stastics" - }, - { - "role": "assistant", - "content": "{\"required_functions\": [\"get_networking_device_sumary\"], \"clarification\": \"Which device would you like to retrieve network statistics for?\"}" - }, - { - "role": "user", - "content": "device id 114332" + "content": "I want to check for my account balance" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + + +### Scenario: Single-Turn. Check for all 3 required parameters. +### Tools Used: apply_loan and check_account_balance +### Expected output(s): "{\"required_functions\": [\"apply_loan\"], \"clarification\": \"Could you please provide me with the loan amount, the term duration in months, and your credit score?\"}", +### Notes: Tested 10 times. 8/10 times the language was consistent and precise. 2/10 the language was to abrupt and not clear and in those turns not all the required parameters were asked for. +### Status: Needs Work +### Tested By: Salman Paracha +POST {{archfc_endpoint}}/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"apply_loan\", \"description\": \"Apply for a personal loan\", \"parameters\": {\"type\": \"object\", \"properties\": {\"amount\": {\"type\": \"number\", \"description\": \"Loan amount in USD\"}, \"term_months\": {\"type\": \"integer\", \"description\": \"Term duration in months\"}, \"credit_score\": {\"type\": \"integer\", \"description\": \"Applicant's credit score\"}}, \"required\": [\"amount\", \"term_months\", \"credit_score\"]}}\n{\"name\": \"check_account_balance\", \"description\": \"Check balance of a financial account\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"string\", \"description\": \"User's account number or identifier\"}}, \"required\": [\"account_id\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "want to apply for a loan" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +### Scenario: Multi-Turn(2) Chat History Context. +### Tools Used: apply_loan, get_stock_price and check_account_balance +### Expected output(s): "content": "{\"response\": \"You're welcome! If you need any more information, feel free to ask.\"}", +### Notes: Tested 10 times. Worked every time. The assistant was able to keep track of the chat history and provide the right function call based on the user query. +### Status: Approved +### Tested By: Salman Paracha + +POST {{archfc_endpoint}}/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"get_stock_price\", \"description\": \"Get the current stock price\", \"parameters\": {\"type\": \"object\", \"properties\": {\"symbol\": {\"type\": \"string\", \"description\": \"The stock symbol\"}}, \"required\": [\"symbol\"]}}\n{\"name\": \"apply_loan\", \"description\": \"Apply for a personal loan\", \"parameters\": {\"type\": \"object\", \"properties\": {\"amount\": {\"type\": \"number\", \"description\": \"Loan amount in USD\"}, \"term_months\": {\"type\": \"integer\", \"description\": \"Term duration in months\"}, \"credit_score\": {\"type\": \"integer\", \"description\": \"Applicant's credit score\"}}, \"required\": [\"amount\", \"term_months\", \"credit_score\"]}}\n{\"name\": \"check_account_balance\", \"description\": \"Check balance of a financial account\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"string\", \"description\": \"User's account number or identifier\"}}, \"required\": [\"account_id\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "can you check the stock price for Amazon?" + }, + { + "role": "assistant", + "content": "{\"tool_calls\": [{\"name\": \"get_stock_price\", \"arguments\": {\"symbol\": \"AMZN\"}}]}" + }, + { + "role": "user", + "content": "\n{'name': 'get_stock_price', 'result': '$247.66'}\n" + }, + { + "role": "assistant", + "content": "{\"response\": \"The current stock price for AMZN is $247.66.\"}" + }, + { + "role": "user", + "content": "what about Apple" + }, + { + "role": "assistant", + "content": "{\"tool_calls\": [{\"name\": \"get_stock_price\", \"arguments\": {\"symbol\": \"AAPL\"}}]}" + }, + { + "role": "user", + "content": "\n{'name': 'get_stock_price', 'result': '$196.66'}\n" + }, + { + "role": "assistant", + "content": "{\"response\": \"The current stock price for AAPL is $196.66.\"}" + }, + { + "role": "user", + "content": "Thanks" + } +], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + + +### Scenario: Multi-Turn (5) Chat History Context. +### Tools Used: apply_loan, get_stock_price and check_account_balance +### Expected output(s): "content": "{\"tool_calls\": [{\"name\": \"get_stock_price\", \"arguments\": {\"symbol\": \"ORCL\"}}]}", +### Notes: Tested 10 times. The assistant was able to keep track of the chat history and provide the right function call based on the user query. +### Tested By: Salman Paracha +### Status: Approved + +POST {{archfc_endpoint}}/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"get_stock_price\", \"description\": \"Get the current stock price\", \"parameters\": {\"type\": \"object\", \"properties\": {\"symbol\": {\"type\": \"string\", \"description\": \"The stock symbol\"}}, \"required\": [\"symbol\"]}}\n{\"name\": \"apply_loan\", \"description\": \"Apply for a personal loan\", \"parameters\": {\"type\": \"object\", \"properties\": {\"amount\": {\"type\": \"number\", \"description\": \"Loan amount in USD\"}, \"term_months\": {\"type\": \"integer\", \"description\": \"Term duration in months\"}, \"credit_score\": {\"type\": \"integer\", \"description\": \"Applicant's credit score\"}}, \"required\": [\"amount\", \"term_months\", \"credit_score\"]}}\n{\"name\": \"check_account_balance\", \"description\": \"Check balance of a financial account\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"string\", \"description\": \"User's account number or identifier\"}}, \"required\": [\"account_id\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "can you check the stock price for Amazon?" + }, + { + "role": "assistant", + "content": "{\"tool_calls\": [{\"name\": \"get_stock_price\", \"arguments\": {\"symbol\": \"AMZN\"}}]}" + }, + { + "role": "user", + "content": "\n{'name': 'get_stock_price', 'result': '$247.66'}\n" + }, + { + "role": "assistant", + "content": "{\"response\": \"The current stock price for AMZN is $247.66.\"}" + }, + { + "role": "user", + "content": "what about Apple" + }, + { + "role": "assistant", + "content": "{\"tool_calls\": [{\"name\": \"get_stock_price\", \"arguments\": {\"symbol\": \"AAPL\"}}]}" + }, + { + "role": "user", + "content": "\n{'name': 'get_stock_price', 'result': '$196.66'}\n" + }, + { + "role": "assistant", + "content": "{\"response\": \"The current stock price for AAPL is $196.66.\"}" + }, + { + "role": "user", + "content": "How about Toyota" + }, + { + "role": "assistant", + "content": "{\"tool_calls\": [{\"name\": \"get_stock_price\", \"arguments\": {\"symbol\": \"TM\"}}]}" + }, + { + "role": "user", + "content": "\n{'name': 'get_stock_price', 'result': '$24.66'}\n" + }, + { + "role": "assistant", + "content": "{\"response\": \"The current stock price for TM is $196.66.\"}" + }, + { + "role": "user", + "content": "How about Nike" + }, + { + "role": "assistant", + "content": "{\"tool_calls\": [{\"name\": \"get_stock_price\", \"arguments\": {\"symbol\": \"NKE\"}}]}" + }, + { + "role": "user", + "content": "\n{'name': 'get_stock_price', 'result': '$102.66'}\n" + }, + { + "role": "assistant", + "content": "{\"response\": \"The current stock price for NKE is $102.66.\"}" + }, + { + "role": "user", + "content": "and what about Oracle" + } +], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + + +### Scenario: Multi-Turn(2) Progressive Information Disclosure. +### Tools Used: apply_loan, get_stock_price and check_account_balance +### Expected output(s): "content": "{\"required_functions\": [\"apply_loan\"], \"clarification\": \"Could you please confirm the term duration and your credit score?\"}", +### Notes: Tested 10 times. Worked every time. The assistant was able to keep track of the chat history and provide the right function call based on the user query. +### Status: Approved +### Tested By: Salman Paracha + +POST {{archfc_endpoint}}/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"get_stock_price\", \"description\": \"Get the current stock price\", \"parameters\": {\"type\": \"object\", \"properties\": {\"symbol\": {\"type\": \"string\", \"description\": \"The stock symbol\"}}, \"required\": [\"symbol\"]}}\n{\"name\": \"apply_loan\", \"description\": \"Apply for a personal loan\", \"parameters\": {\"type\": \"object\", \"properties\": {\"amount\": {\"type\": \"number\", \"description\": \"Loan amount in USD\"}, \"term_months\": {\"type\": \"integer\", \"description\": \"Term duration in months\"}, \"credit_score\": {\"type\": \"integer\", \"description\": \"Applicant's credit score\"}}, \"required\": [\"amount\", \"term_months\", \"credit_score\"]}}\n{\"name\": \"check_account_balance\", \"description\": \"Check balance of a financial account\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"string\", \"description\": \"User's account number or identifier\"}}, \"required\": [\"account_id\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I want to apply for a loan" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"apply_loan\"], \"clarification\": \"Could you please provide me with the amount you want to loan, the term duration in months, and your credit score?\"}" + }, + { + "role": "user", + "content": "$5000" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"apply_loan\"], \"clarification\": \"Could you please confirm the term duration and your credit score?\"}" + }, + { + "role": "user", + "content": "actually make the loan for $6000" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + + +### Scenario: Multi-Turn(5) Progressive Information Disclosure. +### Tools Used: apply_loan, get_stock_price and check_account_balance +### Expected output(s): "content": "{\"response\": \"Have a great day\"}", +### Notes: Tested 10 times. Worked every time. The assistant was able to keep track of the chat history and provide the right function call based on the user query. +### Status: Approved +### Tested By: Salman Paracha + +POST {{archfc_endpoint}}/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"apply_loan\", \"description\": \"Apply for a personal loan\", \"parameters\": {\"type\": \"object\", \"properties\": {\"amount\": {\"type\": \"number\", \"description\": \"Loan amount in USD\"}, \"term_months\": {\"type\": \"integer\", \"description\": \"Term duration in months\"}, \"credit_score\": {\"type\": \"integer\", \"description\": \"Applicant's credit score\"}}, \"required\": [\"amount\", \"term_months\", \"credit_score\"]}}\n{\"name\": \"check_account_balance\", \"description\": \"Check balance of a financial account\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"string\", \"description\": \"User's account number or identifier\"}}, \"required\": [\"account_id\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I want to apply for a loan" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"apply_loan\"], \"clarification\": \"Could you please provide me with the amount you want to loan, the term duration in months, and your credit score?\"}" + }, + { + "role": "user", + "content": "$5000" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"apply_loan\"], \"clarification\": \"Could you please provide me with the term duration in months and your credit score?\"}" + }, + { + "role": "user", + "content": "18 months" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"apply_loan\"], \"clarification\": \"Could you please provide me with your credit score?\"}" + }, + { + "role": "user", + "content": "700" + }, + { + "role": "assistant", + "content": "{\"tool_calls\": [{\"name\": \"apply_loan\", \"arguments\": {\"amount\": 5000, \"term_months\": 18, \"credit_score\": 700}}]}" + }, + { + "role": "user", + "content": "\n{'name': 'apply_loan', 'approved': 'Done'}\n" + }, + { + "role": "assistant", + "content": "your loan has been approved" + }, + { + "role": "user", + "content": "Thanks" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + + +### Scenario: Multi-Turn Progressive Information Disclosure with multiple function calling. +### Tools Used: apply_loan, get_stock_price and check_account_balance +### Tested By: Salman Paracha +### Expected output(s): "content": "{\"tool_calls\": [{\"name\": \"apply_loan\", \"arguments\": {\"amount\": 6000, \"term_months\": 18, \"credit_score\": 700}}]}", +### Notes: The assistant doesn't engage in gathering information for the second function call. The assistant should ask for the missing parameter for the second function call. +### Status: Rejected +POST {{archfc_endpoint}}/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"apply_loan\", \"description\": \"Apply for a personal loan\", \"parameters\": {\"type\": \"object\", \"properties\": {\"amount\": {\"type\": \"number\", \"description\": \"Loan amount in USD\"}, \"term_months\": {\"type\": \"integer\", \"description\": \"Term duration in months\"}, \"credit_score\": {\"type\": \"integer\", \"description\": \"Applicant's credit score\"}}, \"required\": [\"amount\", \"term_months\", \"credit_score\"]}}\n{\"name\": \"check_account_balance\", \"description\": \"Check balance of a financial account\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"string\", \"description\": \"User's account number or identifier\"}}, \"required\": [\"account_id\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I want to apply for a loan and check my account balance" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"apply_loan\", \"check_account_balance\"], \"clarification\": \"Could you please provide me with the amount you want to borrow, the term of the loan in months, and your credit score?\"}" + }, + { + "role": "user", + "content": "amount $5000, 18 months term and credit score of 780" + }, + { + "role": "assistant", + "content": "{\"tool_calls\": [{\"name\": \"apply_loan\", \"arguments\": {\"amount\": 5000, \"term_months\": 18, \"credit_score\": 700}}]}" + }, + { + "role": "user", + "content": "\n{'name': 'apply_loan', 'result': 'Status: Ok'}\n" + }, + { + "role": "assistant", + "content": "{\"response\": \"Status: Ok. You have sucessfully applied for the loan\"}" + }, + { + "role": "user", + "content": "Ok thanks" + } + ], + "temperature": 0.6, + "top_p": 1.0, + "top_k": 10 +} + +### Scenario: Multi-Turn(3) Progressive Information Disclosure; Changing parameter values. +### Tools Used: apply_loan, get_stock_price and check_account_balance +### Expected output(s): "content": "{\"tool_calls\": [{\"name\": \"apply_loan\", \"arguments\": {\"amount\": 6000, \"term_months\": 18, \"credit_score\": 700}}]}", +### Notes: Tested 10 times. Worked every time. The assistant was able to keep track of the chat history and provide the right function call based on the user query. +### Status: Approved +### Tested By: Salman Paracha + +POST {{archfc_endpoint}}/v1/chat/completions HTTP/1.1 +Content-Type: application/json + +{ + "model": "Arch-Function", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant designed to assist with the user query by making one or more function calls if needed.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"get_stock_price\", \"description\": \"Get the current stock price\", \"parameters\": {\"type\": \"object\", \"properties\": {\"symbol\": {\"type\": \"string\", \"description\": \"The stock symbol\"}}, \"required\": [\"symbol\"]}}\n{\"name\": \"apply_loan\", \"description\": \"Apply for a personal loan\", \"parameters\": {\"type\": \"object\", \"properties\": {\"amount\": {\"type\": \"number\", \"description\": \"Loan amount in USD\"}, \"term_months\": {\"type\": \"integer\", \"description\": \"Term duration in months\"}, \"credit_score\": {\"type\": \"integer\", \"description\": \"Applicant's credit score\"}}, \"required\": [\"amount\", \"term_months\", \"credit_score\"]}}\n{\"name\": \"check_account_balance\", \"description\": \"Check balance of a financial account\", \"parameters\": {\"type\": \"object\", \"properties\": {\"account_id\": {\"type\": \"string\", \"description\": \"User's account number or identifier\"}}, \"required\": [\"account_id\"]}}\n\n\nYour task is to decide which functions are needed and collect missing parameters if necessary.\n\nBased on your analysis, provide your response in one of the following JSON formats:\n1. If no functions are needed:\n```\n{\"response\": \"Your response text here\"}\n```\n2. If functions are needed but some required parameters are missing:\n```\n{\"required_functions\": [\"func_name1\", \"func_name2\", ...], \"clarification\": \"Text asking for missing parameters\"}\n```\n3. If functions are needed and all required parameters are available:\n```\n{\"tool_calls\": [{\"name\": \"func_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},... (more tool calls as required)]}\n```" + }, + { + "role": "user", + "content": "I want to apply for a loan" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"apply_loan\"], \"clarification\": \"Could you please provide me with the amount you want to loan, the term duration in months, and your credit score?\"}" + }, + { + "role": "user", + "content": "$5000" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"apply_loan\"], \"clarification\": \"Could you please confirm the term duration and your credit score?\"}" + }, + { + "role": "user", + "content": "actually make the loan for $6000" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"apply_loan\"], \"clarification\": \"Please confirm the term duration and your credit score.\"}" + }, + { + "role": "user", + "content": "for 18 months" + }, + { + "role": "assistant", + "content": "{\"required_functions\": [\"apply_loan\"], \"clarification\": \"Please confirm your credit score.\"}" + }, + { + "role": "user", + "content": "700" } ], "temperature": 0.6,