mirror of
https://github.com/katanemo/plano.git
synced 2026-04-25 08:46:24 +02:00
265 lines
9.4 KiB
YAML
265 lines
9.4 KiB
YAML
|
test_cases:
|
||
|
|
- id: "[WEATHER AGENT] - single turn, single tool, prompt prefilling"
|
||
|
|
input:
|
||
|
|
messages:
|
||
|
|
- role: "user"
|
||
|
|
content: "what is the weather forecast for seattle?"
|
||
|
|
tools:
|
||
|
|
- id: "weather-112"
|
||
|
|
type: "function"
|
||
|
|
function:
|
||
|
|
name: "get_current_weather"
|
||
|
|
description: "Get current weather at a location."
|
||
|
|
parameters:
|
||
|
|
type: "object"
|
||
|
|
properties:
|
||
|
|
location:
|
||
|
|
type: "str"
|
||
|
|
description: "The location to get the weather for"
|
||
|
|
format: "City, State"
|
||
|
|
days:
|
||
|
|
type: "int"
|
||
|
|
description: "the number of days for the request."
|
||
|
|
required: ["location", "days"]
|
||
|
|
expected:
|
||
|
|
- type: "metadata"
|
||
|
|
hallucination: false
|
||
|
|
prompt_prefilling: true
|
||
|
|
|
||
|
|
- id: "[WEATHER AGENT] - single turn, single tool, hallucination"
|
||
|
|
input:
|
||
|
|
messages:
|
||
|
|
- role: "user"
|
||
|
|
content: "what is the weather in Seattle in days?"
|
||
|
|
tools:
|
||
|
|
- id: "weather-112"
|
||
|
|
type: "function"
|
||
|
|
function:
|
||
|
|
name: "get_current_weather"
|
||
|
|
description: "Get current weather at a location."
|
||
|
|
parameters:
|
||
|
|
type: "object"
|
||
|
|
properties:
|
||
|
|
location:
|
||
|
|
type: "str"
|
||
|
|
description: "The location to get the weather for"
|
||
|
|
format: "City, State"
|
||
|
|
days:
|
||
|
|
type: "int"
|
||
|
|
description: "the number of days for the request."
|
||
|
|
required: ["location", "days"]
|
||
|
|
expected:
|
||
|
|
- type: "metadata"
|
||
|
|
hallucination: true
|
||
|
|
prompt_prefilling: true
|
||
|
|
|
||
|
|
- id: "[WEATHER AGENT] - multi turn, single tool, all params passed"
|
||
|
|
input:
|
||
|
|
messages:
|
||
|
|
- role: "user"
|
||
|
|
content: "how is the weather in chicago for next 5 days?"
|
||
|
|
- role: "assistant"
|
||
|
|
content: "Can you tell me your location and how many days you want?"
|
||
|
|
- role: "user"
|
||
|
|
content: "Seattle"
|
||
|
|
- role: "assistant"
|
||
|
|
content: "Can you please provide me the days for the weather forecast?"
|
||
|
|
- role: "user"
|
||
|
|
content: "los angeles in 5 days"
|
||
|
|
tools:
|
||
|
|
- type: "function"
|
||
|
|
function:
|
||
|
|
name: "get_current_weather"
|
||
|
|
description: "Get current weather at a location."
|
||
|
|
parameters:
|
||
|
|
type: "object"
|
||
|
|
properties:
|
||
|
|
location:
|
||
|
|
type: "str"
|
||
|
|
description: "The location to get the weather for"
|
||
|
|
format: "City, State"
|
||
|
|
days:
|
||
|
|
type: "int"
|
||
|
|
description: "the number of days for the request."
|
||
|
|
required: ["location", "days"]
|
||
|
|
expected:
|
||
|
|
- type: "metadata"
|
||
|
|
hallucination: true
|
||
|
|
prompt_prefilling: true
|
||
|
|
|
||
|
|
|
||
|
|
- id: "[WEATHER AGENT] - multi turn, single tool, clarification"
|
||
|
|
input:
|
||
|
|
messages:
|
||
|
|
- role: "user"
|
||
|
|
content: "how is the weather for next 5 days?"
|
||
|
|
- role: "assistant"
|
||
|
|
content: "Can you tell me your location and how many days you want?"
|
||
|
|
- role: "user"
|
||
|
|
content: "Seattle"
|
||
|
|
- role: "assistant"
|
||
|
|
content: "Can you please provide me the days for the weather forecast?"
|
||
|
|
- role: "user"
|
||
|
|
content: "Sorry, the location is actually los angeles in 5 days"
|
||
|
|
tools:
|
||
|
|
- type: "function"
|
||
|
|
function:
|
||
|
|
name: "get_current_weather"
|
||
|
|
description: "Get current weather at a location."
|
||
|
|
parameters:
|
||
|
|
type: "object"
|
||
|
|
properties:
|
||
|
|
location:
|
||
|
|
type: "str"
|
||
|
|
description: "The location to get the weather for"
|
||
|
|
format: "City, State"
|
||
|
|
days:
|
||
|
|
type: "int"
|
||
|
|
description: "the number of days for the request."
|
||
|
|
required: ["location", "days"]
|
||
|
|
expected:
|
||
|
|
- type: "metadata"
|
||
|
|
hallucination: false
|
||
|
|
prompt_prefilling: true
|
||
|
|
- id: "[SALE AGENT] - single turn, single tool, hallucination region"
|
||
|
|
input:
|
||
|
|
messages:
|
||
|
|
- role: "user"
|
||
|
|
content: "get me sales opportunities of tech"
|
||
|
|
tools:
|
||
|
|
- type: "function"
|
||
|
|
function:
|
||
|
|
name: "sales_opportunity"
|
||
|
|
description: "Retrieve potential sales opportunities based for a particular industry type in a region."
|
||
|
|
parameters:
|
||
|
|
type: "object"
|
||
|
|
properties:
|
||
|
|
region:
|
||
|
|
type: "str"
|
||
|
|
description: "Geographical region to identify sales opportunities."
|
||
|
|
industry:
|
||
|
|
type: "str"
|
||
|
|
description: "Industry type."
|
||
|
|
max_results:
|
||
|
|
type: "int"
|
||
|
|
description: "Maximum number of sales opportunities to retrieve."
|
||
|
|
default: 20
|
||
|
|
required: ["region", "industry"]
|
||
|
|
expected:
|
||
|
|
- type: "metadata"
|
||
|
|
hallucination: true
|
||
|
|
prompt_prefilling: true
|
||
|
|
|
||
|
|
- id: "[SALE AGENT] - single turn, single tool, hallucination industry"
|
||
|
|
input:
|
||
|
|
messages:
|
||
|
|
- role: "user"
|
||
|
|
content: "get me sales opportunities in NA"
|
||
|
|
tools:
|
||
|
|
- type: "function"
|
||
|
|
function:
|
||
|
|
name: "sales_opportunity"
|
||
|
|
description: "Retrieve potential sales opportunities based for a particular industry type in a region."
|
||
|
|
parameters:
|
||
|
|
type: "object"
|
||
|
|
properties:
|
||
|
|
region:
|
||
|
|
type: "str"
|
||
|
|
description: "Geographical region to identify sales opportunities."
|
||
|
|
industry:
|
||
|
|
type: "str"
|
||
|
|
description: "Industry type."
|
||
|
|
max_results:
|
||
|
|
type: "int"
|
||
|
|
description: "Maximum number of sales opportunities to retrieve."
|
||
|
|
default: 20
|
||
|
|
required: ["region", "industry"]
|
||
|
|
expected:
|
||
|
|
- type: "metadata"
|
||
|
|
hallucination: true
|
||
|
|
prompt_prefilling: true
|
||
|
|
|
||
|
|
- id: "[PRODUCT AGENT] - single turn, single tool, hallucination industry"
|
||
|
|
input:
|
||
|
|
messages:
|
||
|
|
- role: "user"
|
||
|
|
content: "get me sales opportunities in NA"
|
||
|
|
tools:
|
||
|
|
- type: "function"
|
||
|
|
function:
|
||
|
|
name: "product_recommendation"
|
||
|
|
description: "Place an order for an iphone with user_id 195 and location is 1600 pensylvania ave"
|
||
|
|
parameters:
|
||
|
|
type: "object"
|
||
|
|
properties:
|
||
|
|
user_id:
|
||
|
|
type: "str"
|
||
|
|
description: "Unique identifier for the user."
|
||
|
|
category:
|
||
|
|
type: "str"
|
||
|
|
description: "Product category for recommendations."
|
||
|
|
max_results:
|
||
|
|
type: "int"
|
||
|
|
description: "Maximum number of recommended products to show."
|
||
|
|
default: 10
|
||
|
|
required: ["user_id", "category"]
|
||
|
|
- type: "function"
|
||
|
|
function:
|
||
|
|
name: "place_order"
|
||
|
|
description: "Place and pay for an order for one or more products to ship to the an address."
|
||
|
|
parameters:
|
||
|
|
type: "object"
|
||
|
|
properties:
|
||
|
|
user_id:
|
||
|
|
type: "str"
|
||
|
|
description: "Unique identifier for the user placing the order."
|
||
|
|
product_ids:
|
||
|
|
type: "array"
|
||
|
|
description: "List of product IDs to include in the order."
|
||
|
|
shipping_address:
|
||
|
|
type: "str"
|
||
|
|
description: "Shipping address for the order."
|
||
|
|
payment_method:
|
||
|
|
type: "str"
|
||
|
|
description: "Payment method for the order."
|
||
|
|
required: ["user_id", "product_ids", "shipping_address", "payment_method"]
|
||
|
|
- type: "function"
|
||
|
|
function:
|
||
|
|
name: "sales_opportunity"
|
||
|
|
description: "Retrieve potential sales opportunities based for a particular industry type in a region."
|
||
|
|
parameters:
|
||
|
|
type: "object"
|
||
|
|
properties:
|
||
|
|
region:
|
||
|
|
type: "str"
|
||
|
|
description: "Geographical region to identify sales opportunities."
|
||
|
|
industry:
|
||
|
|
type: "str"
|
||
|
|
description: "Industry type."
|
||
|
|
max_results:
|
||
|
|
type: "int"
|
||
|
|
description: "Maximum number of sales opportunities to retrieve."
|
||
|
|
default: 20
|
||
|
|
required: ["region", "industry"]
|
||
|
|
- type: "function"
|
||
|
|
function:
|
||
|
|
name: "query_database"
|
||
|
|
description: "Perform a database query to retrieve or update information."
|
||
|
|
parameters:
|
||
|
|
type: "object"
|
||
|
|
properties:
|
||
|
|
query:
|
||
|
|
type: "str"
|
||
|
|
description: "SQL query string to execute against the database."
|
||
|
|
parameters:
|
||
|
|
type: "array"
|
||
|
|
description: "List of parameters to safely inject into the SQL query (to prevent SQL injection)."
|
||
|
|
operation:
|
||
|
|
type: "str"
|
||
|
|
description: "Type of operation."
|
||
|
|
required: ["query", "operation"]
|
||
|
|
expected:
|
||
|
|
- type: "metadata"
|
||
|
|
hallucination: true
|
||
|
|
prompt_prefilling: true
|