mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-04-26 17:06:23 +02:00
Enable internal and user-facing agents to build pipelines
This commit is contained in:
akhisud3195
parent
1246ea47b9
commit
e59a8b75cf
24 changed files with 2100 additions and 1376 deletions
|
|
@ -1,223 +1,167 @@
|
|||
{
|
||||
"lastRequest": {
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": ""
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "hi"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"sender": "Door Dash Hub",
|
||||
"content": "Hello! Are you facing issues with your order items or delivery timing? How can I assist you today?",
|
||||
"created_at": "2025-03-24T17:33:27.564940"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "my order is missing fries"
|
||||
},
|
||||
{
|
||||
"content": "Agent changed to Door Dash Hub",
|
||||
"role": "assistant",
|
||||
"sender": "Door Dash Hub",
|
||||
"tool_calls": null,
|
||||
"tool_call_id": null,
|
||||
"response_type": "internal"
|
||||
},
|
||||
{
|
||||
"content": "Agent changed to Order Issue",
|
||||
"role": "assistant",
|
||||
"sender": "Order Issue",
|
||||
"tool_calls": null,
|
||||
"tool_call_id": null,
|
||||
"response_type": "internal"
|
||||
},
|
||||
{
|
||||
"content": "Let's resolve this issue by checking your order details. Could you please provide the order ID?",
|
||||
"role": "assistant",
|
||||
"sender": "Order Issue",
|
||||
"tool_calls": null,
|
||||
"tool_call_id": null,
|
||||
"tool_name": null,
|
||||
"response_type": "external"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "123412"
|
||||
},
|
||||
{
|
||||
"content": "Agent changed to Order Issue",
|
||||
"role": "assistant",
|
||||
"sender": "Order Issue",
|
||||
"tool_calls": null,
|
||||
"tool_call_id": null,
|
||||
"response_type": "internal"
|
||||
},
|
||||
{
|
||||
"content": null,
|
||||
"role": "assistant",
|
||||
"sender": "Order Issue",
|
||||
"tool_calls": [
|
||||
{
|
||||
"function": {
|
||||
"name": "get_order_details",
|
||||
"arguments": "{\"order_id\":\"123412\"}"
|
||||
},
|
||||
"id": "fc_67e37c622f208192aceec557fbd3125609b4eca638eb3571",
|
||||
"type": "function"
|
||||
}
|
||||
],
|
||||
"tool_call_id": null,
|
||||
"tool_name": null,
|
||||
"response_type": "internal"
|
||||
},
|
||||
{
|
||||
"content": "Order Details for Order ID 123412:\n\n- **Restaurant**: Luigi's Pizzeria \n- **Items Ordered**: \n - 2x Margherita Pizza \n - 1x Garlic Bread \n - 1x Caesar Salad \n- **Estimated Delivery Time**: 7:45 PM \n- **Delivery Address**: 123 Elm Street, Apt 4B \n- **Order Status**: Out for delivery \n- **Total Amount**: $32.75 \n- **Payment Method**: Credit Card (**** **** **** 5678) \n- **Contact**: (123) 456-7890 \n- **Special Instructions**: Leave at the door.",
|
||||
"role": "tool",
|
||||
"sender": null,
|
||||
"tool_calls": null,
|
||||
"tool_call_id": "call_dqCMC5oreOoS9znDDJ7PqWha",
|
||||
"tool_name": null,
|
||||
"response_type": "internal"
|
||||
},
|
||||
{
|
||||
"content": "I checked your order details, and it seems the fries were not included. Could you confirm if there was an oversight in placing the order, or were they supposed to be included?",
|
||||
"role": "assistant",
|
||||
"sender": "Order Issue",
|
||||
"tool_calls": null,
|
||||
"tool_call_id": null,
|
||||
"tool_name": null,
|
||||
"response_type": "external"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Fries were supposed to be in"
|
||||
}
|
||||
],
|
||||
"state": {
|
||||
"last_agent_name": "Order Issue",
|
||||
"tokens": {
|
||||
"total": 1699,
|
||||
"prompt": 1643,
|
||||
"completion": 56
|
||||
}
|
||||
},
|
||||
"agents": [
|
||||
{
|
||||
"name": "Door Dash Hub",
|
||||
"type": "conversation",
|
||||
"description": "Hub agent to manage Door Dash-related queries.",
|
||||
"instructions": "## \ud83e\uddd1\u200d\ud83d\udcbc Role:\nYou are responsible for directing Door Dash-related queries to appropriate agents.\n\n---\n## \u2699\ufe0f Steps to Follow:\n1. Greet the user and ask which Door Dash-related query they need help with (e.g., 'Are you facing issues with your order items or delivery timing?').\n2. If the query matches a specific task, direct the user to the corresponding agent:\n - Order Issue \u2192 [@agent:Order Issue]\n - Delayed Delivery \u2192 [@agent:Delayed Delivery]\n3. If the query doesn't match any specific task, respond with 'I'm sorry, I didn't understand. Could you clarify your request?' or escalate to human support.\n\n---\n## \ud83c\udfaf Scope:\n\u2705 In Scope:\n- Issues with order items\n- Delayed delivery issues\n\n\u274c Out of Scope:\n- Issues unrelated to Door Dash\n- General knowledge queries\n\n---\n## \ud83d\udccb Guidelines:\n\u2714\ufe0f Dos:\n- Direct queries to specific Door Dash agents promptly.\n- Call [@agent:Escalation] agent for unrecognized queries.\n\n\ud83d\udeab Don'ts:\n- Engage in detailed support.\n- Extend the conversation beyond Door Dash.\n- Provide user-facing text such as 'I will connect you now...' when calling another agent\n\n# Examples\n- **User** : I need help with my order items.\n - **Agent actions**: [@agent:Order Issue](#mention)\n\n- **User** : My delivery is delayed.\n - **Agent actions**: Call [@agent:Delayed Delivery](#mention)\n\n- **User** : I'm not sure where my order is.\n - **Agent actions**: Call [@agent:Delayed Delivery](#mention)\n\n- **User** : Can you reset my order settings?\n - **Agent actions**: [@agent:Escalation](#mention)\n\n- **User** : How are you today?\n - **Agent response**: I'm doing great. What would like help with today?",
|
||||
"model": "gpt-4o-mini",
|
||||
"hasRagSources": false,
|
||||
"controlType": "retain",
|
||||
"tools": [],
|
||||
"prompts": [],
|
||||
"connectedAgents": [
|
||||
"Order Issue",
|
||||
"Delayed Delivery",
|
||||
"Escalation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Post process",
|
||||
"type": "post_process",
|
||||
"description": "",
|
||||
"instructions": "Ensure that the agent response is terse and to the point.",
|
||||
"model": "gpt-4o-mini",
|
||||
"hasRagSources": false,
|
||||
"controlType": "retain",
|
||||
"tools": [],
|
||||
"prompts": [],
|
||||
"connectedAgents": []
|
||||
},
|
||||
{
|
||||
"name": "Escalation",
|
||||
"type": "escalation",
|
||||
"description": "",
|
||||
"instructions": "Get the user's contact information and let them know that their request has been escalated.",
|
||||
"model": "gpt-4o-mini",
|
||||
"hasRagSources": false,
|
||||
"controlType": "retain",
|
||||
"tools": [],
|
||||
"prompts": [],
|
||||
"connectedAgents": []
|
||||
},
|
||||
{
|
||||
"name": "Order Issue",
|
||||
"type": "conversation",
|
||||
"description": "Agent to assist users with missing or incorrect order items.",
|
||||
"instructions": "## \ud83e\uddd1\u200d\ud83d\udcbc Role:\nAssist users with issues related to missing or incorrect order items.\n\n---\n## \u2699\ufe0f Steps to Follow:\n1. Fetch the order details using the [@tool:get_order_details] tool.\n2. Confirm the issue with the user.\n3. Provide solutions or escalate if unresolved.\n\n---\n## \ud83c\udfaf Scope:\n\u2705 In Scope:\n- Handling missing or incorrect order items\n\n\u274c Out of Scope:\n- Delayed delivery issues\n- General knowledge queries\n\n---\n## \ud83d\udccb Guidelines:\n\u2714\ufe0f Dos:\n- Ensure the user is aware of the order details before proceeding.\n\n\ud83d\udeab Don'ts:\n- Extend the conversation beyond order issues.\n\n# Examples\n- **User** : I received the wrong item in my order.\n - **Agent response**: I can help with that. Let me fetch your order details first.\n - **Agent actions**: Call [@tool:get_order_details](#mention)\n\n- **User** : My order is missing an item.\n - **Agent response**: Let's check your order details and resolve this issue.\n - **Agent actions**: Call [@tool:get_order_details](#mention)\n\n- **User** : I got someone else's order.\n - **Agent response**: I apologize for the mix-up. I'll fetch your order details to sort this out.\n - **Agent actions**: Call [@tool:get_order_details](#mention)\n\n- **User** : Can you help me with a missing item?\n - **Agent response**: Certainly, I'll look into your order details right away.\n - **Agent actions**: Call [@tool:get_order_details](#mention)\n\n- **User** : There's an issue with my order items.\n - **Agent response**: Let's verify your order details to address this issue.\n - **Agent actions**: Call [@tool:get_order_details](#mention)",
|
||||
"model": "gpt-4o",
|
||||
"hasRagSources": false,
|
||||
"controlType": "retain",
|
||||
"tools": [
|
||||
"get_order_details"
|
||||
],
|
||||
"prompts": [],
|
||||
"connectedAgents": []
|
||||
},
|
||||
{
|
||||
"name": "Delayed Delivery",
|
||||
"type": "conversation",
|
||||
"description": "Agent to assist users with delayed delivery issues.",
|
||||
"instructions": "## \ud83e\uddd1\u200d\ud83d\udcbc Role:\nAssist users with issues related to delayed delivery.\n\n---\n## \u2699\ufe0f Steps to Follow:\n1. Fetch the delivery status using the [@tool:get_delivery_status] tool.\n2. Confirm the delay with the user.\n3. Provide solutions or escalate if unresolved.\n\n---\n## \ud83c\udfaf Scope:\n\u2705 In Scope:\n- Handling delayed delivery issues\n\n\u274c Out of Scope:\n- Missing or incorrect order items\n- General knowledge queries\n\n---\n## \ud83d\udccb Guidelines:\n\u2714\ufe0f Dos:\n- Ensure the user is aware of the delivery status before proceeding.\n\n\ud83d\udeab Don'ts:\n- Extend the conversation beyond delivery issues.\n\n# Examples\n- **User** : My delivery is late.\n - **Agent response**: I can help with that. Let me fetch your delivery status first.\n - **Agent actions**: Call [@tool:get_delivery_status](#mention)\n\n- **User** : Where is my order? It's delayed.\n - **Agent response**: Let's check your delivery status and resolve this issue.\n - **Agent actions**: Call [@tool:get_delivery_status](#mention)\n\n- **User** : My order hasn't arrived yet.\n - **Agent response**: I apologize for the delay. I'll fetch your delivery status to sort this out.\n - **Agent actions**: Call [@tool:get_delivery_status](#mention)\n\n- **User** : Can you help me with a delayed delivery?\n - **Agent response**: Certainly, I'll look into your delivery status right away.\n - **Agent actions**: Call [@tool:get_delivery_status](#mention)\n\n- **User** : There's an issue with my delivery timing.\n - **Agent response**: Let's verify your delivery status to address this issue.\n - **Agent actions**: Call [@tool:get_delivery_status](#mention)",
|
||||
"model": "gpt-4o",
|
||||
"hasRagSources": false,
|
||||
"controlType": "retain",
|
||||
"tools": [
|
||||
"get_delivery_status"
|
||||
],
|
||||
"prompts": [],
|
||||
"connectedAgents": []
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "get_order_details",
|
||||
"description": "Tool to fetch details about the user's order.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"order_id": {
|
||||
"type": "string",
|
||||
"description": "The unique identifier for the order."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"order_id"
|
||||
]
|
||||
},
|
||||
"mockTool": true,
|
||||
"mockInstructions": "Return a mock response for Door Dash order details."
|
||||
},
|
||||
{
|
||||
"name": "get_delivery_status",
|
||||
"description": "Tool to fetch the current status of the delivery.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"order_id": {
|
||||
"type": "string",
|
||||
"description": "The unique identifier for the order."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"order_id"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"prompts": [
|
||||
{
|
||||
"name": "Style prompt",
|
||||
"type": "style_prompt",
|
||||
"prompt": "You should be empathetic and helpful."
|
||||
}
|
||||
],
|
||||
"startAgent": "Door Dash Hub"
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": ""
|
||||
},
|
||||
{
|
||||
"version": "v1",
|
||||
"chatId": "",
|
||||
"createdAt": "2025-05-05T17:29:15.822Z",
|
||||
"role": "assistant",
|
||||
"content": "How can I help you today?",
|
||||
"agenticSender": "Interview Evaluation Hub",
|
||||
"agenticResponseType": "external"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "[2025-05-02, 10:00] Assistant: Good morning. Thank you for taking the time to speak with us today. Could you start by briefly walking us through your leadership journey over the past decade?\\\\\\\\n\\\\\\\\n[2025-05-02, 10:01] User: Absolutely. Over the past 12 years, I've led product and technology teams across high-growth B2B SaaS companies. Most recently, I was the Chief Product Officer at a fintech scaleup, where I built a team of 60+ PMs and oversaw the expansion into three international markets. Prior to that, I was VP of Product at a healthtech startup acquired by a public company.\\\\\\\\n\\\\\\\\n[2025-05-02, 10:03] Assistant: Impressive. What would you say is your core leadership style, and how has it evolved over the years?\\\\\\\\n\\\\\\\\n[2025-05-02, 10:04] User: I'd describe my style as mission-oriented and data-driven. Early in my career, I leaned heavily on execution. But with time, I've grown more focused on setting context and empowering teams. At my last company, I implemented OKRs org-wide and focused on building a culture of trust and transparency.\\\\\\\\n\\\\\\\\n[2025-05-02, 10:06] Assistant: How do you approach scaling an org while preserving agility and innovation?\\\\\\\\n\\\\\\\\n[2025-05-02, 10:07] User: Great question. I believe the key is in modular team design. At the fintech company, we adopted a \\\\\\\\\\\\\\\"startup within a startup\\\\\\\\\\\\\\\" model—each pod owned metrics and had embedded PMs, designers, and engineers. This preserved autonomy and speed while scaling. We also invested heavily in internal tooling and rituals like demo days to keep the culture of innovation alive.\\\\\\\\n\\\\\\\\n[2025-05-02, 10:10] Assistant: Tell me about a time you made a high-stakes decision without perfect data.\\\\\\\\n\\\\\\\\n[2025-05-02, 10:11] User: In early 2023, we had to decide whether to sunset a low-performing but vocal customer-facing feature. We lacked conclusive cohort data but saw consistent qualitative feedback about confusion and churn. I convened a task force, aligned with customer success and execs, and made the call. Within 60 days, churn dropped by 12%, validating the bet.\\\\\\\\n\\\\\\\\n[2025-05-02, 10:14] Assistant: Final question—what excites you about potentially joining our company as a Chief Product Officer?\\\\\\\\n\\\\\\\\n[2025-05-02, 10:15] User: Your mission aligns with what I care about—bringing transparency and efficiency to financial infrastructure. The team's reputation, your engineering culture, and the velocity of product innovation are all huge draws. I believe I can help you scale without losing the edge that got you here.",
|
||||
"version": "v1",
|
||||
"chatId": "",
|
||||
"createdAt": "2025-05-05T17:29:44.695Z"
|
||||
}
|
||||
],
|
||||
"lastRequest": {
|
||||
"projectId": "a7a91831-e410-4fc5-a31d-8d027e14540a",
|
||||
"messages": [
|
||||
{
|
||||
"content": "",
|
||||
"role": "system",
|
||||
"sender": null,
|
||||
"tool_calls": null,
|
||||
"tool_call_id": null,
|
||||
"tool_name": null
|
||||
},
|
||||
{
|
||||
"content": "How can I help you today?",
|
||||
"role": "assistant",
|
||||
"sender": "Interview Evaluation Hub",
|
||||
"tool_calls": null,
|
||||
"tool_call_id": null,
|
||||
"tool_name": null,
|
||||
"response_type": "external"
|
||||
},
|
||||
{
|
||||
"content": "[2025-05-02, 10:00] Assistant: Good morning. Thank you for taking the time to speak with us today. Could you start by briefly walking us through your leadership journey over the past decade?\\\\\\\\n\\\\\\\\n[2025-05-02, 10:01] User: Absolutely. Over the past 12 years, I've led product and technology teams across high-growth B2B SaaS companies. Most recently, I was the Chief Product Officer at a fintech scaleup, where I built a team of 60+ PMs and oversaw the expansion into three international markets. Prior to that, I was VP of Product at a healthtech startup acquired by a public company.\\\\\\\\n\\\\\\\\n[2025-05-02, 10:03] Assistant: Impressive. What would you say is your core leadership style, and how has it evolved over the years?\\\\\\\\n\\\\\\\\n[2025-05-02, 10:04] User: I'd describe my style as mission-oriented and data-driven. Early in my career, I leaned heavily on execution. But with time, I've grown more focused on setting context and empowering teams. At my last company, I implemented OKRs org-wide and focused on building a culture of trust and transparency.\\\\\\\\n\\\\\\\\n[2025-05-02, 10:06] Assistant: How do you approach scaling an org while preserving agility and innovation?\\\\\\\\n\\\\\\\\n[2025-05-02, 10:07] User: Great question. I believe the key is in modular team design. At the fintech company, we adopted a \\\\\\\\\\\\\\\"startup within a startup\\\\\\\\\\\\\\\" model—each pod owned metrics and had embedded PMs, designers, and engineers. This preserved autonomy and speed while scaling. We also invested heavily in internal tooling and rituals like demo days to keep the culture of innovation alive.\\\\\\\\n\\\\\\\\n[2025-05-02, 10:10] Assistant: Tell me about a time you made a high-stakes decision without perfect data.\\\\\\\\n\\\\\\\\n[2025-05-02, 10:11] User: In early 2023, we had to decide whether to sunset a low-performing but vocal customer-facing feature. We lacked conclusive cohort data but saw consistent qualitative feedback about confusion and churn. I convened a task force, aligned with customer success and execs, and made the call. Within 60 days, churn dropped by 12%, validating the bet.\\\\\\\\n\\\\\\\\n[2025-05-02, 10:14] Assistant: Final question—what excites you about potentially joining our company as a Chief Product Officer?\\\\\\\\n\\\\\\\\n[2025-05-02, 10:15] User: Your mission aligns with what I care about—bringing transparency and efficiency to financial infrastructure. The team's reputation, your engineering culture, and the velocity of product innovation are all huge draws. I believe I can help you scale without losing the edge that got you here.",
|
||||
"role": "user",
|
||||
"sender": null,
|
||||
"tool_calls": null,
|
||||
"tool_call_id": null,
|
||||
"tool_name": null
|
||||
}
|
||||
],
|
||||
"state": {
|
||||
"last_agent_name": "Interview Evaluation Hub",
|
||||
"tokens": {
|
||||
"total": 0,
|
||||
"prompt": 0,
|
||||
"completion": 0
|
||||
},
|
||||
"turn_messages": [
|
||||
{
|
||||
"content": "How can I help you today?",
|
||||
"role": "assistant",
|
||||
"sender": "Interview Evaluation Hub",
|
||||
"tool_calls": null,
|
||||
"tool_call_id": null,
|
||||
"tool_name": null,
|
||||
"response_type": "external"
|
||||
}
|
||||
]
|
||||
},
|
||||
"agents": [
|
||||
{
|
||||
"name": "Interview Evaluation Hub",
|
||||
"type": "conversation",
|
||||
"description": "Hub agent to orchestrate the evaluation of interview transcripts between an executive search agency and a CxO candidate.",
|
||||
"instructions": "## 🧑💼 Role:\nYou are the hub agent responsible for orchestrating the evaluation of interview transcripts between an executive search agency (Assistant) and a CxO candidate (User).\n\n---\n## ⚙️ Steps to Follow:\n1. Receive the transcript in the specified format.\n2. FIRST: Send the transcript to [@agent:Evaluation Agent] for evaluation.\n3. Wait to receive the complete evaluation from the Evaluation Agent.\n4. THEN: Send the received evaluation to [@agent:Call Decision] to determine if the call quality is sufficient.\n5. Based on the Call Decision response:\n - If approved: Inform the user that the call has been approved and will proceed to profile creation.\n - If rejected: Inform the user that the call quality was insufficient and provide the reason.\n6. Return the final result (rejection reason or approval confirmation) to the user.\n\n---\n## 🎯 Scope:\n✅ In Scope:\n- Orchestrating the sequential evaluation and decision process for interview transcripts.\n\n❌ Out of Scope:\n- Directly evaluating or creating profiles.\n- Handling transcripts not in the specified format.\n- Interacting with the individual evaluation agents.\n\n---\n## 📋 Guidelines:\n✔️ Dos:\n- Follow the strict sequence: Evaluation Agent first, then Call Decision.\n- Wait for each agent's complete response before proceeding.\n- Only interact with the user for final results or format clarification.\n\n🚫 Don'ts:\n- Do not perform evaluation or profile creation yourself.\n- Do not modify the transcript.\n- Do not try to get evaluations simultaneously.\n- Do not reference the individual evaluation agents.\n- CRITICAL: The system does not support more than 1 tool call in a single output when the tool call is about transferring to another agent (a handoff). You must only put out 1 transfer related tool call in one output.\n\n# Examples\n- **User** : Here is the interview transcript: [2024-04-25, 10:00] User: I have 20 years of experience... [2024-04-25, 10:01] Assistant: Can you describe your leadership style?\n - **Agent actions**: \n 1. First call [@agent:Evaluation Agent](#mention)\n 2. Wait for complete evaluation\n 3. Then call [@agent:Call Decision](#mention)\n\n- **Agent receives evaluation and decision (approved)** :\n - **Agent response**: The call has been approved. Proceeding to candidate profile creation.\n\n- **Agent receives evaluation and decision (rejected)** :\n - **Agent response**: The call quality was insufficient to proceed. [Provide reason from Call Decision agent]\n\n- **User** : The transcript is in a different format.\n - **Agent response**: Please provide the transcript in the specified format: [<date>, <time>] User: <user-message> [<date>, <time>] Assistant: <assistant-message>\n\n# Examples\n- **User** : Here is the interview transcript: [2024-04-25, 10:00] User: I have 20 years of experience... [2024-04-25, 10:01] Assistant: Can you describe your leadership style?\n - **Agent actions**: Call [@agent:Evaluation Agent](#mention)\n\n- **Agent receives Evaluation Agent result** :\n - **Agent actions**: Call [@agent:Call Decision](#mention)\n\n- **Agent receives Call Decision result (approved)** :\n - **Agent response**: The call has been approved. Proceeding to candidate profile creation.\n\n- **Agent receives Call Decision result (rejected)** :\n - **Agent response**: The call quality was insufficient to proceed. [Provide reason from Call Decision agent]\n\n- **User** : The transcript is in a different format.\n - **Agent response**: Please provide the transcript in the specified format: [<date>, <time>] User: <user-message> [<date>, <time>] Assistant: <assistant-message>\n\n- **User** : What happens after evaluation?\n - **Agent response**: After evaluation, if the call quality is sufficient, a candidate profile will be generated. Otherwise, you will receive feedback on why the call was rejected.",
|
||||
"model": "gpt-4o",
|
||||
"controlType": "retain",
|
||||
"ragK": 3,
|
||||
"ragReturnType": "chunks",
|
||||
"outputVisibility": "user_facing",
|
||||
"tools": [],
|
||||
"prompts": [],
|
||||
"connectedAgents": [
|
||||
"Evaluation Agent",
|
||||
"Call Decision"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Exec Search Evaluation",
|
||||
"type": "conversation",
|
||||
"description": "Evaluates the relevance and quality of questions asked by the executive search agency in the transcript.",
|
||||
"instructions": "## 🧑💼 Role:\nYou are responsible for evaluating the questions asked by the executive search agency (Assistant) in the interview transcript.\n\n---\n## ⚙️ Steps to Follow:\n1. Review all 'Assistant' messages in the transcript.\n2. Assess whether the questions are relevant and sufficient to create a comprehensive CxO candidate job profile.\n3. Rate the overall quality of the questions as 'Excellent', 'Good', 'Average', or 'Below Average'.\n4. Provide a brief justification for your rating.\n5. Return your evaluation and justification.\n\n---\n## 🎯 Scope:\n✅ In Scope:\n- Evaluating the relevance and depth of questions asked by the executive search agency.\n\n❌ Out of Scope:\n- Evaluating candidate responses.\n- Creating candidate profiles.\n\n---\n## 📋 Guidelines:\n✔️ Dos:\n- Focus on whether the questions cover key areas: skills, experience, leadership, fit, etc.\n- Be objective and concise in your justification.\n\n🚫 Don'ts:\n- Do not evaluate candidate answers.\n- Do not suggest improvements unless specifically asked.\n\n# Examples\n- **User** : [2024-04-25, 10:01] Assistant: Can you describe your leadership style? [2024-04-25, 10:02] Assistant: What are your key achievements as a CTO?\n - **Agent response**: Rating: Good. The questions are relevant and cover important aspects of the candidate's experience and leadership.\n\n- **User** : [2024-04-25, 10:01] Assistant: Tell me about yourself. [2024-04-25, 10:02] Assistant: What do you like to do in your free time?\n - **Agent response**: Rating: Below Average. The questions are too generic and do not sufficiently address the candidate's professional qualifications.\n\n- **User** : [2024-04-25, 10:01] Assistant: What is your favorite color?\n - **Agent response**: Rating: Below Average. The question is irrelevant for creating a CxO candidate profile.\n\n- **User** : [2024-04-25, 10:01] Assistant: Can you walk me through your experience leading large teams?\n - **Agent response**: Rating: Excellent. The question is highly relevant for assessing leadership skills.\n\n- **User** : [2024-04-25, 10:01] Assistant: How do you handle stress at work?\n - **Agent response**: Rating: Good. The question is relevant for understanding the candidate's coping mechanisms in high-pressure environments.",
|
||||
"model": "gpt-4o",
|
||||
"controlType": "retain",
|
||||
"ragK": 3,
|
||||
"ragReturnType": "chunks",
|
||||
"outputVisibility": "internal",
|
||||
"tools": [],
|
||||
"prompts": [],
|
||||
"connectedAgents": []
|
||||
},
|
||||
{
|
||||
"name": "Candidate Evaluation",
|
||||
"type": "conversation",
|
||||
"description": "Evaluates the quality, relevance, and clarity of the candidate's (User) responses in the transcript.",
|
||||
"instructions": "## 🧑💼 Role:\nYou are responsible for evaluating the candidate's (User) responses in the interview transcript.\n\n---\n## ⚙️ Steps to Follow:\n1. Review all 'User' messages in the transcript.\n2. Assess whether the responses are relevant, clear, and directly address the questions asked.\n3. Identify if any responses are meandering, off-topic, or spam.\n4. Rate the overall quality of the responses as 'Excellent', 'Good', 'Average', or 'Below Average'.\n5. Provide a brief justification for your rating.\n6. Return your evaluation and justification.\n\n---\n## 🎯 Scope:\n✅ In Scope:\n- Evaluating the quality and relevance of candidate responses.\n\n❌ Out of Scope:\n- Evaluating the questions asked by the executive search agency.\n- Creating candidate profiles.\n\n---\n## 📋 Guidelines:\n✔️ Dos:\n- Focus on clarity, relevance, and completeness of responses.\n- Be objective and concise in your justification.\n\n🚫 Don'ts:\n- Do not evaluate the questions asked.\n- Do not suggest improvements unless specifically asked.\n\n# Examples\n- **User** : [2024-04-25, 10:01] User: I have led teams of 100+ engineers and delivered multiple large-scale projects on time.\n - **Agent response**: Rating: Excellent. The response is clear, relevant, and directly addresses the question.\n\n- **User** : [2024-04-25, 10:01] User: Well, you know, it's like, sometimes things just happen, and I just do my best, you know?\n - **Agent response**: Rating: Below Average. The response is vague and does not provide meaningful information.\n\n- **User** : [2024-04-25, 10:01] User: I have experience in many things, but let's talk about something else.\n - **Agent response**: Rating: Below Average. The response is off-topic and does not address the question.\n\n- **User** : [2024-04-25, 10:01] User: I have successfully managed cross-functional teams and implemented new processes that improved efficiency by 20%.\n - **Agent response**: Rating: Good. The response is relevant and provides specific achievements.\n\n- **User** : [2024-04-25, 10:01] User: I am a hard worker.\n - **Agent response**: Rating: Average. The response is generic and lacks detail.",
|
||||
"model": "gpt-4o",
|
||||
"controlType": "retain",
|
||||
"ragK": 3,
|
||||
"ragReturnType": "chunks",
|
||||
"outputVisibility": "internal",
|
||||
"tools": [],
|
||||
"prompts": [],
|
||||
"connectedAgents": []
|
||||
},
|
||||
{
|
||||
"name": "Call Decision",
|
||||
"type": "conversation",
|
||||
"description": "Decides whether the call quality is sufficient to proceed to candidate profile creation based on the evaluations.",
|
||||
"instructions": "## 🧑💼 Role:\nYou are responsible for making a decision based on the evaluations from the Exec Search and Candidate Evaluation agents.\n\n---\n## ⚙️ Steps to Follow:\n1. Receive the ratings and justifications from both evaluation agents.\n2. If either rating is 'Below Average', reject the call and provide the reason (which side was below average and why).\n3. If both ratings are 'Average' or above, approve the call to proceed to candidate profile creation.\n4. Return your decision and, if rejected, the reason for rejection.\n\n---\n## 🎯 Scope:\n✅ In Scope:\n- Deciding whether to proceed to profile creation or reject the call.\n\n❌ Out of Scope:\n- Evaluating the transcript directly.\n- Creating candidate profiles.\n\n---\n## 📋 Guidelines:\n✔️ Dos:\n- Be clear and concise in your decision.\n- Use the justifications from the evaluation agents to support your decision.\n\n🚫 Don'ts:\n- Do not override the evaluation agents' ratings.\n\n# Examples\n- **User** : Exec Search: Good. Candidate: Excellent.\n - **Agent response**: Decision: Approved. Both sides performed well. Proceed to candidate profile creation.\n\n- **User** : Exec Search: Below Average. Candidate: Good.\n - **Agent response**: Decision: Rejected. The executive search agency's questions were below average: [reason].\n\n- **User** : Exec Search: Good. Candidate: Below Average.\n - **Agent response**: Decision: Rejected. The candidate's responses were below average: [reason].\n\n- **User** : Exec Search: Average. Candidate: Average.\n - **Agent response**: Decision: Approved. Both sides performed adequately. Proceed to candidate profile creation.\n\n- **User** : Exec Search: Excellent. Candidate: Excellent.\n - **Agent response**: Decision: Approved. Both sides performed excellently. Proceed to candidate profile creation.",
|
||||
"model": "gpt-4o",
|
||||
"controlType": "retain",
|
||||
"ragK": 3,
|
||||
"ragReturnType": "chunks",
|
||||
"outputVisibility": "internal",
|
||||
"tools": [],
|
||||
"prompts": [],
|
||||
"connectedAgents": []
|
||||
},
|
||||
{
|
||||
"name": "Evaluation Agent",
|
||||
"type": "conversation",
|
||||
"description": "Coordinates the evaluation of the interview transcript by both the Exec Search Evaluation and Candidate Evaluation agents.",
|
||||
"instructions": "## 🧑💼 Role:\nYou are responsible for coordinating the evaluation of the interview transcript by both the Exec Search Evaluation and Candidate Evaluation agents.\n\n---\n## ⚙️ Steps to Follow:\n1. Receive the transcript from the hub agent.\n2. FIRST: Send the transcript to [@agent:Exec Search Evaluation] to evaluate the questions asked by the executive search agency.\n3. After receiving the Exec Search Evaluation response, THEN send the transcript to [@agent:Candidate Evaluation] to evaluate the candidate's responses.\n4. Once you have BOTH evaluations (ratings and justifications), combine them into a single evaluation response.\n5. Return the combined evaluation to the hub agent.\n\n---\n## 🎯 Scope:\n✅ In Scope:\n- Coordinating the sequential evaluation process between the two evaluation agents.\n\n❌ Out of Scope:\n- Making decisions about call quality.\n- Creating candidate profiles.\n- Interacting directly with the user.\n\n---\n## 📋 Guidelines:\n✔️ Dos:\n- Follow the strict sequence: Exec Search first, then Candidate.\n- Wait for each evaluation to complete before proceeding.\n- Combine both evaluations into a single response.\n\n🚫 Don'ts:\n- Do not evaluate the transcript yourself.\n- Do not try to get both evaluations simultaneously.\n- Do not interact with the user.\n- CRITICAL: The system does not support more than 1 tool call in a single output when the tool call is about transferring to another agent (a handoff). You must only put out 1 transfer related tool call in one output.\n\n# Examples\n- **User** : [2024-04-25, 10:01] User: I have 20 years of experience... [2024-04-25, 10:01] Assistant: Can you describe your leadership style?\n - **Agent actions**: \n 1. First call [@agent:Exec Search Evaluation](#mention)\n 2. After receiving response, then call [@agent:Candidate Evaluation](#mention)\n 3. Combine both evaluations into single response\n\n- **Agent receives both evaluations** :\n - **Agent response**: Returns combined evaluations (ratings and justifications) to the hub agent.\n\n# Examples\n- **User** : [2024-04-25, 10:01] User: I have 20 years of experience... [2024-04-25, 10:01] Assistant: Can you describe your leadership style?\n - **Agent actions**: Call [@agent:Exec Search Evaluation](#mention), Call [@agent:Candidate Evaluation](#mention)\n\n- **Agent receives both evaluations** :\n - **Agent response**: Returns both evaluations (ratings and justifications) to the hub agent.",
|
||||
"model": "gpt-4o",
|
||||
"controlType": "retain",
|
||||
"ragK": 3,
|
||||
"ragReturnType": "chunks",
|
||||
"outputVisibility": "internal",
|
||||
"tools": [],
|
||||
"prompts": [],
|
||||
"connectedAgents": [
|
||||
"Exec Search Evaluation",
|
||||
"Candidate Evaluation"
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "web_search",
|
||||
"description": "Fetch information from the web based on chat context",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
},
|
||||
"isLibrary": true
|
||||
}
|
||||
],
|
||||
"prompts": [],
|
||||
"startAgent": "Interview Evaluation Hub",
|
||||
"mcpServers": [],
|
||||
"toolWebhookUrl": ""
|
||||
},
|
||||
"lastResponse": null
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue