mock tool responses in api as well

apps/rowboat/app/actions/actions.ts

@@ -29,7 +29,7 @@ import { embeddingModel } from "../lib/embedding";
 import { apiV1 } from "rowboat-shared";
 import { zodToJsonSchema } from 'zod-to-json-schema';
 import { Claims, getSession } from "@auth0/nextjs-auth0";
-import { callClientToolWebhook, getAgenticApiResponse, runRAGToolCall } from "../lib/utils";
+import { callClientToolWebhook, getAgenticApiResponse, mockToolResponse, runRAGToolCall } from "../lib/utils";
 import { assert } from "node:console";
 import { check_query_limit } from "../lib/rate_limiting";
 import { QueryLimitError } from "../lib/client_utils";
@@ -267,41 +267,7 @@ export async function suggestToolResponse(toolId: string, projectId: string, mes
         throw new QueryLimitError();
     }
 
-    const prompt = `
-# Your Specific Task:
-Here is a chat between a user and a customer support assistant.
-The assistant has requested a tool call with ID {{toolID}}.
-Your job is to come up with an example of the data that the tool call should return.
-The current date is {{date}}.
-
-CONVERSATION:
-{{messages}}
-`
-        .replace('{{toolID}}', toolId)
-        .replace(`{{date}}`, new Date().toISOString())
-        .replace('{{messages}}', JSON.stringify(messages.map((m) => {
-            let tool_calls;
-            if ('tool_calls' in m && m.role == 'assistant') {
-                tool_calls = m.tool_calls;
-            }
-            let { role, content } = m;
-            return {
-                role,
-                content,
-                tool_calls,
-            }
-        })));
-    // console.log(prompt);
-
-    const { object } = await generateObject({
-        model: openai("gpt-4o"),
-        prompt: prompt,
-        schema: z.object({
-            result: z.any(),
-        }),
-    });
-
-    return JSON.stringify(object);
+    return await mockToolResponse(toolId, messages);
 }
 
 export async function getInformationTool(

apps/rowboat/app/api/v1/[projectId]/chat/route.ts

@@ -5,7 +5,7 @@ import { ObjectId } from "mongodb";
 import { authCheck } from "../../utils";
 import { ApiRequest, ApiResponse } from "../../../../lib/types/types";
 import { AgenticAPIChatRequest, AgenticAPIChatMessage, convertFromAgenticApiToApiMessages, convertFromApiToAgenticApiMessages, convertWorkflowToAgenticAPI } from "../../../../lib/types/agents_api_types";
-import { getAgenticApiResponse, callClientToolWebhook, runRAGToolCall } from "../../../../lib/utils";
+import { getAgenticApiResponse, callClientToolWebhook, runRAGToolCall, mockToolResponse } from "../../../../lib/utils";
 import { check_query_limit } from "../../../../lib/rate_limiting";
 import { apiV1 } from "rowboat-shared";
 import { PrefixLogger } from "../../../../lib/utils";
@@ -136,18 +136,27 @@ export async function POST(
                         }
                     } else {
                         logger.log(`Running client tool webhook for tool ${toolCall.function.name}`);
-                        // run other tool calls by calling the client tool webhook
+
                         try {
+                            // if tool is supposed to be mocked, mock it
+                            const workflowTool = workflow.tools.find(t => t.name === toolCall.function.name);
+                            if (workflowTool?.mockInPlayground) {
+                                logger.log(`Mocking tool call ${toolCall.function.name}`);
+                                result = await mockToolResponse(toolCall.id, currentMessages);
+                            } else {
+                                // else run the tool call by calling the client tool webhook
+                                logger.log(`Running client tool webhook for tool ${toolCall.function.name}`);
                                 result = await callClientToolWebhook(
                                     toolCall,
                                     currentMessages,
                                     projectId,
                                 );
-                            logger.log(`Client tool webhook call completed for tool ${toolCall.function.name}`);
-                        } catch (e) {
-                            logger.log(`Error calling client tool webhook: ${e}`);
-                            return Response.json({ error: "Error calling client tool webhook" }, { status: 500 });
                             }
+                        } catch (e) {
+                            logger.log(`Error in tool call ${toolCall.function.name}: ${e}`);
+                            return Response.json({ error: `Error in tool call ${toolCall.function.name}` }, { status: 500 });
+                        }
+                        logger.log(`Tool call ${toolCall.function.name} completed`);
                     }
 
                     toolCallResultMessages.push({

apps/rowboat/app/lib/utils.ts

@@ -13,10 +13,11 @@ import { SignJWT } from "jose";
 import crypto from "crypto";
 import { ObjectId } from "mongodb";
 import { embeddingModel } from "./embedding";
-import { embed } from "ai";
+import { embed, generateObject } from "ai";
 import { qdrantClient } from "./qdrant";
 import { EmbeddingRecord } from "./types/datasource_types";
 import { ApiMessage } from "./types/types";
+import { openai } from "@ai-sdk/openai";
 
 export async function callClientToolWebhook(
     toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number],
@@ -218,3 +219,41 @@ export class PrefixLogger {
         return new PrefixLogger(childPrefix, this);
     }
 }
+
+export async function mockToolResponse(toolId: string, messages: z.infer<typeof ApiMessage>[]): Promise<string> {
+    const prompt = `
+# Your Specific Task:
+Here is a chat between a user and a customer support assistant.
+The assistant has requested a tool call with ID {{toolID}}.
+Your job is to come up with an example of the data that the tool call should return.
+The current date is {{date}}.
+
+CONVERSATION:
+{{messages}}
+`
+        .replace('{{toolID}}', toolId)
+        .replace(`{{date}}`, new Date().toISOString())
+        .replace('{{messages}}', JSON.stringify(messages.map((m) => {
+            let tool_calls;
+            if ('tool_calls' in m && m.role == 'assistant') {
+                tool_calls = m.tool_calls;
+            }
+            let { role, content } = m;
+            return {
+                role,
+                content,
+                tool_calls,
+            }
+        })));
+    // console.log(prompt);
+
+    const { object } = await generateObject({
+        model: openai("gpt-4o"),
+        prompt: prompt,
+        schema: z.object({
+            result: z.any(),
+        }),
+    });
+
+    return JSON.stringify(object);
+}

apps/rowboat/app/projects/[projectId]/menu.tsx

@@ -57,7 +57,7 @@ export default function Menu({
         />
         <NavLink
             href={`/projects/${projectId}/simulation`}
-            label="Test"
+            label="Test (wip)"
             collapsed={collapsed}
             icon={<PlayIcon size={16} />}
             selected={pathname.startsWith(`/projects/${projectId}/simulation`)}

apps/rowboat/app/projects/[projectId]/workflow/tool_config.tsx

@@ -252,7 +252,7 @@ export function ToolConfig({
                                 label: "text-sm font-normal"
                             }}
                         >
-                            Mock tool responses in playground
+                            Mock tool responses
                         </Radio>
                         <Radio 
                             value="api"