From 768c5749a01c55064a85418d52158c9916435304 Mon Sep 17 00:00:00 2001 From: ramnique <30795890+ramnique@users.noreply.github.com> Date: Thu, 27 Feb 2025 23:42:04 +0530 Subject: [PATCH] Add test profiles and overhaul testing --- apps/rowboat/app/actions/actions.ts | 74 +- apps/rowboat/app/actions/project_actions.ts | 19 +- .../rowboat/app/actions/simulation_actions.ts | 270 ------- apps/rowboat/app/actions/testing_actions.ts | 575 +++++++++++++++ .../app/api/v1/[projectId]/chat/route.ts | 43 +- .../components/selectors/profile-selector.tsx | 100 +++ .../selectors/scenario-selector.tsx | 98 +++ .../selectors/simulation-selector.tsx | 140 ++++ .../selectors/workflow-selector.tsx | 106 +++ apps/rowboat/app/lib/mongodb.ts | 12 +- apps/rowboat/app/lib/types/project_types.ts | 2 + apps/rowboat/app/lib/types/testing_types.ts | 70 +- apps/rowboat/app/lib/types/types.ts | 4 +- apps/rowboat/app/lib/utils.ts | 29 +- .../rowboat/app/projects/[projectId]/menu.tsx | 4 +- .../projects/[projectId]/playground/app.tsx | 100 ++- .../projects/[projectId]/playground/chat.tsx | 51 +- .../[projectId]/playground/messages.tsx | 112 +-- .../projects/[projectId]/simulation/app.tsx | 459 ------------ .../simulation/components/RunComponents.tsx | 399 ---------- .../components/ScenarioComponents.tsx | 204 ------ .../projects/[projectId]/simulation/page.tsx | 10 - .../[projectId]/test/[[...slug]]/app.tsx | 68 ++ .../[projectId]/test/[[...slug]]/page.tsx | 8 + .../test/[[...slug]]/profiles_app.tsx | 535 ++++++++++++++ .../[projectId]/test/[[...slug]]/runs_app.tsx | 456 ++++++++++++ .../test/[[...slug]]/scenarios_app.tsx | 463 ++++++++++++ .../test/[[...slug]]/simulations_app.tsx | 690 ++++++++++++++++++ .../app/projects/[projectId]/workflow/app.tsx | 12 +- .../[projectId]/workflow/workflow_editor.tsx | 4 + 30 files changed, 3473 insertions(+), 1644 deletions(-) delete mode 100644 apps/rowboat/app/actions/simulation_actions.ts create mode 100644 apps/rowboat/app/actions/testing_actions.ts create mode 100644 apps/rowboat/app/lib/components/selectors/profile-selector.tsx create mode 100644 apps/rowboat/app/lib/components/selectors/scenario-selector.tsx create mode 100644 apps/rowboat/app/lib/components/selectors/simulation-selector.tsx create mode 100644 apps/rowboat/app/lib/components/selectors/workflow-selector.tsx delete mode 100644 apps/rowboat/app/projects/[projectId]/simulation/app.tsx delete mode 100644 apps/rowboat/app/projects/[projectId]/simulation/components/RunComponents.tsx delete mode 100644 apps/rowboat/app/projects/[projectId]/simulation/components/ScenarioComponents.tsx delete mode 100644 apps/rowboat/app/projects/[projectId]/simulation/page.tsx create mode 100644 apps/rowboat/app/projects/[projectId]/test/[[...slug]]/app.tsx create mode 100644 apps/rowboat/app/projects/[projectId]/test/[[...slug]]/page.tsx create mode 100644 apps/rowboat/app/projects/[projectId]/test/[[...slug]]/profiles_app.tsx create mode 100644 apps/rowboat/app/projects/[projectId]/test/[[...slug]]/runs_app.tsx create mode 100644 apps/rowboat/app/projects/[projectId]/test/[[...slug]]/scenarios_app.tsx create mode 100644 apps/rowboat/app/projects/[projectId]/test/[[...slug]]/simulations_app.tsx diff --git a/apps/rowboat/app/actions/actions.ts b/apps/rowboat/app/actions/actions.ts index 7862305f..6ebd5fd4 100644 --- a/apps/rowboat/app/actions/actions.ts +++ b/apps/rowboat/app/actions/actions.ts @@ -6,7 +6,6 @@ import { EmbeddingRecord } from "../lib/types/datasource_types"; import { WebpageCrawlResponse } from "../lib/types/tool_types"; import { GetInformationToolResult } from "../lib/types/tool_types"; import { EmbeddingDoc } from "../lib/types/datasource_types"; -import { SimulationData } from "../lib/types/testing_types"; import { generateObject, generateText, embed } from "ai"; import { dataSourceDocsCollection, dataSourcesCollection, embeddingsCollection, webpagesCollection } from "../lib/mongodb"; import { z } from 'zod'; @@ -21,6 +20,7 @@ import { QueryLimitError } from "../lib/client_utils"; import { projectAuthCheck } from "./project_actions"; import { qdrantClient } from "../lib/qdrant"; import { ObjectId } from "mongodb"; +import { TestProfile } from "../lib/types/testing_types"; const crawler = new FirecrawlApp({ apiKey: process.env.FIRECRAWL_API_KEY || '' }); @@ -99,13 +99,13 @@ export async function getAssistantResponse( }; } -export async function suggestToolResponse(toolId: string, projectId: string, messages: z.infer[]): Promise { +export async function suggestToolResponse(toolId: string, projectId: string, messages: z.infer[], testProfile: z.infer): Promise { await projectAuthCheck(projectId); if (!await check_query_limit(projectId)) { throw new QueryLimitError(); } - return await mockToolResponse(toolId, messages); + return await mockToolResponse(toolId, messages, testProfile); } export async function getInformationTool( @@ -123,39 +123,13 @@ export async function getInformationTool( export async function simulateUserResponse( projectId: string, messages: z.infer[], - simulationData: z.infer + scenario: string, ): Promise { await projectAuthCheck(projectId); if (!await check_query_limit(projectId)) { throw new QueryLimitError(); } - const articlePrompt = ` -# Your Specific Task: - -## Context: - -Here is a help article: - -Content: - -Title: {{title}} -{{content}} - - -## Task definition: - -Pretend to be a user reaching out to customer support. Chat with the -customer support assistant, assuming your issue or query is from this article. -Ask follow-up questions and make it real-world like. Don't do dummy -conversations. Your conversation should be a maximum of 5 user turns. - -As output, simply provide your (user) turn of conversation. - -After you are done with the chat, keep replying with a single word EXIT -in all capitals. -`; - const scenarioPrompt = ` # Your Specific Task: @@ -181,30 +155,6 @@ After you are done with the chat, keep replying with a single word EXIT in all capitals. `; - const previousChatPrompt = ` -# Your Specific Task: - -## Context: - -Here is a chat between a user and a customer support assistant: - -Chat: - -{{messages}} - - -## Task definition: - -Pretend to be a user reaching out to customer support. Chat with the -customer support assistant, assuming your issue based on this previous chat. -Ask follow-up questions and make it real-world like. Don't do dummy -conversations. Your conversation should be a maximum of 5 user turns. - -As output, simply provide your (user) turn of conversation. - -After you are done with the chat, keep replying with a single word EXIT -in all capitals. -`; await projectAuthCheck(projectId); // flip message assistant / user message @@ -219,19 +169,9 @@ in all capitals. // simulate user call let prompt; - if ('articleUrl' in simulationData) { - prompt = articlePrompt - .replace('{{title}}', simulationData.articleTitle || '') - .replace('{{content}}', simulationData.articleContent || ''); - } - if ('scenario' in simulationData) { - prompt = scenarioPrompt - .replace('{{scenario}}', simulationData.scenario); - } - if ('chatMessages' in simulationData) { - prompt = previousChatPrompt - .replace('{{messages}}', simulationData.chatMessages); - } + prompt = scenarioPrompt + .replace('{{scenario}}', scenario); + const { text } = await generateText({ model: openai("gpt-4o"), system: prompt || '', diff --git a/apps/rowboat/app/actions/project_actions.ts b/apps/rowboat/app/actions/project_actions.ts index e78c40e8..c5016b70 100644 --- a/apps/rowboat/app/actions/project_actions.ts +++ b/apps/rowboat/app/actions/project_actions.ts @@ -1,7 +1,7 @@ 'use server'; import { redirect } from "next/navigation"; import { ObjectId } from "mongodb"; -import { dataSourcesCollection, embeddingsCollection, projectsCollection, agentWorkflowsCollection, scenariosCollection, projectMembersCollection, apiKeysCollection, dataSourceDocsCollection } from "../lib/mongodb"; +import { dataSourcesCollection, embeddingsCollection, projectsCollection, agentWorkflowsCollection, testScenariosCollection, projectMembersCollection, apiKeysCollection, dataSourceDocsCollection, testProfilesCollection } from "../lib/mongodb"; import { z } from 'zod'; import crypto from 'crypto'; import { revalidatePath } from "next/cache"; @@ -41,6 +41,7 @@ export async function createProject(formData: FormData) { const projectId = crypto.randomUUID(); const chatClientId = crypto.randomBytes(16).toString('base64url'); const secret = crypto.randomBytes(32).toString('hex'); + const defaultTestProfileId = new ObjectId(); // create project await projectsCollection.insertOne({ @@ -52,12 +53,13 @@ export async function createProject(formData: FormData) { chatClientId, secret, nextWorkflowNumber: 1, + testRunCounter: 0, + defaultTestProfileId: defaultTestProfileId.toString(), }); // add first workflow version const { agents, prompts, tools, startAgent } = templates[templateKey]; await agentWorkflowsCollection.insertOne({ - _id: new ObjectId(), projectId, agents, prompts, @@ -68,6 +70,17 @@ export async function createProject(formData: FormData) { name: `Version 1`, }); + // add default test profile + await testProfilesCollection.insertOne({ + _id: defaultTestProfileId, + projectId, + name: "Default", + context: "", + mockTools: false, + createdAt: (new Date()).toISOString(), + lastUpdatedAt: (new Date()).toISOString(), + }); + // add user to project await projectMembersCollection.insertOne({ userId: user.sub, @@ -198,7 +211,7 @@ export async function deleteProject(projectId: string) { }); // delete scenarios - await scenariosCollection.deleteMany({ + await testScenariosCollection.deleteMany({ projectId, }); diff --git a/apps/rowboat/app/actions/simulation_actions.ts b/apps/rowboat/app/actions/simulation_actions.ts deleted file mode 100644 index 661315b2..00000000 --- a/apps/rowboat/app/actions/simulation_actions.ts +++ /dev/null @@ -1,270 +0,0 @@ -'use server'; - -import { ObjectId } from "mongodb"; -import { scenariosCollection, simulationRunsCollection, simulationResultsCollection } from "../lib/mongodb"; -import { z } from 'zod'; -import { projectAuthCheck } from "./project_actions"; -import { type WithStringId } from "../lib/types/types"; -import { Scenario, SimulationRun, SimulationResult, SimulationAggregateResult } from "../lib/types/testing_types"; -import { SimulationScenarioData } from "../lib/types/testing_types"; - -export async function getScenarios(projectId: string): Promise>[]> { - await projectAuthCheck(projectId); - - const scenarios = await scenariosCollection.find({ projectId }).toArray(); - return scenarios.map(s => ({ - ...s, - _id: s._id.toString(), - })); -} - -export async function getScenario(projectId: string, scenarioId: string): Promise>> { - await projectAuthCheck(projectId); - - // fetch scenario - const scenario = await scenariosCollection.findOne({ - _id: new ObjectId(scenarioId), - projectId, - }); - if (!scenario) { - throw new Error('Scenario not found'); - } - const { _id, description, ...rest } = scenario; - return { - ...rest, - _id: _id.toString(), - scenario: description, - }; -} - -export async function createScenario(projectId: string, name: string, description: string): Promise { - await projectAuthCheck(projectId); - - const now = new Date().toISOString(); - const result = await scenariosCollection.insertOne({ - projectId, - name, - description, - context: '', - criteria: '', - lastUpdatedAt: now, - createdAt: now, - }); - - return result.insertedId.toString(); -} - -export async function updateScenario( - projectId: string, - scenarioId: string, - updates: { - name?: string; - description?: string; - context?: string; - criteria?: string; - } -): Promise { - await projectAuthCheck(projectId); - - const updateData: any = { - ...updates, - lastUpdatedAt: new Date().toISOString(), - }; - - await scenariosCollection.updateOne( - { - _id: new ObjectId(scenarioId), - projectId, - }, - { - $set: updateData, - } - ); -} - -export async function deleteScenario(projectId: string, scenarioId: string): Promise { - await projectAuthCheck(projectId); - - await scenariosCollection.deleteOne({ - _id: new ObjectId(scenarioId), - projectId, - }); -} - -export async function getRuns(projectId: string): Promise>[]> { - await projectAuthCheck(projectId); - - const runs = await simulationRunsCollection - .find({ projectId }) - .sort({ startedAt: -1 }) // Most recent first - .toArray(); - - return runs.map(run => ({ - ...run, - _id: run._id.toString(), - })); -} - -export async function getRun(projectId: string, runId: string): Promise>> { - await projectAuthCheck(projectId); - - const run = await simulationRunsCollection.findOne({ - _id: new ObjectId(runId), - projectId, - }); - - if (!run) { - throw new Error('Run not found'); - } - - return { - ...run, - _id: run._id.toString(), - }; -} - -export async function createRun( - projectId: string, - scenarioIds: string[], - workflowId: string -): Promise>> { - await projectAuthCheck(projectId); - - const run = { - projectId, - status: 'pending' as const, - scenarioIds, - workflowId, - startedAt: new Date().toISOString(), - }; - - const result = await simulationRunsCollection.insertOne(run); - - return { - ...run, - _id: result.insertedId.toString(), - }; -} - -export async function updateRunStatus( - projectId: string, - runId: string, - status: z.infer['status'], - completedAt?: string -): Promise { - await projectAuthCheck(projectId); - - const updateData: Partial> = { - status, - }; - - if (completedAt) { - updateData.completedAt = completedAt; - } - - await simulationRunsCollection.updateOne( - { - _id: new ObjectId(runId), - projectId, - }, - { - $set: updateData, - } - ); -} - -export async function getRunResults( - projectId: string, - runId: string -): Promise>[]> { - await projectAuthCheck(projectId); - - const results = await simulationResultsCollection - .find({ - runId, - projectId, - }) - .toArray(); - - return results.map(result => ({ - ...result, - _id: result._id.toString(), - })); -} - -export async function createRunResult( - projectId: string, - runId: string, - scenarioId: string, - result: z.infer['result'], - details: string -): Promise { - await projectAuthCheck(projectId); - - const resultDoc = { - projectId, - runId, - scenarioId, - result, - details, - }; - - const insertResult = await simulationResultsCollection.insertOne(resultDoc); - return insertResult.insertedId.toString(); -} - -export async function createAggregateResult( - projectId: string, - runId: string, - total: number, - pass: number, - fail: number -): Promise { - await projectAuthCheck(projectId); - - await simulationRunsCollection.updateOne( - { _id: new ObjectId(runId), projectId }, - { - $set: { - aggregateResults: { total, pass, fail } - } - } - ); -} - -export async function getAggregateResult( - projectId: string, - runId: string -): Promise | null> { - await projectAuthCheck(projectId); - - const run = await simulationRunsCollection.findOne({ - _id: new ObjectId(runId), - projectId, - }); - - if (!run || !run.aggregateResults) return null; - - return run.aggregateResults; -} - -export async function deleteRun(projectId: string, runId: string) { - try { - // Delete the run using the collection directly - await simulationRunsCollection.deleteOne({ - _id: new ObjectId(runId), - projectId: projectId - }); - - // Delete associated results using the collection directly - await simulationResultsCollection.deleteMany({ - runId: runId, - projectId: projectId - }); - - return { success: true }; - } catch (error) { - console.error('Error deleting run:', error); - throw new Error('Failed to delete run'); - } -} \ No newline at end of file diff --git a/apps/rowboat/app/actions/testing_actions.ts b/apps/rowboat/app/actions/testing_actions.ts new file mode 100644 index 00000000..45b525a1 --- /dev/null +++ b/apps/rowboat/app/actions/testing_actions.ts @@ -0,0 +1,575 @@ +'use server'; +import { ObjectId } from "mongodb"; +import { testScenariosCollection, testSimulationsCollection, testProfilesCollection, testRunsCollection, testResultsCollection, projectsCollection } from "../lib/mongodb"; +import { z } from 'zod'; +import { projectAuthCheck } from "./project_actions"; +import { type WithStringId } from "../lib/types/types"; +import { TestScenario, TestSimulation, TestProfile, TestRun, TestResult } from "../lib/types/testing_types"; + +export async function listScenarios( + projectId: string, + page: number = 1, + pageSize: number = 10 +): Promise<{ + scenarios: WithStringId>[]; + total: number; +}> { + await projectAuthCheck(projectId); + + // Calculate skip value for pagination + const skip = (page - 1) * pageSize; + + // Get total count for pagination + const total = await testScenariosCollection.countDocuments({ projectId }); + + // Get paginated scenarios + const scenarios = await testScenariosCollection + .find({ projectId }) + .skip(skip) + .limit(pageSize) + .toArray(); + + return { + scenarios: scenarios.map(scenario => ({ + ...scenario, + _id: scenario._id.toString(), + })), + total, + }; +} + +export async function getScenario(projectId: string, scenarioId: string): Promise> | null> { + await projectAuthCheck(projectId); + + // fetch scenario + const scenario = await testScenariosCollection.findOne({ + _id: new ObjectId(scenarioId), + projectId, + }); + if (!scenario) { + return null; + } + const { _id, ...rest } = scenario; + return { + ...rest, + _id: _id.toString(), + }; +} + +export async function deleteScenario(projectId: string, scenarioId: string): Promise { + await projectAuthCheck(projectId); + + await testScenariosCollection.deleteOne({ + _id: new ObjectId(scenarioId), + projectId, + }); +} + +export async function createScenario( + projectId: string, + data: { + name: string; + description: string; + } +): Promise>> { + await projectAuthCheck(projectId); + + const doc = { + ...data, + projectId, + createdAt: new Date().toISOString(), + lastUpdatedAt: new Date().toISOString(), + }; + const result = await testScenariosCollection.insertOne(doc); + return { + ...doc, + _id: result.insertedId.toString(), + }; +} + +export async function updateScenario( + projectId: string, + scenarioId: string, + updates: { + name?: string; + description?: string; + } +): Promise { + await projectAuthCheck(projectId); + + const updateData: any = { + ...updates, + lastUpdatedAt: new Date().toISOString(), + }; + + await testScenariosCollection.updateOne( + { + _id: new ObjectId(scenarioId), + projectId, + }, + { + $set: updateData, + } + ); +} + +export async function listSimulations( + projectId: string, + page: number = 1, + pageSize: number = 10 +): Promise<{ + simulations: WithStringId>[]; + total: number; +}> { + await projectAuthCheck(projectId); + const skip = (page - 1) * pageSize; + const total = await testSimulationsCollection.countDocuments({ projectId }); + + const simulations = await testSimulationsCollection + .find({ projectId }) + .skip(skip) + .limit(pageSize) + .toArray(); + + return { + simulations: simulations.map(simulation => ({ + ...simulation, + _id: simulation._id.toString(), + })), + total, + }; +} + +export async function getSimulation(projectId: string, simulationId: string): Promise> | null> { + await projectAuthCheck(projectId); + + const simulation = await testSimulationsCollection.findOne({ + _id: new ObjectId(simulationId), + projectId, + }); + if (!simulation) { + return null; + } + const { _id, ...rest } = simulation; + return { + ...rest, + _id: _id.toString(), + }; +} + +export async function deleteSimulation(projectId: string, simulationId: string): Promise { + await projectAuthCheck(projectId); + + await testSimulationsCollection.deleteOne({ + _id: new ObjectId(simulationId), + projectId, + }); +} + +export async function createSimulation( + projectId: string, + data: { + name: string; + scenarioId: string; + profileId: string; + passCriteria: string; + } +): Promise>> { + await projectAuthCheck(projectId); + + const doc = { + ...data, + projectId, + createdAt: new Date().toISOString(), + lastUpdatedAt: new Date().toISOString(), + }; + const result = await testSimulationsCollection.insertOne(doc); + return { + ...doc, + _id: result.insertedId.toString(), + }; +} + +export async function updateSimulation( + projectId: string, + simulationId: string, + updates: { + name?: string; + scenarioId?: string; + profileId?: string; + passCriteria?: string; + } +): Promise { + await projectAuthCheck(projectId); + + const updateData: any = { + ...updates, + lastUpdatedAt: new Date().toISOString(), + }; + + await testSimulationsCollection.updateOne( + { + _id: new ObjectId(simulationId), + projectId, + }, + { + $set: updateData, + } + ); +} + +export async function listProfiles( + projectId: string, + page: number = 1, + pageSize: number = 10 +): Promise<{ + profiles: WithStringId>[]; + total: number; +}> { + await projectAuthCheck(projectId); + const skip = (page - 1) * pageSize; + const total = await testProfilesCollection.countDocuments({ projectId }); + + const profiles = await testProfilesCollection + .find({ projectId }) + .skip(skip) + .limit(pageSize) + .toArray(); + + return { + profiles: profiles.map(profile => ({ + ...profile, + _id: profile._id.toString(), + })), + total, + }; +} + +export async function getDefaultProfile(projectId: string): Promise> | null> { + await projectAuthCheck(projectId); + const project = await projectsCollection.findOne({ _id: projectId }); + if (!project) { + return null; + } + if (!project.defaultTestProfileId) { + // create a default profile + const profile = await createProfile(projectId, { + name: 'Default', + context: '', + mockTools: false, + mockPrompt: '', + }); + await setDefaultProfile(projectId, profile._id); + return profile; + } + return getProfile(projectId, project.defaultTestProfileId); +} + +export async function setDefaultProfile(projectId: string, profileId: string): Promise { + await projectAuthCheck(projectId); + await projectsCollection.updateOne( + { _id: projectId }, + { $set: { defaultTestProfileId: profileId } } + ); +} + +export async function getProfile(projectId: string, profileId: string): Promise> | null> { + await projectAuthCheck(projectId); + + const profile = await testProfilesCollection.findOne({ + _id: new ObjectId(profileId), + projectId, + }); + if (!profile) { + return null; + } + const { _id, ...rest } = profile; + return { + ...rest, + _id: _id.toString(), + }; +} + +export async function deleteProfile(projectId: string, profileId: string): Promise { + await projectAuthCheck(projectId); + + await testProfilesCollection.deleteOne({ + _id: new ObjectId(profileId), + projectId, + default: false, + }); +} + +export async function createProfile( + projectId: string, + data: { + name: string; + context: string; + mockTools: boolean; + mockPrompt?: string; + } +): Promise>> { + await projectAuthCheck(projectId); + + const doc = { + ...data, + projectId, + createdAt: new Date().toISOString(), + lastUpdatedAt: new Date().toISOString(), + }; + const result = await testProfilesCollection.insertOne(doc); + return { + ...doc, + _id: result.insertedId.toString(), + }; +} + +export async function updateProfile( + projectId: string, + profileId: string, + updates: { + name?: string; + context?: string; + mockTools?: boolean; + mockPrompt?: string; + } +): Promise { + await projectAuthCheck(projectId); + + const updateData: any = { + ...updates, + lastUpdatedAt: new Date().toISOString(), + }; + + await testProfilesCollection.updateOne( + { + _id: new ObjectId(profileId), + projectId, + }, + { + $set: updateData, + } + ); +} + +export async function listRuns( + projectId: string, + page: number = 1, + pageSize: number = 10 +): Promise<{ + runs: WithStringId>[]; + total: number; +}> { + await projectAuthCheck(projectId); + const skip = (page - 1) * pageSize; + const total = await testRunsCollection.countDocuments({ projectId }); + + const runs = await testRunsCollection + .find({ projectId }) + .sort({ startedAt: -1 }) // Sort by most recent first + .skip(skip) + .limit(pageSize) + .toArray(); + + return { + runs: runs.map(run => ({ + ...run, + _id: run._id.toString(), + })), + total, + }; +} + +export async function getRun(projectId: string, runId: string): Promise> | null> { + await projectAuthCheck(projectId); + + const run = await testRunsCollection.findOne({ + _id: new ObjectId(runId), + projectId, + }); + if (!run) { + return null; + } + const { _id, ...rest } = run; + return { + ...rest, + _id: _id.toString(), + }; +} + +export async function deleteRun(projectId: string, runId: string): Promise { + await projectAuthCheck(projectId); + + await testRunsCollection.deleteOne({ + _id: new ObjectId(runId), + projectId, + }); +} + +export async function createRun( + projectId: string, + data: { + simulationIds: string[]; + workflowId: string; + } +): Promise>> { + await projectAuthCheck(projectId); + + // Increment the testRunCounter and get the new value + const result = await projectsCollection.findOneAndUpdate( + { _id: projectId }, + { $inc: { testRunCounter: 1 } }, + { returnDocument: 'after' } + ); + + if (!result) { + throw new Error("Project not found"); + } + + const runNumber = result.testRunCounter || 1; + + const doc = { + ...data, + projectId, + name: `Run #${runNumber}`, + status: 'pending' as const, + startedAt: new Date().toISOString(), + aggregateResults: { + total: 0, + pass: 0, + fail: 0, + }, + }; + const insertResult = await testRunsCollection.insertOne(doc); + return { + ...doc, + _id: insertResult.insertedId.toString(), + }; +} + +export async function updateRun( + projectId: string, + runId: string, + updates: { + status?: 'pending' | 'running' | 'completed' | 'cancelled' | 'failed' | 'error'; + completedAt?: string; + aggregateResults?: { + total: number; + pass: number; + fail: number; + }; + } +): Promise { + await projectAuthCheck(projectId); + + const updateData: any = { + ...updates, + }; + + await testRunsCollection.updateOne( + { + _id: new ObjectId(runId), + projectId, + }, + { + $set: updateData, + } + ); +} + +export async function listResults( + projectId: string, + runId: string, + page: number = 1, + pageSize: number = 10 +): Promise<{ + results: WithStringId>[]; + total: number; +}> { + await projectAuthCheck(projectId); + const skip = (page - 1) * pageSize; + const total = await testResultsCollection.countDocuments({ projectId, runId }); + + const results = await testResultsCollection + .find({ projectId, runId }) + .skip(skip) + .limit(pageSize) + .toArray(); + + return { + results: results.map(result => ({ + ...result, + _id: result._id.toString(), + })), + total, + }; +} + +export async function getResult(projectId: string, resultId: string): Promise> | null> { + await projectAuthCheck(projectId); + + const result = await testResultsCollection.findOne({ + _id: new ObjectId(resultId), + projectId, + }); + if (!result) { + return null; + } + const { _id, ...rest } = result; + return { + ...rest, + _id: _id.toString(), + }; +} + +export async function deleteResult(projectId: string, resultId: string): Promise { + await projectAuthCheck(projectId); + + await testResultsCollection.deleteOne({ + _id: new ObjectId(resultId), + projectId, + }); +} + +export async function createResult( + projectId: string, + data: { + runId: string; + simulationId: string; + result: 'pass' | 'fail'; + details: string; + } +): Promise>> { + await projectAuthCheck(projectId); + + const doc = { + ...data, + projectId, + }; + const result = await testResultsCollection.insertOne(doc); + return { + ...doc, + _id: result.insertedId.toString(), + }; +} + +export async function updateResult( + projectId: string, + resultId: string, + updates: { + result?: 'pass' | 'fail'; + details?: string; + } +): Promise { + await projectAuthCheck(projectId); + + await testResultsCollection.updateOne( + { + _id: new ObjectId(resultId), + projectId, + }, + { + $set: updates, + } + ); +} \ No newline at end of file diff --git a/apps/rowboat/app/api/v1/[projectId]/chat/route.ts b/apps/rowboat/app/api/v1/[projectId]/chat/route.ts index 47fc1b45..8b241cbe 100644 --- a/apps/rowboat/app/api/v1/[projectId]/chat/route.ts +++ b/apps/rowboat/app/api/v1/[projectId]/chat/route.ts @@ -1,5 +1,5 @@ import { NextRequest } from "next/server"; -import { agentWorkflowsCollection, db, projectsCollection } from "../../../../lib/mongodb"; +import { agentWorkflowsCollection, db, projectsCollection, testProfilesCollection } from "../../../../lib/mongodb"; import { z } from "zod"; import { ObjectId } from "mongodb"; import { authCheck } from "../../utils"; @@ -9,6 +9,7 @@ import { getAgenticApiResponse, callClientToolWebhook, runRAGToolCall, mockToolR import { check_query_limit } from "../../../../lib/rate_limiting"; import { apiV1 } from "rowboat-shared"; import { PrefixLogger } from "../../../../lib/utils"; +import { TestProfile } from "@/app/lib/types/testing_types"; // get next turn / agent response export async function POST( @@ -68,9 +69,43 @@ export async function POST( logger.log(`Workflow ${workflowId} not found for project ${projectId}`); return Response.json({ error: "Workflow not found" }, { status: 404 }); } + + // if test profile is provided in the request, use it + let profile: z.infer = { + projectId: projectId, + name: 'Default', + createdAt: new Date().toISOString(), + lastUpdatedAt: new Date().toISOString(), + context: '', + mockTools: false, + mockPrompt: '', + }; + if (result.data.testProfileId) { + const testProfile = await testProfilesCollection.findOne({ + projectId: projectId, + _id: new ObjectId(result.data.testProfileId), + }); + if (!testProfile) { + logger.log(`Test profile ${result.data.testProfileId} not found for project ${projectId}`); + return Response.json({ error: "Test profile not found" }, { status: 404 }); + } + profile = testProfile; + } + + // if profile has a context available, overwrite the system message in the request (if there is one) + let currentMessages = reqMessages; + if (profile.context) { + // if there is a system message, overwrite it + const systemMessageIndex = reqMessages.findIndex(m => m.role === "system"); + if (systemMessageIndex !== -1) { + currentMessages[systemMessageIndex].content = profile.context; + } else { + // if there is no system message, add one + currentMessages.unshift({ role: "system", content: profile.context }); + } + } const MAX_TURNS = result.data.maxTurns ?? 3; - let currentMessages = reqMessages; let currentState: unknown = reqState ?? { last_agent_name: workflow.agents[0].name }; let turns = 0; let hasToolCalls = false; @@ -140,9 +175,9 @@ export async function POST( try { // if tool is supposed to be mocked, mock it const workflowTool = workflow.tools.find(t => t.name === toolCall.function.name); - if (workflowTool?.mockInPlayground) { + if (profile.mockTools) { logger.log(`Mocking tool call ${toolCall.function.name}`); - result = await mockToolResponse(toolCall.id, currentMessages); + result = await mockToolResponse(toolCall.id, currentMessages, profile); } else { // else run the tool call by calling the client tool webhook logger.log(`Running client tool webhook for tool ${toolCall.function.name}`); diff --git a/apps/rowboat/app/lib/components/selectors/profile-selector.tsx b/apps/rowboat/app/lib/components/selectors/profile-selector.tsx new file mode 100644 index 00000000..7bca32de --- /dev/null +++ b/apps/rowboat/app/lib/components/selectors/profile-selector.tsx @@ -0,0 +1,100 @@ +import { WithStringId } from "@/app/lib/types/types"; +import { TestProfile } from "@/app/lib/types/testing_types"; +import { useCallback, useEffect, useState } from "react"; +import { listProfiles } from "@/app/actions/testing_actions"; +import { Button, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react"; +import { z } from "zod"; + +interface ProfileSelectorProps { + projectId: string; + isOpen: boolean; + onOpenChange: (open: boolean) => void; + onSelect: (profile: WithStringId>) => void; +} + +export function ProfileSelector({ projectId, isOpen, onOpenChange, onSelect }: ProfileSelectorProps) { + const [page, setPage] = useState(1); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [profiles, setProfiles] = useState>[]>([]); + const [totalPages, setTotalPages] = useState(0); + const pageSize = 10; + + const fetchProfiles = useCallback(async (page: number) => { + setLoading(true); + setError(null); + try { + const result = await listProfiles(projectId, page, pageSize); + setProfiles(result.profiles); + setTotalPages(Math.ceil(result.total / pageSize)); + } catch (error) { + setError(`Unable to fetch profiles: ${error}`); + } finally { + setLoading(false); + } + }, [projectId]); + + useEffect(() => { + if (isOpen) { + fetchProfiles(page); + } + }, [page, isOpen, fetchProfiles]); + + return ( + + + {(onClose) => ( + <> + Select a Profile + + {loading &&
+ + Loading... +
} + {error &&
+ {error} + +
} + {!loading && !error && <> + {profiles.length === 0 &&
No profiles found
} + {profiles.length > 0 &&
+
+
Name
+
Context
+
Mock Tools
+
+ + {profiles.map((p) => ( +
{ + onSelect(p); + onClose(); + }} + > +
{p.name}
+
{p.context}
+
{p.mockTools ? "Yes" : "No"}
+
+ ))} +
} + {totalPages > 1 && } + } + + + + + + )} + + + ); +} \ No newline at end of file diff --git a/apps/rowboat/app/lib/components/selectors/scenario-selector.tsx b/apps/rowboat/app/lib/components/selectors/scenario-selector.tsx new file mode 100644 index 00000000..d52726a3 --- /dev/null +++ b/apps/rowboat/app/lib/components/selectors/scenario-selector.tsx @@ -0,0 +1,98 @@ +import { WithStringId } from "@/app/lib/types/types"; +import { TestScenario } from "@/app/lib/types/testing_types"; +import { useCallback, useEffect, useState } from "react"; +import { listScenarios } from "@/app/actions/testing_actions"; +import { Button, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react"; +import { z } from "zod"; + +interface ScenarioSelectorProps { + projectId: string; + isOpen: boolean; + onOpenChange: (open: boolean) => void; + onSelect: (scenario: WithStringId>) => void; +} + +export function ScenarioSelector({ projectId, isOpen, onOpenChange, onSelect }: ScenarioSelectorProps) { + const [page, setPage] = useState(1); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [scenarios, setScenarios] = useState>[]>([]); + const [totalPages, setTotalPages] = useState(0); + const pageSize = 10; + + const fetchScenarios = useCallback(async (page: number) => { + setLoading(true); + setError(null); + try { + const result = await listScenarios(projectId, page, pageSize); + setScenarios(result.scenarios); + setTotalPages(Math.ceil(result.total / pageSize)); + } catch (error) { + setError(`Unable to fetch scenarios: ${error}`); + } finally { + setLoading(false); + } + }, [projectId]); + + useEffect(() => { + if (isOpen) { + fetchScenarios(page); + } + }, [page, isOpen, fetchScenarios]); + + return ( + + + {(onClose) => ( + <> + Select a Scenario + + {loading &&
+ + Loading... +
} + {error &&
+ {error} + +
} + {!loading && !error && <> + {scenarios.length === 0 &&
No scenarios found
} + {scenarios.length > 0 &&
+
+
Name
+
Description
+
+ + {scenarios.map((s) => ( +
{ + onSelect(s); + onClose(); + }} + > +
{s.name}
+
{s.description}
+
+ ))} +
} + {totalPages > 1 && } + } + + + + + + )} + + + ); +} \ No newline at end of file diff --git a/apps/rowboat/app/lib/components/selectors/simulation-selector.tsx b/apps/rowboat/app/lib/components/selectors/simulation-selector.tsx new file mode 100644 index 00000000..bb5b789c --- /dev/null +++ b/apps/rowboat/app/lib/components/selectors/simulation-selector.tsx @@ -0,0 +1,140 @@ +import { WithStringId } from "@/app/lib/types/types"; +import { TestSimulation } from "@/app/lib/types/testing_types"; +import { useCallback, useEffect, useState } from "react"; +import { listSimulations } from "@/app/actions/testing_actions"; +import { Button, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter, Chip } from "@nextui-org/react"; +import { z } from "zod"; +import { RelativeTime } from "@primer/react"; + +interface SimulationSelectorProps { + projectId: string; + isOpen: boolean; + onOpenChange: (open: boolean) => void; + onSelect: (simulations: WithStringId>[]) => void; + initialSelected?: WithStringId>[]; +} + +export function SimulationSelector({ projectId, isOpen, onOpenChange, onSelect, initialSelected = [] }: SimulationSelectorProps) { + const [page, setPage] = useState(1); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [simulations, setSimulations] = useState>[]>([]); + const [totalPages, setTotalPages] = useState(0); + const [selectedSimulations, setSelectedSimulations] = useState>[]>(initialSelected); + const pageSize = 3; + + const fetchSimulations = useCallback(async (page: number) => { + setLoading(true); + setError(null); + try { + const result = await listSimulations(projectId, page, pageSize); + setSimulations(result.simulations); + setTotalPages(Math.ceil(result.total / pageSize)); + } catch (error) { + setError(`Unable to fetch simulations: ${error}`); + } finally { + setLoading(false); + } + }, [projectId]); + + useEffect(() => { + if (isOpen) { + fetchSimulations(page); + } + }, [page, isOpen, fetchSimulations]); + + const handleSelect = (simulation: WithStringId>) => { + const isSelected = selectedSimulations.some(s => s._id === simulation._id); + let newSelected; + if (isSelected) { + newSelected = selectedSimulations.filter(s => s._id !== simulation._id); + } else { + newSelected = [...selectedSimulations, simulation]; + } + setSelectedSimulations(newSelected); + onSelect(newSelected); + }; + + const handleRemove = (simulationId: string) => { + const newSelected = selectedSimulations.filter(s => s._id !== simulationId); + setSelectedSimulations(newSelected); + onSelect(newSelected); + }; + + return ( + + + {(onClose) => ( + <> + Select Simulations + + {selectedSimulations.length > 0 && ( +
+ {selectedSimulations.map((sim) => ( + handleRemove(sim._id)} + variant="flat" + className="py-1" + > + {sim.name} + + ))} +
+ )} + + {loading &&
+ + Loading... +
} + {error &&
+ {error} + +
} + {!loading && !error && <> + {simulations.length === 0 &&
No simulations found
} + {simulations.length > 0 &&
+
+
Name
+
Pass Criteria
+
Last Updated
+
+ + {simulations.map((sim) => { + const isSelected = selectedSimulations.some(s => s._id === sim._id); + return ( +
handleSelect(sim)} + > +
{sim.name}
+
{sim.passCriteria || '-'}
+
+ +
+
+ ); + })} +
} + {totalPages > 1 && } + } + + + + + + )} + + + ); +} \ No newline at end of file diff --git a/apps/rowboat/app/lib/components/selectors/workflow-selector.tsx b/apps/rowboat/app/lib/components/selectors/workflow-selector.tsx new file mode 100644 index 00000000..a914f864 --- /dev/null +++ b/apps/rowboat/app/lib/components/selectors/workflow-selector.tsx @@ -0,0 +1,106 @@ +import { WithStringId } from "@/app/lib/types/types"; +import { Workflow } from "@/app/lib/types/workflow_types"; +import { useCallback, useEffect, useState } from "react"; +import { listWorkflows } from "@/app/actions/workflow_actions"; +import { Button, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react"; +import { z } from "zod"; +import { RelativeTime } from "@primer/react"; +import { WorkflowIcon } from "../icons"; +import { PublishedBadge } from "@/app/projects/[projectId]/workflow/published_badge"; + +interface WorkflowSelectorProps { + projectId: string; + isOpen: boolean; + onOpenChange: (open: boolean) => void; + onSelect: (workflow: WithStringId>) => void; +} + +export function WorkflowSelector({ projectId, isOpen, onOpenChange, onSelect }: WorkflowSelectorProps) { + const [page, setPage] = useState(1); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [workflows, setWorkflows] = useState>[]>([]); + const [totalPages, setTotalPages] = useState(0); + const [publishedWorkflowId, setPublishedWorkflowId] = useState(null); + const pageSize = 10; + + const fetchWorkflows = useCallback(async (page: number) => { + setLoading(true); + setError(null); + try { + const result = await listWorkflows(projectId, page, pageSize); + setWorkflows(result.workflows); + setTotalPages(Math.ceil(result.total / pageSize)); + setPublishedWorkflowId(result.publishedWorkflowId); + } catch (error) { + setError(`Unable to fetch workflows: ${error}`); + } finally { + setLoading(false); + } + }, [projectId]); + + useEffect(() => { + if (isOpen) { + fetchWorkflows(page); + } + }, [page, isOpen, fetchWorkflows]); + + return ( + + + {(onClose) => ( + <> + Select a Workflow + + {loading &&
+ + Loading... +
} + {error &&
+ {error} + +
} + {!loading && !error && <> + {workflows.length === 0 &&
No workflows found
} + {workflows.length > 0 &&
+ {workflows.map((workflow) => ( +
{ + onSelect(workflow); + onClose(); + }} + > +
+
+ + {workflow.name || 'Unnamed workflow'} + {publishedWorkflowId === workflow._id && } +
+
+ Updated +
+
+
+ ))} +
} + {totalPages > 1 && } + } + + + + + + )} + + + ); +} \ No newline at end of file diff --git a/apps/rowboat/app/lib/mongodb.ts b/apps/rowboat/app/lib/mongodb.ts index 7444cc8f..5225b89a 100644 --- a/apps/rowboat/app/lib/mongodb.ts +++ b/apps/rowboat/app/lib/mongodb.ts @@ -1,5 +1,5 @@ import { MongoClient } from "mongodb"; -import { PlaygroundChat, Webpage, ChatClientId } from "./types/types"; +import { Webpage } from "./types/types"; import { Workflow } from "./types/workflow_types"; import { ApiKey } from "./types/project_types"; import { ProjectMember } from "./types/project_types"; @@ -7,7 +7,7 @@ import { Project } from "./types/project_types"; import { EmbeddingDoc } from "./types/datasource_types"; import { DataSourceDoc } from "./types/datasource_types"; import { DataSource } from "./types/datasource_types"; -import { Scenario, SimulationResult, SimulationRun, SimulationAggregateResult } from "./types/testing_types"; +import { TestScenario, TestResult, TestRun, TestProfile, TestSimulation } from "./types/testing_types"; import { z } from 'zod'; const client = new MongoClient(process.env["MONGODB_CONNECTION_STRING"] || "mongodb://localhost:27017"); @@ -20,7 +20,9 @@ export const projectsCollection = db.collection>("projec export const projectMembersCollection = db.collection>("project_members"); export const webpagesCollection = db.collection>('webpages'); export const agentWorkflowsCollection = db.collection>("agent_workflows"); -export const scenariosCollection = db.collection>("scenarios"); export const apiKeysCollection = db.collection>("api_keys"); -export const simulationRunsCollection = db.collection>("simulation_runs"); -export const simulationResultsCollection = db.collection>("simulation_results"); \ No newline at end of file +export const testScenariosCollection = db.collection>("test_scenarios"); +export const testProfilesCollection = db.collection>("test_profiles"); +export const testSimulationsCollection = db.collection>("test_simulations"); +export const testRunsCollection = db.collection>("test_runs"); +export const testResultsCollection = db.collection>("test_results"); \ No newline at end of file diff --git a/apps/rowboat/app/lib/types/project_types.ts b/apps/rowboat/app/lib/types/project_types.ts index 62e91f75..b426ed6c 100644 --- a/apps/rowboat/app/lib/types/project_types.ts +++ b/apps/rowboat/app/lib/types/project_types.ts @@ -10,6 +10,8 @@ export const Project = z.object({ webhookUrl: z.string().optional(), publishedWorkflowId: z.string().optional(), nextWorkflowNumber: z.number().optional(), + testRunCounter: z.number().default(0), + defaultTestProfileId: z.string().optional(), });export const ProjectMember = z.object({ userId: z.string(), projectId: z.string(), diff --git a/apps/rowboat/app/lib/types/testing_types.ts b/apps/rowboat/app/lib/types/testing_types.ts index 79c6f6e5..1d3036bd 100644 --- a/apps/rowboat/app/lib/types/testing_types.ts +++ b/apps/rowboat/app/lib/types/testing_types.ts @@ -1,53 +1,37 @@ import { z } from "zod"; -// Base type - -export const Scenario = z.object({ +export const TestScenario = z.object({ projectId: z.string(), name: z.string().min(1, "Name cannot be empty"), description: z.string().min(1, "Description cannot be empty"), - criteria: z.string().default(''), - context: z.string().default(''), createdAt: z.string().datetime(), lastUpdatedAt: z.string().datetime(), }); -// Relevant to new simulation features - -export const SimulationScenarioData = z.object({ - scenario: z.string(), - context: z.string().default(''), -}); - -// Legacy - -export const SimulationArticleData = z.object({ - articleUrl: z.string(), - articleTitle: z.string().default('').optional(), - articleContent: z.string().default('').optional(), -}); - -export const SimulationChatMessagesData = z.object({ - chatMessages: z.string(), -}); - -// Relevant to new simulation features - -export const SimulationData = z.union([ - SimulationScenarioData, - SimulationArticleData, - SimulationChatMessagesData -]); - -export const SimulationAggregateResult = z.object({ - total: z.number(), - pass: z.number(), - fail: z.number(), -}); - -export const SimulationRun = z.object({ +export const TestProfile = z.object({ projectId: z.string(), - scenarioIds: z.array(z.string()), + name: z.string().min(1, "Name cannot be empty"), + context: z.string(), + createdAt: z.string().datetime(), + lastUpdatedAt: z.string().datetime(), + mockTools: z.boolean(), + mockPrompt: z.string().optional(), +}); + +export const TestSimulation = z.object({ + projectId: z.string(), + name: z.string().min(1, "Name cannot be empty"), + scenarioId: z.string(), + profileId: z.string(), + passCriteria: z.string(), + createdAt: z.string().datetime(), + lastUpdatedAt: z.string().datetime(), +}); + +export const TestRun = z.object({ + projectId: z.string(), + name: z.string(), + simulationIds: z.array(z.string()), workflowId: z.string(), status: z.enum(['pending', 'running', 'completed', 'cancelled', 'failed', 'error']), startedAt: z.string(), @@ -57,12 +41,12 @@ export const SimulationRun = z.object({ pass: z.number(), fail: z.number(), }).optional(), - }); +}); -export const SimulationResult = z.object({ +export const TestResult = z.object({ projectId: z.string(), runId: z.string(), - scenarioId: z.string(), + simulationId: z.string(), result: z.union([z.literal('pass'), z.literal('fail')]), details: z.string() }); \ No newline at end of file diff --git a/apps/rowboat/app/lib/types/types.ts b/apps/rowboat/app/lib/types/types.ts index b412208b..b3f59ce9 100644 --- a/apps/rowboat/app/lib/types/types.ts +++ b/apps/rowboat/app/lib/types/types.ts @@ -1,7 +1,6 @@ import { CoreMessage, ToolCallPart } from "ai"; import { z } from "zod"; import { apiV1 } from "rowboat-shared"; -import { SimulationData } from "./testing_types"; export const PlaygroundChat = z.object({ createdAt: z.string().datetime(), @@ -9,7 +8,7 @@ export const PlaygroundChat = z.object({ title: z.string().optional(), messages: z.array(apiV1.ChatMessage), simulated: z.boolean().default(false).optional(), - simulationData: SimulationData.optional(), + simulationScenario: z.string().optional(), simulationComplete: z.boolean().default(false).optional(), agenticState: z.unknown().optional(), systemMessage: z.string().optional(), @@ -110,6 +109,7 @@ export const ApiRequest = z.object({ skipToolCalls: z.boolean().nullable().optional(), maxTurns: z.number().nullable().optional(), workflowId: z.string().nullable().optional(), + testProfileId: z.string().nullable().optional(), }); export const ApiResponse = z.object({ diff --git a/apps/rowboat/app/lib/utils.ts b/apps/rowboat/app/lib/utils.ts index cc08d626..f371eb74 100644 --- a/apps/rowboat/app/lib/utils.ts +++ b/apps/rowboat/app/lib/utils.ts @@ -18,6 +18,7 @@ import { qdrantClient } from "./qdrant"; import { EmbeddingRecord } from "./types/datasource_types"; import { ApiMessage } from "./types/types"; import { openai } from "@ai-sdk/openai"; +import { TestProfile } from "./types/testing_types"; export async function callClientToolWebhook( toolCall: z.infer['tool_calls'][number], @@ -220,19 +221,33 @@ export class PrefixLogger { } } -export async function mockToolResponse(toolId: string, messages: z.infer[]): Promise { - const prompt = ` -# Your Specific Task: -Here is a chat between a user and a customer support assistant. +export async function mockToolResponse(toolId: string, messages: z.infer[], testProfile: z.infer): Promise { + const prompt = `Given below is a chat between a user and a customer support assistant. The assistant has requested a tool call with ID {{toolID}}. -Your job is to come up with an example of the data that the tool call should return. -The current date is {{date}}. -CONVERSATION: +Your job is to come up with the data that the tool call should return. + +In order to help you mock the responses, the user has provided some contextual information, +and also some instructions on how to mock the tool call. + +>>>CHAT_HISTORY {{messages}} +<<>>CONTEXT +{{context}} +<<>>MOCK_INSTRUCTIONS +{{mockInstructions}} +<< { let tool_calls; if ('tool_calls' in m && m.role == 'assistant') { diff --git a/apps/rowboat/app/projects/[projectId]/menu.tsx b/apps/rowboat/app/projects/[projectId]/menu.tsx index 99042acd..987a86e7 100644 --- a/apps/rowboat/app/projects/[projectId]/menu.tsx +++ b/apps/rowboat/app/projects/[projectId]/menu.tsx @@ -62,11 +62,11 @@ export default function Menu({ selected={pathname.startsWith(`/projects/${projectId}/workflow`)} /> } - selected={pathname.startsWith(`/projects/${projectId}/simulation`)} + selected={pathname.startsWith(`/projects/${projectId}/test`)} /> {useDataSources && ( Simulatebeta; } @@ -25,18 +24,16 @@ export function App({ projectId, workflow, messageSubscriber, + initialTestProfile, }: { hidden?: boolean; projectId: string; workflow: z.infer; messageSubscriber?: (messages: z.infer[]) => void; + initialTestProfile: z.infer; }) { - const searchParams = useSearchParams(); - const router = useRouter(); - const initialChatId = useMemo(() => searchParams.get('chatId'), [searchParams]); - const [existingChatId, setExistingChatId] = useState(initialChatId); - const [loadingChat, setLoadingChat] = useState(false); const [counter, setCounter] = useState(0); + const [testProfile, setTestProfile] = useState>(initialTestProfile); const [chat, setChat] = useState>({ projectId, createdAt: new Date().toISOString(), @@ -45,49 +42,45 @@ export function App({ systemMessage: defaultSystemMessage, }); - const beginSimulation = useCallback((data: z.infer) => { - setExistingChatId(null); - setLoadingChat(true); + function handleTestProfileChange(profile: WithStringId>) { + setTestProfile(profile); setCounter(counter + 1); - setChat({ - projectId, - createdAt: new Date().toISOString(), - messages: [], - simulated: true, - simulationData: data, - systemMessage: 'context' in data ? data.context : '', - }); - }, [counter, projectId]); + } - useEffect(() => { - const scenarioId = localStorage.getItem('pendingScenarioId'); - if (scenarioId && projectId) { - console.log('Scenario Effect triggered:', { scenarioId, projectId }); - getScenario(projectId, scenarioId).then((scenario) => { - console.log('Scenario data received:', scenario); - beginSimulation({ - ...scenario, - systemMessage: scenario.context || '', - } as z.infer); - localStorage.removeItem('pendingScenarioId'); - }).catch(error => { - console.error('Error fetching scenario:', error); - localStorage.removeItem('pendingScenarioId'); - }); - } - }, [projectId, beginSimulation]); + // const beginSimulation = useCallback((scenario: string) => { + // setExistingChatId(null); + // setLoadingChat(true); + // setCounter(counter + 1); + // setChat({ + // projectId, + // createdAt: new Date().toISOString(), + // messages: [], + // simulated: true, + // simulationScenario: scenario, + // systemMessage: '', + // }); + // }, [counter, projectId]); + + // useEffect(() => { + // const scenarioId = localStorage.getItem('pendingScenarioId'); + // if (scenarioId && projectId) { + // console.log('Scenario Effect triggered:', { scenarioId, projectId }); + // getScenario(projectId, scenarioId).then((scenario) => { + // console.log('Scenario data received:', scenario); + // beginSimulation(scenario.description); + // localStorage.removeItem('pendingScenarioId'); + // }).catch(error => { + // console.error('Error fetching scenario:', error); + // localStorage.removeItem('pendingScenarioId'); + // }); + // } + // }, [projectId, beginSimulation]); if (hidden) { return <>; } - function handleSimulateButtonClick() { - router.push(`/projects/${projectId}/simulation`); - } - function handleNewChatButtonClick() { - setExistingChatId(null); - setLoadingChat(true); setCounter(counter + 1); setChat({ projectId, @@ -110,27 +103,18 @@ export function App({ > New chat , - } - onClick={handleSimulateButtonClick} - > - Simulate - , ]} >
- {loadingChat &&
- -
} - {!loadingChat && } + onTestProfileChange={handleTestProfileChange} + />
); diff --git a/apps/rowboat/app/projects/[projectId]/playground/chat.tsx b/apps/rowboat/app/projects/[projectId]/playground/chat.tsx index 41c2ec1b..e1fa473a 100644 --- a/apps/rowboat/app/projects/[projectId]/playground/chat.tsx +++ b/apps/rowboat/app/projects/[projectId]/playground/chat.tsx @@ -9,24 +9,28 @@ import { convertWorkflowToAgenticAPI } from "../../../lib/types/agents_api_types import { AgenticAPIChatRequest } from "../../../lib/types/agents_api_types"; import { Workflow } from "../../../lib/types/workflow_types"; import { ComposeBox } from "./compose-box"; -import { Button, Spinner } from "@nextui-org/react"; +import { Button, Spinner, Tooltip } from "@nextui-org/react"; import { apiV1 } from "rowboat-shared"; import { CopyAsJsonButton } from "./copy-as-json-button"; +import { TestProfile } from "@/app/lib/types/testing_types"; +import { ProfileSelector } from "@/app/lib/components/selectors/profile-selector"; +import { WithStringId } from "@/app/lib/types/types"; export function Chat({ chat, - initialChatId = null, projectId, workflow, messageSubscriber, + testProfile, + onTestProfileChange, }: { chat: z.infer; - initialChatId?: string | null; projectId: string; workflow: z.infer; messageSubscriber?: (messages: z.infer[]) => void; + testProfile: z.infer; + onTestProfileChange: (profile: WithStringId>) => void; }) { - const [chatId, setChatId] = useState(initialChatId); const [messages, setMessages] = useState[]>(chat.messages); const [loadingAssistantResponse, setLoadingAssistantResponse] = useState(false); const [loadingUserResponse, setLoadingUserResponse] = useState(false); @@ -34,11 +38,11 @@ export function Chat({ const [agenticState, setAgenticState] = useState(chat.agenticState || { last_agent_name: workflow.startAgent, }); - const [showCopySuccess, setShowCopySuccess] = useState(false); const [fetchResponseError, setFetchResponseError] = useState(null); const [lastAgenticRequest, setLastAgenticRequest] = useState(null); const [lastAgenticResponse, setLastAgenticResponse] = useState(null); - const [systemMessage, setSystemMessage] = useState(chat.systemMessage); + const [systemMessage, setSystemMessage] = useState(testProfile.context); + const [isProfileSelectorOpen, setIsProfileSelectorOpen] = useState(false); // collect published tool call results const toolCallResults: Record> = {}; @@ -53,7 +57,7 @@ export function Chat({ role: 'user', content: prompt, version: 'v1', - chatId: chatId ?? '', + chatId: '', createdAt: new Date().toISOString(), }]; setMessages(updatedMessages); @@ -64,7 +68,7 @@ export function Chat({ setMessages([...messages, ...results.map((result) => ({ ...result, version: 'v1' as const, - chatId: chatId ?? '', + chatId: '', createdAt: new Date().toISOString(), }))]); } @@ -97,7 +101,7 @@ export function Chat({ role: 'system', content: systemMessage || '', version: 'v1' as const, - chatId: chatId ?? '', + chatId: '', createdAt: new Date().toISOString(), }, ...messages]), state: agenticState, @@ -122,7 +126,7 @@ export function Chat({ setMessages([...messages, ...response.messages.map((message) => ({ ...message, version: 'v1' as const, - chatId: chatId ?? '', + chatId: '', createdAt: new Date().toISOString(), }))]); setAgenticState(response.state); @@ -157,14 +161,14 @@ export function Chat({ return () => { ignore = true; }; - }, [chatId, chat.simulated, messages, projectId, agenticState, workflow, fetchResponseError, systemMessage, simulationComplete]); + }, [chat.simulated, messages, projectId, agenticState, workflow, fetchResponseError, systemMessage, simulationComplete]); // simulate user turn useEffect(() => { let ignore = false; async function process() { - if (chat.simulationData === undefined) { + if (chat.simulationScenario === undefined) { return; } @@ -172,7 +176,7 @@ export function Chat({ setLoadingUserResponse(true); try { - const response = await simulateUserResponse(projectId, messages, chat.simulationData) + const response = await simulateUserResponse(projectId, messages, chat.simulationScenario) if (ignore) { return; } @@ -187,7 +191,7 @@ export function Chat({ role: 'user', content: response, version: 'v1' as const, - chatId: chatId ?? '', + chatId: '', createdAt: new Date().toISOString(), }]); setFetchResponseError(null); @@ -226,7 +230,7 @@ export function Chat({ return () => { ignore = true; }; - }, [chatId, chat.simulated, messages, projectId, simulationComplete, chat.simulationData]); + }, [chat.simulated, messages, projectId, simulationComplete, chat.simulationScenario]); // save chat on every assistant message // useEffect(() => { @@ -275,6 +279,22 @@ export function Chat({ return
+
+ + + +
+
diff --git a/apps/rowboat/app/projects/[projectId]/playground/messages.tsx b/apps/rowboat/app/projects/[projectId]/playground/messages.tsx index b7e62c0b..d03586c9 100644 --- a/apps/rowboat/app/projects/[projectId]/playground/messages.tsx +++ b/apps/rowboat/app/projects/[projectId]/playground/messages.tsx @@ -12,6 +12,7 @@ import Link from "next/link"; import { apiV1 } from "rowboat-shared"; import { EditableField } from "../../../lib/components/editable-field"; import { MessageSquareIcon, EllipsisIcon, CircleCheckIcon, ChevronsDownIcon, ChevronsRightIcon, ChevronRightIcon, ChevronDownIcon, ExternalLinkIcon, XIcon } from "lucide-react"; +import { TestProfile } from "@/app/lib/types/testing_types"; function UserMessage({ content }: { content: string }) { return
@@ -93,6 +94,7 @@ function ToolCalls({ messages, sender, workflow, + testProfile, }: { toolCalls: z.infer['tool_calls']; results: Record>; @@ -101,6 +103,7 @@ function ToolCalls({ messages: z.infer[]; sender: string | null | undefined; workflow: z.infer; + testProfile: z.infer; }) { const resultsMap: Record> = {}; @@ -123,6 +126,7 @@ function ToolCalls({ messages={messages} sender={sender} workflow={workflow} + testProfile={testProfile} /> })}
; @@ -136,6 +140,7 @@ function ToolCall({ messages, sender, workflow, + testProfile, }: { toolCall: z.infer['tool_calls'][number]; result: z.infer | undefined; @@ -144,6 +149,7 @@ function ToolCall({ messages: z.infer[]; sender: string | null | undefined; workflow: z.infer; + testProfile: z.infer; }) { let matchingWorkflowTool: z.infer | undefined; for (const tool of workflow.tools) { @@ -154,15 +160,6 @@ function ToolCall({ } switch (toolCall.function.name) { - case 'retrieve_url_info': - return ; case 'getArticleInfo': return ; } - if (matchingWorkflowTool && !matchingWorkflowTool.mockInPlayground) { + if (matchingWorkflowTool && !testProfile.mockTools) { return ; } } @@ -310,85 +308,6 @@ function GetInformationToolCall({
; } -function RetrieveUrlInfoToolCall({ - toolCall, - result: availableResult, - handleResult, - projectId, - messages, - sender, -}: { - toolCall: z.infer['tool_calls'][number]; - result: z.infer | undefined; - handleResult: (result: z.infer) => void; - projectId: string; - messages: z.infer[]; - sender: string | null | undefined; -}) { - const [result, setResult] = useState | undefined>(availableResult); - const args = JSON.parse(toolCall.function.arguments) as { url: string }; - let typedResult: z.infer | undefined; - if (result) { - typedResult = JSON.parse(result.content) as z.infer; - } - - useEffect(() => { - if (result) { - return; - } - let ignore = false; - - function process() { - // parse args - - scrapeWebpage(args.url) - .then(page => { - if (ignore) { - return; - } - const result: z.infer = { - role: 'tool', - tool_call_id: toolCall.id, - tool_name: toolCall.function.name, - content: JSON.stringify(page), - }; - setResult(result); - handleResult(result); - }); - } - process(); - - return () => { - ignore = true; - }; - }, [result, toolCall.id, toolCall.function.name, projectId, args.url, handleResult]); - - return
- {sender &&
{sender}
} -
- - -
- - {result && ( - - )} -
-
-
; -} - function TransferToAgentToolCall({ toolCall, result: availableResult, @@ -495,6 +414,7 @@ function MockToolCall({ messages, sender, autoSubmit = false, + testProfile, }: { toolCall: z.infer['tool_calls'][number]; result: z.infer | undefined; @@ -503,6 +423,7 @@ function MockToolCall({ messages: z.infer[]; sender: string | null | undefined; autoSubmit?: boolean; + testProfile: z.infer; }) { const [result, setResult] = useState | undefined>(availableResult); const [response, setResponse] = useState(''); @@ -538,7 +459,7 @@ function MockToolCall({ async function process() { setGeneratingResponse(true); - const response = await suggestToolResponse(toolCall.id, projectId, messages); + const response = await suggestToolResponse(toolCall.id, projectId, messages, testProfile); if (ignore) { return; } @@ -550,7 +471,7 @@ function MockToolCall({ return () => { ignore = true; }; - }, [result, response, toolCall.id, projectId, messages]); + }, [result, response, toolCall.id, projectId, messages, testProfile]); // auto submit if autoSubmitMockedResponse is true useEffect(() => { @@ -682,6 +603,7 @@ export function Messages({ loadingAssistantResponse, loadingUserResponse, workflow, + testProfile, onSystemMessageChange, }: { projectId: string; @@ -692,13 +614,12 @@ export function Messages({ loadingAssistantResponse: boolean; loadingUserResponse: boolean; workflow: z.infer; + testProfile: z.infer; onSystemMessageChange: (message: string) => void; }) { const messagesEndRef = useRef(null); let lastUserMessageTimestamp = 0; - const systemMessageLocked = messages.length > 0; - // scroll to bottom on new messages useEffect(() => { messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }) @@ -707,9 +628,9 @@ export function Messages({ return
{messages.map((message, index) => { if (message.role === 'assistant') { @@ -723,6 +644,7 @@ export function Messages({ messages={messages} sender={message.agenticSender} workflow={workflow} + testProfile={testProfile} />; } else { // the assistant message createdAt is an ISO string timestamp diff --git a/apps/rowboat/app/projects/[projectId]/simulation/app.tsx b/apps/rowboat/app/projects/[projectId]/simulation/app.tsx deleted file mode 100644 index c2564b76..00000000 --- a/apps/rowboat/app/projects/[projectId]/simulation/app.tsx +++ /dev/null @@ -1,459 +0,0 @@ -'use client'; - -import { useState, useEffect, useCallback } from 'react'; -import { PlusIcon, PencilIcon, XMarkIcon, EllipsisVerticalIcon, TrashIcon, ChevronRightIcon, PlayIcon, ChevronDownIcon, ChevronLeftIcon } from '@heroicons/react/24/outline'; -import { useParams, useRouter, useSearchParams } from 'next/navigation'; -import { - getScenarios, - createScenario, - updateScenario, - deleteScenario, - getRuns, - getRun, - getRunResults, - createRun, - createRunResult, - updateRunStatus, - createAggregateResult, - deleteRun, -} from '../../../actions/simulation_actions'; -import { type WithStringId } from '../../../lib/types/types'; -import { Scenario, SimulationRun, SimulationResult } from "../../../lib/types/testing_types"; -import { Workflow } from "../../../lib/types/workflow_types"; -import { z } from 'zod'; -import { SimulationResultCard, ScenarioResultCard } from './components/RunComponents'; -import { ScenarioList, ScenarioViewer } from './components/ScenarioComponents'; -import { fetchWorkflow } from '../../../actions/workflow_actions'; -import { StructuredPanel, ActionButton } from "../../../lib/components/structured-panel"; -import { - ResizableHandle, - ResizablePanel, - ResizablePanelGroup, -} from "../../../../components/ui/resizable" -import { Pagination } from "../../../lib/components/pagination"; - -type ScenarioType = WithStringId>; -type SimulationRunType = WithStringId>; -type SimulationResultType = WithStringId>; - -type SimulationReport = { - totalScenarios: number; - passedScenarios: number; - failedScenarios: number; - results: z.infer[]; - timestamp: Date; -}; - -const dummySimulator = async (scenario: ScenarioType, runId: string, projectId: string): Promise> => { - await new Promise(resolve => setTimeout(resolve, 500)); - const passed = Math.random() > 0.5; - - const result: z.infer = { - projectId: projectId, - runId: runId, - scenarioId: scenario._id, - result: passed ? 'pass' : 'fail' as const, - details: passed - ? "The bot successfully completed the conversation" - : "The bot could not handle the conversation", - }; - - await createRunResult( - projectId, - runId, - scenario._id, - result.result, - result.details - ); - - return result; -}; - -export default function SimulationApp() { - const { projectId } = useParams(); - const router = useRouter(); - const searchParams = useSearchParams(); - const [scenarios, setScenarios] = useState([]); - const [selectedScenario, setSelectedScenario] = useState(null); - const [isEditing, setIsEditing] = useState(false); - const [menuOpenScenarioId, setMenuOpenScenarioId] = useState(null); - const [isRunning, setIsRunning] = useState(false); - const [simulationReport, setSimulationReport] = useState(null); - const [expandedResults, setExpandedResults] = useState>(new Set()); - const [runs, setRuns] = useState([]); - const [activeRun, setActiveRun] = useState(null); - const [runResults, setRunResults] = useState([]); - const [isLoadingRuns, setIsLoadingRuns] = useState(true); - const [allRunResults, setAllRunResults] = useState>({}); - const [workflowVersions, setWorkflowVersions] = useState>>>({}); - const [menuOpenId, setMenuOpenIdState] = useState(null); - const runsPerPage = 10; - const currentPage = Number(searchParams.get('page')) || 1; - - const setMenuOpenId = useCallback((id: string | null) => { - setMenuOpenIdState(id); - }, []); - - // Load scenarios on mount - useEffect(() => { - if (!projectId) return; - getScenarios(projectId as string).then(setScenarios); - }, [projectId]); - - useEffect(() => { - if (menuOpenScenarioId) { - const closeMenu = () => setMenuOpenScenarioId(null); - window.addEventListener('click', closeMenu); - return () => window.removeEventListener('click', closeMenu); - } - }, [menuOpenScenarioId]); - - // Modify the fetchRuns function to also fetch results - const fetchRuns = useCallback(async () => { - if (!projectId) return; - setIsLoadingRuns(true); - try { - const runsData = await getRuns(projectId as string); - setRuns(runsData); - - // Fetch results for all runs - const resultsPromises = runsData.map(run => - getRunResults(projectId as string, run._id) - ); - const allResults = await Promise.all(resultsPromises); - - // Create a map of run ID to results - const resultsMap = runsData.reduce((acc, run, index) => ({ - ...acc, - [run._id]: allResults[index] - }), {}); - - setAllRunResults(resultsMap); - } catch (error) { - console.error('Error fetching runs:', error); - } finally { - setIsLoadingRuns(false); - } - }, [projectId]); - - // Update the useEffect hooks to include fetchRuns - useEffect(() => { - if (!projectId) return; - fetchRuns(); - }, [projectId, fetchRuns]); - - useEffect(() => { - if (!projectId || !activeRun || activeRun.status === 'completed' || activeRun.status === 'cancelled') return; - - const interval = setInterval(async () => { - try { - const updatedRun = await getRun(projectId as string, activeRun._id); - setActiveRun(updatedRun); - - if (updatedRun.status === 'completed') { - const results = await getRunResults(projectId as string, activeRun._id); - setRunResults(results); - fetchRuns(); // Refresh the runs list - } - } catch (error) { - console.error('Error polling run status:', error); - } - }, 2000); - - return () => clearInterval(interval); - }, [activeRun, projectId, fetchRuns]); - - const createNewScenario = async () => { - if (!projectId) return; - const newScenarioId = await createScenario( - projectId as string, - 'New Scenario', - '' - ); - // Refresh scenarios list - const updatedScenarios = await getScenarios(projectId as string); - setScenarios(updatedScenarios); - const newScenario = updatedScenarios.find(s => s._id === newScenarioId); - if (newScenario) { - setSelectedScenario(newScenario); - setIsEditing(true); - } - }; - - const handleUpdateScenario = async (updatedScenario: ScenarioType) => { - if (!projectId) return; - - // First verify the scenario exists and get its current state - const currentScenarios = await getScenarios(projectId as string); - const existingScenario = currentScenarios.find(s => s._id === updatedScenario._id); - - if (!existingScenario) { - console.error('Scenario not found'); - return; - } - - // Only update the specific fields that have changed - await updateScenario( - projectId as string, - updatedScenario._id, - { - name: updatedScenario.name, - description: updatedScenario.description, - criteria: updatedScenario.criteria, - context: updatedScenario.context, - } - ); - - // Just refresh the scenarios list without setting selected scenario - const updatedScenarios = await getScenarios(projectId as string); - setScenarios(updatedScenarios); - setIsEditing(false); - }; - - const handleCloseScenario = () => { - setSelectedScenario(null); - setIsEditing(false); - }; - - const handleDeleteScenario = async (scenarioId: string) => { - if (!projectId) return; - await deleteScenario(projectId as string, scenarioId); - const updatedScenarios = await getScenarios(projectId as string); - setScenarios(updatedScenarios); - if (selectedScenario?._id === scenarioId) { - setSelectedScenario(null); - setIsEditing(false); - } - setMenuOpenScenarioId(null); - }; - - const runAllScenarios = async () => { - if (!projectId) return; - setIsRunning(true); - setSimulationReport(null); - - try { - // Get workflowId from localStorage - const workflowId = localStorage.getItem(`lastWorkflowId_${projectId}`); - if (!workflowId) { - throw new Error('No workflow selected. Please select a workflow first.'); - } - - // First verify the workflow exists before creating the run - let workflow; - try { - workflow = await fetchWorkflow(projectId as string, workflowId); - } catch (error) { - // If workflow doesn't exist, clear localStorage and throw error - localStorage.removeItem(`lastWorkflowId_${projectId}`); - throw new Error('Selected workflow no longer exists. Please select a new workflow.'); - } - - const newRun = await createRun( - projectId as string, - scenarios.map(s => s._id), - workflowId - ); - setActiveRun(newRun); - - // Store workflow version - setWorkflowVersions(prev => ({ - ...prev, - [workflowId]: workflow - })); - - const shouldMock = process.env.NEXT_PUBLIC_MOCK_SIMULATION_RESULTS === 'true'; - - if (shouldMock) { - console.log('Using mock simulation...'); - - await updateRunStatus(projectId as string, newRun._id, 'running'); - - // Run all scenarios and collect results - const mockResults = await Promise.all( - scenarios.map(scenario => - dummySimulator(scenario, newRun._id, projectId as string) - ) - ); - - // Calculate and store aggregate results before marking as complete - const total = scenarios.length; - const pass = mockResults.filter(r => r.result === 'pass').length; - const fail = mockResults.filter(r => r.result === 'fail').length; - - await createAggregateResult( - projectId as string, - newRun._id, - total, - pass, - fail - ); - - await updateRunStatus( - projectId as string, - newRun._id, - 'completed', - new Date().toISOString() - ); - - const results = await getRunResults(projectId as string, newRun._id); - setRunResults(results); - - const updatedRun = await getRun(projectId as string, newRun._id); - setActiveRun(updatedRun); - } - - await fetchRuns(); - } catch (error) { - console.error('Error starting scenarios:', error); - alert(error instanceof Error ? error.message : 'An error occurred while starting scenarios'); - } finally { - setIsRunning(false); - } - }; - - const runSingleScenario = (scenario: ScenarioType) => { - // Store scenario ID in localStorage instead of URL parameter - localStorage.setItem('pendingScenarioId', scenario._id); - // Navigate to the playground without query parameter - router.push(`/projects/${projectId}/workflow`); - setMenuOpenScenarioId(null); - }; - - // Update the workflow versions fetching effect - useEffect(() => { - if (!projectId || !runs.length) return; - - const fetchWorkflowVersions = async () => { - const workflowIds = Array.from(new Set(runs.map(run => run.workflowId))); - const versions: Record>> = {}; - - for (const workflowId of workflowIds) { - try { - const workflow = await fetchWorkflow(projectId as string, workflowId); - versions[workflowId] = workflow; - } catch (error) { - console.error(`Error fetching workflow ${workflowId}:`, error); - // Add a placeholder for deleted/invalid workflows - versions[workflowId] = { - _id: workflowId, - name: "Deleted/Invalid Workflow", - projectId: projectId as string, - agents: [], - prompts: [], - tools: [], - startAgent: "", - createdAt: new Date().toISOString(), - lastUpdatedAt: new Date().toISOString(), - }; - } - } - - setWorkflowVersions(versions); - }; - - fetchWorkflowVersions(); - }, [projectId, runs]); - - const handleCancelRun = async (runId: string) => { - if (!projectId) return; - try { - await updateRunStatus(projectId as string, runId, 'cancelled'); - await fetchRuns(); // Refresh the runs list - } catch (error) { - console.error('Error cancelling run:', error); - } - }; - - const handleDeleteRun = async (runId: string) => { - if (!projectId) return; - try { - await deleteRun(projectId as string, runId); - await fetchRuns(); // Refresh the runs list - } catch (error) { - console.error('Error deleting run:', error); - } - }; - - const indexOfLastRun = currentPage * runsPerPage; - const indexOfFirstRun = indexOfLastRun - runsPerPage; - const currentRuns = runs.slice(indexOfFirstRun, indexOfLastRun); - const totalPages = Math.ceil(runs.length / runsPerPage); - - return ( - - - setSelectedScenario(scenarios.find(s => s._id === id) ?? null)} - onAdd={createNewScenario} - onRunScenario={(id) => { - const scenario = scenarios.find(s => s._id === id); - if (scenario) runSingleScenario(scenario); - }} - onDeleteScenario={(id) => handleDeleteScenario(id)} - /> - - - - {selectedScenario ? ( - - ) : ( - void runAllScenarios()} - disabled={isRunning} - icon={} - primary - > - Run All Scenarios - - ]} - > -
- {/* Runs list */} - {isLoadingRuns ? ( -
Loading runs...
- ) : ( -
- {currentRuns.map((run) => ( - - ))} -
- )} - - {/* Pagination */} - {runs.length > runsPerPage && ( -
- -
- )} -
- - )} - - - ); -} diff --git a/apps/rowboat/app/projects/[projectId]/simulation/components/RunComponents.tsx b/apps/rowboat/app/projects/[projectId]/simulation/components/RunComponents.tsx deleted file mode 100644 index fbaef1d9..00000000 --- a/apps/rowboat/app/projects/[projectId]/simulation/components/RunComponents.tsx +++ /dev/null @@ -1,399 +0,0 @@ -'use client'; - -import { useState, useEffect } from 'react'; -import { ChevronDownIcon, ChevronRightIcon, NoSymbolIcon, EllipsisVerticalIcon, ArrowDownTrayIcon, TrashIcon } from '@heroicons/react/24/outline'; -import { WithStringId } from '../../../../lib/types/types'; -import { Scenario, SimulationRun, SimulationResult, SimulationAggregateResult } from "../../../../lib/types/testing_types"; -import { z } from 'zod'; -import { Workflow } from "../../../../lib/types/workflow_types"; -import { clsx } from 'clsx'; - -type ScenarioType = WithStringId>; -type SimulationRunType = WithStringId>; -type SimulationResultType = WithStringId>; - -interface SimulationResultCardProps { - run: SimulationRunType; - results: SimulationResultType[]; - scenarios: ScenarioType[]; - workflow?: WithStringId>; - onCancelRun?: (runId: string) => void; - onDeleteRun?: (runId: string) => Promise; - menuOpenId: string | null; - setMenuOpenId: (id: string | null) => void; -} - -export const SimulationResultCard = ({ run, results, scenarios, workflow, onCancelRun, onDeleteRun, menuOpenId, setMenuOpenId }: SimulationResultCardProps) => { - const [isExpanded, setIsExpanded] = useState(false); - const [expandedScenarios, setExpandedScenarios] = useState>(new Set()); - const [showDeleteConfirm, setShowDeleteConfirm] = useState(false); - - const totalScenarios = run.aggregateResults?.total ?? run.scenarioIds.length; - const passedScenarios = run.aggregateResults?.pass ?? 0; - const failedScenarios = run.aggregateResults?.fail ?? 0; - - const getStatusClass = (status: string) => { - const baseClass = "w-[110px] px-3 py-1 rounded text-xs text-center uppercase font-semibold inline-block"; - switch (status) { - case 'completed': - case 'pass': - return `${baseClass} bg-green-50 dark:bg-green-900/20 text-green-800 dark:text-green-400`; - case 'failed': - case 'fail': - return `${baseClass} bg-red-50 dark:bg-red-900/20 text-red-800 dark:text-red-400`; - case 'error': - return `${baseClass} bg-orange-50 dark:bg-orange-900/20 text-orange-800 dark:text-orange-400`; - case 'cancelled': - return `${baseClass} bg-gray-50 dark:bg-neutral-800 text-gray-800 dark:text-neutral-400`; - case 'running': - case 'pending': - default: - return `${baseClass} bg-yellow-50 dark:bg-yellow-900/20 text-yellow-800 dark:text-yellow-400`; - } - }; - - const formatMainTitle = (date: string) => { - return `Run from ${new Date(date).toLocaleString('en-US', { - month: 'numeric', - day: 'numeric', - year: 'numeric', - hour: 'numeric', - minute: '2-digit', - second: '2-digit', - hour12: true - })}`; - }; - - const formatDateTime = (date: string) => { - return new Date(date).toLocaleString('en-US', { - month: 'numeric', - day: 'numeric', - year: 'numeric', - hour: 'numeric', - minute: '2-digit', - second: '2-digit', - hour12: true - }); - }; - - const getDuration = () => { - if (!run.completedAt) return 'In Progress'; - const start = new Date(run.startedAt); - const end = new Date(run.completedAt); - const diff = end.getTime() - start.getTime(); - return `${(diff / 1000).toFixed(1)}s`; - }; - - const toggleScenario = (scenarioId: string, e: React.MouseEvent) => { - e.stopPropagation(); // Prevent triggering parent's onClick - setExpandedScenarios(prev => { - const newSet = new Set(prev); - if (newSet.has(scenarioId)) { - newSet.delete(scenarioId); - } else { - newSet.add(scenarioId); - } - return newSet; - }); - }; - - useEffect(() => { - if (menuOpenId) { - const closeMenu = () => setMenuOpenId(null); - window.addEventListener('click', closeMenu); - return () => window.removeEventListener('click', closeMenu); - } - }, [menuOpenId, setMenuOpenId]); - - return ( -
-
setIsExpanded(!isExpanded)} - > -
- {isExpanded ? ( - - ) : ( - - )} -
- {formatMainTitle(run.startedAt)} -
-
-
- - {run.status} - -
- - - {menuOpenId === run._id && ( -
-
- {(run.status === 'running' || run.status === 'pending') && onCancelRun && ( - - )} - - -
-
- )} -
-
-
- - {isExpanded && ( -
- {run.status === 'error' ? ( -
- Your simulation could not be completed. Please run a new simulation again. -
- ) : ( - <> - {/* Workflow and timing information in a grid */} -
- {workflow && ( -
-
Workflow Version
-
{workflow.name}
-
- )} -
-
Completed
-
- {run.completedAt ? formatDateTime(run.completedAt) : 'Not completed'} -
-
-
-
Duration
-
{getDuration()}
-
-
- - {/* Results statistics */} -
-
-
Total Scenarios
-
{totalScenarios}
-
-
-
Passed
-
{passedScenarios}
-
-
-
Failed
-
{failedScenarios}
-
-
- -
- {run.scenarioIds.map(scenarioId => { - const scenario = scenarios.find(s => s._id === scenarioId); - const result = results.find(r => r.scenarioId === scenarioId); - const isScenarioExpanded = expandedScenarios.has(scenarioId); - - return scenario && ( -
-
toggleScenario(scenarioId, e)} - > -
- {isScenarioExpanded ? ( - - ) : ( - - )} - {scenario.name} -
- {result && ( - - {result.result} - - )} -
- - {isScenarioExpanded && ( -
-
-
Description
-
- {scenario.description} -
-
-
-
Criteria
-
- {scenario.criteria || 'No criteria specified'} -
-
-
-
Context
-
- {scenario.context || 'No context provided'} -
-
- {result && ( -
-
Result Details
-
- {result.details} -
-
- )} -
- )} -
- ); - })} -
- - )} -
- )} - - {showDeleteConfirm && ( -
-
-
-

- Are you sure you want to delete this run? -

-
- - -
-
-
-
- )} -
- ); -}; - -interface ScenarioResultCardProps { - scenario: ScenarioType; - result?: SimulationResultType; -} - -export const ScenarioResultCard = ({ scenario, result }: ScenarioResultCardProps) => { - const [isExpanded, setIsExpanded] = useState(false); - - return ( -
-
setIsExpanded(!isExpanded)} - > -
- {isExpanded ? ( - - ) : ( - - )} - {scenario.name} -
- {result && ( - - {result.result} - - )} -
- - {isExpanded && ( -
-
-
Description
-
{scenario.description}
-
-
-
Criteria
-
{scenario.criteria}
-
-
-
Context
-
{scenario.context}
-
- {result && ( -
-
Result Details
-
{result.details}
-
- )} -
- )} -
- ); -}; \ No newline at end of file diff --git a/apps/rowboat/app/projects/[projectId]/simulation/components/ScenarioComponents.tsx b/apps/rowboat/app/projects/[projectId]/simulation/components/ScenarioComponents.tsx deleted file mode 100644 index a56eea9e..00000000 --- a/apps/rowboat/app/projects/[projectId]/simulation/components/ScenarioComponents.tsx +++ /dev/null @@ -1,204 +0,0 @@ -'use client'; - -import { useState, useEffect, useCallback } from 'react'; -import { Save, EllipsisVerticalIcon, PlayIcon, TrashIcon, X } from "lucide-react"; -import { WithStringId } from '../../../../lib/types/types'; -import { Scenario } from "../../../../lib/types/testing_types"; -import { z } from 'zod'; -import { EditableField } from '../../../../lib/components/editable-field'; -import { FormSection } from '../../../../lib/components/form-section'; -import { StructuredPanel, ActionButton } from "../../../../lib/components/structured-panel"; -import clsx from "clsx"; -import { Dropdown, DropdownItem, DropdownTrigger, DropdownMenu } from "@nextui-org/react"; -import { SectionHeader, ListItem } from "../../../../lib/components/structured-list"; - -type ScenarioType = WithStringId>; - -interface ScenarioViewerProps { - scenario: ScenarioType; - onSave: (scenario: ScenarioType) => void; - onClose: () => void; -} - -export function ScenarioViewer({ scenario, onSave, onClose }: ScenarioViewerProps) { - const [editedScenario, setEditedScenario] = useState(scenario); - const [isDirty, setIsDirty] = useState(false); - const [nameError, setNameError] = useState(null); - - useEffect(() => { - setEditedScenario(scenario); - setIsDirty(false); - }, [scenario]); - - const handleChange = useCallback((field: keyof ScenarioType, value: string) => { - if (field === 'name') { - setNameError(value.trim() ? null : 'Name is required'); - } - setEditedScenario(prev => ({ - ...prev, - [field]: value, - })); - setIsDirty(true); - }, []); - - const handleSave = useCallback(() => { - if (!editedScenario.name.trim()) { - setNameError('Name is required'); - return; - } - onSave(editedScenario); - onClose(); - }, [editedScenario, onSave, onClose]); - - return ( - } - primary - > - Save - - ), - } - > - Close - - ].filter(Boolean)} - > -
- - handleChange('name', value)} - multiline={false} - className="w-full" - showSaveButton={false} - placeholder="Enter an identifiable scenario name" - error={nameError} - /> - - - - handleChange('description', value)} - multiline={true} - className="w-full" - showSaveButton={false} - placeholder="Describe the user scenario to be simulated" - /> - - - - handleChange('criteria', value)} - multiline={true} - className="w-full" - showSaveButton={false} - placeholder="Enter success criteria for this scenario to pass in a simulation" - /> - - - - handleChange('context', value)} - multiline={true} - className="w-full" - showSaveButton={false} - placeholder="Provide context about the user to the assistant at the start of chat" - /> - -
- - ); -} - -function ScenarioDropdown({ - name, - onRun, - onDelete, -}: { - name: string; - onRun: () => void; - onDelete: () => void; -}) { - return ( - - - - - { - if (key === 'run') onRun(); - if (key === 'delete') onDelete(); - }} - > - } - > - Run scenario - - } - > - Delete - - - - ); -} - -export function ScenarioList({ - scenarios, - selectedId, - onSelect, - onAdd, - onRunScenario, - onDeleteScenario, -}: { - scenarios: ScenarioType[]; - selectedId: string | null; - onSelect: (id: string) => void; - onAdd: () => void; - onRunScenario: (id: string) => void; - onDeleteScenario: (id: string) => void; -}) { - return ( - -
- - {scenarios.map((scenario) => ( - onSelect(scenario._id)} - rightElement={ - onRunScenario(scenario._id)} - onDelete={() => onDeleteScenario(scenario._id)} - /> - } - /> - ))} -
- - ); -} \ No newline at end of file diff --git a/apps/rowboat/app/projects/[projectId]/simulation/page.tsx b/apps/rowboat/app/projects/[projectId]/simulation/page.tsx deleted file mode 100644 index 63c25119..00000000 --- a/apps/rowboat/app/projects/[projectId]/simulation/page.tsx +++ /dev/null @@ -1,10 +0,0 @@ -import { Metadata } from "next"; -import SimulationApp from "./app"; - -export const metadata: Metadata = { - title: "Project simulation", -}; - -export default function SimulationPage() { - return ; -} \ No newline at end of file diff --git a/apps/rowboat/app/projects/[projectId]/test/[[...slug]]/app.tsx b/apps/rowboat/app/projects/[projectId]/test/[[...slug]]/app.tsx new file mode 100644 index 00000000..06aa1475 --- /dev/null +++ b/apps/rowboat/app/projects/[projectId]/test/[[...slug]]/app.tsx @@ -0,0 +1,68 @@ +"use client"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { ScenariosApp } from "./scenarios_app"; +import { ProfilesApp } from "./profiles_app"; +import { SimulationsApp } from "./simulations_app"; +import { cn } from "@/lib/utils"; +import { usePathname } from "next/navigation"; +import { RunsApp } from "./runs_app"; + +export function App({ + projectId, + slug +}: { + projectId: string, + slug?: string[] +}) { + const router = useRouter(); + const pathname = usePathname(); + let selection: "scenarios" | "profiles" | "criteria" | "simulations" | "runs" = "runs"; + if (!slug || slug.length === 0) { + router.push(`/projects/${projectId}/test/runs`); + } else if (slug[0] === "scenarios") { + selection = "scenarios"; + } else if (slug[0] === "profiles") { + selection = "profiles"; + } else if (slug[0] === "criteria") { + selection = "criteria"; + } else if (slug[0] === "simulations") { + selection = "simulations"; + } else if (slug[0] === "runs") { + selection = "runs"; + } + let innerSlug: string[] = []; + if (slug && slug.length > 1) { + innerSlug = slug.slice(1); + } + + const menuItems = [ + { label: "Scenarios", href: `/projects/${projectId}/test/scenarios` }, + { label: "Profiles", href: `/projects/${projectId}/test/profiles` }, + { label: "Simulations", href: `/projects/${projectId}/test/simulations` }, + { label: "Test Runs", href: `/projects/${projectId}/test/runs` }, + ]; + + return
+
+
    + {menuItems.map((item) => ( +
  • + {item.label} +
    • + ))} +
+
+
+ {selection === "scenarios" && } + {selection === "profiles" && } + {selection === "simulations" && } + {selection === "runs" && } +
+
; +} diff --git a/apps/rowboat/app/projects/[projectId]/test/[[...slug]]/page.tsx b/apps/rowboat/app/projects/[projectId]/test/[[...slug]]/page.tsx new file mode 100644 index 00000000..f6f323bd --- /dev/null +++ b/apps/rowboat/app/projects/[projectId]/test/[[...slug]]/page.tsx @@ -0,0 +1,8 @@ +import { App } from "./app"; + +export default function Page({ params }: { params: { projectId: string, slug?: string[] } }) { + return ; +} \ No newline at end of file diff --git a/apps/rowboat/app/projects/[projectId]/test/[[...slug]]/profiles_app.tsx b/apps/rowboat/app/projects/[projectId]/test/[[...slug]]/profiles_app.tsx new file mode 100644 index 00000000..5001f164 --- /dev/null +++ b/apps/rowboat/app/projects/[projectId]/test/[[...slug]]/profiles_app.tsx @@ -0,0 +1,535 @@ +import Link from "next/link"; +import { WithStringId } from "@/app/lib/types/types"; +import { TestProfile } from "@/app/lib/types/testing_types"; +import { useEffect, useState, useRef } from "react"; +import { createProfile, getProfile, listProfiles, updateProfile, deleteProfile, setDefaultProfile } from "@/app/actions/testing_actions"; +import { Button, Input, Pagination, Spinner, Switch, Textarea, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter, Tooltip } from "@nextui-org/react"; +import { useRouter, useSearchParams } from "next/navigation"; +import { z } from "zod"; +import { PlusIcon, ArrowLeftIcon, StarIcon } from "lucide-react"; +import { FormStatusButton } from "@/app/lib/components/form-status-button"; +import { RelativeTime } from "@primer/react" +import { getProjectConfig } from "@/app/actions/project_actions"; + +function EditProfile({ + projectId, + profileId, +}: { + projectId: string, + profileId: string, +}) { + const router = useRouter(); + const [profile, setProfile] = useState> | null>(null); + const [loading, setLoading] = useState(true); + const [mockTools, setMockTools] = useState(false); + const [error, setError] = useState(null); + const formRef = useRef(null); + + useEffect(() => { + async function fetchProfile() { + setError(null); + try { + const profile = await getProfile(projectId, profileId); + setProfile(profile); + setMockTools(profile?.mockTools || false); + } catch (error) { + setError(`Unable to fetch profile: ${error}`); + } finally { + setLoading(false); + } + } + fetchProfile(); + }, [profileId, projectId]); + + async function handleSubmit(formData: FormData) { + setError(null); + try { + const name = formData.get("name") as string; + const context = formData.get("context") as string; + const mockPrompt = formData.get("mockPrompt") as string; + await updateProfile(projectId, profileId, { + name, + context, + mockTools, + mockPrompt: mockPrompt || undefined + }); + router.push(`/projects/${projectId}/test/profiles/${profileId}`); + } catch (error) { + setError(`Unable to update profile: ${error}`); + } + } + + return
+

Edit Profile

+ {loading &&
+ + Loading... +
} + {error &&
+ {error} + +
} + {!loading && profile && ( +
+ +