apunkt/rowboat
1
0
Fork 0
mirror of https://github.com/rowboatlabs/rowboat.git synced 2026-05-10 15:52:38 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add test profiles and overhaul testing

Browse source
This commit is contained in:
ramnique 2025-02-27 23:42:04 +05:30
parent 8c1b5346f3
commit 768c5749a0
30 changed files with 3473 additions and 1644 deletions
Download patch file Download diff file Expand all files Collapse all files

72
apps/rowboat/app/actions/actions.ts
View file

@ -6,7 +6,6 @@ import { EmbeddingRecord } from "../lib/types/datasource_types";
import { WebpageCrawlResponse } from "../lib/types/tool_types";
import { GetInformationToolResult } from "../lib/types/tool_types";
import { EmbeddingDoc } from "../lib/types/datasource_types";
import { SimulationData } from "../lib/types/testing_types";
import { generateObject, generateText, embed } from "ai";
import { dataSourceDocsCollection, dataSourcesCollection, embeddingsCollection, webpagesCollection } from "../lib/mongodb";
import { z } from 'zod';
@ -21,6 +20,7 @@ import { QueryLimitError } from "../lib/client_utils";
import { projectAuthCheck } from "./project_actions";
import { qdrantClient } from "../lib/qdrant";
import { ObjectId } from "mongodb";
import { TestProfile } from "../lib/types/testing_types";
const crawler = new FirecrawlApp({ apiKey: process.env.FIRECRAWL_API_KEY || '' });
@ -99,13 +99,13 @@ export async function getAssistantResponse(
};
}
export async function suggestToolResponse(toolId: string, projectId: string, messages: z.infer<typeof apiV1.ChatMessage>[]): Promise<string> {
export async function suggestToolResponse(toolId: string, projectId: string, messages: z.infer<typeof apiV1.ChatMessage>[], testProfile: z.infer<typeof TestProfile>): Promise<string> {
await projectAuthCheck(projectId);
if (!await check_query_limit(projectId)) {
throw new QueryLimitError();
}
return await mockToolResponse(toolId, messages);
return await mockToolResponse(toolId, messages, testProfile);
}
export async function getInformationTool(
@ -123,39 +123,13 @@ export async function getInformationTool(
export async function simulateUserResponse(
projectId: string,
messages: z.infer<typeof apiV1.ChatMessage>[],
simulationData: z.infer<typeof SimulationData>
scenario: string,
): Promise<string> {
await projectAuthCheck(projectId);
if (!await check_query_limit(projectId)) {
throw new QueryLimitError();
}
const articlePrompt = `
# Your Specific Task:
## Context:
Here is a help article:
Content:
<START_ARTICLE_CONTENT>
Title: {{title}}
{{content}}
<END_ARTICLE_CONTENT>
## Task definition:
Pretend to be a user reaching out to customer support. Chat with the
customer support assistant, assuming your issue or query is from this article.
Ask follow-up questions and make it real-world like. Don't do dummy
conversations. Your conversation should be a maximum of 5 user turns.
As output, simply provide your (user) turn of conversation.
After you are done with the chat, keep replying with a single word EXIT
in all capitals.
`;
const scenarioPrompt = `
# Your Specific Task:
@ -181,30 +155,6 @@ After you are done with the chat, keep replying with a single word EXIT
in all capitals.
`;
const previousChatPrompt = `
# Your Specific Task:
## Context:
Here is a chat between a user and a customer support assistant:
Chat:
<PREVIOUS_CHAT>
{{messages}}
<END_PREVIOUS_CHAT>
## Task definition:
Pretend to be a user reaching out to customer support. Chat with the
customer support assistant, assuming your issue based on this previous chat.
Ask follow-up questions and make it real-world like. Don't do dummy
conversations. Your conversation should be a maximum of 5 user turns.
As output, simply provide your (user) turn of conversation.
After you are done with the chat, keep replying with a single word EXIT
in all capitals.
`;
await projectAuthCheck(projectId);
// flip message assistant / user message
@ -219,19 +169,9 @@ in all capitals.
// simulate user call
let prompt;
if ('articleUrl' in simulationData) {
prompt = articlePrompt
.replace('{{title}}', simulationData.articleTitle || '')
.replace('{{content}}', simulationData.articleContent || '');
}
if ('scenario' in simulationData) {
prompt = scenarioPrompt
.replace('{{scenario}}', simulationData.scenario);
}
if ('chatMessages' in simulationData) {
prompt = previousChatPrompt
.replace('{{messages}}', simulationData.chatMessages);
}
.replace('{{scenario}}', scenario);
const { text } = await generateText({
model: openai("gpt-4o"),
system: prompt || '',

19
apps/rowboat/app/actions/project_actions.ts
View file

@ -1,7 +1,7 @@
'use server';
import { redirect } from "next/navigation";
import { ObjectId } from "mongodb";
import { dataSourcesCollection, embeddingsCollection, projectsCollection, agentWorkflowsCollection, scenariosCollection, projectMembersCollection, apiKeysCollection, dataSourceDocsCollection } from "../lib/mongodb";
import { dataSourcesCollection, embeddingsCollection, projectsCollection, agentWorkflowsCollection, testScenariosCollection, projectMembersCollection, apiKeysCollection, dataSourceDocsCollection, testProfilesCollection } from "../lib/mongodb";
import { z } from 'zod';
import crypto from 'crypto';
import { revalidatePath } from "next/cache";
@ -41,6 +41,7 @@ export async function createProject(formData: FormData) {
const projectId = crypto.randomUUID();
const chatClientId = crypto.randomBytes(16).toString('base64url');
const secret = crypto.randomBytes(32).toString('hex');
const defaultTestProfileId = new ObjectId();
// create project
await projectsCollection.insertOne({
@ -52,12 +53,13 @@ export async function createProject(formData: FormData) {
chatClientId,
secret,
nextWorkflowNumber: 1,
testRunCounter: 0,
defaultTestProfileId: defaultTestProfileId.toString(),
});
// add first workflow version
const { agents, prompts, tools, startAgent } = templates[templateKey];
await agentWorkflowsCollection.insertOne({
_id: new ObjectId(),
projectId,
agents,
prompts,
@ -68,6 +70,17 @@ export async function createProject(formData: FormData) {
name: `Version 1`,
});
// add default test profile
await testProfilesCollection.insertOne({
_id: defaultTestProfileId,
projectId,
name: "Default",
context: "",
mockTools: false,
createdAt: (new Date()).toISOString(),
lastUpdatedAt: (new Date()).toISOString(),
});
// add user to project
await projectMembersCollection.insertOne({
userId: user.sub,
@ -198,7 +211,7 @@ export async function deleteProject(projectId: string) {
});
// delete scenarios
await scenariosCollection.deleteMany({
await testScenariosCollection.deleteMany({
projectId,
});

270
apps/rowboat/app/actions/simulation_actions.ts
View file

@ -1,270 +0,0 @@
'use server';
import { ObjectId } from "mongodb";
import { scenariosCollection, simulationRunsCollection, simulationResultsCollection } from "../lib/mongodb";
import { z } from 'zod';
import { projectAuthCheck } from "./project_actions";
import { type WithStringId } from "../lib/types/types";
import { Scenario, SimulationRun, SimulationResult, SimulationAggregateResult } from "../lib/types/testing_types";
import { SimulationScenarioData } from "../lib/types/testing_types";
export async function getScenarios(projectId: string): Promise<WithStringId<z.infer<typeof Scenario>>[]> {
await projectAuthCheck(projectId);
const scenarios = await scenariosCollection.find({ projectId }).toArray();
return scenarios.map(s => ({
...s,
_id: s._id.toString(),
}));
}
export async function getScenario(projectId: string, scenarioId: string): Promise<WithStringId<z.infer<typeof SimulationScenarioData>>> {
await projectAuthCheck(projectId);
// fetch scenario
const scenario = await scenariosCollection.findOne({
_id: new ObjectId(scenarioId),
projectId,
});
if (!scenario) {
throw new Error('Scenario not found');
}
const { _id, description, ...rest } = scenario;
return {
...rest,
_id: _id.toString(),
scenario: description,
};
}
export async function createScenario(projectId: string, name: string, description: string): Promise<string> {
await projectAuthCheck(projectId);
const now = new Date().toISOString();
const result = await scenariosCollection.insertOne({
projectId,
name,
description,
context: '',
criteria: '',
lastUpdatedAt: now,
createdAt: now,
});
return result.insertedId.toString();
}
export async function updateScenario(
projectId: string,
scenarioId: string,
updates: {
name?: string;
description?: string;
context?: string;
criteria?: string;
}
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: any = {
...updates,
lastUpdatedAt: new Date().toISOString(),
};
await scenariosCollection.updateOne(
{
_id: new ObjectId(scenarioId),
projectId,
},
{
$set: updateData,
}
);
}
export async function deleteScenario(projectId: string, scenarioId: string): Promise<void> {
await projectAuthCheck(projectId);
await scenariosCollection.deleteOne({
_id: new ObjectId(scenarioId),
projectId,
});
}
export async function getRuns(projectId: string): Promise<WithStringId<z.infer<typeof SimulationRun>>[]> {
await projectAuthCheck(projectId);
const runs = await simulationRunsCollection
.find({ projectId })
.sort({ startedAt: -1 }) // Most recent first
.toArray();
return runs.map(run => ({
...run,
_id: run._id.toString(),
}));
}
export async function getRun(projectId: string, runId: string): Promise<WithStringId<z.infer<typeof SimulationRun>>> {
await projectAuthCheck(projectId);
const run = await simulationRunsCollection.findOne({
_id: new ObjectId(runId),
projectId,
});
if (!run) {
throw new Error('Run not found');
}
return {
...run,
_id: run._id.toString(),
};
}
export async function createRun(
projectId: string,
scenarioIds: string[],
workflowId: string
): Promise<WithStringId<z.infer<typeof SimulationRun>>> {
await projectAuthCheck(projectId);
const run = {
projectId,
status: 'pending' as const,
scenarioIds,
workflowId,
startedAt: new Date().toISOString(),
};
const result = await simulationRunsCollection.insertOne(run);
return {
...run,
_id: result.insertedId.toString(),
};
}
export async function updateRunStatus(
projectId: string,
runId: string,
status: z.infer<typeof SimulationRun>['status'],
completedAt?: string
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: Partial<z.infer<typeof SimulationRun>> = {
status,
};
if (completedAt) {
updateData.completedAt = completedAt;
}
await simulationRunsCollection.updateOne(
{
_id: new ObjectId(runId),
projectId,
},
{
$set: updateData,
}
);
}
export async function getRunResults(
projectId: string,
runId: string
): Promise<WithStringId<z.infer<typeof SimulationResult>>[]> {
await projectAuthCheck(projectId);
const results = await simulationResultsCollection
.find({
runId,
projectId,
})
.toArray();
return results.map(result => ({
...result,
_id: result._id.toString(),
}));
}
export async function createRunResult(
projectId: string,
runId: string,
scenarioId: string,
result: z.infer<typeof SimulationResult>['result'],
details: string
): Promise<string> {
await projectAuthCheck(projectId);
const resultDoc = {
projectId,
runId,
scenarioId,
result,
details,
};
const insertResult = await simulationResultsCollection.insertOne(resultDoc);
return insertResult.insertedId.toString();
}
export async function createAggregateResult(
projectId: string,
runId: string,
total: number,
pass: number,
fail: number
): Promise<void> {
await projectAuthCheck(projectId);
await simulationRunsCollection.updateOne(
{ _id: new ObjectId(runId), projectId },
{
$set: {
aggregateResults: { total, pass, fail }
}
}
);
}
export async function getAggregateResult(
projectId: string,
runId: string
): Promise<z.infer<typeof SimulationAggregateResult> | null> {
await projectAuthCheck(projectId);
const run = await simulationRunsCollection.findOne({
_id: new ObjectId(runId),
projectId,
});
if (!run || !run.aggregateResults) return null;
return run.aggregateResults;
}
export async function deleteRun(projectId: string, runId: string) {
try {
// Delete the run using the collection directly
await simulationRunsCollection.deleteOne({
_id: new ObjectId(runId),
projectId: projectId
});
// Delete associated results using the collection directly
await simulationResultsCollection.deleteMany({
runId: runId,
projectId: projectId
});
return { success: true };
} catch (error) {
console.error('Error deleting run:', error);
throw new Error('Failed to delete run');
}
}

575
apps/rowboat/app/actions/testing_actions.ts Normal file
View file

@ -0,0 +1,575 @@
'use server';
import { ObjectId } from "mongodb";
import { testScenariosCollection, testSimulationsCollection, testProfilesCollection, testRunsCollection, testResultsCollection, projectsCollection } from "../lib/mongodb";
import { z } from 'zod';
import { projectAuthCheck } from "./project_actions";
import { type WithStringId } from "../lib/types/types";
import { TestScenario, TestSimulation, TestProfile, TestRun, TestResult } from "../lib/types/testing_types";
export async function listScenarios(
projectId: string,
page: number = 1,
pageSize: number = 10
): Promise<{
scenarios: WithStringId<z.infer<typeof TestScenario>>[];
total: number;
}> {
await projectAuthCheck(projectId);
// Calculate skip value for pagination
const skip = (page - 1) * pageSize;
// Get total count for pagination
const total = await testScenariosCollection.countDocuments({ projectId });
// Get paginated scenarios
const scenarios = await testScenariosCollection
.find({ projectId })
.skip(skip)
.limit(pageSize)
.toArray();
return {
scenarios: scenarios.map(scenario => ({
...scenario,
_id: scenario._id.toString(),
})),
total,
};
}
export async function getScenario(projectId: string, scenarioId: string): Promise<WithStringId<z.infer<typeof TestScenario>> | null> {
await projectAuthCheck(projectId);
// fetch scenario
const scenario = await testScenariosCollection.findOne({
_id: new ObjectId(scenarioId),
projectId,
});
if (!scenario) {
return null;
}
const { _id, ...rest } = scenario;
return {
...rest,
_id: _id.toString(),
};
}
export async function deleteScenario(projectId: string, scenarioId: string): Promise<void> {
await projectAuthCheck(projectId);
await testScenariosCollection.deleteOne({
_id: new ObjectId(scenarioId),
projectId,
});
}
export async function createScenario(
projectId: string,
data: {
name: string;
description: string;
}
): Promise<WithStringId<z.infer<typeof TestScenario>>> {
await projectAuthCheck(projectId);
const doc = {
...data,
projectId,
createdAt: new Date().toISOString(),
lastUpdatedAt: new Date().toISOString(),
};
const result = await testScenariosCollection.insertOne(doc);
return {
...doc,
_id: result.insertedId.toString(),
};
}
export async function updateScenario(
projectId: string,
scenarioId: string,
updates: {
name?: string;
description?: string;
}
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: any = {
...updates,
lastUpdatedAt: new Date().toISOString(),
};
await testScenariosCollection.updateOne(
{
_id: new ObjectId(scenarioId),
projectId,
},
{
$set: updateData,
}
);
}
export async function listSimulations(
projectId: string,
page: number = 1,
pageSize: number = 10
): Promise<{
simulations: WithStringId<z.infer<typeof TestSimulation>>[];
total: number;
}> {
await projectAuthCheck(projectId);
const skip = (page - 1) * pageSize;
const total = await testSimulationsCollection.countDocuments({ projectId });
const simulations = await testSimulationsCollection
.find({ projectId })
.skip(skip)
.limit(pageSize)
.toArray();
return {
simulations: simulations.map(simulation => ({
...simulation,
_id: simulation._id.toString(),
})),
total,
};
}
export async function getSimulation(projectId: string, simulationId: string): Promise<WithStringId<z.infer<typeof TestSimulation>> | null> {
await projectAuthCheck(projectId);
const simulation = await testSimulationsCollection.findOne({
_id: new ObjectId(simulationId),
projectId,
});
if (!simulation) {
return null;
}
const { _id, ...rest } = simulation;
return {
...rest,
_id: _id.toString(),
};
}
export async function deleteSimulation(projectId: string, simulationId: string): Promise<void> {
await projectAuthCheck(projectId);
await testSimulationsCollection.deleteOne({
_id: new ObjectId(simulationId),
projectId,
});
}
export async function createSimulation(
projectId: string,
data: {
name: string;
scenarioId: string;
profileId: string;
passCriteria: string;
}
): Promise<WithStringId<z.infer<typeof TestSimulation>>> {
await projectAuthCheck(projectId);
const doc = {
...data,
projectId,
createdAt: new Date().toISOString(),
lastUpdatedAt: new Date().toISOString(),
};
const result = await testSimulationsCollection.insertOne(doc);
return {
...doc,
_id: result.insertedId.toString(),
};
}
export async function updateSimulation(
projectId: string,
simulationId: string,
updates: {
name?: string;
scenarioId?: string;
profileId?: string;
passCriteria?: string;
}
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: any = {
...updates,
lastUpdatedAt: new Date().toISOString(),
};
await testSimulationsCollection.updateOne(
{
_id: new ObjectId(simulationId),
projectId,
},
{
$set: updateData,
}
);
}
export async function listProfiles(
projectId: string,
page: number = 1,
pageSize: number = 10
): Promise<{
profiles: WithStringId<z.infer<typeof TestProfile>>[];
total: number;
}> {
await projectAuthCheck(projectId);
const skip = (page - 1) * pageSize;
const total = await testProfilesCollection.countDocuments({ projectId });
const profiles = await testProfilesCollection
.find({ projectId })
.skip(skip)
.limit(pageSize)
.toArray();
return {
profiles: profiles.map(profile => ({
...profile,
_id: profile._id.toString(),
})),
total,
};
}
export async function getDefaultProfile(projectId: string): Promise<WithStringId<z.infer<typeof TestProfile>> | null> {
await projectAuthCheck(projectId);
const project = await projectsCollection.findOne({ _id: projectId });
if (!project) {
return null;
}
if (!project.defaultTestProfileId) {
// create a default profile
const profile = await createProfile(projectId, {
name: 'Default',
context: '',
mockTools: false,
mockPrompt: '',
});
await setDefaultProfile(projectId, profile._id);
return profile;
}
return getProfile(projectId, project.defaultTestProfileId);
}
export async function setDefaultProfile(projectId: string, profileId: string): Promise<void> {
await projectAuthCheck(projectId);
await projectsCollection.updateOne(
{ _id: projectId },
{ $set: { defaultTestProfileId: profileId } }
);
}
export async function getProfile(projectId: string, profileId: string): Promise<WithStringId<z.infer<typeof TestProfile>> | null> {
await projectAuthCheck(projectId);
const profile = await testProfilesCollection.findOne({
_id: new ObjectId(profileId),
projectId,
});
if (!profile) {
return null;
}
const { _id, ...rest } = profile;
return {
...rest,
_id: _id.toString(),
};
}
export async function deleteProfile(projectId: string, profileId: string): Promise<void> {
await projectAuthCheck(projectId);
await testProfilesCollection.deleteOne({
_id: new ObjectId(profileId),
projectId,
default: false,
});
}
export async function createProfile(
projectId: string,
data: {
name: string;
context: string;
mockTools: boolean;
mockPrompt?: string;
}
): Promise<WithStringId<z.infer<typeof TestProfile>>> {
await projectAuthCheck(projectId);
const doc = {
...data,
projectId,
createdAt: new Date().toISOString(),
lastUpdatedAt: new Date().toISOString(),
};
const result = await testProfilesCollection.insertOne(doc);
return {
...doc,
_id: result.insertedId.toString(),
};
}
export async function updateProfile(
projectId: string,
profileId: string,
updates: {
name?: string;
context?: string;
mockTools?: boolean;
mockPrompt?: string;
}
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: any = {
...updates,
lastUpdatedAt: new Date().toISOString(),
};
await testProfilesCollection.updateOne(
{
_id: new ObjectId(profileId),
projectId,
},
{
$set: updateData,
}
);
}
export async function listRuns(
projectId: string,
page: number = 1,
pageSize: number = 10
): Promise<{
runs: WithStringId<z.infer<typeof TestRun>>[];
total: number;
}> {
await projectAuthCheck(projectId);
const skip = (page - 1) * pageSize;
const total = await testRunsCollection.countDocuments({ projectId });
const runs = await testRunsCollection
.find({ projectId })
.sort({ startedAt: -1 }) // Sort by most recent first
.skip(skip)
.limit(pageSize)
.toArray();
return {
runs: runs.map(run => ({
...run,
_id: run._id.toString(),
})),
total,
};
}
export async function getRun(projectId: string, runId: string): Promise<WithStringId<z.infer<typeof TestRun>> | null> {
await projectAuthCheck(projectId);
const run = await testRunsCollection.findOne({
_id: new ObjectId(runId),
projectId,
});
if (!run) {
return null;
}
const { _id, ...rest } = run;
return {
...rest,
_id: _id.toString(),
};
}
export async function deleteRun(projectId: string, runId: string): Promise<void> {
await projectAuthCheck(projectId);
await testRunsCollection.deleteOne({
_id: new ObjectId(runId),
projectId,
});
}
export async function createRun(
projectId: string,
data: {
simulationIds: string[];
workflowId: string;
}
): Promise<WithStringId<z.infer<typeof TestRun>>> {
await projectAuthCheck(projectId);
// Increment the testRunCounter and get the new value
const result = await projectsCollection.findOneAndUpdate(
{ _id: projectId },
{ $inc: { testRunCounter: 1 } },
{ returnDocument: 'after' }
);
if (!result) {
throw new Error("Project not found");
}
const runNumber = result.testRunCounter || 1;
const doc = {
...data,
projectId,
name: `Run #${runNumber}`,
status: 'pending' as const,
startedAt: new Date().toISOString(),
aggregateResults: {
total: 0,
pass: 0,
fail: 0,
},
};
const insertResult = await testRunsCollection.insertOne(doc);
return {
...doc,
_id: insertResult.insertedId.toString(),
};
}
export async function updateRun(
projectId: string,
runId: string,
updates: {
status?: 'pending' | 'running' | 'completed' | 'cancelled' | 'failed' | 'error';
completedAt?: string;
aggregateResults?: {
total: number;
pass: number;
fail: number;
};
}
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: any = {
...updates,
};
await testRunsCollection.updateOne(
{
_id: new ObjectId(runId),
projectId,
},
{
$set: updateData,
}
);
}
export async function listResults(
projectId: string,
runId: string,
page: number = 1,
pageSize: number = 10
): Promise<{
results: WithStringId<z.infer<typeof TestResult>>[];
total: number;
}> {
await projectAuthCheck(projectId);
const skip = (page - 1) * pageSize;
const total = await testResultsCollection.countDocuments({ projectId, runId });
const results = await testResultsCollection
.find({ projectId, runId })
.skip(skip)
.limit(pageSize)
.toArray();
return {
results: results.map(result => ({
...result,
_id: result._id.toString(),
})),
total,
};
}
export async function getResult(projectId: string, resultId: string): Promise<WithStringId<z.infer<typeof TestResult>> | null> {
await projectAuthCheck(projectId);
const result = await testResultsCollection.findOne({
_id: new ObjectId(resultId),
projectId,
});
if (!result) {
return null;
}
const { _id, ...rest } = result;
return {
...rest,
_id: _id.toString(),
};
}
export async function deleteResult(projectId: string, resultId: string): Promise<void> {
await projectAuthCheck(projectId);
await testResultsCollection.deleteOne({
_id: new ObjectId(resultId),
projectId,
});
}
export async function createResult(
projectId: string,
data: {
runId: string;
simulationId: string;
result: 'pass' | 'fail';
details: string;
}
): Promise<WithStringId<z.infer<typeof TestResult>>> {
await projectAuthCheck(projectId);
const doc = {
...data,
projectId,
};
const result = await testResultsCollection.insertOne(doc);
return {
...doc,
_id: result.insertedId.toString(),
};
}
export async function updateResult(
projectId: string,
resultId: string,
updates: {
result?: 'pass' | 'fail';
details?: string;
}
): Promise<void> {
await projectAuthCheck(projectId);
await testResultsCollection.updateOne(
{
_id: new ObjectId(resultId),
projectId,
},
{
$set: updates,
}
);
}

43
apps/rowboat/app/api/v1/[projectId]/chat/route.ts
View file

@ -1,5 +1,5 @@
import { NextRequest } from "next/server";
import { agentWorkflowsCollection, db, projectsCollection } from "../../../../lib/mongodb";
import { agentWorkflowsCollection, db, projectsCollection, testProfilesCollection } from "../../../../lib/mongodb";
import { z } from "zod";
import { ObjectId } from "mongodb";
import { authCheck } from "../../utils";
@ -9,6 +9,7 @@ import { getAgenticApiResponse, callClientToolWebhook, runRAGToolCall, mockToolR
import { check_query_limit } from "../../../../lib/rate_limiting";
import { apiV1 } from "rowboat-shared";
import { PrefixLogger } from "../../../../lib/utils";
import { TestProfile } from "@/app/lib/types/testing_types";
// get next turn / agent response
export async function POST(
@ -69,8 +70,42 @@ export async function POST(
return Response.json({ error: "Workflow not found" }, { status: 404 });
}
const MAX_TURNS = result.data.maxTurns ?? 3;
// if test profile is provided in the request, use it
let profile: z.infer<typeof TestProfile> = {
projectId: projectId,
name: 'Default',
createdAt: new Date().toISOString(),
lastUpdatedAt: new Date().toISOString(),
context: '',
mockTools: false,
mockPrompt: '',
};
if (result.data.testProfileId) {
const testProfile = await testProfilesCollection.findOne({
projectId: projectId,
_id: new ObjectId(result.data.testProfileId),
});
if (!testProfile) {
logger.log(`Test profile ${result.data.testProfileId} not found for project ${projectId}`);
return Response.json({ error: "Test profile not found" }, { status: 404 });
}
profile = testProfile;
}
// if profile has a context available, overwrite the system message in the request (if there is one)
let currentMessages = reqMessages;
if (profile.context) {
// if there is a system message, overwrite it
const systemMessageIndex = reqMessages.findIndex(m => m.role === "system");
if (systemMessageIndex !== -1) {
currentMessages[systemMessageIndex].content = profile.context;
} else {
// if there is no system message, add one
currentMessages.unshift({ role: "system", content: profile.context });
}
}
const MAX_TURNS = result.data.maxTurns ?? 3;
let currentState: unknown = reqState ?? { last_agent_name: workflow.agents[0].name };
let turns = 0;
let hasToolCalls = false;
@ -140,9 +175,9 @@ export async function POST(
try {
// if tool is supposed to be mocked, mock it
const workflowTool = workflow.tools.find(t => t.name === toolCall.function.name);
if (workflowTool?.mockInPlayground) {
if (profile.mockTools) {
logger.log(`Mocking tool call ${toolCall.function.name}`);
result = await mockToolResponse(toolCall.id, currentMessages);
result = await mockToolResponse(toolCall.id, currentMessages, profile);
} else {
// else run the tool call by calling the client tool webhook
logger.log(`Running client tool webhook for tool ${toolCall.function.name}`);

100
apps/rowboat/app/lib/components/selectors/profile-selector.tsx Normal file
View file

@ -0,0 +1,100 @@
import { WithStringId } from "@/app/lib/types/types";
import { TestProfile } from "@/app/lib/types/testing_types";
import { useCallback, useEffect, useState } from "react";
import { listProfiles } from "@/app/actions/testing_actions";
import { Button, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react";
import { z } from "zod";
interface ProfileSelectorProps {
projectId: string;
isOpen: boolean;
onOpenChange: (open: boolean) => void;
onSelect: (profile: WithStringId<z.infer<typeof TestProfile>>) => void;
}
export function ProfileSelector({ projectId, isOpen, onOpenChange, onSelect }: ProfileSelectorProps) {
const [page, setPage] = useState(1);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [profiles, setProfiles] = useState<WithStringId<z.infer<typeof TestProfile>>[]>([]);
const [totalPages, setTotalPages] = useState(0);
const pageSize = 10;
const fetchProfiles = useCallback(async (page: number) => {
setLoading(true);
setError(null);
try {
const result = await listProfiles(projectId, page, pageSize);
setProfiles(result.profiles);
setTotalPages(Math.ceil(result.total / pageSize));
} catch (error) {
setError(`Unable to fetch profiles: ${error}`);
} finally {
setLoading(false);
}
}, [projectId]);
useEffect(() => {
if (isOpen) {
fetchProfiles(page);
}
}, [page, isOpen, fetchProfiles]);
return (
<Modal isOpen={isOpen} onOpenChange={onOpenChange} size="xl">
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Select a Profile</ModalHeader>
<ModalBody>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => fetchProfiles(page)}>Retry</Button>
</div>}
{!loading && !error && <>
{profiles.length === 0 && <div className="text-gray-600 text-center">No profiles found</div>}
{profiles.length > 0 && <div className="flex flex-col w-full">
<div className="grid grid-cols-6 py-2 bg-gray-100 font-semibold text-sm">
<div className="col-span-2 px-4">Name</div>
<div className="col-span-3 px-4">Context</div>
<div className="col-span-1 px-4">Mock Tools</div>
</div>
{profiles.map((p) => (
<div
key={p._id}
className="grid grid-cols-6 py-2 border-b hover:bg-gray-50 text-sm cursor-pointer"
onClick={() => {
onSelect(p);
onClose();
}}
>
<div className="col-span-2 px-4 truncate">{p.name}</div>
<div className="col-span-3 px-4 truncate">{p.context}</div>
<div className="col-span-1 px-4">{p.mockTools ? "Yes" : "No"}</div>
</div>
))}
</div>}
{totalPages > 1 && <Pagination
total={totalPages}
page={page}
onChange={setPage}
className="self-center"
/>}
</>}
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Cancel
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
);
}

98
apps/rowboat/app/lib/components/selectors/scenario-selector.tsx Normal file
View file

@ -0,0 +1,98 @@
import { WithStringId } from "@/app/lib/types/types";
import { TestScenario } from "@/app/lib/types/testing_types";
import { useCallback, useEffect, useState } from "react";
import { listScenarios } from "@/app/actions/testing_actions";
import { Button, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react";
import { z } from "zod";
interface ScenarioSelectorProps {
projectId: string;
isOpen: boolean;
onOpenChange: (open: boolean) => void;
onSelect: (scenario: WithStringId<z.infer<typeof TestScenario>>) => void;
}
export function ScenarioSelector({ projectId, isOpen, onOpenChange, onSelect }: ScenarioSelectorProps) {
const [page, setPage] = useState(1);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [scenarios, setScenarios] = useState<WithStringId<z.infer<typeof TestScenario>>[]>([]);
const [totalPages, setTotalPages] = useState(0);
const pageSize = 10;
const fetchScenarios = useCallback(async (page: number) => {
setLoading(true);
setError(null);
try {
const result = await listScenarios(projectId, page, pageSize);
setScenarios(result.scenarios);
setTotalPages(Math.ceil(result.total / pageSize));
} catch (error) {
setError(`Unable to fetch scenarios: ${error}`);
} finally {
setLoading(false);
}
}, [projectId]);
useEffect(() => {
if (isOpen) {
fetchScenarios(page);
}
}, [page, isOpen, fetchScenarios]);
return (
<Modal isOpen={isOpen} onOpenChange={onOpenChange} size="xl">
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Select a Scenario</ModalHeader>
<ModalBody>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => fetchScenarios(page)}>Retry</Button>
</div>}
{!loading && !error && <>
{scenarios.length === 0 && <div className="text-gray-600 text-center">No scenarios found</div>}
{scenarios.length > 0 && <div className="flex flex-col w-full">
<div className="grid grid-cols-5 py-2 bg-gray-100 font-semibold text-sm">
<div className="col-span-2 px-4">Name</div>
<div className="col-span-3 px-4">Description</div>
</div>
{scenarios.map((s) => (
<div
key={s._id}
className="grid grid-cols-5 py-2 border-b hover:bg-gray-50 text-sm cursor-pointer"
onClick={() => {
onSelect(s);
onClose();
}}
>
<div className="col-span-2 px-4 truncate">{s.name}</div>
<div className="col-span-3 px-4 truncate">{s.description}</div>
</div>
))}
</div>}
{totalPages > 1 && <Pagination
total={totalPages}
page={page}
onChange={setPage}
className="self-center"
/>}
</>}
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Cancel
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
);
}

140
apps/rowboat/app/lib/components/selectors/simulation-selector.tsx Normal file
View file

@ -0,0 +1,140 @@
import { WithStringId } from "@/app/lib/types/types";
import { TestSimulation } from "@/app/lib/types/testing_types";
import { useCallback, useEffect, useState } from "react";
import { listSimulations } from "@/app/actions/testing_actions";
import { Button, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter, Chip } from "@nextui-org/react";
import { z } from "zod";
import { RelativeTime } from "@primer/react";
interface SimulationSelectorProps {
projectId: string;
isOpen: boolean;
onOpenChange: (open: boolean) => void;
onSelect: (simulations: WithStringId<z.infer<typeof TestSimulation>>[]) => void;
initialSelected?: WithStringId<z.infer<typeof TestSimulation>>[];
}
export function SimulationSelector({ projectId, isOpen, onOpenChange, onSelect, initialSelected = [] }: SimulationSelectorProps) {
const [page, setPage] = useState(1);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [simulations, setSimulations] = useState<WithStringId<z.infer<typeof TestSimulation>>[]>([]);
const [totalPages, setTotalPages] = useState(0);
const [selectedSimulations, setSelectedSimulations] = useState<WithStringId<z.infer<typeof TestSimulation>>[]>(initialSelected);
const pageSize = 3;
const fetchSimulations = useCallback(async (page: number) => {
setLoading(true);
setError(null);
try {
const result = await listSimulations(projectId, page, pageSize);
setSimulations(result.simulations);
setTotalPages(Math.ceil(result.total / pageSize));
} catch (error) {
setError(`Unable to fetch simulations: ${error}`);
} finally {
setLoading(false);
}
}, [projectId]);
useEffect(() => {
if (isOpen) {
fetchSimulations(page);
}
}, [page, isOpen, fetchSimulations]);
const handleSelect = (simulation: WithStringId<z.infer<typeof TestSimulation>>) => {
const isSelected = selectedSimulations.some(s => s._id === simulation._id);
let newSelected;
if (isSelected) {
newSelected = selectedSimulations.filter(s => s._id !== simulation._id);
} else {
newSelected = [...selectedSimulations, simulation];
}
setSelectedSimulations(newSelected);
onSelect(newSelected);
};
const handleRemove = (simulationId: string) => {
const newSelected = selectedSimulations.filter(s => s._id !== simulationId);
setSelectedSimulations(newSelected);
onSelect(newSelected);
};
return (
<Modal isOpen={isOpen} onOpenChange={onOpenChange} size="xl">
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Select Simulations</ModalHeader>
<ModalBody>
{selectedSimulations.length > 0 && (
<div className="flex flex-wrap gap-2 mb-4">
{selectedSimulations.map((sim) => (
<Chip
key={sim._id}
onClose={() => handleRemove(sim._id)}
variant="flat"
className="py-1"
>
{sim.name}
</Chip>
))}
</div>
)}
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => fetchSimulations(page)}>Retry</Button>
</div>}
{!loading && !error && <>
{simulations.length === 0 && <div className="text-gray-600 text-center">No simulations found</div>}
{simulations.length > 0 && <div className="flex flex-col w-full">
<div className="grid grid-cols-8 py-2 bg-gray-100 font-semibold text-sm">
<div className="col-span-3 px-4">Name</div>
<div className="col-span-3 px-4">Pass Criteria</div>
<div className="col-span-2 px-4">Last Updated</div>
</div>
{simulations.map((sim) => {
const isSelected = selectedSimulations.some(s => s._id === sim._id);
return (
<div
key={sim._id}
className={`grid grid-cols-8 py-2 border-b hover:bg-gray-50 text-sm cursor-pointer ${
isSelected ? 'bg-blue-50 hover:bg-blue-100' : ''
}`}
onClick={() => handleSelect(sim)}
>
<div className="col-span-3 px-4 truncate">{sim.name}</div>
<div className="col-span-3 px-4 truncate">{sim.passCriteria || '-'}</div>
<div className="col-span-2 px-4 truncate">
<RelativeTime date={new Date(sim.lastUpdatedAt)} />
</div>
</div>
);
})}
</div>}
{totalPages > 1 && <Pagination
total={totalPages}
page={page}
onChange={setPage}
className="self-center"
/>}
</>}
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Done
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
);
}

106
apps/rowboat/app/lib/components/selectors/workflow-selector.tsx Normal file
View file

@ -0,0 +1,106 @@
import { WithStringId } from "@/app/lib/types/types";
import { Workflow } from "@/app/lib/types/workflow_types";
import { useCallback, useEffect, useState } from "react";
import { listWorkflows } from "@/app/actions/workflow_actions";
import { Button, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react";
import { z } from "zod";
import { RelativeTime } from "@primer/react";
import { WorkflowIcon } from "../icons";
import { PublishedBadge } from "@/app/projects/[projectId]/workflow/published_badge";
interface WorkflowSelectorProps {
projectId: string;
isOpen: boolean;
onOpenChange: (open: boolean) => void;
onSelect: (workflow: WithStringId<z.infer<typeof Workflow>>) => void;
}
export function WorkflowSelector({ projectId, isOpen, onOpenChange, onSelect }: WorkflowSelectorProps) {
const [page, setPage] = useState(1);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [workflows, setWorkflows] = useState<WithStringId<z.infer<typeof Workflow>>[]>([]);
const [totalPages, setTotalPages] = useState(0);
const [publishedWorkflowId, setPublishedWorkflowId] = useState<string | null>(null);
const pageSize = 10;
const fetchWorkflows = useCallback(async (page: number) => {
setLoading(true);
setError(null);
try {
const result = await listWorkflows(projectId, page, pageSize);
setWorkflows(result.workflows);
setTotalPages(Math.ceil(result.total / pageSize));
setPublishedWorkflowId(result.publishedWorkflowId);
} catch (error) {
setError(`Unable to fetch workflows: ${error}`);
} finally {
setLoading(false);
}
}, [projectId]);
useEffect(() => {
if (isOpen) {
fetchWorkflows(page);
}
}, [page, isOpen, fetchWorkflows]);
return (
<Modal isOpen={isOpen} onOpenChange={onOpenChange} size="xl">
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Select a Workflow</ModalHeader>
<ModalBody>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => fetchWorkflows(page)}>Retry</Button>
</div>}
{!loading && !error && <>
{workflows.length === 0 && <div className="text-gray-600 text-center">No workflows found</div>}
{workflows.length > 0 && <div className="flex flex-col gap-2">
{workflows.map((workflow) => (
<div
key={workflow._id}
className="flex items-center justify-between p-3 border rounded-md hover:bg-gray-50 cursor-pointer"
onClick={() => {
onSelect(workflow);
onClose();
}}
>
<div className="flex flex-col gap-1">
<div className="flex items-center gap-2">
<WorkflowIcon />
<span className="font-medium">{workflow.name || 'Unnamed workflow'}</span>
{publishedWorkflowId === workflow._id && <PublishedBadge />}
</div>
<div className="text-xs text-gray-500">
Updated <RelativeTime date={new Date(workflow.lastUpdatedAt)} />
</div>
</div>
</div>
))}
</div>}
{totalPages > 1 && <Pagination
total={totalPages}
page={page}
onChange={setPage}
className="self-center"
/>}
</>}
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Cancel
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
);
}

12
apps/rowboat/app/lib/mongodb.ts
View file

@ -1,5 +1,5 @@
import { MongoClient } from "mongodb";
import { PlaygroundChat, Webpage, ChatClientId } from "./types/types";
import { Webpage } from "./types/types";
import { Workflow } from "./types/workflow_types";
import { ApiKey } from "./types/project_types";
import { ProjectMember } from "./types/project_types";
@ -7,7 +7,7 @@ import { Project } from "./types/project_types";
import { EmbeddingDoc } from "./types/datasource_types";
import { DataSourceDoc } from "./types/datasource_types";
import { DataSource } from "./types/datasource_types";
import { Scenario, SimulationResult, SimulationRun, SimulationAggregateResult } from "./types/testing_types";
import { TestScenario, TestResult, TestRun, TestProfile, TestSimulation } from "./types/testing_types";
import { z } from 'zod';
const client = new MongoClient(process.env["MONGODB_CONNECTION_STRING"] || "mongodb://localhost:27017");
@ -20,7 +20,9 @@ export const projectsCollection = db.collection<z.infer<typeof Project>>("projec
export const projectMembersCollection = db.collection<z.infer<typeof ProjectMember>>("project_members");
export const webpagesCollection = db.collection<z.infer<typeof Webpage>>('webpages');
export const agentWorkflowsCollection = db.collection<z.infer<typeof Workflow>>("agent_workflows");
export const scenariosCollection = db.collection<z.infer<typeof Scenario>>("scenarios");
export const apiKeysCollection = db.collection<z.infer<typeof ApiKey>>("api_keys");
export const simulationRunsCollection = db.collection<z.infer<typeof SimulationRun>>("simulation_runs");
export const simulationResultsCollection = db.collection<z.infer<typeof SimulationResult>>("simulation_results");
export const testScenariosCollection = db.collection<z.infer<typeof TestScenario>>("test_scenarios");
export const testProfilesCollection = db.collection<z.infer<typeof TestProfile>>("test_profiles");
export const testSimulationsCollection = db.collection<z.infer<typeof TestSimulation>>("test_simulations");
export const testRunsCollection = db.collection<z.infer<typeof TestRun>>("test_runs");
export const testResultsCollection = db.collection<z.infer<typeof TestResult>>("test_results");

2
apps/rowboat/app/lib/types/project_types.ts
View file

@ -10,6 +10,8 @@ export const Project = z.object({
webhookUrl: z.string().optional(),
publishedWorkflowId: z.string().optional(),
nextWorkflowNumber: z.number().optional(),
testRunCounter: z.number().default(0),
defaultTestProfileId: z.string().optional(),
});export const ProjectMember = z.object({
userId: z.string(),
projectId: z.string(),

70
apps/rowboat/app/lib/types/testing_types.ts
View file

@ -1,53 +1,37 @@
import { z } from "zod";
// Base type
export const Scenario = z.object({
export const TestScenario = z.object({
projectId: z.string(),
name: z.string().min(1, "Name cannot be empty"),
description: z.string().min(1, "Description cannot be empty"),
criteria: z.string().default(''),
context: z.string().default(''),
createdAt: z.string().datetime(),
lastUpdatedAt: z.string().datetime(),
});
// Relevant to new simulation features
export const SimulationScenarioData = z.object({
scenario: z.string(),
context: z.string().default(''),
});
// Legacy
export const SimulationArticleData = z.object({
articleUrl: z.string(),
articleTitle: z.string().default('').optional(),
articleContent: z.string().default('').optional(),
});
export const SimulationChatMessagesData = z.object({
chatMessages: z.string(),
});
// Relevant to new simulation features
export const SimulationData = z.union([
SimulationScenarioData,
SimulationArticleData,
SimulationChatMessagesData
]);
export const SimulationAggregateResult = z.object({
total: z.number(),
pass: z.number(),
fail: z.number(),
});
export const SimulationRun = z.object({
export const TestProfile = z.object({
projectId: z.string(),
scenarioIds: z.array(z.string()),
name: z.string().min(1, "Name cannot be empty"),
context: z.string(),
createdAt: z.string().datetime(),
lastUpdatedAt: z.string().datetime(),
mockTools: z.boolean(),
mockPrompt: z.string().optional(),
});
export const TestSimulation = z.object({
projectId: z.string(),
name: z.string().min(1, "Name cannot be empty"),
scenarioId: z.string(),
profileId: z.string(),
passCriteria: z.string(),
createdAt: z.string().datetime(),
lastUpdatedAt: z.string().datetime(),
});
export const TestRun = z.object({
projectId: z.string(),
name: z.string(),
simulationIds: z.array(z.string()),
workflowId: z.string(),
status: z.enum(['pending', 'running', 'completed', 'cancelled', 'failed', 'error']),
startedAt: z.string(),
@ -57,12 +41,12 @@ export const SimulationRun = z.object({
pass: z.number(),
fail: z.number(),
}).optional(),
});
});
export const SimulationResult = z.object({
export const TestResult = z.object({
projectId: z.string(),
runId: z.string(),
scenarioId: z.string(),
simulationId: z.string(),
result: z.union([z.literal('pass'), z.literal('fail')]),
details: z.string()
});

4
apps/rowboat/app/lib/types/types.ts
View file

@ -1,7 +1,6 @@
import { CoreMessage, ToolCallPart } from "ai";
import { z } from "zod";
import { apiV1 } from "rowboat-shared";
import { SimulationData } from "./testing_types";
export const PlaygroundChat = z.object({
createdAt: z.string().datetime(),
@ -9,7 +8,7 @@ export const PlaygroundChat = z.object({
title: z.string().optional(),
messages: z.array(apiV1.ChatMessage),
simulated: z.boolean().default(false).optional(),
simulationData: SimulationData.optional(),
simulationScenario: z.string().optional(),
simulationComplete: z.boolean().default(false).optional(),
agenticState: z.unknown().optional(),
systemMessage: z.string().optional(),
@ -110,6 +109,7 @@ export const ApiRequest = z.object({
skipToolCalls: z.boolean().nullable().optional(),
maxTurns: z.number().nullable().optional(),
workflowId: z.string().nullable().optional(),
testProfileId: z.string().nullable().optional(),
});
export const ApiResponse = z.object({

29
apps/rowboat/app/lib/utils.ts
View file

@ -18,6 +18,7 @@ import { qdrantClient } from "./qdrant";
import { EmbeddingRecord } from "./types/datasource_types";
import { ApiMessage } from "./types/types";
import { openai } from "@ai-sdk/openai";
import { TestProfile } from "./types/testing_types";
export async function callClientToolWebhook(
toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number],
@ -220,19 +221,33 @@ export class PrefixLogger {
}
}
export async function mockToolResponse(toolId: string, messages: z.infer<typeof ApiMessage>[]): Promise<string> {
const prompt = `
# Your Specific Task:
Here is a chat between a user and a customer support assistant.
export async function mockToolResponse(toolId: string, messages: z.infer<typeof ApiMessage>[], testProfile: z.infer<typeof TestProfile>): Promise<string> {
const prompt = `Given below is a chat between a user and a customer support assistant.
The assistant has requested a tool call with ID {{toolID}}.
Your job is to come up with an example of the data that the tool call should return.
The current date is {{date}}.
CONVERSATION:
Your job is to come up with the data that the tool call should return.
In order to help you mock the responses, the user has provided some contextual information,
and also some instructions on how to mock the tool call.
>>>CHAT_HISTORY
{{messages}}
<<<END_OF_CHAT_HISTORY
>>>CONTEXT
{{context}}
<<<END_OF_CONTEXT
>>>MOCK_INSTRUCTIONS
{{mockInstructions}}
<<<END_OF_MOCK_INSTRUCTIONS
The current date is {{date}}.
`
.replace('{{toolID}}', toolId)
.replace(`{{date}}`, new Date().toISOString())
.replace('{{context}}', testProfile.context)
.replace('{{mockInstructions}}', testProfile.mockPrompt || '')
.replace('{{messages}}', JSON.stringify(messages.map((m) => {
let tool_calls;
if ('tool_calls' in m && m.role == 'assistant') {

4
apps/rowboat/app/projects/[projectId]/menu.tsx
View file

@ -62,11 +62,11 @@ export default function Menu({
selected={pathname.startsWith(`/projects/${projectId}/workflow`)}
/>
<NavLink
href={`/projects/${projectId}/simulation`}
href={`/projects/${projectId}/test`}
label="Test"
collapsed={collapsed}
icon={<PlayIcon size={16} />}
selected={pathname.startsWith(`/projects/${projectId}/simulation`)}
selected={pathname.startsWith(`/projects/${projectId}/test`)}
/>
{useDataSources && (
<NavLink

100
apps/rowboat/app/projects/[projectId]/playground/app.tsx
View file

@ -4,16 +4,15 @@ import { useEffect, useState, useMemo, useCallback } from "react";
import { z } from "zod";
import { PlaygroundChat } from "../../../lib/types/types";
import { Workflow } from "../../../lib/types/workflow_types";
import { SimulationData } from "../../../lib/types/testing_types";
import { SimulationScenarioData } from "../../../lib/types/testing_types";
import { Chat } from "./chat";
import { useSearchParams, useRouter } from "next/navigation";
import { ActionButton, Pane } from "../workflow/pane";
import { apiV1 } from "rowboat-shared";
import { EllipsisVerticalIcon, MessageSquarePlusIcon, PlayIcon } from "lucide-react";
import { getScenario } from "../../../actions/simulation_actions";
import { getScenario } from "../../../actions/testing_actions";
import clsx from "clsx";
import { TestProfile, TestScenario } from "@/app/lib/types/testing_types";
import { WithStringId } from "@/app/lib/types/types";
function SimulateLabel() {
return <span>Simulate<sup className="pl-1">beta</sup></span>;
}
@ -25,18 +24,16 @@ export function App({
projectId,
workflow,
messageSubscriber,
initialTestProfile,
}: {
hidden?: boolean;
projectId: string;
workflow: z.infer<typeof Workflow>;
messageSubscriber?: (messages: z.infer<typeof apiV1.ChatMessage>[]) => void;
initialTestProfile: z.infer<typeof TestProfile>;
}) {
const searchParams = useSearchParams();
const router = useRouter();
const initialChatId = useMemo(() => searchParams.get('chatId'), [searchParams]);
const [existingChatId, setExistingChatId] = useState<string | null>(initialChatId);
const [loadingChat, setLoadingChat] = useState<boolean>(false);
const [counter, setCounter] = useState<number>(0);
const [testProfile, setTestProfile] = useState<z.infer<typeof TestProfile>>(initialTestProfile);
const [chat, setChat] = useState<z.infer<typeof PlaygroundChat>>({
projectId,
createdAt: new Date().toISOString(),
@ -45,49 +42,45 @@ export function App({
systemMessage: defaultSystemMessage,
});
const beginSimulation = useCallback((data: z.infer<typeof SimulationData>) => {
setExistingChatId(null);
setLoadingChat(true);
function handleTestProfileChange(profile: WithStringId<z.infer<typeof TestProfile>>) {
setTestProfile(profile);
setCounter(counter + 1);
setChat({
projectId,
createdAt: new Date().toISOString(),
messages: [],
simulated: true,
simulationData: data,
systemMessage: 'context' in data ? data.context : '',
});
}, [counter, projectId]);
useEffect(() => {
const scenarioId = localStorage.getItem('pendingScenarioId');
if (scenarioId && projectId) {
console.log('Scenario Effect triggered:', { scenarioId, projectId });
getScenario(projectId, scenarioId).then((scenario) => {
console.log('Scenario data received:', scenario);
beginSimulation({
...scenario,
systemMessage: scenario.context || '',
} as z.infer<typeof SimulationScenarioData>);
localStorage.removeItem('pendingScenarioId');
}).catch(error => {
console.error('Error fetching scenario:', error);
localStorage.removeItem('pendingScenarioId');
});
}
}, [projectId, beginSimulation]);
// const beginSimulation = useCallback((scenario: string) => {
// setExistingChatId(null);
// setLoadingChat(true);
// setCounter(counter + 1);
// setChat({
// projectId,
// createdAt: new Date().toISOString(),
// messages: [],
// simulated: true,
// simulationScenario: scenario,
// systemMessage: '',
// });
// }, [counter, projectId]);
// useEffect(() => {
// const scenarioId = localStorage.getItem('pendingScenarioId');
// if (scenarioId && projectId) {
// console.log('Scenario Effect triggered:', { scenarioId, projectId });
// getScenario(projectId, scenarioId).then((scenario) => {
// console.log('Scenario data received:', scenario);
// beginSimulation(scenario.description);
// localStorage.removeItem('pendingScenarioId');
// }).catch(error => {
// console.error('Error fetching scenario:', error);
// localStorage.removeItem('pendingScenarioId');
// });
// }
// }, [projectId, beginSimulation]);
if (hidden) {
return <></>;
}
function handleSimulateButtonClick() {
router.push(`/projects/${projectId}/simulation`);
}
function handleNewChatButtonClick() {
setExistingChatId(null);
setLoadingChat(true);
setCounter(counter + 1);
setChat({
projectId,
@ -110,27 +103,18 @@ export function App({
>
New chat
</ActionButton>,
<ActionButton
key="simulate"
icon={<PlayIcon size={16} />}
onClick={handleSimulateButtonClick}
>
Simulate
</ActionButton>,
]}
>
<div className="h-full overflow-auto">
{loadingChat && <div className="flex justify-center items-center h-full">
<Spinner />
</div>}
{!loadingChat && <Chat
key={existingChatId || 'chat-' + counter}
<Chat
key={`chat-${counter}`}
chat={chat}
initialChatId={existingChatId || null}
projectId={projectId}
workflow={workflow}
testProfile={testProfile}
messageSubscriber={messageSubscriber}
/>}
onTestProfileChange={handleTestProfileChange}
/>
</div>
</Pane>
);

51
apps/rowboat/app/projects/[projectId]/playground/chat.tsx
View file

@ -9,24 +9,28 @@ import { convertWorkflowToAgenticAPI } from "../../../lib/types/agents_api_types
import { AgenticAPIChatRequest } from "../../../lib/types/agents_api_types";
import { Workflow } from "../../../lib/types/workflow_types";
import { ComposeBox } from "./compose-box";
import { Button, Spinner } from "@nextui-org/react";
import { Button, Spinner, Tooltip } from "@nextui-org/react";
import { apiV1 } from "rowboat-shared";
import { CopyAsJsonButton } from "./copy-as-json-button";
import { TestProfile } from "@/app/lib/types/testing_types";
import { ProfileSelector } from "@/app/lib/components/selectors/profile-selector";
import { WithStringId } from "@/app/lib/types/types";
export function Chat({
chat,
initialChatId = null,
projectId,
workflow,
messageSubscriber,
testProfile,
onTestProfileChange,
}: {
chat: z.infer<typeof PlaygroundChat>;
initialChatId?: string | null;
projectId: string;
workflow: z.infer<typeof Workflow>;
messageSubscriber?: (messages: z.infer<typeof apiV1.ChatMessage>[]) => void;
testProfile: z.infer<typeof TestProfile>;
onTestProfileChange: (profile: WithStringId<z.infer<typeof TestProfile>>) => void;
}) {
const [chatId, setChatId] = useState<string | null>(initialChatId);
const [messages, setMessages] = useState<z.infer<typeof apiV1.ChatMessage>[]>(chat.messages);
const [loadingAssistantResponse, setLoadingAssistantResponse] = useState<boolean>(false);
const [loadingUserResponse, setLoadingUserResponse] = useState<boolean>(false);
@ -34,11 +38,11 @@ export function Chat({
const [agenticState, setAgenticState] = useState<unknown>(chat.agenticState || {
last_agent_name: workflow.startAgent,
});
const [showCopySuccess, setShowCopySuccess] = useState(false);
const [fetchResponseError, setFetchResponseError] = useState<string | null>(null);
const [lastAgenticRequest, setLastAgenticRequest] = useState<unknown | null>(null);
const [lastAgenticResponse, setLastAgenticResponse] = useState<unknown | null>(null);
const [systemMessage, setSystemMessage] = useState<string | undefined>(chat.systemMessage);
const [systemMessage, setSystemMessage] = useState<string | undefined>(testProfile.context);
const [isProfileSelectorOpen, setIsProfileSelectorOpen] = useState(false);
// collect published tool call results
const toolCallResults: Record<string, z.infer<typeof apiV1.ToolMessage>> = {};
@ -53,7 +57,7 @@ export function Chat({
role: 'user',
content: prompt,
version: 'v1',
chatId: chatId ?? '',
chatId: '',
createdAt: new Date().toISOString(),
}];
setMessages(updatedMessages);
@ -64,7 +68,7 @@ export function Chat({
setMessages([...messages, ...results.map((result) => ({
...result,
version: 'v1' as const,
chatId: chatId ?? '',
chatId: '',
createdAt: new Date().toISOString(),
}))]);
}
@ -97,7 +101,7 @@ export function Chat({
role: 'system',
content: systemMessage || '',
version: 'v1' as const,
chatId: chatId ?? '',
chatId: '',
createdAt: new Date().toISOString(),
}, ...messages]),
state: agenticState,
@ -122,7 +126,7 @@ export function Chat({
setMessages([...messages, ...response.messages.map((message) => ({
...message,
version: 'v1' as const,
chatId: chatId ?? '',
chatId: '',
createdAt: new Date().toISOString(),
}))]);
setAgenticState(response.state);
@ -157,14 +161,14 @@ export function Chat({
return () => {
ignore = true;
};
}, [chatId, chat.simulated, messages, projectId, agenticState, workflow, fetchResponseError, systemMessage, simulationComplete]);
}, [chat.simulated, messages, projectId, agenticState, workflow, fetchResponseError, systemMessage, simulationComplete]);
// simulate user turn
useEffect(() => {
let ignore = false;
async function process() {
if (chat.simulationData === undefined) {
if (chat.simulationScenario === undefined) {
return;
}
@ -172,7 +176,7 @@ export function Chat({
setLoadingUserResponse(true);
try {
const response = await simulateUserResponse(projectId, messages, chat.simulationData)
const response = await simulateUserResponse(projectId, messages, chat.simulationScenario)
if (ignore) {
return;
}
@ -187,7 +191,7 @@ export function Chat({
role: 'user',
content: response,
version: 'v1' as const,
chatId: chatId ?? '',
chatId: '',
createdAt: new Date().toISOString(),
}]);
setFetchResponseError(null);
@ -226,7 +230,7 @@ export function Chat({
return () => {
ignore = true;
};
}, [chatId, chat.simulated, messages, projectId, simulationComplete, chat.simulationData]);
}, [chat.simulated, messages, projectId, simulationComplete, chat.simulationScenario]);
// save chat on every assistant message
// useEffect(() => {
@ -275,6 +279,22 @@ export function Chat({
return <div className="relative h-full flex flex-col gap-8 pt-8 overflow-auto">
<CopyAsJsonButton onCopy={handleCopyChat} />
<div className="absolute top-0 left-0">
<Tooltip content={"Change profile"} placement="right">
<button
className="border border-gray-200 dark:border-gray-800 p-2 rounded-lg text-xs hover:bg-gray-100 dark:hover:bg-gray-800"
onClick={() => setIsProfileSelectorOpen(true)}
>
{`Test profile: ${testProfile.name}`}
</button>
</Tooltip>
</div>
<ProfileSelector
projectId={projectId}
isOpen={isProfileSelectorOpen}
onOpenChange={setIsProfileSelectorOpen}
onSelect={onTestProfileChange}
/>
<Messages
projectId={projectId}
messages={messages}
@ -284,6 +304,7 @@ export function Chat({
loadingAssistantResponse={loadingAssistantResponse}
loadingUserResponse={loadingUserResponse}
workflow={workflow}
testProfile={testProfile}
onSystemMessageChange={handleSystemMessageChange}
/>
<div className="shrink-0">

112
apps/rowboat/app/projects/[projectId]/playground/messages.tsx
View file

@ -12,6 +12,7 @@ import Link from "next/link";
import { apiV1 } from "rowboat-shared";
import { EditableField } from "../../../lib/components/editable-field";
import { MessageSquareIcon, EllipsisIcon, CircleCheckIcon, ChevronsDownIcon, ChevronsRightIcon, ChevronRightIcon, ChevronDownIcon, ExternalLinkIcon, XIcon } from "lucide-react";
import { TestProfile } from "@/app/lib/types/testing_types";
function UserMessage({ content }: { content: string }) {
return <div className="self-end ml-[30%] flex flex-col">
@ -93,6 +94,7 @@ function ToolCalls({
messages,
sender,
workflow,
testProfile,
}: {
toolCalls: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'];
results: Record<string, z.infer<typeof apiV1.ToolMessage>>;
@ -101,6 +103,7 @@ function ToolCalls({
messages: z.infer<typeof apiV1.ChatMessage>[];
sender: string | null | undefined;
workflow: z.infer<typeof Workflow>;
testProfile: z.infer<typeof TestProfile>;
}) {
const resultsMap: Record<string, z.infer<typeof apiV1.ToolMessage>> = {};
@ -123,6 +126,7 @@ function ToolCalls({
messages={messages}
sender={sender}
workflow={workflow}
testProfile={testProfile}
/>
})}
</div>;
@ -136,6 +140,7 @@ function ToolCall({
messages,
sender,
workflow,
testProfile,
}: {
toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number];
result: z.infer<typeof apiV1.ToolMessage> | undefined;
@ -144,6 +149,7 @@ function ToolCall({
messages: z.infer<typeof apiV1.ChatMessage>[];
sender: string | null | undefined;
workflow: z.infer<typeof Workflow>;
testProfile: z.infer<typeof TestProfile>;
}) {
let matchingWorkflowTool: z.infer<typeof WorkflowTool> | undefined;
for (const tool of workflow.tools) {
@ -154,15 +160,6 @@ function ToolCall({
}
switch (toolCall.function.name) {
case 'retrieve_url_info':
return <RetrieveUrlInfoToolCall
toolCall={toolCall}
result={result}
handleResult={handleResult}
projectId={projectId}
messages={messages}
sender={sender}
/>;
case 'getArticleInfo':
return <GetInformationToolCall
toolCall={toolCall}
@ -184,7 +181,7 @@ function ToolCall({
sender={sender}
/>;
}
if (matchingWorkflowTool && !matchingWorkflowTool.mockInPlayground) {
if (matchingWorkflowTool && !testProfile.mockTools) {
return <ClientToolCall
toolCall={toolCall}
result={result}
@ -202,6 +199,7 @@ function ToolCall({
messages={messages}
sender={sender}
autoSubmit={matchingWorkflowTool?.autoSubmitMockedResponse}
testProfile={testProfile}
/>;
}
}
@ -310,85 +308,6 @@ function GetInformationToolCall({
</div>;
}
function RetrieveUrlInfoToolCall({
toolCall,
result: availableResult,
handleResult,
projectId,
messages,
sender,
}: {
toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number];
result: z.infer<typeof apiV1.ToolMessage> | undefined;
handleResult: (result: z.infer<typeof apiV1.ToolMessage>) => void;
projectId: string;
messages: z.infer<typeof apiV1.ChatMessage>[];
sender: string | null | undefined;
}) {
const [result, setResult] = useState<z.infer<typeof apiV1.ToolMessage> | undefined>(availableResult);
const args = JSON.parse(toolCall.function.arguments) as { url: string };
let typedResult: z.infer<typeof WebpageCrawlResponse> | undefined;
if (result) {
typedResult = JSON.parse(result.content) as z.infer<typeof WebpageCrawlResponse>;
}
useEffect(() => {
if (result) {
return;
}
let ignore = false;
function process() {
// parse args
scrapeWebpage(args.url)
.then(page => {
if (ignore) {
return;
}
const result: z.infer<typeof apiV1.ToolMessage> = {
role: 'tool',
tool_call_id: toolCall.id,
tool_name: toolCall.function.name,
content: JSON.stringify(page),
};
setResult(result);
handleResult(result);
});
}
process();
return () => {
ignore = true;
};
}, [result, toolCall.id, toolCall.function.name, projectId, args.url, handleResult]);
return <div className="flex flex-col gap-1">
{sender && <div className='text-gray-500 text-sm ml-3'>{sender}</div>}
<div className='border border-gray-300 p-2 rounded-lg rounded-bl-none flex flex-col gap-2 mr-[30%]'>
<ToolCallHeader toolCall={toolCall} result={result} />
<div className='mt-1 flex flex-col gap-2'>
<div className="flex gap-1">
URL: <a className="inline-flex items-center gap-1" target="_blank" href={args.url}>
<span className='underline'>
{args.url}
</span>
<ExternalLinkIcon size={16} />
</a>
</div>
{result && (
<ExpandableContent
label='Content'
content={typedResult}
expanded={false}
/>
)}
</div>
</div>
</div>;
}
function TransferToAgentToolCall({
toolCall,
result: availableResult,
@ -495,6 +414,7 @@ function MockToolCall({
messages,
sender,
autoSubmit = false,
testProfile,
}: {
toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number];
result: z.infer<typeof apiV1.ToolMessage> | undefined;
@ -503,6 +423,7 @@ function MockToolCall({
messages: z.infer<typeof apiV1.ChatMessage>[];
sender: string | null | undefined;
autoSubmit?: boolean;
testProfile: z.infer<typeof TestProfile>;
}) {
const [result, setResult] = useState<z.infer<typeof apiV1.ToolMessage> | undefined>(availableResult);
const [response, setResponse] = useState('');
@ -538,7 +459,7 @@ function MockToolCall({
async function process() {
setGeneratingResponse(true);
const response = await suggestToolResponse(toolCall.id, projectId, messages);
const response = await suggestToolResponse(toolCall.id, projectId, messages, testProfile);
if (ignore) {
return;
}
@ -550,7 +471,7 @@ function MockToolCall({
return () => {
ignore = true;
};
}, [result, response, toolCall.id, projectId, messages]);
}, [result, response, toolCall.id, projectId, messages, testProfile]);
// auto submit if autoSubmitMockedResponse is true
useEffect(() => {
@ -682,6 +603,7 @@ export function Messages({
loadingAssistantResponse,
loadingUserResponse,
workflow,
testProfile,
onSystemMessageChange,
}: {
projectId: string;
@ -692,13 +614,12 @@ export function Messages({
loadingAssistantResponse: boolean;
loadingUserResponse: boolean;
workflow: z.infer<typeof Workflow>;
testProfile: z.infer<typeof TestProfile>;
onSystemMessageChange: (message: string) => void;
}) {
const messagesEndRef = useRef<HTMLDivElement>(null);
let lastUserMessageTimestamp = 0;
const systemMessageLocked = messages.length > 0;
// scroll to bottom on new messages
useEffect(() => {
messagesEndRef.current?.scrollIntoView({ behavior: "smooth" })
@ -707,9 +628,9 @@ export function Messages({
return <div className="grow pt-4 overflow-auto">
<div className="max-w-[768px] mx-auto flex flex-col gap-8">
<SystemMessage
content={systemMessage || ''}
content={testProfile.context}
onChange={onSystemMessageChange}
locked={systemMessageLocked}
locked={true}
/>
{messages.map((message, index) => {
if (message.role === 'assistant') {
@ -723,6 +644,7 @@ export function Messages({
messages={messages}
sender={message.agenticSender}
workflow={workflow}
testProfile={testProfile}
/>;
} else {
// the assistant message createdAt is an ISO string timestamp

459
apps/rowboat/app/projects/[projectId]/simulation/app.tsx
View file

@ -1,459 +0,0 @@
'use client';
import { useState, useEffect, useCallback } from 'react';
import { PlusIcon, PencilIcon, XMarkIcon, EllipsisVerticalIcon, TrashIcon, ChevronRightIcon, PlayIcon, ChevronDownIcon, ChevronLeftIcon } from '@heroicons/react/24/outline';
import { useParams, useRouter, useSearchParams } from 'next/navigation';
import {
getScenarios,
createScenario,
updateScenario,
deleteScenario,
getRuns,
getRun,
getRunResults,
createRun,
createRunResult,
updateRunStatus,
createAggregateResult,
deleteRun,
} from '../../../actions/simulation_actions';
import { type WithStringId } from '../../../lib/types/types';
import { Scenario, SimulationRun, SimulationResult } from "../../../lib/types/testing_types";
import { Workflow } from "../../../lib/types/workflow_types";
import { z } from 'zod';
import { SimulationResultCard, ScenarioResultCard } from './components/RunComponents';
import { ScenarioList, ScenarioViewer } from './components/ScenarioComponents';
import { fetchWorkflow } from '../../../actions/workflow_actions';
import { StructuredPanel, ActionButton } from "../../../lib/components/structured-panel";
import {
ResizableHandle,
ResizablePanel,
ResizablePanelGroup,
} from "../../../../components/ui/resizable"
import { Pagination } from "../../../lib/components/pagination";
type ScenarioType = WithStringId<z.infer<typeof Scenario>>;
type SimulationRunType = WithStringId<z.infer<typeof SimulationRun>>;
type SimulationResultType = WithStringId<z.infer<typeof SimulationResult>>;
type SimulationReport = {
totalScenarios: number;
passedScenarios: number;
failedScenarios: number;
results: z.infer<typeof SimulationResult>[];
timestamp: Date;
};
const dummySimulator = async (scenario: ScenarioType, runId: string, projectId: string): Promise<z.infer<typeof SimulationResult>> => {
await new Promise(resolve => setTimeout(resolve, 500));
const passed = Math.random() > 0.5;
const result: z.infer<typeof SimulationResult> = {
projectId: projectId,
runId: runId,
scenarioId: scenario._id,
result: passed ? 'pass' : 'fail' as const,
details: passed
? "The bot successfully completed the conversation"
: "The bot could not handle the conversation",
};
await createRunResult(
projectId,
runId,
scenario._id,
result.result,
result.details
);
return result;
};
export default function SimulationApp() {
const { projectId } = useParams();
const router = useRouter();
const searchParams = useSearchParams();
const [scenarios, setScenarios] = useState<ScenarioType[]>([]);
const [selectedScenario, setSelectedScenario] = useState<ScenarioType | null>(null);
const [isEditing, setIsEditing] = useState(false);
const [menuOpenScenarioId, setMenuOpenScenarioId] = useState<string | null>(null);
const [isRunning, setIsRunning] = useState(false);
const [simulationReport, setSimulationReport] = useState<SimulationReport | null>(null);
const [expandedResults, setExpandedResults] = useState<Set<string>>(new Set());
const [runs, setRuns] = useState<SimulationRunType[]>([]);
const [activeRun, setActiveRun] = useState<SimulationRunType | null>(null);
const [runResults, setRunResults] = useState<SimulationResultType[]>([]);
const [isLoadingRuns, setIsLoadingRuns] = useState(true);
const [allRunResults, setAllRunResults] = useState<Record<string, SimulationResultType[]>>({});
const [workflowVersions, setWorkflowVersions] = useState<Record<string, WithStringId<z.infer<typeof Workflow>>>>({});
const [menuOpenId, setMenuOpenIdState] = useState<string | null>(null);
const runsPerPage = 10;
const currentPage = Number(searchParams.get('page')) || 1;
const setMenuOpenId = useCallback((id: string | null) => {
setMenuOpenIdState(id);
}, []);
// Load scenarios on mount
useEffect(() => {
if (!projectId) return;
getScenarios(projectId as string).then(setScenarios);
}, [projectId]);
useEffect(() => {
if (menuOpenScenarioId) {
const closeMenu = () => setMenuOpenScenarioId(null);
window.addEventListener('click', closeMenu);
return () => window.removeEventListener('click', closeMenu);
}
}, [menuOpenScenarioId]);
// Modify the fetchRuns function to also fetch results
const fetchRuns = useCallback(async () => {
if (!projectId) return;
setIsLoadingRuns(true);
try {
const runsData = await getRuns(projectId as string);
setRuns(runsData);
// Fetch results for all runs
const resultsPromises = runsData.map(run =>
getRunResults(projectId as string, run._id)
);
const allResults = await Promise.all(resultsPromises);
// Create a map of run ID to results
const resultsMap = runsData.reduce((acc, run, index) => ({
...acc,
[run._id]: allResults[index]
}), {});
setAllRunResults(resultsMap);
} catch (error) {
console.error('Error fetching runs:', error);
} finally {
setIsLoadingRuns(false);
}
}, [projectId]);
// Update the useEffect hooks to include fetchRuns
useEffect(() => {
if (!projectId) return;
fetchRuns();
}, [projectId, fetchRuns]);
useEffect(() => {
if (!projectId || !activeRun || activeRun.status === 'completed' || activeRun.status === 'cancelled') return;
const interval = setInterval(async () => {
try {
const updatedRun = await getRun(projectId as string, activeRun._id);
setActiveRun(updatedRun);
if (updatedRun.status === 'completed') {
const results = await getRunResults(projectId as string, activeRun._id);
setRunResults(results);
fetchRuns(); // Refresh the runs list
}
} catch (error) {
console.error('Error polling run status:', error);
}
}, 2000);
return () => clearInterval(interval);
}, [activeRun, projectId, fetchRuns]);
const createNewScenario = async () => {
if (!projectId) return;
const newScenarioId = await createScenario(
projectId as string,
'New Scenario',
''
);
// Refresh scenarios list
const updatedScenarios = await getScenarios(projectId as string);
setScenarios(updatedScenarios);
const newScenario = updatedScenarios.find(s => s._id === newScenarioId);
if (newScenario) {
setSelectedScenario(newScenario);
setIsEditing(true);
}
};
const handleUpdateScenario = async (updatedScenario: ScenarioType) => {
if (!projectId) return;
// First verify the scenario exists and get its current state
const currentScenarios = await getScenarios(projectId as string);
const existingScenario = currentScenarios.find(s => s._id === updatedScenario._id);
if (!existingScenario) {
console.error('Scenario not found');
return;
}
// Only update the specific fields that have changed
await updateScenario(
projectId as string,
updatedScenario._id,
{
name: updatedScenario.name,
description: updatedScenario.description,
criteria: updatedScenario.criteria,
context: updatedScenario.context,
}
);
// Just refresh the scenarios list without setting selected scenario
const updatedScenarios = await getScenarios(projectId as string);
setScenarios(updatedScenarios);
setIsEditing(false);
};
const handleCloseScenario = () => {
setSelectedScenario(null);
setIsEditing(false);
};
const handleDeleteScenario = async (scenarioId: string) => {
if (!projectId) return;
await deleteScenario(projectId as string, scenarioId);
const updatedScenarios = await getScenarios(projectId as string);
setScenarios(updatedScenarios);
if (selectedScenario?._id === scenarioId) {
setSelectedScenario(null);
setIsEditing(false);
}
setMenuOpenScenarioId(null);
};
const runAllScenarios = async () => {
if (!projectId) return;
setIsRunning(true);
setSimulationReport(null);
try {
// Get workflowId from localStorage
const workflowId = localStorage.getItem(`lastWorkflowId_${projectId}`);
if (!workflowId) {
throw new Error('No workflow selected. Please select a workflow first.');
}
// First verify the workflow exists before creating the run
let workflow;
try {
workflow = await fetchWorkflow(projectId as string, workflowId);
} catch (error) {
// If workflow doesn't exist, clear localStorage and throw error
localStorage.removeItem(`lastWorkflowId_${projectId}`);
throw new Error('Selected workflow no longer exists. Please select a new workflow.');
}
const newRun = await createRun(
projectId as string,
scenarios.map(s => s._id),
workflowId
);
setActiveRun(newRun);
// Store workflow version
setWorkflowVersions(prev => ({
...prev,
[workflowId]: workflow
}));
const shouldMock = process.env.NEXT_PUBLIC_MOCK_SIMULATION_RESULTS === 'true';
if (shouldMock) {
console.log('Using mock simulation...');
await updateRunStatus(projectId as string, newRun._id, 'running');
// Run all scenarios and collect results
const mockResults = await Promise.all(
scenarios.map(scenario =>
dummySimulator(scenario, newRun._id, projectId as string)
)
);
// Calculate and store aggregate results before marking as complete
const total = scenarios.length;
const pass = mockResults.filter(r => r.result === 'pass').length;
const fail = mockResults.filter(r => r.result === 'fail').length;
await createAggregateResult(
projectId as string,
newRun._id,
total,
pass,
fail
);
await updateRunStatus(
projectId as string,
newRun._id,
'completed',
new Date().toISOString()
);
const results = await getRunResults(projectId as string, newRun._id);
setRunResults(results);
const updatedRun = await getRun(projectId as string, newRun._id);
setActiveRun(updatedRun);
}
await fetchRuns();
} catch (error) {
console.error('Error starting scenarios:', error);
alert(error instanceof Error ? error.message : 'An error occurred while starting scenarios');
} finally {
setIsRunning(false);
}
};
const runSingleScenario = (scenario: ScenarioType) => {
// Store scenario ID in localStorage instead of URL parameter
localStorage.setItem('pendingScenarioId', scenario._id);
// Navigate to the playground without query parameter
router.push(`/projects/${projectId}/workflow`);
setMenuOpenScenarioId(null);
};
// Update the workflow versions fetching effect
useEffect(() => {
if (!projectId || !runs.length) return;
const fetchWorkflowVersions = async () => {
const workflowIds = Array.from(new Set(runs.map(run => run.workflowId)));
const versions: Record<string, WithStringId<z.infer<typeof Workflow>>> = {};
for (const workflowId of workflowIds) {
try {
const workflow = await fetchWorkflow(projectId as string, workflowId);
versions[workflowId] = workflow;
} catch (error) {
console.error(`Error fetching workflow ${workflowId}:`, error);
// Add a placeholder for deleted/invalid workflows
versions[workflowId] = {
_id: workflowId,
name: "Deleted/Invalid Workflow",
projectId: projectId as string,
agents: [],
prompts: [],
tools: [],
startAgent: "",
createdAt: new Date().toISOString(),
lastUpdatedAt: new Date().toISOString(),
};
}
}
setWorkflowVersions(versions);
};
fetchWorkflowVersions();
}, [projectId, runs]);
const handleCancelRun = async (runId: string) => {
if (!projectId) return;
try {
await updateRunStatus(projectId as string, runId, 'cancelled');
await fetchRuns(); // Refresh the runs list
} catch (error) {
console.error('Error cancelling run:', error);
}
};
const handleDeleteRun = async (runId: string) => {
if (!projectId) return;
try {
await deleteRun(projectId as string, runId);
await fetchRuns(); // Refresh the runs list
} catch (error) {
console.error('Error deleting run:', error);
}
};
const indexOfLastRun = currentPage * runsPerPage;
const indexOfFirstRun = indexOfLastRun - runsPerPage;
const currentRuns = runs.slice(indexOfFirstRun, indexOfLastRun);
const totalPages = Math.ceil(runs.length / runsPerPage);
return (
<ResizablePanelGroup direction="horizontal" className="h-screen gap-1">
<ResizablePanel minSize={10} defaultSize={15}>
<ScenarioList
scenarios={scenarios}
selectedId={selectedScenario?._id ?? null}
onSelect={(id) => setSelectedScenario(scenarios.find(s => s._id === id) ?? null)}
onAdd={createNewScenario}
onRunScenario={(id) => {
const scenario = scenarios.find(s => s._id === id);
if (scenario) runSingleScenario(scenario);
}}
onDeleteScenario={(id) => handleDeleteScenario(id)}
/>
</ResizablePanel>
<ResizableHandle />
<ResizablePanel minSize={20} defaultSize={85} className="overflow-auto">
{selectedScenario ? (
<ScenarioViewer
scenario={selectedScenario}
onSave={handleUpdateScenario}
onClose={handleCloseScenario}
/>
) : (
<StructuredPanel
title="SIMULATION RUNS"
tooltip="Run and view simulations"
actions={[
<ActionButton
key="run-all"
onClick={() => void runAllScenarios()}
disabled={isRunning}
icon={<PlayIcon className="w-4 h-4" />}
primary
>
Run All Scenarios
</ActionButton>
]}
>
<div className="p-6">
{/* Runs list */}
{isLoadingRuns ? (
<div>Loading runs...</div>
) : (
<div className="space-y-4">
{currentRuns.map((run) => (
<SimulationResultCard
key={run._id}
run={run}
results={allRunResults[run._id] || []}
scenarios={scenarios}
workflow={workflowVersions[run.workflowId]}
onCancelRun={handleCancelRun}
onDeleteRun={handleDeleteRun}
menuOpenId={menuOpenId}
setMenuOpenId={setMenuOpenId}
/>
))}
</div>
)}
{/* Pagination */}
{runs.length > runsPerPage && (
<div className="flex justify-center mt-4">
<Pagination
total={totalPages}
page={currentPage}
/>
</div>
)}
</div>
</StructuredPanel>
)}
</ResizablePanel>
</ResizablePanelGroup>
);
}

399
apps/rowboat/app/projects/[projectId]/simulation/components/RunComponents.tsx
View file

@ -1,399 +0,0 @@
'use client';
import { useState, useEffect } from 'react';
import { ChevronDownIcon, ChevronRightIcon, NoSymbolIcon, EllipsisVerticalIcon, ArrowDownTrayIcon, TrashIcon } from '@heroicons/react/24/outline';
import { WithStringId } from '../../../../lib/types/types';
import { Scenario, SimulationRun, SimulationResult, SimulationAggregateResult } from "../../../../lib/types/testing_types";
import { z } from 'zod';
import { Workflow } from "../../../../lib/types/workflow_types";
import { clsx } from 'clsx';
type ScenarioType = WithStringId<z.infer<typeof Scenario>>;
type SimulationRunType = WithStringId<z.infer<typeof SimulationRun>>;
type SimulationResultType = WithStringId<z.infer<typeof SimulationResult>>;
interface SimulationResultCardProps {
run: SimulationRunType;
results: SimulationResultType[];
scenarios: ScenarioType[];
workflow?: WithStringId<z.infer<typeof Workflow>>;
onCancelRun?: (runId: string) => void;
onDeleteRun?: (runId: string) => Promise<void>;
menuOpenId: string | null;
setMenuOpenId: (id: string | null) => void;
}
export const SimulationResultCard = ({ run, results, scenarios, workflow, onCancelRun, onDeleteRun, menuOpenId, setMenuOpenId }: SimulationResultCardProps) => {
const [isExpanded, setIsExpanded] = useState(false);
const [expandedScenarios, setExpandedScenarios] = useState<Set<string>>(new Set());
const [showDeleteConfirm, setShowDeleteConfirm] = useState(false);
const totalScenarios = run.aggregateResults?.total ?? run.scenarioIds.length;
const passedScenarios = run.aggregateResults?.pass ?? 0;
const failedScenarios = run.aggregateResults?.fail ?? 0;
const getStatusClass = (status: string) => {
const baseClass = "w-[110px] px-3 py-1 rounded text-xs text-center uppercase font-semibold inline-block";
switch (status) {
case 'completed':
case 'pass':
return `${baseClass} bg-green-50 dark:bg-green-900/20 text-green-800 dark:text-green-400`;
case 'failed':
case 'fail':
return `${baseClass} bg-red-50 dark:bg-red-900/20 text-red-800 dark:text-red-400`;
case 'error':
return `${baseClass} bg-orange-50 dark:bg-orange-900/20 text-orange-800 dark:text-orange-400`;
case 'cancelled':
return `${baseClass} bg-gray-50 dark:bg-neutral-800 text-gray-800 dark:text-neutral-400`;
case 'running':
case 'pending':
default:
return `${baseClass} bg-yellow-50 dark:bg-yellow-900/20 text-yellow-800 dark:text-yellow-400`;
}
};
const formatMainTitle = (date: string) => {
return `Run from ${new Date(date).toLocaleString('en-US', {
month: 'numeric',
day: 'numeric',
year: 'numeric',
hour: 'numeric',
minute: '2-digit',
second: '2-digit',
hour12: true
})}`;
};
const formatDateTime = (date: string) => {
return new Date(date).toLocaleString('en-US', {
month: 'numeric',
day: 'numeric',
year: 'numeric',
hour: 'numeric',
minute: '2-digit',
second: '2-digit',
hour12: true
});
};
const getDuration = () => {
if (!run.completedAt) return 'In Progress';
const start = new Date(run.startedAt);
const end = new Date(run.completedAt);
const diff = end.getTime() - start.getTime();
return `${(diff / 1000).toFixed(1)}s`;
};
const toggleScenario = (scenarioId: string, e: React.MouseEvent) => {
e.stopPropagation(); // Prevent triggering parent's onClick
setExpandedScenarios(prev => {
const newSet = new Set(prev);
if (newSet.has(scenarioId)) {
newSet.delete(scenarioId);
} else {
newSet.add(scenarioId);
}
return newSet;
});
};
useEffect(() => {
if (menuOpenId) {
const closeMenu = () => setMenuOpenId(null);
window.addEventListener('click', closeMenu);
return () => window.removeEventListener('click', closeMenu);
}
}, [menuOpenId, setMenuOpenId]);
return (
<div className="border dark:border-neutral-800 rounded-lg mb-4 shadow-sm">
<div
className={clsx(
"p-4 flex items-center justify-between cursor-pointer",
"transition-colors duration-200",
"hover:bg-neutral-100 dark:hover:bg-neutral-800",
"border-b border-transparent",
isExpanded && "border-b-neutral-200 dark:border-b-neutral-800"
)}
onClick={() => setIsExpanded(!isExpanded)}
>
<div className="flex items-center space-x-2">
{isExpanded ? (
<ChevronDownIcon className="h-5 w-5 text-gray-400 dark:text-neutral-500" />
) : (
<ChevronRightIcon className="h-5 w-5 text-gray-400 dark:text-neutral-500" />
)}
<div className="text-sm truncate dark:text-neutral-200">
{formatMainTitle(run.startedAt)}
</div>
</div>
<div className="flex items-center gap-2">
<span className={getStatusClass(run.status)}>
{run.status}
</span>
<div className="relative">
<button
onClick={(e) => {
e.stopPropagation();
setMenuOpenId(menuOpenId === run._id ? null : run._id);
}}
className="p-1 rounded-full hover:bg-gray-100 dark:hover:bg-neutral-700"
>
<EllipsisVerticalIcon className="h-5 w-5 text-gray-600 dark:text-neutral-400" />
</button>
{menuOpenId === run._id && (
<div className="absolute right-0 mt-1 w-48 rounded-md shadow-lg bg-white dark:bg-neutral-900 ring-1 ring-black ring-opacity-5 dark:ring-neutral-700 z-10">
<div className="py-1">
{(run.status === 'running' || run.status === 'pending') && onCancelRun && (
<button
onClick={(e) => {
e.stopPropagation();
onCancelRun(run._id);
setMenuOpenId(null);
}}
className="flex items-center px-4 py-2 text-sm text-red-600 dark:text-red-400 hover:bg-gray-100 dark:hover:bg-neutral-800 w-full"
>
<NoSymbolIcon className="h-4 w-4 mr-2" />
Cancel Run
</button>
)}
<button
disabled
className="flex items-center px-4 py-2 text-sm text-gray-400 dark:text-neutral-500 w-full cursor-not-allowed whitespace-nowrap"
>
<ArrowDownTrayIcon className="h-4 w-4 mr-2" />
Download transcripts
</button>
<button
onClick={(e) => {
e.stopPropagation();
setShowDeleteConfirm(true);
setMenuOpenId(null);
}}
className="flex items-center px-4 py-2 text-sm text-red-600 dark:text-red-400 hover:bg-gray-100 dark:hover:bg-neutral-800 w-full"
>
<TrashIcon className="h-4 w-4 mr-2" />
Delete run
</button>
</div>
</div>
)}
</div>
</div>
</div>
{isExpanded && (
<div className="p-4 border-t dark:border-neutral-800">
{run.status === 'error' ? (
<div className="text-orange-800 bg-orange-50 p-4 rounded-lg">
Your simulation could not be completed. Please run a new simulation again.
</div>
) : (
<>
{/* Workflow and timing information in a grid */}
<div className="grid grid-cols-3 gap-4 mb-6">
{workflow && (
<div className="bg-gray-50 dark:bg-neutral-800 p-4 rounded-lg">
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Workflow Version</div>
<div className="font-medium dark:text-neutral-200">{workflow.name}</div>
</div>
)}
<div className="bg-gray-50 dark:bg-neutral-800 p-4 rounded-lg">
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Completed</div>
<div className="text-sm dark:text-neutral-300">
{run.completedAt ? formatDateTime(run.completedAt) : 'Not completed'}
</div>
</div>
<div className="bg-gray-50 dark:bg-neutral-800 p-4 rounded-lg">
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Duration</div>
<div className="text-sm dark:text-neutral-300">{getDuration()}</div>
</div>
</div>
{/* Results statistics */}
<div className="grid grid-cols-3 gap-4 mb-6">
<div className="p-4 rounded-lg bg-gray-50 dark:bg-neutral-800">
<div className="text-sm text-gray-600 dark:text-neutral-400">Total Scenarios</div>
<div className="text-2xl font-semibold dark:text-neutral-200">{totalScenarios}</div>
</div>
<div className="p-4 rounded-lg bg-green-50 dark:bg-green-900/20">
<div className="text-sm text-green-600 dark:text-green-400">Passed</div>
<div className="text-2xl font-semibold text-green-700 dark:text-green-400">{passedScenarios}</div>
</div>
<div className="p-4 rounded-lg bg-red-50 dark:bg-red-900/20">
<div className="text-sm text-red-600 dark:text-red-400">Failed</div>
<div className="text-2xl font-semibold text-red-700 dark:text-red-400">{failedScenarios}</div>
</div>
</div>
<div className="space-y-2">
{run.scenarioIds.map(scenarioId => {
const scenario = scenarios.find(s => s._id === scenarioId);
const result = results.find(r => r.scenarioId === scenarioId);
const isScenarioExpanded = expandedScenarios.has(scenarioId);
return scenario && (
<div
key={scenarioId}
className={clsx(
"border dark:border-neutral-800 rounded-lg overflow-hidden",
"transition-colors duration-200",
result?.result === 'pass'
? 'bg-green-50/50 dark:bg-green-900/10 border-green-200 dark:border-green-900/50'
: result?.result === 'fail'
? 'bg-red-50/50 dark:bg-red-900/10 border-red-200 dark:border-red-900/50'
: 'bg-gray-50/50 dark:bg-neutral-900/50 border-gray-200 dark:border-neutral-800'
)}
>
<div
className={clsx(
"p-3 flex items-center justify-between cursor-pointer",
"hover:bg-white/50 dark:hover:bg-neutral-800/50"
)}
onClick={(e) => toggleScenario(scenarioId, e)}
>
<div className="flex items-center space-x-2">
{isScenarioExpanded ? (
<ChevronDownIcon className="h-4 w-4 text-gray-600 dark:text-neutral-400" />
) : (
<ChevronRightIcon className="h-4 w-4 text-gray-600 dark:text-neutral-400" />
)}
<span className="font-medium text-gray-900 dark:text-neutral-200">{scenario.name}</span>
</div>
{result && (
<span className={getStatusClass(result.result)}>
{result.result}
</span>
)}
</div>
{isScenarioExpanded && (
<div className="p-3 border-t border-opacity-50 dark:border-neutral-800 space-y-4">
<div>
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Description</div>
<div className="text-sm text-gray-700 dark:text-neutral-300">
{scenario.description}
</div>
</div>
<div>
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Criteria</div>
<div className="text-sm text-gray-700 dark:text-neutral-300">
{scenario.criteria || 'No criteria specified'}
</div>
</div>
<div>
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Context</div>
<div className="text-sm text-gray-700 dark:text-neutral-300">
{scenario.context || 'No context provided'}
</div>
</div>
{result && (
<div>
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Result Details</div>
<div className="text-sm text-gray-700 dark:text-neutral-300">
{result.details}
</div>
</div>
)}
</div>
)}
</div>
);
})}
</div>
</>
)}
</div>
)}
{showDeleteConfirm && (
<div className="fixed inset-0 bg-gray-600 bg-opacity-50 overflow-y-auto h-full w-full z-50">
<div className="relative top-20 mx-auto p-5 border w-96 shadow-lg rounded-md bg-white">
<div className="mt-3 text-center">
<h3 className="text-lg leading-6 font-medium text-gray-900 whitespace-nowrap">
Are you sure you want to delete this run?
</h3>
<div className="mt-6 flex justify-center space-x-4">
<button
onClick={() => setShowDeleteConfirm(false)}
className="px-4 py-2 bg-white text-gray-600 text-sm font-medium border rounded-md hover:bg-gray-50"
>
Retain
</button>
<button
onClick={async () => {
if (onDeleteRun) {
await onDeleteRun(run._id);
setShowDeleteConfirm(false);
}
}}
className="px-4 py-2 bg-red-600 text-white text-sm font-medium rounded-md hover:bg-red-700"
>
Delete
</button>
</div>
</div>
</div>
</div>
)}
</div>
);
};
interface ScenarioResultCardProps {
scenario: ScenarioType;
result?: SimulationResultType;
}
export const ScenarioResultCard = ({ scenario, result }: ScenarioResultCardProps) => {
const [isExpanded, setIsExpanded] = useState(false);
return (
<div className="border rounded-lg mb-2 last:mb-0">
<div
className="p-3 flex items-center justify-between cursor-pointer hover:bg-gray-50"
onClick={() => setIsExpanded(!isExpanded)}
>
<div className="flex items-center space-x-2">
{isExpanded ? (
<ChevronDownIcon className="h-4 w-4 text-gray-400" />
) : (
<ChevronRightIcon className="h-4 w-4 text-gray-400" />
)}
<span className="font-medium">{scenario.name}</span>
</div>
{result && (
<span className={`px-2 py-1 rounded-full text-xs ${
result.result === 'pass' ? 'bg-green-100 text-green-800' : 'bg-red-100 text-red-800'
}`}>
{result.result}
</span>
)}
</div>
{isExpanded && (
<div className="p-3 border-t space-y-2 bg-gray-50">
<div>
<div className="text-sm font-medium text-gray-600">Description</div>
<div className="text-sm">{scenario.description}</div>
</div>
<div>
<div className="text-sm font-medium text-gray-600">Criteria</div>
<div className="text-sm">{scenario.criteria}</div>
</div>
<div>
<div className="text-sm font-medium text-gray-600">Context</div>
<div className="text-sm">{scenario.context}</div>
</div>
{result && (
<div>
<div className="text-sm font-medium text-gray-600">Result Details</div>
<div className="text-sm">{result.details}</div>
</div>
)}
</div>
)}
</div>
);
};

204
apps/rowboat/app/projects/[projectId]/simulation/components/ScenarioComponents.tsx
View file

@ -1,204 +0,0 @@
'use client';
import { useState, useEffect, useCallback } from 'react';
import { Save, EllipsisVerticalIcon, PlayIcon, TrashIcon, X } from "lucide-react";
import { WithStringId } from '../../../../lib/types/types';
import { Scenario } from "../../../../lib/types/testing_types";
import { z } from 'zod';
import { EditableField } from '../../../../lib/components/editable-field';
import { FormSection } from '../../../../lib/components/form-section';
import { StructuredPanel, ActionButton } from "../../../../lib/components/structured-panel";
import clsx from "clsx";
import { Dropdown, DropdownItem, DropdownTrigger, DropdownMenu } from "@nextui-org/react";
import { SectionHeader, ListItem } from "../../../../lib/components/structured-list";
type ScenarioType = WithStringId<z.infer<typeof Scenario>>;
interface ScenarioViewerProps {
scenario: ScenarioType;
onSave: (scenario: ScenarioType) => void;
onClose: () => void;
}
export function ScenarioViewer({ scenario, onSave, onClose }: ScenarioViewerProps) {
const [editedScenario, setEditedScenario] = useState<ScenarioType>(scenario);
const [isDirty, setIsDirty] = useState(false);
const [nameError, setNameError] = useState<string | null>(null);
useEffect(() => {
setEditedScenario(scenario);
setIsDirty(false);
}, [scenario]);
const handleChange = useCallback((field: keyof ScenarioType, value: string) => {
if (field === 'name') {
setNameError(value.trim() ? null : 'Name is required');
}
setEditedScenario(prev => ({
...prev,
[field]: value,
}));
setIsDirty(true);
}, []);
const handleSave = useCallback(() => {
if (!editedScenario.name.trim()) {
setNameError('Name is required');
return;
}
onSave(editedScenario);
onClose();
}, [editedScenario, onSave, onClose]);
return (
<StructuredPanel
title="SCENARIO DETAILS"
actions={[
isDirty && (
<ActionButton
key="save"
onClick={handleSave}
icon={<Save className="w-4 h-4" />}
primary
>
Save
</ActionButton>
),
<ActionButton
key="close"
onClick={onClose}
icon={<X className="w-4 h-4" />}
>
Close
</ActionButton>
].filter(Boolean)}
>
<div className="flex flex-col gap-4">
<FormSection label="Name" showDivider>
<EditableField
value={editedScenario.name}
onChange={(value) => handleChange('name', value)}
multiline={false}
className="w-full"
showSaveButton={false}
placeholder="Enter an identifiable scenario name"
error={nameError}
/>
</FormSection>
<FormSection label="Description" showDivider>
<EditableField
value={editedScenario.description}
onChange={(value) => handleChange('description', value)}
multiline={true}
className="w-full"
showSaveButton={false}
placeholder="Describe the user scenario to be simulated"
/>
</FormSection>
<FormSection label="Criteria" showDivider>
<EditableField
value={editedScenario.criteria}
onChange={(value) => handleChange('criteria', value)}
multiline={true}
className="w-full"
showSaveButton={false}
placeholder="Enter success criteria for this scenario to pass in a simulation"
/>
</FormSection>
<FormSection label="Context">
<EditableField
value={editedScenario.context}
onChange={(value) => handleChange('context', value)}
multiline={true}
className="w-full"
showSaveButton={false}
placeholder="Provide context about the user to the assistant at the start of chat"
/>
</FormSection>
</div>
</StructuredPanel>
);
}
function ScenarioDropdown({
name,
onRun,
onDelete,
}: {
name: string;
onRun: () => void;
onDelete: () => void;
}) {
return (
<Dropdown>
<DropdownTrigger>
<EllipsisVerticalIcon size={16} />
</DropdownTrigger>
<DropdownMenu
onAction={(key) => {
if (key === 'run') onRun();
if (key === 'delete') onDelete();
}}
>
<DropdownItem
key="run"
startContent={<PlayIcon className="w-4 h-4" />}
>
Run scenario
</DropdownItem>
<DropdownItem
key="delete"
className="text-danger"
startContent={<TrashIcon className="w-4 h-4" />}
>
Delete
</DropdownItem>
</DropdownMenu>
</Dropdown>
);
}
export function ScenarioList({
scenarios,
selectedId,
onSelect,
onAdd,
onRunScenario,
onDeleteScenario,
}: {
scenarios: ScenarioType[];
selectedId: string | null;
onSelect: (id: string) => void;
onAdd: () => void;
onRunScenario: (id: string) => void;
onDeleteScenario: (id: string) => void;
}) {
return (
<StructuredPanel
title="TESTS"
tooltip="Browse and manage your test scenarios"
>
<div className="overflow-auto flex flex-col gap-1 justify-start">
<SectionHeader title="Scenarios" onAdd={onAdd} />
{scenarios.map((scenario) => (
<ListItem
key={scenario._id}
name={scenario.name}
isSelected={selectedId === scenario._id}
onClick={() => onSelect(scenario._id)}
rightElement={
<ScenarioDropdown
name={scenario.name}
onRun={() => onRunScenario(scenario._id)}
onDelete={() => onDeleteScenario(scenario._id)}
/>
}
/>
))}
</div>
</StructuredPanel>
);
}

10
apps/rowboat/app/projects/[projectId]/simulation/page.tsx
View file

@ -1,10 +0,0 @@
import { Metadata } from "next";
import SimulationApp from "./app";
export const metadata: Metadata = {
title: "Project simulation",
};
export default function SimulationPage() {
return <SimulationApp />;
}

68
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/app.tsx Normal file
View file

@ -0,0 +1,68 @@
"use client";
import Link from "next/link";
import { useRouter } from "next/navigation";
import { ScenariosApp } from "./scenarios_app";
import { ProfilesApp } from "./profiles_app";
import { SimulationsApp } from "./simulations_app";
import { cn } from "@/lib/utils";
import { usePathname } from "next/navigation";
import { RunsApp } from "./runs_app";
export function App({
projectId,
slug
}: {
projectId: string,
slug?: string[]
}) {
const router = useRouter();
const pathname = usePathname();
let selection: "scenarios" | "profiles" | "criteria" | "simulations" | "runs" = "runs";
if (!slug || slug.length === 0) {
router.push(`/projects/${projectId}/test/runs`);
} else if (slug[0] === "scenarios") {
selection = "scenarios";
} else if (slug[0] === "profiles") {
selection = "profiles";
} else if (slug[0] === "criteria") {
selection = "criteria";
} else if (slug[0] === "simulations") {
selection = "simulations";
} else if (slug[0] === "runs") {
selection = "runs";
}
let innerSlug: string[] = [];
if (slug && slug.length > 1) {
innerSlug = slug.slice(1);
}
const menuItems = [
{ label: "Scenarios", href: `/projects/${projectId}/test/scenarios` },
{ label: "Profiles", href: `/projects/${projectId}/test/profiles` },
{ label: "Simulations", href: `/projects/${projectId}/test/simulations` },
{ label: "Test Runs", href: `/projects/${projectId}/test/runs` },
];
return <div className="flex h-full">
<div className="w-40 shrink-0 p-2">
<ul>
{menuItems.map((item) => (
<li key={item.label}>
<Link
className={cn(
"block p-2 rounded-md text-sm",
pathname.startsWith(item.href) ? "bg-gray-100" : "hover:bg-gray-100"
)}
href={item.href}>{item.label}</Link>
</li>
))}
</ul>
</div>
<div className="grow border-l border-gray-200 p-2">
{selection === "scenarios" && <ScenariosApp projectId={projectId} slug={innerSlug} />}
{selection === "profiles" && <ProfilesApp projectId={projectId} slug={innerSlug} />}
{selection === "simulations" && <SimulationsApp projectId={projectId} slug={innerSlug} />}
{selection === "runs" && <RunsApp projectId={projectId} slug={innerSlug} />}
</div>
</div>;
}

8
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/page.tsx Normal file
View file

@ -0,0 +1,8 @@
import { App } from "./app";
export default function Page({ params }: { params: { projectId: string, slug?: string[] } }) {
return <App
projectId={params.projectId}
slug={params.slug}
/>;
}

535
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/profiles_app.tsx Normal file
View file

@ -0,0 +1,535 @@
import Link from "next/link";
import { WithStringId } from "@/app/lib/types/types";
import { TestProfile } from "@/app/lib/types/testing_types";
import { useEffect, useState, useRef } from "react";
import { createProfile, getProfile, listProfiles, updateProfile, deleteProfile, setDefaultProfile } from "@/app/actions/testing_actions";
import { Button, Input, Pagination, Spinner, Switch, Textarea, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter, Tooltip } from "@nextui-org/react";
import { useRouter, useSearchParams } from "next/navigation";
import { z } from "zod";
import { PlusIcon, ArrowLeftIcon, StarIcon } from "lucide-react";
import { FormStatusButton } from "@/app/lib/components/form-status-button";
import { RelativeTime } from "@primer/react"
import { getProjectConfig } from "@/app/actions/project_actions";
function EditProfile({
projectId,
profileId,
}: {
projectId: string,
profileId: string,
}) {
const router = useRouter();
const [profile, setProfile] = useState<WithStringId<z.infer<typeof TestProfile>> | null>(null);
const [loading, setLoading] = useState(true);
const [mockTools, setMockTools] = useState(false);
const [error, setError] = useState<string | null>(null);
const formRef = useRef<HTMLFormElement>(null);
useEffect(() => {
async function fetchProfile() {
setError(null);
try {
const profile = await getProfile(projectId, profileId);
setProfile(profile);
setMockTools(profile?.mockTools || false);
} catch (error) {
setError(`Unable to fetch profile: ${error}`);
} finally {
setLoading(false);
}
}
fetchProfile();
}, [profileId, projectId]);
async function handleSubmit(formData: FormData) {
setError(null);
try {
const name = formData.get("name") as string;
const context = formData.get("context") as string;
const mockPrompt = formData.get("mockPrompt") as string;
await updateProfile(projectId, profileId, {
name,
context,
mockTools,
mockPrompt: mockPrompt || undefined
});
router.push(`/projects/${projectId}/test/profiles/${profileId}`);
} catch (error) {
setError(`Unable to update profile: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">Edit Profile</h1>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => formRef.current?.requestSubmit()}>Retry</Button>
</div>}
{!loading && profile && (
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<Input
type="text"
name="name"
label="Name"
placeholder="Enter a name for the profile"
defaultValue={profile.name}
required
/>
<Textarea
name="context"
label="Context"
placeholder="Enter the context for this profile"
defaultValue={profile.context}
required
/>
<Switch
name="mockTools"
isSelected={mockTools}
onValueChange={(value) => {
setMockTools(value);
}}
className="self-start"
>
Mock Tools
</Switch>
{mockTools && <Textarea
name="mockPrompt"
label="Mock Prompt (Optional)"
placeholder="Enter a mock prompt"
defaultValue={profile.mockPrompt}
/>}
<div className="flex gap-2 items-center">
<FormStatusButton
props={{
className: "self-start",
children: "Update",
size: "sm",
type: "submit",
}}
/>
<Button
size="sm"
variant="flat"
as={Link}
href={`/projects/${projectId}/test/profiles/${profileId}`}
>
Cancel
</Button>
</div>
</form>
)}
</div>;
}
function ViewProfile({
projectId,
profileId,
}: {
projectId: string,
profileId: string,
}) {
const router = useRouter();
const [profile, setProfile] = useState<WithStringId<z.infer<typeof TestProfile>> | null>(null);
const [defaultProfileId, setDefaultProfileId] = useState<string | null>(null);
const [loading, setLoading] = useState(true);
const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
const [deleteError, setDeleteError] = useState<string | null>(null);
useEffect(() => {
async function fetchProfile() {
const profile = await getProfile(projectId, profileId);
const projectConfig = await getProjectConfig(projectId);
setProfile(profile);
setDefaultProfileId(projectConfig.defaultTestProfileId || null);
setLoading(false);
}
fetchProfile();
}, [projectId, profileId]);
async function handleDelete() {
try {
await deleteProfile(projectId, profileId);
router.push(`/projects/${projectId}/test/profiles`);
} catch (error) {
setDeleteError(`Failed to delete profile: ${error}`);
}
}
async function handleSetDefault() {
await setDefaultProfile(projectId, profileId);
router.push(`/projects/${projectId}/test/profiles`);
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">View Profile</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/profiles`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Profiles
</Button>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{!loading && !profile && <div className="text-gray-600 text-center">Profile not found</div>}
{!loading && profile && (
<>
<div className="flex flex-col gap-1 text-sm">
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Name</div>
<div className="flex-[2]">
<div className="flex flex-col gap-2">
<div>{profile.name}</div>
{defaultProfileId === profile._id && <div className="flex items-center gap-2">
<StarIcon className="w-4 h-4" />
<div className="text-gray-600">This is the default profile</div>
</div>}
</div>
</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Context</div>
<div className="flex-[2] whitespace-pre-wrap">{profile.context}</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Mock Tools</div>
<div className="flex-[2]">{profile.mockTools ? "Yes" : "No"}</div>
</div>
{profile.mockPrompt && <div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Mock Prompt</div>
<div className="flex-[2] whitespace-pre-wrap">{profile.mockPrompt}</div>
</div>}
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Created</div>
<div className="flex-[2]"><RelativeTime date={new Date(profile.createdAt)} /></div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Last Updated</div>
<div className="flex-[2]"><RelativeTime date={new Date(profile.lastUpdatedAt)} /></div>
</div>
</div>
<div className="flex gap-2 mt-4">
<Button
size="sm"
as={Link}
href={`/projects/${projectId}/test/profiles/${profileId}/edit`}
>
Edit
</Button>
{defaultProfileId !== profile._id && <Button
size="sm"
color="danger"
variant="flat"
onClick={() => setIsDeleteModalOpen(true)}
>
Delete
</Button>}
{defaultProfileId !== profile._id && <Button
size="sm"
onClick={() => handleSetDefault()}
startContent={<StarIcon className="w-4 h-4" />}
>
Set as default profile
</Button>}
</div>
<Modal
isOpen={isDeleteModalOpen}
onOpenChange={setIsDeleteModalOpen}
size="sm"
>
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Confirm Deletion</ModalHeader>
<ModalBody>
Are you sure you want to delete this profile?
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Cancel
</Button>
<Button
size="sm"
color="danger"
onPress={() => {
handleDelete();
onClose();
}}
>
Delete
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
<Modal
isOpen={deleteError !== null}
onOpenChange={() => setDeleteError(null)}
size="sm"
>
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Error</ModalHeader>
<ModalBody>
{deleteError}
</ModalBody>
<ModalFooter>
<Button
size="sm"
color="primary"
onPress={onClose}
>
Close
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
</>
)}
</div>;
}
function NewProfile({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const [error, setError] = useState<string | null>(null);
const [mockTools, setMockTools] = useState(false);
const formRef = useRef<HTMLFormElement>(null);
async function handleSubmit(formData: FormData) {
setError(null);
try {
const name = formData.get("name") as string;
const context = formData.get("context") as string;
const mockPrompt = formData.get("mockPrompt") as string;
const profile = await createProfile(projectId, {
name,
context,
mockTools,
mockPrompt: mockPrompt || undefined
});
router.push(`/projects/${projectId}/test/profiles/${profile._id}`);
} catch (error) {
setError(`Unable to create profile: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">New Profile</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/profiles`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Profiles
</Button>
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => formRef.current?.requestSubmit()}>Retry</Button>
</div>}
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<Input
type="text"
name="name"
label="Name"
placeholder="Enter a name for the profile"
required
/>
<Textarea
name="context"
label="Context"
placeholder="Enter the context for this profile"
required
/>
<Switch
name="mockTools"
isSelected={mockTools}
onValueChange={(value) => {
setMockTools(value);
}}
className="self-start"
>
Mock Tools
</Switch>
{mockTools && <Textarea
name="mockPrompt"
label="Mock Prompt (Optional)"
placeholder="Enter a mock prompt"
/>}
<FormStatusButton
props={{
className: "self-start",
children: "Create",
size: "sm",
type: "submit",
}}
/>
</form>
</div>;
}
function ProfileList({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const searchParams = useSearchParams();
const page = parseInt(searchParams.get("page") || "1");
const pageSize = 10;
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [profiles, setProfiles] = useState<WithStringId<z.infer<typeof TestProfile>>[]>([]);
const [defaultProfileId, setDefaultProfileId] = useState<string | null>(null);
const [total, setTotal] = useState(0);
useEffect(() => {
let ignore = false;
async function fetchProfiles() {
setLoading(true);
setError(null);
try {
const profiles = await listProfiles(projectId, page, pageSize);
const projectConfig = await getProjectConfig(projectId);
if (!ignore) {
setProfiles(profiles.profiles);
setTotal(Math.ceil(profiles.total / pageSize));
setDefaultProfileId(projectConfig.defaultTestProfileId || null);
}
} catch (error) {
if (!ignore) {
setError(`Unable to fetch profiles: ${error}`);
}
} finally {
if (!ignore) {
setLoading(false);
}
}
}
if (error == null) {
fetchProfiles();
}
return () => {
ignore = true;
};
}, [page, pageSize, error, projectId]);
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">Profiles</h1>
<Button
size="sm"
onClick={() => router.push(`/projects/${projectId}/test/profiles/new`)}
className="self-end"
startContent={<PlusIcon className="w-4 h-4" />}
>
New Profile
</Button>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => setError(null)}>Retry</Button>
</div>}
{!loading && !error && <>
{profiles.length === 0 && <div className="text-gray-600 text-center">No profiles found</div>}
{profiles.length > 0 && <div className="flex flex-col w-full">
{/* Header */}
<div className="grid grid-cols-8 py-2 bg-gray-100 font-semibold text-sm">
<div className="col-span-2 px-4">Name</div>
<div className="col-span-3 px-4">Context</div>
<div className="col-span-1 px-4">Mock Tools</div>
<div className="col-span-1 px-4">Created</div>
<div className="col-span-1 px-4">Updated</div>
</div>
{/* Rows */}
{profiles.map((profile) => (
<div key={profile._id} className="grid grid-cols-8 py-2 border-b hover:bg-gray-50 text-sm">
<div className="col-span-2 px-4 truncate">
<div className="flex items-center gap-2">
<Link
href={`/projects/${projectId}/test/profiles/${profile._id}`}
className="text-blue-600 hover:underline"
>
{profile.name}
</Link>
{defaultProfileId === profile._id && <Tooltip content="Default Profile">
<StarIcon className="w-4 h-4" />
</Tooltip>}
</div>
</div>
<div className="col-span-3 px-4 truncate">{profile.context}</div>
<div className="col-span-1 px-4">{profile.mockTools ? "Yes" : "No"}</div>
<div className="col-span-1 px-4 text-gray-600 truncate">
<RelativeTime date={new Date(profile.createdAt)} />
</div>
<div className="col-span-1 px-4 text-gray-600 truncate">
<RelativeTime date={new Date(profile.lastUpdatedAt)} />
</div>
</div>
))}
</div>}
{total > 1 && <Pagination
total={total}
page={page}
onChange={(page) => {
router.push(`/projects/${projectId}/test/profiles?page=${page}`);
}}
className="self-center"
/>}
</>}
</div>;
}
export function ProfilesApp({
projectId,
slug
}: {
projectId: string,
slug: string[]
}) {
let selection: "list" | "view" | "new" | "edit" = "list";
let profileId: string | null = null;
if (slug.length > 0) {
if (slug[0] === "new") {
selection = "new";
} else if (slug[slug.length - 1] === "edit") {
selection = "edit";
profileId = slug[0];
} else {
selection = "view";
profileId = slug[0];
}
}
return <>
{selection === "list" && <ProfileList projectId={projectId} />}
{selection === "new" && <NewProfile projectId={projectId} />}
{selection === "view" && profileId && <ViewProfile projectId={projectId} profileId={profileId} />}
{selection === "edit" && profileId && <EditProfile projectId={projectId} profileId={profileId} />}
</>;
}

456
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/runs_app.tsx Normal file
View file

@ -0,0 +1,456 @@
import Link from "next/link";
import { WithStringId } from "@/app/lib/types/types";
import { TestSimulation, TestRun } from "@/app/lib/types/testing_types";
import { useEffect, useState, useRef } from "react";
import { createRun, getRun, getSimulation, listRuns } from "@/app/actions/testing_actions";
import { Button, Input, Pagination, Spinner, Chip } from "@nextui-org/react";
import { useRouter, useSearchParams } from "next/navigation";
import { z } from "zod";
import { ArrowLeftIcon, PlusIcon, WorkflowIcon } from "lucide-react";
import { FormStatusButton } from "@/app/lib/components/form-status-button";
import { RelativeTime } from "@primer/react"
import { SimulationSelector } from "@/app/lib/components/selectors/simulation-selector";
import { WorkflowSelector } from "@/app/lib/components/selectors/workflow-selector";
import { Workflow } from "@/app/lib/types/workflow_types";
import { fetchWorkflow } from "@/app/actions/workflow_actions";
function NewRun({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const [error, setError] = useState<string | null>(null);
const formRef = useRef<HTMLFormElement>(null);
const [selectedSimulations, setSelectedSimulations] = useState<WithStringId<z.infer<typeof TestSimulation>>[]>([]);
const [isSimulationSelectorOpen, setIsSimulationSelectorOpen] = useState(false);
const [selectedWorkflow, setSelectedWorkflow] = useState<WithStringId<z.infer<typeof Workflow>> | null>(null);
const [isWorkflowSelectorOpen, setIsWorkflowSelectorOpen] = useState(false);
async function handleSubmit(formData: FormData) {
setError(null);
const simulationIds = selectedSimulations.map(sim => sim._id);
if (!selectedWorkflow) {
setError("Please select a workflow");
return;
}
if (simulationIds.length === 0) {
setError("Please select at least one simulation");
return;
}
try {
const run = await createRun(projectId, {
workflowId: selectedWorkflow._id,
simulationIds
});
router.push(`/projects/${projectId}/test/runs/${run._id}`);
} catch (error) {
setError(`Unable to create run: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">New Run</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/runs`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Runs
</Button>
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button
size="sm"
color="danger"
onClick={() => {
formRef.current?.requestSubmit();
}}
>
Retry
</Button>
</div>}
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<div className="flex flex-col gap-2">
<label className="text-sm font-medium">Workflow</label>
<div className="flex items-center gap-2">
{selectedWorkflow ? (
<div className="text-sm text-blue-600">{selectedWorkflow.name}</div>
) : (
<div className="text-sm text-gray-500">No workflow selected</div>
)}
<Button
size="sm"
onClick={() => setIsWorkflowSelectorOpen(true)}
type="button"
>
{selectedWorkflow ? "Change" : "Select"} Workflow
</Button>
</div>
</div>
<div className="flex flex-col gap-2">
<Button
size="sm"
onClick={() => setIsSimulationSelectorOpen(true)}
type="button"
className="self-start"
>
Select Simulations
</Button>
{selectedSimulations.length > 0 && (
<div className="flex flex-wrap gap-2">
{selectedSimulations.map((sim) => (
<Chip
key={sim._id}
onClose={() => setSelectedSimulations(prev => prev.filter(s => s._id !== sim._id))}
variant="flat"
className="py-1"
>
{sim.name}
</Chip>
))}
</div>
)}
</div>
<FormStatusButton
props={{
className: "self-start",
children: "Create Run",
size: "sm",
type: "submit",
isDisabled: !selectedWorkflow || selectedSimulations.length === 0,
}}
/>
</form>
<SimulationSelector
projectId={projectId}
isOpen={isSimulationSelectorOpen}
onOpenChange={setIsSimulationSelectorOpen}
onSelect={setSelectedSimulations}
initialSelected={selectedSimulations}
/>
<WorkflowSelector
projectId={projectId}
isOpen={isWorkflowSelectorOpen}
onOpenChange={setIsWorkflowSelectorOpen}
onSelect={setSelectedWorkflow}
/>
</div>;
}
function ViewRun({
projectId,
runId,
}: {
projectId: string,
runId: string,
}) {
const [run, setRun] = useState<WithStringId<z.infer<typeof TestRun>> | null>(null);
const [loading, setLoading] = useState(true);
const [workflow, setWorkflow] = useState<WithStringId<z.infer<typeof Workflow>> | null>(null);
const [simulations, setSimulations] = useState<WithStringId<z.infer<typeof TestSimulation>>[]>([]);
useEffect(() => {
async function fetchRun() {
const run = await getRun(projectId, runId);
setRun(run);
if (run) {
// Fetch workflow and simulations in parallel
const [workflowResult, simulationsResult] = await Promise.all([
fetchWorkflow(projectId, run.workflowId),
Promise.all(run.simulationIds.map(id => getSimulation(projectId, id)))
]);
setWorkflow(workflowResult);
setSimulations(simulationsResult.filter(s => s !== null));
}
setLoading(false);
}
fetchRun();
}, [runId, projectId]);
return <div className="h-full flex flex-col gap-4">
<div className="flex items-center justify-between">
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/runs`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Runs
</Button>
</div>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{!loading && !run && <div className="text-gray-600 text-center">Run not found</div>}
{!loading && run && (
<>
{/* Workflow and timing information in a grid */}
<div className="grid grid-cols-3 gap-4">
{workflow && (
<div className="bg-gray-50 dark:bg-neutral-800 p-4 rounded-lg">
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Workflow Version</div>
<div className="font-medium dark:text-neutral-200">{workflow.name}</div>
</div>
)}
<div className="bg-gray-50 dark:bg-neutral-800 p-4 rounded-lg">
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Completed</div>
<div className="text-sm dark:text-neutral-300">
{run.completedAt ? <RelativeTime date={new Date(run.completedAt)} /> : 'Not completed'}
</div>
</div>
<div className="bg-gray-50 dark:bg-neutral-800 p-4 rounded-lg">
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Duration</div>
<div className="text-sm dark:text-neutral-300">
{run.completedAt ?
`${((new Date(run.completedAt).getTime() - new Date(run.startedAt).getTime()) / 1000).toFixed(1)}s` :
'In Progress'}
</div>
</div>
</div>
{/* Results statistics */}
<div className="grid grid-cols-3 gap-4">
<div className="p-4 rounded-lg bg-gray-50 dark:bg-neutral-800">
<div className="text-sm text-gray-600 dark:text-neutral-400">Total Tests</div>
<div className="text-2xl font-semibold dark:text-neutral-200">{run.aggregateResults?.total || 0}</div>
</div>
<div className="p-4 rounded-lg bg-green-50 dark:bg-green-900/20">
<div className="text-sm text-green-600 dark:text-green-400">Passed</div>
<div className="text-2xl font-semibold text-green-700 dark:text-green-400">{run.aggregateResults?.pass || 0}</div>
</div>
<div className="p-4 rounded-lg bg-red-50 dark:bg-red-900/20">
<div className="text-sm text-red-600 dark:text-red-400">Failed</div>
<div className="text-2xl font-semibold text-red-700 dark:text-red-400">{run.aggregateResults?.fail || 0}</div>
</div>
</div>
{/* Simulations List */}
<div className="mt-4">
<h2 className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-2">Simulations</h2>
<div className="space-y-2">
{simulations.map(sim => (
<div key={sim._id} className="border dark:border-neutral-800 rounded-lg p-3">
<Link
href={`/projects/${projectId}/test/simulations/${sim._id}`}
className="text-blue-600 hover:underline"
>
{sim.name}
</Link>
</div>
))}
</div>
</div>
</>
)}
</div>;
}
function RunList({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const searchParams = useSearchParams();
const page = parseInt(searchParams.get("page") || "1");
const pageSize = 10;
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [runs, setRuns] = useState<WithStringId<z.infer<typeof TestRun>>[]>([]);
const [workflowMap, setWorkflowMap] = useState<Record<string, WithStringId<z.infer<typeof Workflow>>>>({});
const [total, setTotal] = useState(0);
useEffect(() => {
let ignore = false;
async function fetchRuns() {
setLoading(true);
setError(null);
try {
const result = await listRuns(projectId, page, pageSize);
if (!ignore) {
setRuns(result.runs);
setTotal(Math.ceil(result.total / pageSize));
}
} catch (error) {
if (!ignore) {
setError(`Unable to fetch runs: ${error}`);
}
} finally {
if (!ignore) {
setLoading(false);
}
}
}
if (error == null) {
fetchRuns();
}
return () => {
ignore = true;
};
}, [page, pageSize, error, projectId]);
useEffect(() => {
let ignore = false;
async function resolveWorkflows() {
const workflowIds = runs.reduce((acc, run) => {
if (!acc.includes(run.workflowId)) {
acc.push(run.workflowId);
}
return acc;
}, [] as string[]);
const workflows = await Promise.all(workflowIds.map((workflowId) => fetchWorkflow(projectId, workflowId)));
if (ignore) {
return;
}
setWorkflowMap(workflows.filter((workflow) => workflow !== null).reduce((acc, workflow) => {
acc[workflow._id] = workflow;
return acc;
}, {} as Record<string, WithStringId<z.infer<typeof Workflow>>>));
}
if (error == null) {
resolveWorkflows();
}
return () => {
ignore = true;
};
}, [runs, error, projectId]);
return <div className="h-full flex flex-col gap-4">
<div className="flex items-center justify-between">
<h1 className="text-xl font-semibold text-gray-800 dark:text-neutral-200">Test Runs</h1>
<Button
size="sm"
onClick={() => router.push(`/projects/${projectId}/test/runs/new`)}
startContent={<PlusIcon className="w-4 h-4" />}
>
New Run
</Button>
</div>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => setError(null)}>Retry</Button>
</div>}
{!loading && !error && <>
{runs.length === 0 && <div className="text-gray-600 text-center">No test runs found</div>}
{runs.length > 0 && <div className="space-y-4">
{runs.map((run) => (
<div key={run._id} className="border dark:border-neutral-800 rounded-lg shadow-sm">
<div className="p-4 flex items-center justify-between hover:bg-neutral-100 dark:hover:bg-neutral-800">
<div className="flex items-center space-x-4">
<Link
href={`/projects/${projectId}/test/runs/${run._id}`}
className="text-blue-600 hover:underline"
>
{run.name}
</Link>
{workflowMap[run.workflowId] && (
<div className="flex items-center gap-2 text-sm text-gray-600 dark:text-neutral-400">
<WorkflowIcon className="w-4 h-4 shrink-0" />
{workflowMap[run.workflowId].name}
</div>
)}
</div>
<div className="flex items-center gap-4">
<span className={getStatusClass(run.status)}>
{run.status}
</span>
<div className="text-sm text-gray-600 dark:text-neutral-400">
<RelativeTime date={new Date(run.startedAt)} />
</div>
</div>
</div>
{run.aggregateResults && (
<div className="border-t dark:border-neutral-800 px-4 py-2 bg-gray-50 dark:bg-neutral-900/50">
<div className="grid grid-cols-3 gap-4 text-sm">
<div className="text-gray-600 dark:text-neutral-400">
Total: {run.aggregateResults.total}
</div>
<div className="text-green-600 dark:text-green-400">
Passed: {run.aggregateResults.pass}
</div>
<div className="text-red-600 dark:text-red-400">
Failed: {run.aggregateResults.fail}
</div>
</div>
</div>
)}
</div>
))}
</div>}
{total > 1 && <Pagination
total={total}
page={page}
onChange={(page) => {
router.push(`/projects/${projectId}/test/runs?page=${page}`);
}}
className="self-center"
/>}
</>}
</div>;
}
// Helper function for status styling
function getStatusClass(status: string) {
const baseClass = "px-2 py-1 rounded text-xs uppercase font-medium";
switch (status) {
case 'completed':
return `${baseClass} bg-green-100 text-green-800`;
case 'failed':
case 'error':
return `${baseClass} bg-red-100 text-red-800`;
case 'cancelled':
return `${baseClass} bg-gray-100 text-gray-800`;
case 'running':
case 'pending':
default:
return `${baseClass} bg-yellow-100 text-yellow-800`;
}
}
export function RunsApp({
projectId,
slug
}: {
projectId: string,
slug: string[]
}) {
let selection: "list" | "view" | "new" = "list";
let runId: string | null = null;
if (slug.length > 0) {
if (slug[0] === "new") {
selection = "new";
} else {
selection = "view";
runId = slug[0];
}
}
return <>
{selection === "list" && <RunList projectId={projectId} />}
{selection === "new" && <NewRun projectId={projectId} />}
{selection === "view" && runId && <ViewRun projectId={projectId} runId={runId} />}
</>;
}

463
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/scenarios_app.tsx Normal file
View file

@ -0,0 +1,463 @@
import Link from "next/link";
import { WithStringId } from "@/app/lib/types/types";
import { TestScenario } from "@/app/lib/types/testing_types";
import { useEffect, useState, useRef } from "react";
import { createScenario, getScenario, listScenarios, updateScenario, deleteScenario } from "@/app/actions/testing_actions";
import { Button, Input, Pagination, Spinner, Textarea, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react";
import { useRouter, useSearchParams } from "next/navigation";
import { z } from "zod";
import { ArrowLeftIcon, PlusIcon } from "lucide-react";
import { FormStatusButton } from "@/app/lib/components/form-status-button";
import { RelativeTime } from "@primer/react"
function EditScenario({
projectId,
scenarioId,
}: {
projectId: string,
scenarioId: string,
}) {
const router = useRouter();
const [scenario, setScenario] = useState<WithStringId<z.infer<typeof TestScenario>> | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const formRef = useRef<HTMLFormElement>(null);
useEffect(() => {
async function fetchScenario() {
setError(null);
try {
const scenario = await getScenario(projectId, scenarioId);
setScenario(scenario);
} catch (error) {
setError(`Unable to fetch scenario: ${error}`);
} finally {
setLoading(false);
}
}
fetchScenario();
}, [scenarioId, projectId]);
async function handleSubmit(formData: FormData) {
setError(null);
try {
const name = formData.get("name") as string;
const description = formData.get("description") as string;
await updateScenario(projectId, scenarioId, { name, description });
router.push(`/projects/${projectId}/test/scenarios/${scenarioId}`);
} catch (error) {
setError(`Unable to update scenario: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">Edit Scenario</h1>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button
size="sm"
color="danger"
onClick={() => {
formRef.current?.requestSubmit();
}}
>
Retry
</Button>
</div>}
{!loading && scenario && (
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<Input
type="text"
name="name"
label="Name"
placeholder="Enter a name for the scenario"
defaultValue={scenario.name}
required
/>
<Textarea
name="description"
label="Description"
placeholder="Enter a description for the scenario"
defaultValue={scenario.description}
required
/>
<div className="flex gap-2 items-center">
<FormStatusButton
props={{
className: "self-start",
children: "Update",
size: "sm",
type: "submit",
}}
/>
<Button
size="sm"
variant="flat"
as={Link}
href={`/projects/${projectId}/test/scenarios/${scenarioId}`}
>
Cancel
</Button>
</div>
</form>
)}
</div>;
}
function ViewScenario({
projectId,
scenarioId,
}: {
projectId: string,
scenarioId: string,
}) {
const router = useRouter();
const [scenario, setScenario] = useState<WithStringId<z.infer<typeof TestScenario>> | null>(null);
const [loading, setLoading] = useState(true);
const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
const [deleteError, setDeleteError] = useState<string | null>(null);
useEffect(() => {
async function fetchScenario() {
const scenario = await getScenario(projectId, scenarioId);
setScenario(scenario);
setLoading(false);
}
fetchScenario();
}, [scenarioId, projectId]);
async function handleDelete() {
try {
await deleteScenario(projectId, scenarioId);
router.push(`/projects/${projectId}/test/scenarios`);
} catch (error) {
setDeleteError(`Failed to delete scenario: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">View Scenario</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/scenarios`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Scenarios
</Button>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{!loading && !scenario && <div className="text-gray-600 text-center">Scenario not found</div>}
{!loading && scenario && (
<>
<div className="flex flex-col gap-1 text-sm">
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Name</div>
<div className="flex-[2]">{scenario.name}</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Description</div>
<div className="flex-[2]">{scenario.description}</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Created</div>
<div className="flex-[2]"><RelativeTime date={new Date(scenario.createdAt)} /></div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Last Updated</div>
<div className="flex-[2]"><RelativeTime date={new Date(scenario.lastUpdatedAt)} /></div>
</div>
</div>
<div className="flex gap-2 mt-4">
<Button
size="sm"
as={Link}
href={`/projects/${projectId}/test/scenarios/${scenarioId}/edit`}
>
Edit
</Button>
<Button
size="sm"
color="danger"
variant="flat"
onClick={() => setIsDeleteModalOpen(true)}
>
Delete
</Button>
</div>
<Modal
isOpen={isDeleteModalOpen}
onOpenChange={setIsDeleteModalOpen}
size="sm"
>
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Confirm Deletion</ModalHeader>
<ModalBody>
Are you sure you want to delete this scenario?
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Cancel
</Button>
<Button
size="sm"
color="danger"
onPress={() => {
handleDelete();
onClose();
}}
>
Delete
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
<Modal
isOpen={deleteError !== null}
onOpenChange={() => setDeleteError(null)}
size="sm"
>
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Error</ModalHeader>
<ModalBody>
{deleteError}
</ModalBody>
<ModalFooter>
<Button
size="sm"
color="primary"
onPress={onClose}
>
Close
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
</>
)}
</div>;
}
function NewScenario({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const [error, setError] = useState<string | null>(null);
const formRef = useRef<HTMLFormElement>(null);
async function handleSubmit(formData: FormData) {
setError(null);
const name = formData.get("name") as string;
const description = formData.get("description") as string;
try {
const scenario = await createScenario(projectId, { name, description });
router.push(`/projects/${projectId}/test/scenarios/${scenario._id}`);
} catch (error) {
setError(`Unable to create scenario: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">New Scenario</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/scenarios`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Scenarios
</Button>
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button
size="sm"
color="danger"
onClick={() => {
formRef.current?.requestSubmit();
}}
>
Retry
</Button>
</div>}
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<Input
type="text"
name="name"
label="Name"
placeholder="Enter a name for the scenario"
required
/>
<Textarea
name="description"
label="Description"
placeholder="Enter a description for the scenario"
required
/>
<FormStatusButton
props={{
className: "self-start",
children: "Create",
size: "sm",
type: "submit",
}}
/>
</form>
</div>;
}
function ScenarioList({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const searchParams = useSearchParams();
const page = parseInt(searchParams.get("page") || "1");
const pageSize = 10;
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [scenarios, setScenarios] = useState<WithStringId<z.infer<typeof TestScenario>>[]>([]);
const [total, setTotal] = useState(0);
useEffect(() => {
let ignore = false;
async function fetchScenarios() {
setLoading(true);
setError(null);
try {
const scenarios = await listScenarios(projectId, page, pageSize);
if (!ignore) {
setScenarios(scenarios.scenarios);
setTotal(Math.ceil(scenarios.total / pageSize));
}
} catch (error) {
if (!ignore) {
setError(`Unable to fetch scenarios: ${error}`);
}
} finally {
if (!ignore) {
setLoading(false);
}
}
}
if (error == null) {
fetchScenarios();
}
return () => {
ignore = true;
};
}, [page, pageSize, error, projectId]);
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">Scenarios</h1>
<Button
size="sm"
onClick={() => router.push(`/projects/${projectId}/test/scenarios/new`)}
className="self-end"
startContent={<PlusIcon className="w-4 h-4" />}
>
New Scenario
</Button>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => setError(null)}>Retry</Button>
</div>}
{!loading && !error && <>
{scenarios.length === 0 && <div className="text-gray-600 text-center">No scenarios found</div>}
{scenarios.length > 0 && <div className="flex flex-col w-full">
{/* Header */}
<div className="grid grid-cols-7 py-2 bg-gray-100 font-semibold text-sm">
<div className="col-span-2 px-4">Name</div>
<div className="col-span-3 px-4">Description</div>
<div className="col-span-1 px-4">Created</div>
<div className="col-span-1 px-4">Updated</div>
</div>
{/* Rows */}
{scenarios.map((scenario) => (
<div key={scenario._id} className="grid grid-cols-7 py-2 border-b hover:bg-gray-50 text-sm">
<div className="col-span-2 px-4 truncate">
<Link
href={`/projects/${projectId}/test/scenarios/${scenario._id}`}
className="text-blue-600 hover:underline"
>
{scenario.name}
</Link>
</div>
<div className="col-span-3 px-4 truncate">{scenario.description}</div>
<div className="col-span-1 px-4 text-gray-600 truncate">
<RelativeTime date={new Date(scenario.createdAt)} />
</div>
<div className="col-span-1 px-4 text-gray-600 truncate">
<RelativeTime date={new Date(scenario.lastUpdatedAt)} />
</div>
</div>
))}
</div>}
{total > 1 && <Pagination
total={total}
page={page}
onChange={(page) => {
router.push(`/projects/${projectId}/test/scenarios?page=${page}`);
}}
className="self-center"
/>}
</>}
</div>;
}
export function ScenariosApp({
projectId,
slug
}: {
projectId: string,
slug: string[]
}) {
let selection: "list" | "view" | "new" | "edit" = "list";
let scenarioId: string | null = null;
if (slug.length > 0) {
if (slug[0] === "new") {
selection = "new";
} else if (slug[slug.length - 1] === "edit") {
selection = "edit";
scenarioId = slug[0];
} else {
selection = "view";
scenarioId = slug[0];
}
}
return <>
{selection === "list" && <ScenarioList projectId={projectId} />}
{selection === "new" && <NewScenario projectId={projectId} />}
{selection === "view" && scenarioId && <ViewScenario projectId={projectId} scenarioId={scenarioId} />}
{selection === "edit" && scenarioId && <EditScenario projectId={projectId} scenarioId={scenarioId} />}
</>;
}

690
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/simulations_app.tsx Normal file
View file

@ -0,0 +1,690 @@
import Link from "next/link";
import { WithStringId } from "@/app/lib/types/types";
import { TestProfile, TestScenario, TestSimulation } from "@/app/lib/types/testing_types";
import { useEffect, useState, useRef } from "react";
import { createSimulation, getSimulation, listSimulations, updateSimulation, deleteSimulation, listScenarios, getScenario, getProfile } from "@/app/actions/testing_actions";
import { Button, Input, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react";
import { useRouter, useSearchParams } from "next/navigation";
import { z } from "zod";
import { PlusIcon, ArrowLeftIcon } from "lucide-react";
import { FormStatusButton } from "@/app/lib/components/form-status-button";
import { RelativeTime } from "@primer/react"
import { ScenarioSelector } from "@/app/lib/components/selectors/scenario-selector";
import { ProfileSelector } from "@/app/lib/components/selectors/profile-selector";
function EditSimulation({
projectId,
simulationId,
}: {
projectId: string,
simulationId: string,
}) {
const router = useRouter();
const [simulation, setSimulation] = useState<WithStringId<z.infer<typeof TestSimulation>> | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const formRef = useRef<HTMLFormElement>(null);
const [scenario, setScenario] = useState<WithStringId<z.infer<typeof TestScenario>> | null>(null);
const [profile, setProfile] = useState<WithStringId<z.infer<typeof TestProfile>> | null>(null);
const [isScenarioModalOpen, setIsScenarioModalOpen] = useState(false);
const [isProfileModalOpen, setIsProfileModalOpen] = useState(false);
useEffect(() => {
async function fetchSimulation() {
setError(null);
try {
const simulation = await getSimulation(projectId, simulationId);
setSimulation(simulation);
if (simulation) {
const [scenarioResult, profileResult] = await Promise.all([
getScenario(projectId, simulation.scenarioId),
getProfile(projectId, simulation.profileId),
]);
setScenario(scenarioResult);
setProfile(profileResult);
}
} catch (error) {
setError(`Unable to fetch simulation: ${error}`);
} finally {
setLoading(false);
}
}
fetchSimulation();
}, [simulationId, projectId]);
async function handleSubmit(formData: FormData) {
setError(null);
try {
const name = formData.get("name") as string;
const passCriteria = formData.get("passCriteria") as string;
if (!name || !passCriteria) {
throw new Error("Name and Pass Criteria are required");
}
if (!scenario || !profile) {
throw new Error("Please select all required fields");
}
await updateSimulation(projectId, simulationId, {
name,
scenarioId: scenario._id,
profileId: profile._id,
passCriteria
});
router.push(`/projects/${projectId}/test/simulations/${simulationId}`);
} catch (error) {
setError(`Unable to update simulation: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">Edit Simulation</h1>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => formRef.current?.requestSubmit()}>Retry</Button>
</div>}
{!loading && simulation && (
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<Input
type="text"
name="name"
label="Name"
placeholder="Enter a name for the simulation"
defaultValue={simulation.name}
required
/>
<Input
type="text"
name="passCriteria"
label="Pass Criteria"
placeholder="Enter the criteria for passing this simulation"
defaultValue={simulation.passCriteria}
required
/>
<div className="flex flex-col gap-2">
<label className="text-sm font-medium">Scenario</label>
<div className="flex items-center gap-2">
{scenario ? (
<div className="text-sm text-blue-600">{scenario.name}</div>
) : (
<div className="text-sm text-gray-500">No scenario selected</div>
)}
<Button
size="sm"
onClick={() => setIsScenarioModalOpen(true)}
type="button"
>
{scenario ? "Change" : "Select"} Scenario
</Button>
</div>
</div>
<div className="flex flex-col gap-2">
<label className="text-sm font-medium">Profile</label>
<div className="flex items-center gap-2">
{profile ? (
<div className="text-sm text-blue-600">{profile.name}</div>
) : (
<div className="text-sm text-gray-500">No profile selected</div>
)}
<Button
size="sm"
onClick={() => setIsProfileModalOpen(true)}
type="button"
>
{profile ? "Change" : "Select"} Profile
</Button>
</div>
</div>
<div className="flex gap-2 items-center">
<FormStatusButton
props={{
className: "self-start",
children: "Update",
size: "sm",
type: "submit",
isDisabled: !scenario || !profile,
}}
/>
<Button
size="sm"
variant="flat"
as={Link}
href={`/projects/${projectId}/test/simulations/${simulationId}`}
>
Cancel
</Button>
</div>
<ScenarioSelector
projectId={projectId}
isOpen={isScenarioModalOpen}
onOpenChange={setIsScenarioModalOpen}
onSelect={setScenario}
/>
<ProfileSelector
projectId={projectId}
isOpen={isProfileModalOpen}
onOpenChange={setIsProfileModalOpen}
onSelect={setProfile}
/>
</form>
)}
</div>;
}
function ViewSimulation({
projectId,
simulationId,
}: {
projectId: string,
simulationId: string,
}) {
const router = useRouter();
const [simulation, setSimulation] = useState<WithStringId<z.infer<typeof TestSimulation>> | null>(null);
const [loading, setLoading] = useState(true);
const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
const [deleteError, setDeleteError] = useState<string | null>(null);
const [scenario, setScenario] = useState<WithStringId<z.infer<typeof TestScenario>> | null>(null);
const [profile, setProfile] = useState<WithStringId<z.infer<typeof TestProfile>> | null>(null);
useEffect(() => {
async function fetchSimulation() {
const simulation = await getSimulation(projectId, simulationId);
setSimulation(simulation);
if (simulation) {
const [scenarioResult, profileResult] = await Promise.all([
getScenario(projectId, simulation.scenarioId),
getProfile(projectId, simulation.profileId),
]);
setScenario(scenarioResult);
setProfile(profileResult);
}
setLoading(false);
}
fetchSimulation();
}, [simulationId, projectId]);
async function handleDelete() {
try {
await deleteSimulation(projectId, simulationId);
router.push(`/projects/${projectId}/test/simulations`);
} catch (error) {
setDeleteError(`Failed to delete simulation: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">View Simulation</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/simulations`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Simulations
</Button>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{!loading && !simulation && <div className="text-gray-600 text-center">Simulation not found</div>}
{!loading && simulation && (
<>
<div className="flex flex-col gap-1 text-sm">
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Name</div>
<div className="flex-[2]">{simulation.name}</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Scenario</div>
<div className="flex-[2]">
{scenario ? (
<Link href={`/projects/${projectId}/test/scenarios/${scenario._id}`} className="text-blue-600 hover:underline">
{scenario.name}
</Link>
) : (
<div className="text-sm text-gray-500">No scenario selected</div>
)}
</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Profile</div>
<div className="flex-[2]">
{profile ? (
<Link href={`/projects/${projectId}/test/profiles/${profile._id}`} className="text-blue-600 hover:underline">
{profile.name}
</Link>
) : (
<div className="text-sm text-gray-500">No profile selected</div>
)}
</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Pass Criteria</div>
<div className="flex-[2]">{simulation.passCriteria}</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Created</div>
<div className="flex-[2]"><RelativeTime date={new Date(simulation.createdAt)} /></div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Last Updated</div>
<div className="flex-[2]"><RelativeTime date={new Date(simulation.lastUpdatedAt)} /></div>
</div>
</div>
<div className="flex gap-2 mt-4">
<Button
size="sm"
as={Link}
href={`/projects/${projectId}/test/simulations/${simulationId}/edit`}
>
Edit
</Button>
<Button
size="sm"
color="danger"
variant="flat"
onClick={() => setIsDeleteModalOpen(true)}
>
Delete
</Button>
</div>
<Modal
isOpen={isDeleteModalOpen}
onOpenChange={setIsDeleteModalOpen}
size="sm"
>
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Confirm Deletion</ModalHeader>
<ModalBody>
Are you sure you want to delete this simulation?
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Cancel
</Button>
<Button
size="sm"
color="danger"
onPress={() => {
handleDelete();
onClose();
}}
>
Delete
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
<Modal
isOpen={deleteError !== null}
onOpenChange={() => setDeleteError(null)}
size="sm"
>
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Error</ModalHeader>
<ModalBody>
{deleteError}
</ModalBody>
<ModalFooter>
<Button
size="sm"
color="primary"
onPress={onClose}
>
Close
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
</>
)}
</div>;
}
function NewSimulation({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const [error, setError] = useState<string | null>(null);
const [scenario, setScenario] = useState<WithStringId<z.infer<typeof TestScenario>> | null>(null);
const [profile, setProfile] = useState<WithStringId<z.infer<typeof TestProfile>> | null>(null);
const formRef = useRef<HTMLFormElement>(null);
const [isScenarioModalOpen, setIsScenarioModalOpen] = useState(false);
const [isProfileModalOpen, setIsProfileModalOpen] = useState(false);
async function handleSubmit(formData: FormData) {
setError(null);
try {
const name = formData.get("name") as string;
const passCriteria = formData.get("passCriteria") as string;
if (!name || !passCriteria) {
throw new Error("Name and Pass Criteria are required");
}
if (!scenario) {
throw new Error("Please select a scenario");
}
if (!profile) {
throw new Error("Please select a profile");
}
const result = await createSimulation(projectId, {
name,
scenarioId: scenario._id,
profileId: profile._id,
passCriteria,
});
router.push(`/projects/${projectId}/test/simulations/${result._id}`);
} catch (error) {
setError(`Unable to create simulation: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">New Simulation</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/simulations`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Simulations
</Button>
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => formRef.current?.requestSubmit()}>Retry</Button>
</div>}
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<Input
type="text"
name="name"
label="Name"
placeholder="Enter a name for the simulation"
required
/>
<Input
type="text"
name="passCriteria"
label="Pass Criteria"
placeholder="Enter the criteria for passing this simulation"
required
/>
<div className="flex flex-col gap-2">
<label className="text-sm font-medium">Scenario</label>
<div className="flex items-center gap-2">
{scenario ? (
<div className="text-sm text-blue-600">{scenario.name}</div>
) : (
<div className="text-sm text-gray-500">No scenario selected</div>
)}
<Button
size="sm"
onClick={() => setIsScenarioModalOpen(true)}
type="button"
>
{scenario ? "Change" : "Select"} Scenario
</Button>
</div>
</div>
<div className="flex flex-col gap-2">
<label className="text-sm font-medium">Profile</label>
<div className="flex items-center gap-2">
{profile ? (
<div className="text-sm text-blue-600">{profile.name}</div>
) : (
<div className="text-sm text-gray-500">No profile selected</div>
)}
<Button
size="sm"
onClick={() => setIsProfileModalOpen(true)}
type="button"
>
{profile ? "Change" : "Select"} Profile
</Button>
</div>
</div>
<FormStatusButton
props={{
className: "self-start",
children: "Create",
size: "sm",
type: "submit",
isDisabled: !scenario || !profile,
}}
/>
</form>
<ScenarioSelector
projectId={projectId}
isOpen={isScenarioModalOpen}
onOpenChange={setIsScenarioModalOpen}
onSelect={setScenario}
/>
<ProfileSelector
projectId={projectId}
isOpen={isProfileModalOpen}
onOpenChange={setIsProfileModalOpen}
onSelect={setProfile}
/>
</div>;
}
function SimulationList({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const searchParams = useSearchParams();
const page = parseInt(searchParams.get("page") || "1");
const pageSize = 10;
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [simulationList, setSimulationList] = useState<WithStringId<z.infer<typeof TestSimulation>>[]>([]);
const [scenarioMap, setScenarioMap] = useState<Record<string, WithStringId<z.infer<typeof TestScenario>>>>({});
const [profileMap, setProfileMap] = useState<Record<string, WithStringId<z.infer<typeof TestProfile>>>>({});
const [total, setTotal] = useState(0);
useEffect(() => {
let ignore = false;
async function fetchSimulation() {
setLoading(true);
setError(null);
try {
const result = await listSimulations(projectId, page, pageSize);
if (!ignore) {
setSimulationList(result.simulations);
setTotal(Math.ceil(result.total / pageSize));
}
} catch (error) {
if (!ignore) {
setError(`Unable to fetch simulation: ${error}`);
}
} finally {
if (!ignore) {
setLoading(false);
}
}
}
if (error == null) {
fetchSimulation();
}
return () => {
ignore = true;
};
}, [page, pageSize, error, projectId]);
useEffect(() => {
let ignore = false;
async function resolveScenarios() {
const scenarioIds = simulationList.reduce((acc, simulation) => {
if (!acc.includes(simulation.scenarioId)) {
acc.push(simulation.scenarioId);
}
return acc;
}, [] as string[]);
const scenarios = await Promise.all(scenarioIds.map((scenarioId) => getScenario(projectId, scenarioId)));
if (ignore) {
return;
}
setScenarioMap(scenarios.filter((scenario) => scenario !== null).reduce((acc, scenario) => {
acc[scenario._id] = scenario;
return acc;
}, {} as Record<string, WithStringId<z.infer<typeof TestScenario>>>));
}
async function resolveProfiles() {
const profileIds = simulationList.reduce((acc, simulation) => {
if (!acc.includes(simulation.profileId)) {
acc.push(simulation.profileId);
}
return acc;
}, [] as string[]);
const profiles = await Promise.all(profileIds.map((profileId) => getProfile(projectId, profileId)));
if (ignore) {
return;
}
setProfileMap(profiles.filter((profile) => profile !== null).reduce((acc, profile) => {
acc[profile._id] = profile;
return acc;
}, {} as Record<string, WithStringId<z.infer<typeof TestProfile>>>));
}
if (error == null) {
resolveScenarios();
resolveProfiles();
}
return () => {
ignore = true;
};
}, [simulationList, error, projectId]);
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">Simulations</h1>
<Button
size="sm"
onClick={() => router.push(`/projects/${projectId}/test/simulations/new`)}
className="self-end"
startContent={<PlusIcon className="w-4 h-4" />}
>
New Simulation
</Button>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => setError(null)}>Retry</Button>
</div>}
{!loading && !error && <>
{simulationList.length === 0 && <div className="text-gray-600 text-center">No simulation found</div>}
{simulationList.length > 0 && <div className="flex flex-col w-full">
{/* Header */}
<div className="grid grid-cols-9 py-2 bg-gray-100 font-semibold text-sm">
<div className="col-span-2 px-4">Name</div>
<div className="col-span-3 px-4">Scenario</div>
<div className="col-span-1 px-4">Profile</div>
<div className="col-span-1 px-4">Criteria</div>
<div className="col-span-1 px-4">Created</div>
<div className="col-span-1 px-4">Updated</div>
</div>
{/* Rows */}
{simulationList.map((simulation) => (
<div key={simulation._id} className="grid grid-cols-9 py-2 border-b hover:bg-gray-50 text-sm">
<div className="col-span-2 px-4 truncate">
<Link
href={`/projects/${projectId}/test/simulations/${simulation._id}`}
className="text-blue-600 hover:underline"
>
{simulation.name}
</Link>
</div>
<div className="col-span-3 px-4 truncate">
{scenarioMap[simulation.scenarioId]?.name || (
<span className="text-gray-500 font-mono text-xs">{simulation.scenarioId}</span>
)}
</div>
<div className="col-span-1 px-4 truncate">
{profileMap[simulation.profileId]?.name || (
<span className="text-gray-500 font-mono text-xs">{simulation.profileId}</span>
)}
</div>
<div className="col-span-1 px-4 truncate">
{simulation.passCriteria}
</div>
<div className="col-span-1 px-4 text-gray-600 truncate">
<RelativeTime date={new Date(simulation.createdAt)} />
</div>
<div className="col-span-1 px-4 text-gray-600 truncate">
<RelativeTime date={new Date(simulation.lastUpdatedAt)} />
</div>
</div>
))}
</div>}
{total > 1 && <Pagination
total={total}
page={page}
onChange={(page) => {
router.push(`/projects/${projectId}/test/simulations?page=${page}`);
}}
className="self-center"
/>}
</>}
</div>;
}
export function SimulationsApp({
projectId,
slug
}: {
projectId: string,
slug: string[]
}) {
let selection: "list" | "view" | "new" | "edit" = "list";
let simulationId: string | null = null;
if (slug.length > 0) {
if (slug[0] === "new") {
selection = "new";
} else if (slug[slug.length - 1] === "edit") {
selection = "edit";
simulationId = slug[0];
} else {
selection = "view";
simulationId = slug[0];
}
}
return <>
{selection === "list" && <SimulationList projectId={projectId} />}
{selection === "new" && <NewSimulation projectId={projectId} />}
{selection === "view" && simulationId && <ViewSimulation projectId={projectId} simulationId={simulationId} />}
{selection === "edit" && simulationId && <EditSimulation projectId={projectId} simulationId={simulationId} />}
</>;
}

12
apps/rowboat/app/projects/[projectId]/workflow/app.tsx
View file

@ -9,6 +9,8 @@ import { WorkflowSelector } from "./workflow_selector";
import { Spinner } from "@nextui-org/react";
import { cloneWorkflow, createWorkflow, fetchPublishedWorkflowId, fetchWorkflow } from "../../../actions/workflow_actions";
import { listDataSources } from "../../../actions/datasource_actions";
import { TestProfile } from "@/app/lib/types/testing_types";
import { getDefaultProfile } from "../../../actions/testing_actions";
export function App({
projectId,
@ -19,6 +21,7 @@ export function App({
const [workflow, setWorkflow] = useState<WithStringId<z.infer<typeof Workflow>> | null>(null);
const [publishedWorkflowId, setPublishedWorkflowId] = useState<string | null>(null);
const [dataSources, setDataSources] = useState<WithStringId<z.infer<typeof DataSource>>[] | null>(null);
const [defaultTestProfile, setDefaultTestProfile] = useState<z.infer<typeof TestProfile> | null>(null);
const [loading, setLoading] = useState(false);
const [autoSelectIfOnlyOneWorkflow, setAutoSelectIfOnlyOneWorkflow] = useState(true);
@ -27,11 +30,13 @@ export function App({
const workflow = await fetchWorkflow(projectId, workflowId);
const publishedWorkflowId = await fetchPublishedWorkflowId(projectId);
const dataSources = await listDataSources(projectId);
const defaultTestProfile = await getDefaultProfile(projectId);
// Store the selected workflow ID in local storage
localStorage.setItem(`lastWorkflowId_${projectId}`, workflowId);
setWorkflow(workflow);
setPublishedWorkflowId(publishedWorkflowId);
setDataSources(dataSources);
setDefaultTestProfile(defaultTestProfile);
setLoading(false);
}, [projectId]);
@ -47,11 +52,13 @@ export function App({
const workflow = await createWorkflow(projectId);
const publishedWorkflowId = await fetchPublishedWorkflowId(projectId);
const dataSources = await listDataSources(projectId);
const testProfile = await getDefaultProfile(projectId);
// Store the selected workflow ID in local storage
localStorage.setItem(`lastWorkflowId_${projectId}`, workflow._id);
setWorkflow(workflow);
setPublishedWorkflowId(publishedWorkflowId);
setDataSources(dataSources);
setDefaultTestProfile(testProfile);
setLoading(false);
}
@ -60,11 +67,13 @@ export function App({
const workflow = await cloneWorkflow(projectId, workflowId);
const publishedWorkflowId = await fetchPublishedWorkflowId(projectId);
const dataSources = await listDataSources(projectId);
const testProfile = await getDefaultProfile(projectId);
// Store the selected workflow ID in local storage
localStorage.setItem(`lastWorkflowId_${projectId}`, workflow._id);
setWorkflow(workflow);
setPublishedWorkflowId(publishedWorkflowId);
setDataSources(dataSources);
setDefaultTestProfile(testProfile);
setLoading(false);
}
@ -98,10 +107,11 @@ export function App({
handleCreateNewVersion={handleCreateNewVersion}
autoSelectIfOnlyOneWorkflow={autoSelectIfOnlyOneWorkflow}
/>}
{!loading && workflow && (dataSources !== null) && <WorkflowEditor
{!loading && workflow && (dataSources !== null) && (defaultTestProfile !== null) && <WorkflowEditor
key={workflow._id}
workflow={workflow}
dataSources={dataSources}
initialTestProfile={defaultTestProfile}
publishedWorkflowId={publishedWorkflowId}
handleShowSelector={handleShowSelector}
handleCloneVersion={handleCloneVersion}

4
apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx
View file

@ -29,6 +29,7 @@ import { BackIcon, HamburgerIcon, WorkflowIcon } from "../../../lib/components/i
import { CopyIcon, Layers2Icon, RadioIcon, RedoIcon, Sparkles, UndoIcon } from "lucide-react";
import { EntityList } from "./entity_list";
import { CopilotMessage } from "../../../lib/types/copilot_types";
import { TestProfile } from "@/app/lib/types/testing_types";
enablePatches();
@ -533,12 +534,14 @@ export function WorkflowEditor({
publishedWorkflowId,
handleShowSelector,
handleCloneVersion,
initialTestProfile,
}: {
dataSources: WithStringId<z.infer<typeof DataSource>>[];
workflow: WithStringId<z.infer<typeof Workflow>>;
publishedWorkflowId: string | null;
handleShowSelector: () => void;
handleCloneVersion: (workflowId: string) => void;
initialTestProfile: z.infer<typeof TestProfile>;
}) {
const [state, dispatch] = useReducer<Reducer<State, Action>>(reducer, {
patches: [],
@ -859,6 +862,7 @@ export function WorkflowEditor({
projectId={state.present.workflow.projectId}
workflow={state.present.workflow}
messageSubscriber={updateChatMessages}
initialTestProfile={initialTestProfile}
/>
{state.present.selection?.type === "agent" && <AgentConfig
key={state.present.selection.name}