apunkt/rowboat
1
0
Fork 0
mirror of https://github.com/rowboatlabs/rowboat.git synced 2026-06-06 19:35:44 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #37 from rowboatlabs/profiles

Browse source 
Add test profiles and overhaul testing
This commit is contained in:
Ramnique Singh 2025-03-03 11:09:45 +05:30 • committed by GitHub
commit 8840da2261
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 3804 additions and 1802 deletions
Download patch file Download diff file Expand all files Collapse all files

17
apps/python-sdk/README.md
View file

@ -101,6 +101,23 @@ chat = StatefulChat(
) )
``` ```
#### Using a test profile
You can specify a test profile ID to use a specific test configuration:
```python
response_messages, state = client.chat(
messages=messages,
test_profile_id="<TEST_PROFILE_ID>"
)
# or
chat = StatefulChat(
client,
test_profile_id="<TEST_PROFILE_ID>"
)
```
#### Skip tool call runs #### Skip tool call runs
This will surface the tool calls to the SDK instead of running them automatically on the Rowboat server. This will surface the tool calls to the SDK instead of running them automatically on the Rowboat server.

2
apps/python-sdk/pyproject.toml
View file

@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project] [project]
name = "rowboat" name = "rowboat"
version = "1.0.4" version = "1.0.6"
authors = [ authors = [
{ name = "Your Name", email = "your.email@example.com" }, { name = "Your Name", email = "your.email@example.com" },
] ]

19
apps/python-sdk/src/rowboat/client.py
View file

@ -27,14 +27,16 @@ class Client:
state: Optional[Dict[str, Any]] = None, state: Optional[Dict[str, Any]] = None,
skip_tool_calls: bool = False, skip_tool_calls: bool = False,
max_turns: int = 3, max_turns: int = 3,
workflow_id: Optional[str] = None workflow_id: Optional[str] = None,
test_profile_id: Optional[str] = None
) -> ApiResponse: ) -> ApiResponse:
request = ApiRequest( request = ApiRequest(
messages=messages, messages=messages,
state=state, state=state,
skipToolCalls=skip_tool_calls, skipToolCalls=skip_tool_calls,
maxTurns=max_turns, maxTurns=max_turns,
workflowId=workflow_id workflowId=workflow_id,
testProfileId=test_profile_id
) )
response = requests.post(self.base_url, headers=self.headers, data=request.model_dump_json()) response = requests.post(self.base_url, headers=self.headers, data=request.model_dump_json())
@ -83,7 +85,8 @@ class Client:
state: Optional[Dict[str, Any]] = None, state: Optional[Dict[str, Any]] = None,
max_turns: int = 3, max_turns: int = 3,
skip_tool_calls: bool = False, skip_tool_calls: bool = False,
workflow_id: Optional[str] = None workflow_id: Optional[str] = None,
test_profile_id: Optional[str] = None
) -> Tuple[List[ApiMessage], Optional[Dict[str, Any]]]: ) -> Tuple[List[ApiMessage], Optional[Dict[str, Any]]]:
"""Stateless chat method that handles a single conversation turn with multiple tool call rounds""" """Stateless chat method that handles a single conversation turn with multiple tool call rounds"""
@ -102,7 +105,8 @@ class Client:
state=current_state, state=current_state,
skip_tool_calls=skip_tool_calls, skip_tool_calls=skip_tool_calls,
max_turns=max_turns, max_turns=max_turns,
workflow_id=workflow_id workflow_id=workflow_id,
test_profile_id=test_profile_id
) )
current_messages.extend(response_data.messages) current_messages.extend(response_data.messages)
@ -141,7 +145,8 @@ class StatefulChat:
system_prompt: Optional[str] = None, system_prompt: Optional[str] = None,
max_turns: int = 3, max_turns: int = 3,
skip_tool_calls: bool = False, skip_tool_calls: bool = False,
workflow_id: Optional[str] = None workflow_id: Optional[str] = None,
test_profile_id: Optional[str] = None
) -> None: ) -> None:
self.client = client self.client = client
self.tools = tools self.tools = tools
@ -150,6 +155,7 @@ class StatefulChat:
self.max_turns = max_turns self.max_turns = max_turns
self.skip_tool_calls = skip_tool_calls self.skip_tool_calls = skip_tool_calls
self.workflow_id = workflow_id self.workflow_id = workflow_id
self.test_profile_id = test_profile_id
if system_prompt: if system_prompt:
self.messages.append(SystemMessage(role='system', content=system_prompt)) self.messages.append(SystemMessage(role='system', content=system_prompt))
@ -167,7 +173,8 @@ class StatefulChat:
state=self.state, state=self.state,
max_turns=self.max_turns, max_turns=self.max_turns,
skip_tool_calls=self.skip_tool_calls, skip_tool_calls=self.skip_tool_calls,
workflow_id=self.workflow_id workflow_id=self.workflow_id,
test_profile_id=self.test_profile_id
) )
# Update internal state # Update internal state

1
apps/python-sdk/src/rowboat/schema.py
View file

@ -51,6 +51,7 @@ class ApiRequest(BaseModel):
skipToolCalls: Optional[bool] = None skipToolCalls: Optional[bool] = None
maxTurns: Optional[int] = None maxTurns: Optional[int] = None
workflowId: Optional[str] = None workflowId: Optional[str] = None
testProfileId: Optional[str] = None
class ApiResponse(BaseModel): class ApiResponse(BaseModel):
messages: List[ApiMessage] messages: List[ApiMessage]

74
apps/rowboat/app/actions/actions.ts
View file

@ -6,7 +6,6 @@ import { EmbeddingRecord } from "../lib/types/datasource_types";
import { WebpageCrawlResponse } from "../lib/types/tool_types"; import { WebpageCrawlResponse } from "../lib/types/tool_types";
import { GetInformationToolResult } from "../lib/types/tool_types"; import { GetInformationToolResult } from "../lib/types/tool_types";
import { EmbeddingDoc } from "../lib/types/datasource_types"; import { EmbeddingDoc } from "../lib/types/datasource_types";
import { SimulationData } from "../lib/types/testing_types";
import { generateObject, generateText, embed } from "ai"; import { generateObject, generateText, embed } from "ai";
import { dataSourceDocsCollection, dataSourcesCollection, embeddingsCollection, webpagesCollection } from "../lib/mongodb"; import { dataSourceDocsCollection, dataSourcesCollection, embeddingsCollection, webpagesCollection } from "../lib/mongodb";
import { z } from 'zod'; import { z } from 'zod';
@ -21,6 +20,7 @@ import { QueryLimitError } from "../lib/client_utils";
import { projectAuthCheck } from "./project_actions"; import { projectAuthCheck } from "./project_actions";
import { qdrantClient } from "../lib/qdrant"; import { qdrantClient } from "../lib/qdrant";
import { ObjectId } from "mongodb"; import { ObjectId } from "mongodb";
import { TestProfile } from "../lib/types/testing_types";
const crawler = new FirecrawlApp({ apiKey: process.env.FIRECRAWL_API_KEY || '' }); const crawler = new FirecrawlApp({ apiKey: process.env.FIRECRAWL_API_KEY || '' });
@ -99,13 +99,13 @@ export async function getAssistantResponse(
}; };
} }
export async function suggestToolResponse(toolId: string, projectId: string, messages: z.infer<typeof apiV1.ChatMessage>[]): Promise<string> { export async function suggestToolResponse(toolId: string, projectId: string, messages: z.infer<typeof apiV1.ChatMessage>[], testProfile: z.infer<typeof TestProfile>): Promise<string> {
await projectAuthCheck(projectId); await projectAuthCheck(projectId);
if (!await check_query_limit(projectId)) { if (!await check_query_limit(projectId)) {
throw new QueryLimitError(); throw new QueryLimitError();
} }
return await mockToolResponse(toolId, messages); return await mockToolResponse(toolId, messages, testProfile);
} }
export async function getInformationTool( export async function getInformationTool(
@ -123,39 +123,13 @@ export async function getInformationTool(
export async function simulateUserResponse( export async function simulateUserResponse(
projectId: string, projectId: string,
messages: z.infer<typeof apiV1.ChatMessage>[], messages: z.infer<typeof apiV1.ChatMessage>[],
simulationData: z.infer<typeof SimulationData> scenario: string,
): Promise<string> { ): Promise<string> {
await projectAuthCheck(projectId); await projectAuthCheck(projectId);
if (!await check_query_limit(projectId)) { if (!await check_query_limit(projectId)) {
throw new QueryLimitError(); throw new QueryLimitError();
} }
const articlePrompt = `
# Your Specific Task:
## Context:
Here is a help article:
Content:
<START_ARTICLE_CONTENT>
Title: {{title}}
{{content}}
<END_ARTICLE_CONTENT>
## Task definition:
Pretend to be a user reaching out to customer support. Chat with the
customer support assistant, assuming your issue or query is from this article.
Ask follow-up questions and make it real-world like. Don't do dummy
conversations. Your conversation should be a maximum of 5 user turns.
As output, simply provide your (user) turn of conversation.
After you are done with the chat, keep replying with a single word EXIT
in all capitals.
`;
const scenarioPrompt = ` const scenarioPrompt = `
# Your Specific Task: # Your Specific Task:
@ -181,30 +155,6 @@ After you are done with the chat, keep replying with a single word EXIT
in all capitals. in all capitals.
`; `;
const previousChatPrompt = `
# Your Specific Task:
## Context:
Here is a chat between a user and a customer support assistant:
Chat:
<PREVIOUS_CHAT>
{{messages}}
<END_PREVIOUS_CHAT>
## Task definition:
Pretend to be a user reaching out to customer support. Chat with the
customer support assistant, assuming your issue based on this previous chat.
Ask follow-up questions and make it real-world like. Don't do dummy
conversations. Your conversation should be a maximum of 5 user turns.
As output, simply provide your (user) turn of conversation.
After you are done with the chat, keep replying with a single word EXIT
in all capitals.
`;
await projectAuthCheck(projectId); await projectAuthCheck(projectId);
// flip message assistant / user message // flip message assistant / user message
@ -219,19 +169,9 @@ in all capitals.
// simulate user call // simulate user call
let prompt; let prompt;
if ('articleUrl' in simulationData) { prompt = scenarioPrompt
prompt = articlePrompt .replace('{{scenario}}', scenario);
.replace('{{title}}', simulationData.articleTitle || '')
.replace('{{content}}', simulationData.articleContent || '');
}
if ('scenario' in simulationData) {
prompt = scenarioPrompt
.replace('{{scenario}}', simulationData.scenario);
}
if ('chatMessages' in simulationData) {
prompt = previousChatPrompt
.replace('{{messages}}', simulationData.chatMessages);
}
const { text } = await generateText({ const { text } = await generateText({
model: openai("gpt-4o"), model: openai("gpt-4o"),
system: prompt || '', system: prompt || '',

19
apps/rowboat/app/actions/project_actions.ts
View file

@ -1,7 +1,7 @@
'use server'; 'use server';
import { redirect } from "next/navigation"; import { redirect } from "next/navigation";
import { ObjectId } from "mongodb"; import { ObjectId } from "mongodb";
import { dataSourcesCollection, embeddingsCollection, projectsCollection, agentWorkflowsCollection, scenariosCollection, projectMembersCollection, apiKeysCollection, dataSourceDocsCollection } from "../lib/mongodb"; import { dataSourcesCollection, embeddingsCollection, projectsCollection, agentWorkflowsCollection, testScenariosCollection, projectMembersCollection, apiKeysCollection, dataSourceDocsCollection, testProfilesCollection } from "../lib/mongodb";
import { z } from 'zod'; import { z } from 'zod';
import crypto from 'crypto'; import crypto from 'crypto';
import { revalidatePath } from "next/cache"; import { revalidatePath } from "next/cache";
@ -41,6 +41,7 @@ export async function createProject(formData: FormData) {
const projectId = crypto.randomUUID(); const projectId = crypto.randomUUID();
const chatClientId = crypto.randomBytes(16).toString('base64url'); const chatClientId = crypto.randomBytes(16).toString('base64url');
const secret = crypto.randomBytes(32).toString('hex'); const secret = crypto.randomBytes(32).toString('hex');
const defaultTestProfileId = new ObjectId();
// create project // create project
await projectsCollection.insertOne({ await projectsCollection.insertOne({
@ -52,12 +53,13 @@ export async function createProject(formData: FormData) {
chatClientId, chatClientId,
secret, secret,
nextWorkflowNumber: 1, nextWorkflowNumber: 1,
testRunCounter: 0,
defaultTestProfileId: defaultTestProfileId.toString(),
}); });
// add first workflow version // add first workflow version
const { agents, prompts, tools, startAgent } = templates[templateKey]; const { agents, prompts, tools, startAgent } = templates[templateKey];
await agentWorkflowsCollection.insertOne({ await agentWorkflowsCollection.insertOne({
_id: new ObjectId(),
projectId, projectId,
agents, agents,
prompts, prompts,
@ -68,6 +70,17 @@ export async function createProject(formData: FormData) {
name: `Version 1`, name: `Version 1`,
}); });
// add default test profile
await testProfilesCollection.insertOne({
_id: defaultTestProfileId,
projectId,
name: "Default",
context: "",
mockTools: false,
createdAt: (new Date()).toISOString(),
lastUpdatedAt: (new Date()).toISOString(),
});
// add user to project // add user to project
await projectMembersCollection.insertOne({ await projectMembersCollection.insertOne({
userId: user.sub, userId: user.sub,
@ -198,7 +211,7 @@ export async function deleteProject(projectId: string) {
}); });
// delete scenarios // delete scenarios
await scenariosCollection.deleteMany({ await testScenariosCollection.deleteMany({
projectId, projectId,
}); });

270
apps/rowboat/app/actions/simulation_actions.ts
View file

@ -1,270 +0,0 @@
'use server';
import { ObjectId } from "mongodb";
import { scenariosCollection, simulationRunsCollection, simulationResultsCollection } from "../lib/mongodb";
import { z } from 'zod';
import { projectAuthCheck } from "./project_actions";
import { type WithStringId } from "../lib/types/types";
import { Scenario, SimulationRun, SimulationResult, SimulationAggregateResult } from "../lib/types/testing_types";
import { SimulationScenarioData } from "../lib/types/testing_types";
export async function getScenarios(projectId: string): Promise<WithStringId<z.infer<typeof Scenario>>[]> {
await projectAuthCheck(projectId);
const scenarios = await scenariosCollection.find({ projectId }).toArray();
return scenarios.map(s => ({
...s,
_id: s._id.toString(),
}));
}
export async function getScenario(projectId: string, scenarioId: string): Promise<WithStringId<z.infer<typeof SimulationScenarioData>>> {
await projectAuthCheck(projectId);
// fetch scenario
const scenario = await scenariosCollection.findOne({
_id: new ObjectId(scenarioId),
projectId,
});
if (!scenario) {
throw new Error('Scenario not found');
}
const { _id, description, ...rest } = scenario;
return {
...rest,
_id: _id.toString(),
scenario: description,
};
}
export async function createScenario(projectId: string, name: string, description: string): Promise<string> {
await projectAuthCheck(projectId);
const now = new Date().toISOString();
const result = await scenariosCollection.insertOne({
projectId,
name,
description,
context: '',
criteria: '',
lastUpdatedAt: now,
createdAt: now,
});
return result.insertedId.toString();
}
export async function updateScenario(
projectId: string,
scenarioId: string,
updates: {
name?: string;
description?: string;
context?: string;
criteria?: string;
}
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: any = {
...updates,
lastUpdatedAt: new Date().toISOString(),
};
await scenariosCollection.updateOne(
{
_id: new ObjectId(scenarioId),
projectId,
},
{
$set: updateData,
}
);
}
export async function deleteScenario(projectId: string, scenarioId: string): Promise<void> {
await projectAuthCheck(projectId);
await scenariosCollection.deleteOne({
_id: new ObjectId(scenarioId),
projectId,
});
}
export async function getRuns(projectId: string): Promise<WithStringId<z.infer<typeof SimulationRun>>[]> {
await projectAuthCheck(projectId);
const runs = await simulationRunsCollection
.find({ projectId })
.sort({ startedAt: -1 }) // Most recent first
.toArray();
return runs.map(run => ({
...run,
_id: run._id.toString(),
}));
}
export async function getRun(projectId: string, runId: string): Promise<WithStringId<z.infer<typeof SimulationRun>>> {
await projectAuthCheck(projectId);
const run = await simulationRunsCollection.findOne({
_id: new ObjectId(runId),
projectId,
});
if (!run) {
throw new Error('Run not found');
}
return {
...run,
_id: run._id.toString(),
};
}
export async function createRun(
projectId: string,
scenarioIds: string[],
workflowId: string
): Promise<WithStringId<z.infer<typeof SimulationRun>>> {
await projectAuthCheck(projectId);
const run = {
projectId,
status: 'pending' as const,
scenarioIds,
workflowId,
startedAt: new Date().toISOString(),
};
const result = await simulationRunsCollection.insertOne(run);
return {
...run,
_id: result.insertedId.toString(),
};
}
export async function updateRunStatus(
projectId: string,
runId: string,
status: z.infer<typeof SimulationRun>['status'],
completedAt?: string
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: Partial<z.infer<typeof SimulationRun>> = {
status,
};
if (completedAt) {
updateData.completedAt = completedAt;
}
await simulationRunsCollection.updateOne(
{
_id: new ObjectId(runId),
projectId,
},
{
$set: updateData,
}
);
}
export async function getRunResults(
projectId: string,
runId: string
): Promise<WithStringId<z.infer<typeof SimulationResult>>[]> {
await projectAuthCheck(projectId);
const results = await simulationResultsCollection
.find({
runId,
projectId,
})
.toArray();
return results.map(result => ({
...result,
_id: result._id.toString(),
}));
}
export async function createRunResult(
projectId: string,
runId: string,
scenarioId: string,
result: z.infer<typeof SimulationResult>['result'],
details: string
): Promise<string> {
await projectAuthCheck(projectId);
const resultDoc = {
projectId,
runId,
scenarioId,
result,
details,
};
const insertResult = await simulationResultsCollection.insertOne(resultDoc);
return insertResult.insertedId.toString();
}
export async function createAggregateResult(
projectId: string,
runId: string,
total: number,
pass: number,
fail: number
): Promise<void> {
await projectAuthCheck(projectId);
await simulationRunsCollection.updateOne(
{ _id: new ObjectId(runId), projectId },
{
$set: {
aggregateResults: { total, pass, fail }
}
}
);
}
export async function getAggregateResult(
projectId: string,
runId: string
): Promise<z.infer<typeof SimulationAggregateResult> | null> {
await projectAuthCheck(projectId);
const run = await simulationRunsCollection.findOne({
_id: new ObjectId(runId),
projectId,
});
if (!run || !run.aggregateResults) return null;
return run.aggregateResults;
}
export async function deleteRun(projectId: string, runId: string) {
try {
// Delete the run using the collection directly
await simulationRunsCollection.deleteOne({
_id: new ObjectId(runId),
projectId: projectId
});
// Delete associated results using the collection directly
await simulationResultsCollection.deleteMany({
runId: runId,
projectId: projectId
});
return { success: true };
} catch (error) {
console.error('Error deleting run:', error);
throw new Error('Failed to delete run');
}
}

575
apps/rowboat/app/actions/testing_actions.ts Normal file
View file

@ -0,0 +1,575 @@
'use server';
import { ObjectId } from "mongodb";
import { testScenariosCollection, testSimulationsCollection, testProfilesCollection, testRunsCollection, testResultsCollection, projectsCollection } from "../lib/mongodb";
import { z } from 'zod';
import { projectAuthCheck } from "./project_actions";
import { type WithStringId } from "../lib/types/types";
import { TestScenario, TestSimulation, TestProfile, TestRun, TestResult } from "../lib/types/testing_types";
export async function listScenarios(
projectId: string,
page: number = 1,
pageSize: number = 10
): Promise<{
scenarios: WithStringId<z.infer<typeof TestScenario>>[];
total: number;
}> {
await projectAuthCheck(projectId);
// Calculate skip value for pagination
const skip = (page - 1) * pageSize;
// Get total count for pagination
const total = await testScenariosCollection.countDocuments({ projectId });
// Get paginated scenarios
const scenarios = await testScenariosCollection
.find({ projectId })
.skip(skip)
.limit(pageSize)
.toArray();
return {
scenarios: scenarios.map(scenario => ({
...scenario,
_id: scenario._id.toString(),
})),
total,
};
}
export async function getScenario(projectId: string, scenarioId: string): Promise<WithStringId<z.infer<typeof TestScenario>> | null> {
await projectAuthCheck(projectId);
// fetch scenario
const scenario = await testScenariosCollection.findOne({
_id: new ObjectId(scenarioId),
projectId,
});
if (!scenario) {
return null;
}
const { _id, ...rest } = scenario;
return {
...rest,
_id: _id.toString(),
};
}
export async function deleteScenario(projectId: string, scenarioId: string): Promise<void> {
await projectAuthCheck(projectId);
await testScenariosCollection.deleteOne({
_id: new ObjectId(scenarioId),
projectId,
});
}
export async function createScenario(
projectId: string,
data: {
name: string;
description: string;
}
): Promise<WithStringId<z.infer<typeof TestScenario>>> {
await projectAuthCheck(projectId);
const doc = {
...data,
projectId,
createdAt: new Date().toISOString(),
lastUpdatedAt: new Date().toISOString(),
};
const result = await testScenariosCollection.insertOne(doc);
return {
...doc,
_id: result.insertedId.toString(),
};
}
export async function updateScenario(
projectId: string,
scenarioId: string,
updates: {
name?: string;
description?: string;
}
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: any = {
...updates,
lastUpdatedAt: new Date().toISOString(),
};
await testScenariosCollection.updateOne(
{
_id: new ObjectId(scenarioId),
projectId,
},
{
$set: updateData,
}
);
}
export async function listSimulations(
projectId: string,
page: number = 1,
pageSize: number = 10
): Promise<{
simulations: WithStringId<z.infer<typeof TestSimulation>>[];
total: number;
}> {
await projectAuthCheck(projectId);
const skip = (page - 1) * pageSize;
const total = await testSimulationsCollection.countDocuments({ projectId });
const simulations = await testSimulationsCollection
.find({ projectId })
.skip(skip)
.limit(pageSize)
.toArray();
return {
simulations: simulations.map(simulation => ({
...simulation,
_id: simulation._id.toString(),
})),
total,
};
}
export async function getSimulation(projectId: string, simulationId: string): Promise<WithStringId<z.infer<typeof TestSimulation>> | null> {
await projectAuthCheck(projectId);
const simulation = await testSimulationsCollection.findOne({
_id: new ObjectId(simulationId),
projectId,
});
if (!simulation) {
return null;
}
const { _id, ...rest } = simulation;
return {
...rest,
_id: _id.toString(),
};
}
export async function deleteSimulation(projectId: string, simulationId: string): Promise<void> {
await projectAuthCheck(projectId);
await testSimulationsCollection.deleteOne({
_id: new ObjectId(simulationId),
projectId,
});
}
export async function createSimulation(
projectId: string,
data: {
name: string;
scenarioId: string;
profileId: string;
passCriteria: string;
}
): Promise<WithStringId<z.infer<typeof TestSimulation>>> {
await projectAuthCheck(projectId);
const doc = {
...data,
projectId,
createdAt: new Date().toISOString(),
lastUpdatedAt: new Date().toISOString(),
};
const result = await testSimulationsCollection.insertOne(doc);
return {
...doc,
_id: result.insertedId.toString(),
};
}
export async function updateSimulation(
projectId: string,
simulationId: string,
updates: {
name?: string;
scenarioId?: string;
profileId?: string;
passCriteria?: string;
}
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: any = {
...updates,
lastUpdatedAt: new Date().toISOString(),
};
await testSimulationsCollection.updateOne(
{
_id: new ObjectId(simulationId),
projectId,
},
{
$set: updateData,
}
);
}
export async function listProfiles(
projectId: string,
page: number = 1,
pageSize: number = 10
): Promise<{
profiles: WithStringId<z.infer<typeof TestProfile>>[];
total: number;
}> {
await projectAuthCheck(projectId);
const skip = (page - 1) * pageSize;
const total = await testProfilesCollection.countDocuments({ projectId });
const profiles = await testProfilesCollection
.find({ projectId })
.skip(skip)
.limit(pageSize)
.toArray();
return {
profiles: profiles.map(profile => ({
...profile,
_id: profile._id.toString(),
})),
total,
};
}
export async function getDefaultProfile(projectId: string): Promise<WithStringId<z.infer<typeof TestProfile>> | null> {
await projectAuthCheck(projectId);
const project = await projectsCollection.findOne({ _id: projectId });
if (!project) {
return null;
}
if (!project.defaultTestProfileId) {
// create a default profile
const profile = await createProfile(projectId, {
name: 'Default',
context: '',
mockTools: false,
mockPrompt: '',
});
await setDefaultProfile(projectId, profile._id);
return profile;
}
return getProfile(projectId, project.defaultTestProfileId);
}
export async function setDefaultProfile(projectId: string, profileId: string): Promise<void> {
await projectAuthCheck(projectId);
await projectsCollection.updateOne(
{ _id: projectId },
{ $set: { defaultTestProfileId: profileId } }
);
}
export async function getProfile(projectId: string, profileId: string): Promise<WithStringId<z.infer<typeof TestProfile>> | null> {
await projectAuthCheck(projectId);
const profile = await testProfilesCollection.findOne({
_id: new ObjectId(profileId),
projectId,
});
if (!profile) {
return null;
}
const { _id, ...rest } = profile;
return {
...rest,
_id: _id.toString(),
};
}
export async function deleteProfile(projectId: string, profileId: string): Promise<void> {
await projectAuthCheck(projectId);
await testProfilesCollection.deleteOne({
_id: new ObjectId(profileId),
projectId,
default: false,
});
}
export async function createProfile(
projectId: string,
data: {
name: string;
context: string;
mockTools: boolean;
mockPrompt?: string;
}
): Promise<WithStringId<z.infer<typeof TestProfile>>> {
await projectAuthCheck(projectId);
const doc = {
...data,
projectId,
createdAt: new Date().toISOString(),
lastUpdatedAt: new Date().toISOString(),
};
const result = await testProfilesCollection.insertOne(doc);
return {
...doc,
_id: result.insertedId.toString(),
};
}
export async function updateProfile(
projectId: string,
profileId: string,
updates: {
name?: string;
context?: string;
mockTools?: boolean;
mockPrompt?: string;
}
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: any = {
...updates,
lastUpdatedAt: new Date().toISOString(),
};
await testProfilesCollection.updateOne(
{
_id: new ObjectId(profileId),
projectId,
},
{
$set: updateData,
}
);
}
export async function listRuns(
projectId: string,
page: number = 1,
pageSize: number = 10
): Promise<{
runs: WithStringId<z.infer<typeof TestRun>>[];
total: number;
}> {
await projectAuthCheck(projectId);
const skip = (page - 1) * pageSize;
const total = await testRunsCollection.countDocuments({ projectId });
const runs = await testRunsCollection
.find({ projectId })
.sort({ startedAt: -1 }) // Sort by most recent first
.skip(skip)
.limit(pageSize)
.toArray();
return {
runs: runs.map(run => ({
...run,
_id: run._id.toString(),
})),
total,
};
}
export async function getRun(projectId: string, runId: string): Promise<WithStringId<z.infer<typeof TestRun>> | null> {
await projectAuthCheck(projectId);
const run = await testRunsCollection.findOne({
_id: new ObjectId(runId),
projectId,
});
if (!run) {
return null;
}
const { _id, ...rest } = run;
return {
...rest,
_id: _id.toString(),
};
}
export async function deleteRun(projectId: string, runId: string): Promise<void> {
await projectAuthCheck(projectId);
await testRunsCollection.deleteOne({
_id: new ObjectId(runId),
projectId,
});
}
export async function createRun(
projectId: string,
data: {
simulationIds: string[];
workflowId: string;
}
): Promise<WithStringId<z.infer<typeof TestRun>>> {
await projectAuthCheck(projectId);
// Increment the testRunCounter and get the new value
const result = await projectsCollection.findOneAndUpdate(
{ _id: projectId },
{ $inc: { testRunCounter: 1 } },
{ returnDocument: 'after' }
);
if (!result) {
throw new Error("Project not found");
}
const runNumber = result.testRunCounter || 1;
const doc = {
...data,
projectId,
name: `Run #${runNumber}`,
status: 'pending' as const,
startedAt: new Date().toISOString(),
aggregateResults: {
total: 0,
passCount: 0,
failCount: 0,
},
};
const insertResult = await testRunsCollection.insertOne(doc);
return {
...doc,
_id: insertResult.insertedId.toString(),
};
}
export async function updateRun(
projectId: string,
runId: string,
updates: {
status?: 'pending' | 'running' | 'completed' | 'cancelled' | 'failed' | 'error';
completedAt?: string;
aggregateResults?: {
total: number;
passCount: number;
failCount: number;
};
}
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: any = {
...updates,
};
await testRunsCollection.updateOne(
{
_id: new ObjectId(runId),
projectId,
},
{
$set: updateData,
}
);
}
export async function listResults(
projectId: string,
runId: string,
page: number = 1,
pageSize: number = 10
): Promise<{
results: WithStringId<z.infer<typeof TestResult>>[];
total: number;
}> {
await projectAuthCheck(projectId);
const skip = (page - 1) * pageSize;
const total = await testResultsCollection.countDocuments({ projectId, runId });
const results = await testResultsCollection
.find({ projectId, runId })
.skip(skip)
.limit(pageSize)
.toArray();
return {
results: results.map(result => ({
...result,
_id: result._id.toString(),
})),
total,
};
}
export async function getResult(projectId: string, resultId: string): Promise<WithStringId<z.infer<typeof TestResult>> | null> {
await projectAuthCheck(projectId);
const result = await testResultsCollection.findOne({
_id: new ObjectId(resultId),
projectId,
});
if (!result) {
return null;
}
const { _id, ...rest } = result;
return {
...rest,
_id: _id.toString(),
};
}
export async function deleteResult(projectId: string, resultId: string): Promise<void> {
await projectAuthCheck(projectId);
await testResultsCollection.deleteOne({
_id: new ObjectId(resultId),
projectId,
});
}
export async function createResult(
projectId: string,
data: {
runId: string;
simulationId: string;
result: 'pass' | 'fail';
details: string;
}
): Promise<WithStringId<z.infer<typeof TestResult>>> {
await projectAuthCheck(projectId);
const doc = {
...data,
projectId,
};
const result = await testResultsCollection.insertOne(doc);
return {
...doc,
_id: result.insertedId.toString(),
};
}
export async function updateResult(
projectId: string,
resultId: string,
updates: {
result?: 'pass' | 'fail';
details?: string;
}
): Promise<void> {
await projectAuthCheck(projectId);
await testResultsCollection.updateOne(
{
_id: new ObjectId(resultId),
projectId,
},
{
$set: updates,
}
);
}

43
apps/rowboat/app/api/v1/[projectId]/chat/route.ts
View file

@ -1,5 +1,5 @@
import { NextRequest } from "next/server"; import { NextRequest } from "next/server";
import { agentWorkflowsCollection, db, projectsCollection } from "../../../../lib/mongodb"; import { agentWorkflowsCollection, db, projectsCollection, testProfilesCollection } from "../../../../lib/mongodb";
import { z } from "zod"; import { z } from "zod";
import { ObjectId } from "mongodb"; import { ObjectId } from "mongodb";
import { authCheck } from "../../utils"; import { authCheck } from "../../utils";
@ -9,6 +9,7 @@ import { getAgenticApiResponse, callClientToolWebhook, runRAGToolCall, mockToolR
import { check_query_limit } from "../../../../lib/rate_limiting"; import { check_query_limit } from "../../../../lib/rate_limiting";
import { apiV1 } from "rowboat-shared"; import { apiV1 } from "rowboat-shared";
import { PrefixLogger } from "../../../../lib/utils"; import { PrefixLogger } from "../../../../lib/utils";
import { TestProfile } from "@/app/lib/types/testing_types";
// get next turn / agent response // get next turn / agent response
export async function POST( export async function POST(
@ -68,9 +69,43 @@ export async function POST(
logger.log(`Workflow ${workflowId} not found for project ${projectId}`); logger.log(`Workflow ${workflowId} not found for project ${projectId}`);
return Response.json({ error: "Workflow not found" }, { status: 404 }); return Response.json({ error: "Workflow not found" }, { status: 404 });
} }
// if test profile is provided in the request, use it
let profile: z.infer<typeof TestProfile> = {
projectId: projectId,
name: 'Default',
createdAt: new Date().toISOString(),
lastUpdatedAt: new Date().toISOString(),
context: '',
mockTools: false,
mockPrompt: '',
};
if (result.data.testProfileId) {
const testProfile = await testProfilesCollection.findOne({
projectId: projectId,
_id: new ObjectId(result.data.testProfileId),
});
if (!testProfile) {
logger.log(`Test profile ${result.data.testProfileId} not found for project ${projectId}`);
return Response.json({ error: "Test profile not found" }, { status: 404 });
}
profile = testProfile;
}
// if profile has a context available, overwrite the system message in the request (if there is one)
let currentMessages = reqMessages;
if (profile.context) {
// if there is a system message, overwrite it
const systemMessageIndex = reqMessages.findIndex(m => m.role === "system");
if (systemMessageIndex !== -1) {
currentMessages[systemMessageIndex].content = profile.context;
} else {
// if there is no system message, add one
currentMessages.unshift({ role: "system", content: profile.context });
}
}
const MAX_TURNS = result.data.maxTurns ?? 3; const MAX_TURNS = result.data.maxTurns ?? 3;
let currentMessages = reqMessages;
let currentState: unknown = reqState ?? { last_agent_name: workflow.agents[0].name }; let currentState: unknown = reqState ?? { last_agent_name: workflow.agents[0].name };
let turns = 0; let turns = 0;
let hasToolCalls = false; let hasToolCalls = false;
@ -140,9 +175,9 @@ export async function POST(
try { try {
// if tool is supposed to be mocked, mock it // if tool is supposed to be mocked, mock it
const workflowTool = workflow.tools.find(t => t.name === toolCall.function.name); const workflowTool = workflow.tools.find(t => t.name === toolCall.function.name);
if (workflowTool?.mockInPlayground) { if (profile.mockTools) {
logger.log(`Mocking tool call ${toolCall.function.name}`); logger.log(`Mocking tool call ${toolCall.function.name}`);
result = await mockToolResponse(toolCall.id, currentMessages); result = await mockToolResponse(toolCall.id, currentMessages, profile);
} else { } else {
// else run the tool call by calling the client tool webhook // else run the tool call by calling the client tool webhook
logger.log(`Running client tool webhook for tool ${toolCall.function.name}`); logger.log(`Running client tool webhook for tool ${toolCall.function.name}`);

100
apps/rowboat/app/lib/components/selectors/profile-selector.tsx Normal file
View file

@ -0,0 +1,100 @@
import { WithStringId } from "@/app/lib/types/types";
import { TestProfile } from "@/app/lib/types/testing_types";
import { useCallback, useEffect, useState } from "react";
import { listProfiles } from "@/app/actions/testing_actions";
import { Button, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react";
import { z } from "zod";
interface ProfileSelectorProps {
projectId: string;
isOpen: boolean;
onOpenChange: (open: boolean) => void;
onSelect: (profile: WithStringId<z.infer<typeof TestProfile>>) => void;
}
export function ProfileSelector({ projectId, isOpen, onOpenChange, onSelect }: ProfileSelectorProps) {
const [page, setPage] = useState(1);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [profiles, setProfiles] = useState<WithStringId<z.infer<typeof TestProfile>>[]>([]);
const [totalPages, setTotalPages] = useState(0);
const pageSize = 10;
const fetchProfiles = useCallback(async (page: number) => {
setLoading(true);
setError(null);
try {
const result = await listProfiles(projectId, page, pageSize);
setProfiles(result.profiles);
setTotalPages(Math.ceil(result.total / pageSize));
} catch (error) {
setError(`Unable to fetch profiles: ${error}`);
} finally {
setLoading(false);
}
}, [projectId]);
useEffect(() => {
if (isOpen) {
fetchProfiles(page);
}
}, [page, isOpen, fetchProfiles]);
return (
<Modal isOpen={isOpen} onOpenChange={onOpenChange} size="xl">
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Select a Profile</ModalHeader>
<ModalBody>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => fetchProfiles(page)}>Retry</Button>
</div>}
{!loading && !error && <>
{profiles.length === 0 && <div className="text-gray-600 text-center">No profiles found</div>}
{profiles.length > 0 && <div className="flex flex-col w-full">
<div className="grid grid-cols-6 py-2 bg-gray-100 font-semibold text-sm">
<div className="col-span-2 px-4">Name</div>
<div className="col-span-3 px-4">Context</div>
<div className="col-span-1 px-4">Mock Tools</div>
</div>
{profiles.map((p) => (
<div
key={p._id}
className="grid grid-cols-6 py-2 border-b hover:bg-gray-50 text-sm cursor-pointer"
onClick={() => {
onSelect(p);
onClose();
}}
>
<div className="col-span-2 px-4 truncate">{p.name}</div>
<div className="col-span-3 px-4 truncate">{p.context}</div>
<div className="col-span-1 px-4">{p.mockTools ? "Yes" : "No"}</div>
</div>
))}
</div>}
{totalPages > 1 && <Pagination
total={totalPages}
page={page}
onChange={setPage}
className="self-center"
/>}
</>}
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Cancel
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
);
}

98
apps/rowboat/app/lib/components/selectors/scenario-selector.tsx Normal file
View file

@ -0,0 +1,98 @@
import { WithStringId } from "@/app/lib/types/types";
import { TestScenario } from "@/app/lib/types/testing_types";
import { useCallback, useEffect, useState } from "react";
import { listScenarios } from "@/app/actions/testing_actions";
import { Button, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react";
import { z } from "zod";
interface ScenarioSelectorProps {
projectId: string;
isOpen: boolean;
onOpenChange: (open: boolean) => void;
onSelect: (scenario: WithStringId<z.infer<typeof TestScenario>>) => void;
}
export function ScenarioSelector({ projectId, isOpen, onOpenChange, onSelect }: ScenarioSelectorProps) {
const [page, setPage] = useState(1);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [scenarios, setScenarios] = useState<WithStringId<z.infer<typeof TestScenario>>[]>([]);
const [totalPages, setTotalPages] = useState(0);
const pageSize = 10;
const fetchScenarios = useCallback(async (page: number) => {
setLoading(true);
setError(null);
try {
const result = await listScenarios(projectId, page, pageSize);
setScenarios(result.scenarios);
setTotalPages(Math.ceil(result.total / pageSize));
} catch (error) {
setError(`Unable to fetch scenarios: ${error}`);
} finally {
setLoading(false);
}
}, [projectId]);
useEffect(() => {
if (isOpen) {
fetchScenarios(page);
}
}, [page, isOpen, fetchScenarios]);
return (
<Modal isOpen={isOpen} onOpenChange={onOpenChange} size="xl">
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Select a Scenario</ModalHeader>
<ModalBody>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => fetchScenarios(page)}>Retry</Button>
</div>}
{!loading && !error && <>
{scenarios.length === 0 && <div className="text-gray-600 text-center">No scenarios found</div>}
{scenarios.length > 0 && <div className="flex flex-col w-full">
<div className="grid grid-cols-5 py-2 bg-gray-100 font-semibold text-sm">
<div className="col-span-2 px-4">Name</div>
<div className="col-span-3 px-4">Description</div>
</div>
{scenarios.map((s) => (
<div
key={s._id}
className="grid grid-cols-5 py-2 border-b hover:bg-gray-50 text-sm cursor-pointer"
onClick={() => {
onSelect(s);
onClose();
}}
>
<div className="col-span-2 px-4 truncate">{s.name}</div>
<div className="col-span-3 px-4 truncate">{s.description}</div>
</div>
))}
</div>}
{totalPages > 1 && <Pagination
total={totalPages}
page={page}
onChange={setPage}
className="self-center"
/>}
</>}
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Cancel
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
);
}

140
apps/rowboat/app/lib/components/selectors/simulation-selector.tsx Normal file
View file

@ -0,0 +1,140 @@
import { WithStringId } from "@/app/lib/types/types";
import { TestSimulation } from "@/app/lib/types/testing_types";
import { useCallback, useEffect, useState } from "react";
import { listSimulations } from "@/app/actions/testing_actions";
import { Button, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter, Chip } from "@nextui-org/react";
import { z } from "zod";
import { RelativeTime } from "@primer/react";
interface SimulationSelectorProps {
projectId: string;
isOpen: boolean;
onOpenChange: (open: boolean) => void;
onSelect: (simulations: WithStringId<z.infer<typeof TestSimulation>>[]) => void;
initialSelected?: WithStringId<z.infer<typeof TestSimulation>>[];
}
export function SimulationSelector({ projectId, isOpen, onOpenChange, onSelect, initialSelected = [] }: SimulationSelectorProps) {
const [page, setPage] = useState(1);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [simulations, setSimulations] = useState<WithStringId<z.infer<typeof TestSimulation>>[]>([]);
const [totalPages, setTotalPages] = useState(0);
const [selectedSimulations, setSelectedSimulations] = useState<WithStringId<z.infer<typeof TestSimulation>>[]>(initialSelected);
const pageSize = 3;
const fetchSimulations = useCallback(async (page: number) => {
setLoading(true);
setError(null);
try {
const result = await listSimulations(projectId, page, pageSize);
setSimulations(result.simulations);
setTotalPages(Math.ceil(result.total / pageSize));
} catch (error) {
setError(`Unable to fetch simulations: ${error}`);
} finally {
setLoading(false);
}
}, [projectId]);
useEffect(() => {
if (isOpen) {
fetchSimulations(page);
}
}, [page, isOpen, fetchSimulations]);
const handleSelect = (simulation: WithStringId<z.infer<typeof TestSimulation>>) => {
const isSelected = selectedSimulations.some(s => s._id === simulation._id);
let newSelected;
if (isSelected) {
newSelected = selectedSimulations.filter(s => s._id !== simulation._id);
} else {
newSelected = [...selectedSimulations, simulation];
}
setSelectedSimulations(newSelected);
onSelect(newSelected);
};
const handleRemove = (simulationId: string) => {
const newSelected = selectedSimulations.filter(s => s._id !== simulationId);
setSelectedSimulations(newSelected);
onSelect(newSelected);
};
return (
<Modal isOpen={isOpen} onOpenChange={onOpenChange} size="xl">
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Select Simulations</ModalHeader>
<ModalBody>
{selectedSimulations.length > 0 && (
<div className="flex flex-wrap gap-2 mb-4">
{selectedSimulations.map((sim) => (
<Chip
key={sim._id}
onClose={() => handleRemove(sim._id)}
variant="flat"
className="py-1"
>
{sim.name}
</Chip>
))}
</div>
)}
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => fetchSimulations(page)}>Retry</Button>
</div>}
{!loading && !error && <>
{simulations.length === 0 && <div className="text-gray-600 text-center">No simulations found</div>}
{simulations.length > 0 && <div className="flex flex-col w-full">
<div className="grid grid-cols-8 py-2 bg-gray-100 font-semibold text-sm">
<div className="col-span-3 px-4">Name</div>
<div className="col-span-3 px-4">Pass Criteria</div>
<div className="col-span-2 px-4">Last Updated</div>
</div>
{simulations.map((sim) => {
const isSelected = selectedSimulations.some(s => s._id === sim._id);
return (
<div
key={sim._id}
className={`grid grid-cols-8 py-2 border-b hover:bg-gray-50 text-sm cursor-pointer ${
isSelected ? 'bg-blue-50 hover:bg-blue-100' : ''
}`}
onClick={() => handleSelect(sim)}
>
<div className="col-span-3 px-4 truncate">{sim.name}</div>
<div className="col-span-3 px-4 truncate">{sim.passCriteria || '-'}</div>
<div className="col-span-2 px-4 truncate">
<RelativeTime date={new Date(sim.lastUpdatedAt)} />
</div>
</div>
);
})}
</div>}
{totalPages > 1 && <Pagination
total={totalPages}
page={page}
onChange={setPage}
className="self-center"
/>}
</>}
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Done
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
);
}

106
apps/rowboat/app/lib/components/selectors/workflow-selector.tsx Normal file
View file

@ -0,0 +1,106 @@
import { WithStringId } from "@/app/lib/types/types";
import { Workflow } from "@/app/lib/types/workflow_types";
import { useCallback, useEffect, useState } from "react";
import { listWorkflows } from "@/app/actions/workflow_actions";
import { Button, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react";
import { z } from "zod";
import { RelativeTime } from "@primer/react";
import { WorkflowIcon } from "../icons";
import { PublishedBadge } from "@/app/projects/[projectId]/workflow/published_badge";
interface WorkflowSelectorProps {
projectId: string;
isOpen: boolean;
onOpenChange: (open: boolean) => void;
onSelect: (workflow: WithStringId<z.infer<typeof Workflow>>) => void;
}
export function WorkflowSelector({ projectId, isOpen, onOpenChange, onSelect }: WorkflowSelectorProps) {
const [page, setPage] = useState(1);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [workflows, setWorkflows] = useState<WithStringId<z.infer<typeof Workflow>>[]>([]);
const [totalPages, setTotalPages] = useState(0);
const [publishedWorkflowId, setPublishedWorkflowId] = useState<string | null>(null);
const pageSize = 10;
const fetchWorkflows = useCallback(async (page: number) => {
setLoading(true);
setError(null);
try {
const result = await listWorkflows(projectId, page, pageSize);
setWorkflows(result.workflows);
setTotalPages(Math.ceil(result.total / pageSize));
setPublishedWorkflowId(result.publishedWorkflowId);
} catch (error) {
setError(`Unable to fetch workflows: ${error}`);
} finally {
setLoading(false);
}
}, [projectId]);
useEffect(() => {
if (isOpen) {
fetchWorkflows(page);
}
}, [page, isOpen, fetchWorkflows]);
return (
<Modal isOpen={isOpen} onOpenChange={onOpenChange} size="xl">
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Select a Workflow</ModalHeader>
<ModalBody>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => fetchWorkflows(page)}>Retry</Button>
</div>}
{!loading && !error && <>
{workflows.length === 0 && <div className="text-gray-600 text-center">No workflows found</div>}
{workflows.length > 0 && <div className="flex flex-col gap-2">
{workflows.map((workflow) => (
<div
key={workflow._id}
className="flex items-center justify-between p-3 border rounded-md hover:bg-gray-50 cursor-pointer"
onClick={() => {
onSelect(workflow);
onClose();
}}
>
<div className="flex flex-col gap-1">
<div className="flex items-center gap-2">
<WorkflowIcon />
<span className="font-medium">{workflow.name || 'Unnamed workflow'}</span>
{publishedWorkflowId === workflow._id && <PublishedBadge />}
</div>
<div className="text-xs text-gray-500">
Updated <RelativeTime date={new Date(workflow.lastUpdatedAt)} />
</div>
</div>
</div>
))}
</div>}
{totalPages > 1 && <Pagination
total={totalPages}
page={page}
onChange={setPage}
className="self-center"
/>}
</>}
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Cancel
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
);
}

12
apps/rowboat/app/lib/mongodb.ts
View file

@ -1,5 +1,5 @@
import { MongoClient } from "mongodb"; import { MongoClient } from "mongodb";
import { PlaygroundChat, Webpage, ChatClientId } from "./types/types"; import { Webpage } from "./types/types";
import { Workflow } from "./types/workflow_types"; import { Workflow } from "./types/workflow_types";
import { ApiKey } from "./types/project_types"; import { ApiKey } from "./types/project_types";
import { ProjectMember } from "./types/project_types"; import { ProjectMember } from "./types/project_types";
@ -7,7 +7,7 @@ import { Project } from "./types/project_types";
import { EmbeddingDoc } from "./types/datasource_types"; import { EmbeddingDoc } from "./types/datasource_types";
import { DataSourceDoc } from "./types/datasource_types"; import { DataSourceDoc } from "./types/datasource_types";
import { DataSource } from "./types/datasource_types"; import { DataSource } from "./types/datasource_types";
import { Scenario, SimulationResult, SimulationRun, SimulationAggregateResult } from "./types/testing_types"; import { TestScenario, TestResult, TestRun, TestProfile, TestSimulation } from "./types/testing_types";
import { z } from 'zod'; import { z } from 'zod';
const client = new MongoClient(process.env["MONGODB_CONNECTION_STRING"] || "mongodb://localhost:27017"); const client = new MongoClient(process.env["MONGODB_CONNECTION_STRING"] || "mongodb://localhost:27017");
@ -20,7 +20,9 @@ export const projectsCollection = db.collection<z.infer<typeof Project>>("projec
export const projectMembersCollection = db.collection<z.infer<typeof ProjectMember>>("project_members"); export const projectMembersCollection = db.collection<z.infer<typeof ProjectMember>>("project_members");
export const webpagesCollection = db.collection<z.infer<typeof Webpage>>('webpages'); export const webpagesCollection = db.collection<z.infer<typeof Webpage>>('webpages');
export const agentWorkflowsCollection = db.collection<z.infer<typeof Workflow>>("agent_workflows"); export const agentWorkflowsCollection = db.collection<z.infer<typeof Workflow>>("agent_workflows");
export const scenariosCollection = db.collection<z.infer<typeof Scenario>>("scenarios");
export const apiKeysCollection = db.collection<z.infer<typeof ApiKey>>("api_keys"); export const apiKeysCollection = db.collection<z.infer<typeof ApiKey>>("api_keys");
export const simulationRunsCollection = db.collection<z.infer<typeof SimulationRun>>("simulation_runs"); export const testScenariosCollection = db.collection<z.infer<typeof TestScenario>>("test_scenarios");
export const simulationResultsCollection = db.collection<z.infer<typeof SimulationResult>>("simulation_results"); export const testProfilesCollection = db.collection<z.infer<typeof TestProfile>>("test_profiles");
export const testSimulationsCollection = db.collection<z.infer<typeof TestSimulation>>("test_simulations");
export const testRunsCollection = db.collection<z.infer<typeof TestRun>>("test_runs");
export const testResultsCollection = db.collection<z.infer<typeof TestResult>>("test_results");

2
apps/rowboat/app/lib/types/project_types.ts
View file

@ -10,6 +10,8 @@ export const Project = z.object({
webhookUrl: z.string().optional(), webhookUrl: z.string().optional(),
publishedWorkflowId: z.string().optional(), publishedWorkflowId: z.string().optional(),
nextWorkflowNumber: z.number().optional(), nextWorkflowNumber: z.number().optional(),
testRunCounter: z.number().default(0),
defaultTestProfileId: z.string().optional(),
});export const ProjectMember = z.object({ });export const ProjectMember = z.object({
userId: z.string(), userId: z.string(),
projectId: z.string(), projectId: z.string(),

74
apps/rowboat/app/lib/types/testing_types.ts
View file

@ -1,68 +1,52 @@
import { z } from "zod"; import { z } from "zod";
// Base type export const TestScenario = z.object({
export const Scenario = z.object({
projectId: z.string(), projectId: z.string(),
name: z.string().min(1, "Name cannot be empty"), name: z.string().min(1, "Name cannot be empty"),
description: z.string().min(1, "Description cannot be empty"), description: z.string().min(1, "Description cannot be empty"),
criteria: z.string().default(''),
context: z.string().default(''),
createdAt: z.string().datetime(), createdAt: z.string().datetime(),
lastUpdatedAt: z.string().datetime(), lastUpdatedAt: z.string().datetime(),
}); });
// Relevant to new simulation features export const TestProfile = z.object({
export const SimulationScenarioData = z.object({
scenario: z.string(),
context: z.string().default(''),
});
// Legacy
export const SimulationArticleData = z.object({
articleUrl: z.string(),
articleTitle: z.string().default('').optional(),
articleContent: z.string().default('').optional(),
});
export const SimulationChatMessagesData = z.object({
chatMessages: z.string(),
});
// Relevant to new simulation features
export const SimulationData = z.union([
SimulationScenarioData,
SimulationArticleData,
SimulationChatMessagesData
]);
export const SimulationAggregateResult = z.object({
total: z.number(),
pass: z.number(),
fail: z.number(),
});
export const SimulationRun = z.object({
projectId: z.string(), projectId: z.string(),
scenarioIds: z.array(z.string()), name: z.string().min(1, "Name cannot be empty"),
context: z.string(),
createdAt: z.string().datetime(),
lastUpdatedAt: z.string().datetime(),
mockTools: z.boolean(),
mockPrompt: z.string().optional(),
});
export const TestSimulation = z.object({
projectId: z.string(),
name: z.string().min(1, "Name cannot be empty"),
scenarioId: z.string(),
profileId: z.string(),
passCriteria: z.string(),
createdAt: z.string().datetime(),
lastUpdatedAt: z.string().datetime(),
});
export const TestRun = z.object({
projectId: z.string(),
name: z.string(),
simulationIds: z.array(z.string()),
workflowId: z.string(), workflowId: z.string(),
status: z.enum(['pending', 'running', 'completed', 'cancelled', 'failed', 'error']), status: z.enum(['pending', 'running', 'completed', 'cancelled', 'failed', 'error']),
startedAt: z.string(), startedAt: z.string(),
completedAt: z.string().optional(), completedAt: z.string().optional(),
aggregateResults: z.object({ aggregateResults: z.object({
total: z.number(), total: z.number(),
pass: z.number(), passCount: z.number(),
fail: z.number(), failCount: z.number(),
}).optional(), }).optional(),
}); });
export const SimulationResult = z.object({ export const TestResult = z.object({
projectId: z.string(), projectId: z.string(),
runId: z.string(), runId: z.string(),
scenarioId: z.string(), simulationId: z.string(),
result: z.union([z.literal('pass'), z.literal('fail')]), result: z.union([z.literal('pass'), z.literal('fail')]),
details: z.string() details: z.string()
}); });

4
apps/rowboat/app/lib/types/types.ts
View file

@ -1,7 +1,6 @@
import { CoreMessage, ToolCallPart } from "ai"; import { CoreMessage, ToolCallPart } from "ai";
import { z } from "zod"; import { z } from "zod";
import { apiV1 } from "rowboat-shared"; import { apiV1 } from "rowboat-shared";
import { SimulationData } from "./testing_types";
export const PlaygroundChat = z.object({ export const PlaygroundChat = z.object({
createdAt: z.string().datetime(), createdAt: z.string().datetime(),
@ -9,7 +8,7 @@ export const PlaygroundChat = z.object({
title: z.string().optional(), title: z.string().optional(),
messages: z.array(apiV1.ChatMessage), messages: z.array(apiV1.ChatMessage),
simulated: z.boolean().default(false).optional(), simulated: z.boolean().default(false).optional(),
simulationData: SimulationData.optional(), simulationScenario: z.string().optional(),
simulationComplete: z.boolean().default(false).optional(), simulationComplete: z.boolean().default(false).optional(),
agenticState: z.unknown().optional(), agenticState: z.unknown().optional(),
systemMessage: z.string().optional(), systemMessage: z.string().optional(),
@ -110,6 +109,7 @@ export const ApiRequest = z.object({
skipToolCalls: z.boolean().nullable().optional(), skipToolCalls: z.boolean().nullable().optional(),
maxTurns: z.number().nullable().optional(), maxTurns: z.number().nullable().optional(),
workflowId: z.string().nullable().optional(), workflowId: z.string().nullable().optional(),
testProfileId: z.string().nullable().optional(),
}); });
export const ApiResponse = z.object({ export const ApiResponse = z.object({

29
apps/rowboat/app/lib/utils.ts
View file

@ -18,6 +18,7 @@ import { qdrantClient } from "./qdrant";
import { EmbeddingRecord } from "./types/datasource_types"; import { EmbeddingRecord } from "./types/datasource_types";
import { ApiMessage } from "./types/types"; import { ApiMessage } from "./types/types";
import { openai } from "@ai-sdk/openai"; import { openai } from "@ai-sdk/openai";
import { TestProfile } from "./types/testing_types";
export async function callClientToolWebhook( export async function callClientToolWebhook(
toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number], toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number],
@ -220,19 +221,33 @@ export class PrefixLogger {
} }
} }
export async function mockToolResponse(toolId: string, messages: z.infer<typeof ApiMessage>[]): Promise<string> { export async function mockToolResponse(toolId: string, messages: z.infer<typeof ApiMessage>[], testProfile: z.infer<typeof TestProfile>): Promise<string> {
const prompt = ` const prompt = `Given below is a chat between a user and a customer support assistant.
# Your Specific Task:
Here is a chat between a user and a customer support assistant.
The assistant has requested a tool call with ID {{toolID}}. The assistant has requested a tool call with ID {{toolID}}.
Your job is to come up with an example of the data that the tool call should return.
The current date is {{date}}.
CONVERSATION: Your job is to come up with the data that the tool call should return.
In order to help you mock the responses, the user has provided some contextual information,
and also some instructions on how to mock the tool call.
>>>CHAT_HISTORY
{{messages}} {{messages}}
<<<END_OF_CHAT_HISTORY
>>>CONTEXT
{{context}}
<<<END_OF_CONTEXT
>>>MOCK_INSTRUCTIONS
{{mockInstructions}}
<<<END_OF_MOCK_INSTRUCTIONS
The current date is {{date}}.
` `
.replace('{{toolID}}', toolId) .replace('{{toolID}}', toolId)
.replace(`{{date}}`, new Date().toISOString()) .replace(`{{date}}`, new Date().toISOString())
.replace('{{context}}', testProfile.context)
.replace('{{mockInstructions}}', testProfile.mockPrompt || '')
.replace('{{messages}}', JSON.stringify(messages.map((m) => { .replace('{{messages}}', JSON.stringify(messages.map((m) => {
let tool_calls; let tool_calls;
if ('tool_calls' in m && m.role == 'assistant') { if ('tool_calls' in m && m.role == 'assistant') {

4
apps/rowboat/app/projects/[projectId]/menu.tsx
View file

@ -62,11 +62,11 @@ export default function Menu({
selected={pathname.startsWith(`/projects/${projectId}/workflow`)} selected={pathname.startsWith(`/projects/${projectId}/workflow`)}
/> />
<NavLink <NavLink
href={`/projects/${projectId}/simulation`} href={`/projects/${projectId}/test`}
label="Test" label="Test"
collapsed={collapsed} collapsed={collapsed}
icon={<PlayIcon size={16} />} icon={<PlayIcon size={16} />}
selected={pathname.startsWith(`/projects/${projectId}/simulation`)} selected={pathname.startsWith(`/projects/${projectId}/test`)}
/> />
{useDataSources && ( {useDataSources && (
<NavLink <NavLink

100
apps/rowboat/app/projects/[projectId]/playground/app.tsx
View file

@ -4,16 +4,15 @@ import { useEffect, useState, useMemo, useCallback } from "react";
import { z } from "zod"; import { z } from "zod";
import { PlaygroundChat } from "../../../lib/types/types"; import { PlaygroundChat } from "../../../lib/types/types";
import { Workflow } from "../../../lib/types/workflow_types"; import { Workflow } from "../../../lib/types/workflow_types";
import { SimulationData } from "../../../lib/types/testing_types";
import { SimulationScenarioData } from "../../../lib/types/testing_types";
import { Chat } from "./chat"; import { Chat } from "./chat";
import { useSearchParams, useRouter } from "next/navigation"; import { useSearchParams, useRouter } from "next/navigation";
import { ActionButton, Pane } from "../workflow/pane"; import { ActionButton, Pane } from "../workflow/pane";
import { apiV1 } from "rowboat-shared"; import { apiV1 } from "rowboat-shared";
import { EllipsisVerticalIcon, MessageSquarePlusIcon, PlayIcon } from "lucide-react"; import { EllipsisVerticalIcon, MessageSquarePlusIcon, PlayIcon } from "lucide-react";
import { getScenario } from "../../../actions/simulation_actions"; import { getScenario } from "../../../actions/testing_actions";
import clsx from "clsx"; import clsx from "clsx";
import { TestProfile, TestScenario } from "@/app/lib/types/testing_types";
import { WithStringId } from "@/app/lib/types/types";
function SimulateLabel() { function SimulateLabel() {
return <span>Simulate<sup className="pl-1">beta</sup></span>; return <span>Simulate<sup className="pl-1">beta</sup></span>;
} }
@ -25,18 +24,16 @@ export function App({
projectId, projectId,
workflow, workflow,
messageSubscriber, messageSubscriber,
initialTestProfile,
}: { }: {
hidden?: boolean; hidden?: boolean;
projectId: string; projectId: string;
workflow: z.infer<typeof Workflow>; workflow: z.infer<typeof Workflow>;
messageSubscriber?: (messages: z.infer<typeof apiV1.ChatMessage>[]) => void; messageSubscriber?: (messages: z.infer<typeof apiV1.ChatMessage>[]) => void;
initialTestProfile: z.infer<typeof TestProfile>;
}) { }) {
const searchParams = useSearchParams();
const router = useRouter();
const initialChatId = useMemo(() => searchParams.get('chatId'), [searchParams]);
const [existingChatId, setExistingChatId] = useState<string | null>(initialChatId);
const [loadingChat, setLoadingChat] = useState<boolean>(false);
const [counter, setCounter] = useState<number>(0); const [counter, setCounter] = useState<number>(0);
const [testProfile, setTestProfile] = useState<z.infer<typeof TestProfile>>(initialTestProfile);
const [chat, setChat] = useState<z.infer<typeof PlaygroundChat>>({ const [chat, setChat] = useState<z.infer<typeof PlaygroundChat>>({
projectId, projectId,
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
@ -45,49 +42,45 @@ export function App({
systemMessage: defaultSystemMessage, systemMessage: defaultSystemMessage,
}); });
const beginSimulation = useCallback((data: z.infer<typeof SimulationData>) => { function handleTestProfileChange(profile: WithStringId<z.infer<typeof TestProfile>>) {
setExistingChatId(null); setTestProfile(profile);
setLoadingChat(true);
setCounter(counter + 1); setCounter(counter + 1);
setChat({ }
projectId,
createdAt: new Date().toISOString(),
messages: [],
simulated: true,
simulationData: data,
systemMessage: 'context' in data ? data.context : '',
});
}, [counter, projectId]);
useEffect(() => { // const beginSimulation = useCallback((scenario: string) => {
const scenarioId = localStorage.getItem('pendingScenarioId'); // setExistingChatId(null);
if (scenarioId && projectId) { // setLoadingChat(true);
console.log('Scenario Effect triggered:', { scenarioId, projectId }); // setCounter(counter + 1);
getScenario(projectId, scenarioId).then((scenario) => { // setChat({
console.log('Scenario data received:', scenario); // projectId,
beginSimulation({ // createdAt: new Date().toISOString(),
...scenario, // messages: [],
systemMessage: scenario.context || '', // simulated: true,
} as z.infer<typeof SimulationScenarioData>); // simulationScenario: scenario,
localStorage.removeItem('pendingScenarioId'); // systemMessage: '',
}).catch(error => { // });
console.error('Error fetching scenario:', error); // }, [counter, projectId]);
localStorage.removeItem('pendingScenarioId');
}); // useEffect(() => {
} // const scenarioId = localStorage.getItem('pendingScenarioId');
}, [projectId, beginSimulation]); // if (scenarioId && projectId) {
// console.log('Scenario Effect triggered:', { scenarioId, projectId });
// getScenario(projectId, scenarioId).then((scenario) => {
// console.log('Scenario data received:', scenario);
// beginSimulation(scenario.description);
// localStorage.removeItem('pendingScenarioId');
// }).catch(error => {
// console.error('Error fetching scenario:', error);
// localStorage.removeItem('pendingScenarioId');
// });
// }
// }, [projectId, beginSimulation]);
if (hidden) { if (hidden) {
return <></>; return <></>;
} }
function handleSimulateButtonClick() {
router.push(`/projects/${projectId}/simulation`);
}
function handleNewChatButtonClick() { function handleNewChatButtonClick() {
setExistingChatId(null);
setLoadingChat(true);
setCounter(counter + 1); setCounter(counter + 1);
setChat({ setChat({
projectId, projectId,
@ -110,27 +103,18 @@ export function App({
> >
New chat New chat
</ActionButton>, </ActionButton>,
<ActionButton
key="simulate"
icon={<PlayIcon size={16} />}
onClick={handleSimulateButtonClick}
>
Simulate
</ActionButton>,
]} ]}
> >
<div className="h-full overflow-auto"> <div className="h-full overflow-auto">
{loadingChat && <div className="flex justify-center items-center h-full"> <Chat
<Spinner /> key={`chat-${counter}`}
</div>}
{!loadingChat && <Chat
key={existingChatId || 'chat-' + counter}
chat={chat} chat={chat}
initialChatId={existingChatId || null}
projectId={projectId} projectId={projectId}
workflow={workflow} workflow={workflow}
testProfile={testProfile}
messageSubscriber={messageSubscriber} messageSubscriber={messageSubscriber}
/>} onTestProfileChange={handleTestProfileChange}
/>
</div> </div>
</Pane> </Pane>
); );

51
apps/rowboat/app/projects/[projectId]/playground/chat.tsx
View file

@ -9,24 +9,28 @@ import { convertWorkflowToAgenticAPI } from "../../../lib/types/agents_api_types
import { AgenticAPIChatRequest } from "../../../lib/types/agents_api_types"; import { AgenticAPIChatRequest } from "../../../lib/types/agents_api_types";
import { Workflow } from "../../../lib/types/workflow_types"; import { Workflow } from "../../../lib/types/workflow_types";
import { ComposeBox } from "./compose-box"; import { ComposeBox } from "./compose-box";
import { Button, Spinner } from "@nextui-org/react"; import { Button, Spinner, Tooltip } from "@nextui-org/react";
import { apiV1 } from "rowboat-shared"; import { apiV1 } from "rowboat-shared";
import { CopyAsJsonButton } from "./copy-as-json-button"; import { CopyAsJsonButton } from "./copy-as-json-button";
import { TestProfile } from "@/app/lib/types/testing_types";
import { ProfileSelector } from "@/app/lib/components/selectors/profile-selector";
import { WithStringId } from "@/app/lib/types/types";
export function Chat({ export function Chat({
chat, chat,
initialChatId = null,
projectId, projectId,
workflow, workflow,
messageSubscriber, messageSubscriber,
testProfile,
onTestProfileChange,
}: { }: {
chat: z.infer<typeof PlaygroundChat>; chat: z.infer<typeof PlaygroundChat>;
initialChatId?: string | null;
projectId: string; projectId: string;
workflow: z.infer<typeof Workflow>; workflow: z.infer<typeof Workflow>;
messageSubscriber?: (messages: z.infer<typeof apiV1.ChatMessage>[]) => void; messageSubscriber?: (messages: z.infer<typeof apiV1.ChatMessage>[]) => void;
testProfile: z.infer<typeof TestProfile>;
onTestProfileChange: (profile: WithStringId<z.infer<typeof TestProfile>>) => void;
}) { }) {
const [chatId, setChatId] = useState<string | null>(initialChatId);
const [messages, setMessages] = useState<z.infer<typeof apiV1.ChatMessage>[]>(chat.messages); const [messages, setMessages] = useState<z.infer<typeof apiV1.ChatMessage>[]>(chat.messages);
const [loadingAssistantResponse, setLoadingAssistantResponse] = useState<boolean>(false); const [loadingAssistantResponse, setLoadingAssistantResponse] = useState<boolean>(false);
const [loadingUserResponse, setLoadingUserResponse] = useState<boolean>(false); const [loadingUserResponse, setLoadingUserResponse] = useState<boolean>(false);
@ -34,11 +38,11 @@ export function Chat({
const [agenticState, setAgenticState] = useState<unknown>(chat.agenticState || { const [agenticState, setAgenticState] = useState<unknown>(chat.agenticState || {
last_agent_name: workflow.startAgent, last_agent_name: workflow.startAgent,
}); });
const [showCopySuccess, setShowCopySuccess] = useState(false);
const [fetchResponseError, setFetchResponseError] = useState<string | null>(null); const [fetchResponseError, setFetchResponseError] = useState<string | null>(null);
const [lastAgenticRequest, setLastAgenticRequest] = useState<unknown | null>(null); const [lastAgenticRequest, setLastAgenticRequest] = useState<unknown | null>(null);
const [lastAgenticResponse, setLastAgenticResponse] = useState<unknown | null>(null); const [lastAgenticResponse, setLastAgenticResponse] = useState<unknown | null>(null);
const [systemMessage, setSystemMessage] = useState<string | undefined>(chat.systemMessage); const [systemMessage, setSystemMessage] = useState<string | undefined>(testProfile.context);
const [isProfileSelectorOpen, setIsProfileSelectorOpen] = useState(false);
// collect published tool call results // collect published tool call results
const toolCallResults: Record<string, z.infer<typeof apiV1.ToolMessage>> = {}; const toolCallResults: Record<string, z.infer<typeof apiV1.ToolMessage>> = {};
@ -53,7 +57,7 @@ export function Chat({
role: 'user', role: 'user',
content: prompt, content: prompt,
version: 'v1', version: 'v1',
chatId: chatId ?? '', chatId: '',
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
}]; }];
setMessages(updatedMessages); setMessages(updatedMessages);
@ -64,7 +68,7 @@ export function Chat({
setMessages([...messages, ...results.map((result) => ({ setMessages([...messages, ...results.map((result) => ({
...result, ...result,
version: 'v1' as const, version: 'v1' as const,
chatId: chatId ?? '', chatId: '',
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
}))]); }))]);
} }
@ -97,7 +101,7 @@ export function Chat({
role: 'system', role: 'system',
content: systemMessage || '', content: systemMessage || '',
version: 'v1' as const, version: 'v1' as const,
chatId: chatId ?? '', chatId: '',
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
}, ...messages]), }, ...messages]),
state: agenticState, state: agenticState,
@ -122,7 +126,7 @@ export function Chat({
setMessages([...messages, ...response.messages.map((message) => ({ setMessages([...messages, ...response.messages.map((message) => ({
...message, ...message,
version: 'v1' as const, version: 'v1' as const,
chatId: chatId ?? '', chatId: '',
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
}))]); }))]);
setAgenticState(response.state); setAgenticState(response.state);
@ -157,14 +161,14 @@ export function Chat({
return () => { return () => {
ignore = true; ignore = true;
}; };
}, [chatId, chat.simulated, messages, projectId, agenticState, workflow, fetchResponseError, systemMessage, simulationComplete]); }, [chat.simulated, messages, projectId, agenticState, workflow, fetchResponseError, systemMessage, simulationComplete]);
// simulate user turn // simulate user turn
useEffect(() => { useEffect(() => {
let ignore = false; let ignore = false;
async function process() { async function process() {
if (chat.simulationData === undefined) { if (chat.simulationScenario === undefined) {
return; return;
} }
@ -172,7 +176,7 @@ export function Chat({
setLoadingUserResponse(true); setLoadingUserResponse(true);
try { try {
const response = await simulateUserResponse(projectId, messages, chat.simulationData) const response = await simulateUserResponse(projectId, messages, chat.simulationScenario)
if (ignore) { if (ignore) {
return; return;
} }
@ -187,7 +191,7 @@ export function Chat({
role: 'user', role: 'user',
content: response, content: response,
version: 'v1' as const, version: 'v1' as const,
chatId: chatId ?? '', chatId: '',
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
}]); }]);
setFetchResponseError(null); setFetchResponseError(null);
@ -226,7 +230,7 @@ export function Chat({
return () => { return () => {
ignore = true; ignore = true;
}; };
}, [chatId, chat.simulated, messages, projectId, simulationComplete, chat.simulationData]); }, [chat.simulated, messages, projectId, simulationComplete, chat.simulationScenario]);
// save chat on every assistant message // save chat on every assistant message
// useEffect(() => { // useEffect(() => {
@ -275,6 +279,22 @@ export function Chat({
return <div className="relative h-full flex flex-col gap-8 pt-8 overflow-auto"> return <div className="relative h-full flex flex-col gap-8 pt-8 overflow-auto">
<CopyAsJsonButton onCopy={handleCopyChat} /> <CopyAsJsonButton onCopy={handleCopyChat} />
<div className="absolute top-0 left-0">
<Tooltip content={"Change profile"} placement="right">
<button
className="border border-gray-200 dark:border-gray-800 p-2 rounded-lg text-xs hover:bg-gray-100 dark:hover:bg-gray-800"
onClick={() => setIsProfileSelectorOpen(true)}
>
{`Test profile: ${testProfile.name}`}
</button>
</Tooltip>
</div>
<ProfileSelector
projectId={projectId}
isOpen={isProfileSelectorOpen}
onOpenChange={setIsProfileSelectorOpen}
onSelect={onTestProfileChange}
/>
<Messages <Messages
projectId={projectId} projectId={projectId}
messages={messages} messages={messages}
@ -284,6 +304,7 @@ export function Chat({
loadingAssistantResponse={loadingAssistantResponse} loadingAssistantResponse={loadingAssistantResponse}
loadingUserResponse={loadingUserResponse} loadingUserResponse={loadingUserResponse}
workflow={workflow} workflow={workflow}
testProfile={testProfile}
onSystemMessageChange={handleSystemMessageChange} onSystemMessageChange={handleSystemMessageChange}
/> />
<div className="shrink-0"> <div className="shrink-0">

112
apps/rowboat/app/projects/[projectId]/playground/messages.tsx
View file

@ -12,6 +12,7 @@ import Link from "next/link";
import { apiV1 } from "rowboat-shared"; import { apiV1 } from "rowboat-shared";
import { EditableField } from "../../../lib/components/editable-field"; import { EditableField } from "../../../lib/components/editable-field";
import { MessageSquareIcon, EllipsisIcon, CircleCheckIcon, ChevronsDownIcon, ChevronsRightIcon, ChevronRightIcon, ChevronDownIcon, ExternalLinkIcon, XIcon } from "lucide-react"; import { MessageSquareIcon, EllipsisIcon, CircleCheckIcon, ChevronsDownIcon, ChevronsRightIcon, ChevronRightIcon, ChevronDownIcon, ExternalLinkIcon, XIcon } from "lucide-react";
import { TestProfile } from "@/app/lib/types/testing_types";
function UserMessage({ content }: { content: string }) { function UserMessage({ content }: { content: string }) {
return <div className="self-end ml-[30%] flex flex-col"> return <div className="self-end ml-[30%] flex flex-col">
@ -93,6 +94,7 @@ function ToolCalls({
messages, messages,
sender, sender,
workflow, workflow,
testProfile,
}: { }: {
toolCalls: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls']; toolCalls: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'];
results: Record<string, z.infer<typeof apiV1.ToolMessage>>; results: Record<string, z.infer<typeof apiV1.ToolMessage>>;
@ -101,6 +103,7 @@ function ToolCalls({
messages: z.infer<typeof apiV1.ChatMessage>[]; messages: z.infer<typeof apiV1.ChatMessage>[];
sender: string | null | undefined; sender: string | null | undefined;
workflow: z.infer<typeof Workflow>; workflow: z.infer<typeof Workflow>;
testProfile: z.infer<typeof TestProfile>;
}) { }) {
const resultsMap: Record<string, z.infer<typeof apiV1.ToolMessage>> = {}; const resultsMap: Record<string, z.infer<typeof apiV1.ToolMessage>> = {};
@ -123,6 +126,7 @@ function ToolCalls({
messages={messages} messages={messages}
sender={sender} sender={sender}
workflow={workflow} workflow={workflow}
testProfile={testProfile}
/> />
})} })}
</div>; </div>;
@ -136,6 +140,7 @@ function ToolCall({
messages, messages,
sender, sender,
workflow, workflow,
testProfile,
}: { }: {
toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number]; toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number];
result: z.infer<typeof apiV1.ToolMessage> | undefined; result: z.infer<typeof apiV1.ToolMessage> | undefined;
@ -144,6 +149,7 @@ function ToolCall({
messages: z.infer<typeof apiV1.ChatMessage>[]; messages: z.infer<typeof apiV1.ChatMessage>[];
sender: string | null | undefined; sender: string | null | undefined;
workflow: z.infer<typeof Workflow>; workflow: z.infer<typeof Workflow>;
testProfile: z.infer<typeof TestProfile>;
}) { }) {
let matchingWorkflowTool: z.infer<typeof WorkflowTool> | undefined; let matchingWorkflowTool: z.infer<typeof WorkflowTool> | undefined;
for (const tool of workflow.tools) { for (const tool of workflow.tools) {
@ -154,15 +160,6 @@ function ToolCall({
} }
switch (toolCall.function.name) { switch (toolCall.function.name) {
case 'retrieve_url_info':
return <RetrieveUrlInfoToolCall
toolCall={toolCall}
result={result}
handleResult={handleResult}
projectId={projectId}
messages={messages}
sender={sender}
/>;
case 'getArticleInfo': case 'getArticleInfo':
return <GetInformationToolCall return <GetInformationToolCall
toolCall={toolCall} toolCall={toolCall}
@ -184,7 +181,7 @@ function ToolCall({
sender={sender} sender={sender}
/>; />;
} }
if (matchingWorkflowTool && !matchingWorkflowTool.mockInPlayground) { if (matchingWorkflowTool && !testProfile.mockTools) {
return <ClientToolCall return <ClientToolCall
toolCall={toolCall} toolCall={toolCall}
result={result} result={result}
@ -202,6 +199,7 @@ function ToolCall({
messages={messages} messages={messages}
sender={sender} sender={sender}
autoSubmit={matchingWorkflowTool?.autoSubmitMockedResponse} autoSubmit={matchingWorkflowTool?.autoSubmitMockedResponse}
testProfile={testProfile}
/>; />;
} }
} }
@ -310,85 +308,6 @@ function GetInformationToolCall({
</div>; </div>;
} }
function RetrieveUrlInfoToolCall({
toolCall,
result: availableResult,
handleResult,
projectId,
messages,
sender,
}: {
toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number];
result: z.infer<typeof apiV1.ToolMessage> | undefined;
handleResult: (result: z.infer<typeof apiV1.ToolMessage>) => void;
projectId: string;
messages: z.infer<typeof apiV1.ChatMessage>[];
sender: string | null | undefined;
}) {
const [result, setResult] = useState<z.infer<typeof apiV1.ToolMessage> | undefined>(availableResult);
const args = JSON.parse(toolCall.function.arguments) as { url: string };
let typedResult: z.infer<typeof WebpageCrawlResponse> | undefined;
if (result) {
typedResult = JSON.parse(result.content) as z.infer<typeof WebpageCrawlResponse>;
}
useEffect(() => {
if (result) {
return;
}
let ignore = false;
function process() {
// parse args
scrapeWebpage(args.url)
.then(page => {
if (ignore) {
return;
}
const result: z.infer<typeof apiV1.ToolMessage> = {
role: 'tool',
tool_call_id: toolCall.id,
tool_name: toolCall.function.name,
content: JSON.stringify(page),
};
setResult(result);
handleResult(result);
});
}
process();
return () => {
ignore = true;
};
}, [result, toolCall.id, toolCall.function.name, projectId, args.url, handleResult]);
return <div className="flex flex-col gap-1">
{sender && <div className='text-gray-500 text-sm ml-3'>{sender}</div>}
<div className='border border-gray-300 p-2 rounded-lg rounded-bl-none flex flex-col gap-2 mr-[30%]'>
<ToolCallHeader toolCall={toolCall} result={result} />
<div className='mt-1 flex flex-col gap-2'>
<div className="flex gap-1">
URL: <a className="inline-flex items-center gap-1" target="_blank" href={args.url}>
<span className='underline'>
{args.url}
</span>
<ExternalLinkIcon size={16} />
</a>
</div>
{result && (
<ExpandableContent
label='Content'
content={typedResult}
expanded={false}
/>
)}
</div>
</div>
</div>;
}
function TransferToAgentToolCall({ function TransferToAgentToolCall({
toolCall, toolCall,
result: availableResult, result: availableResult,
@ -495,6 +414,7 @@ function MockToolCall({
messages, messages,
sender, sender,
autoSubmit = false, autoSubmit = false,
testProfile,
}: { }: {
toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number]; toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number];
result: z.infer<typeof apiV1.ToolMessage> | undefined; result: z.infer<typeof apiV1.ToolMessage> | undefined;
@ -503,6 +423,7 @@ function MockToolCall({
messages: z.infer<typeof apiV1.ChatMessage>[]; messages: z.infer<typeof apiV1.ChatMessage>[];
sender: string | null | undefined; sender: string | null | undefined;
autoSubmit?: boolean; autoSubmit?: boolean;
testProfile: z.infer<typeof TestProfile>;
}) { }) {
const [result, setResult] = useState<z.infer<typeof apiV1.ToolMessage> | undefined>(availableResult); const [result, setResult] = useState<z.infer<typeof apiV1.ToolMessage> | undefined>(availableResult);
const [response, setResponse] = useState(''); const [response, setResponse] = useState('');
@ -538,7 +459,7 @@ function MockToolCall({
async function process() { async function process() {
setGeneratingResponse(true); setGeneratingResponse(true);
const response = await suggestToolResponse(toolCall.id, projectId, messages); const response = await suggestToolResponse(toolCall.id, projectId, messages, testProfile);
if (ignore) { if (ignore) {
return; return;
} }
@ -550,7 +471,7 @@ function MockToolCall({
return () => { return () => {
ignore = true; ignore = true;
}; };
}, [result, response, toolCall.id, projectId, messages]); }, [result, response, toolCall.id, projectId, messages, testProfile]);
// auto submit if autoSubmitMockedResponse is true // auto submit if autoSubmitMockedResponse is true
useEffect(() => { useEffect(() => {
@ -682,6 +603,7 @@ export function Messages({
loadingAssistantResponse, loadingAssistantResponse,
loadingUserResponse, loadingUserResponse,
workflow, workflow,
testProfile,
onSystemMessageChange, onSystemMessageChange,
}: { }: {
projectId: string; projectId: string;
@ -692,13 +614,12 @@ export function Messages({
loadingAssistantResponse: boolean; loadingAssistantResponse: boolean;
loadingUserResponse: boolean; loadingUserResponse: boolean;
workflow: z.infer<typeof Workflow>; workflow: z.infer<typeof Workflow>;
testProfile: z.infer<typeof TestProfile>;
onSystemMessageChange: (message: string) => void; onSystemMessageChange: (message: string) => void;
}) { }) {
const messagesEndRef = useRef<HTMLDivElement>(null); const messagesEndRef = useRef<HTMLDivElement>(null);
let lastUserMessageTimestamp = 0; let lastUserMessageTimestamp = 0;
const systemMessageLocked = messages.length > 0;
// scroll to bottom on new messages // scroll to bottom on new messages
useEffect(() => { useEffect(() => {
messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }) messagesEndRef.current?.scrollIntoView({ behavior: "smooth" })
@ -707,9 +628,9 @@ export function Messages({
return <div className="grow pt-4 overflow-auto"> return <div className="grow pt-4 overflow-auto">
<div className="max-w-[768px] mx-auto flex flex-col gap-8"> <div className="max-w-[768px] mx-auto flex flex-col gap-8">
<SystemMessage <SystemMessage
content={systemMessage || ''} content={testProfile.context}
onChange={onSystemMessageChange} onChange={onSystemMessageChange}
locked={systemMessageLocked} locked={true}
/> />
{messages.map((message, index) => { {messages.map((message, index) => {
if (message.role === 'assistant') { if (message.role === 'assistant') {
@ -723,6 +644,7 @@ export function Messages({
messages={messages} messages={messages}
sender={message.agenticSender} sender={message.agenticSender}
workflow={workflow} workflow={workflow}
testProfile={testProfile}
/>; />;
} else { } else {
// the assistant message createdAt is an ISO string timestamp // the assistant message createdAt is an ISO string timestamp

459
apps/rowboat/app/projects/[projectId]/simulation/app.tsx
View file

@ -1,459 +0,0 @@
'use client';
import { useState, useEffect, useCallback } from 'react';
import { PlusIcon, PencilIcon, XMarkIcon, EllipsisVerticalIcon, TrashIcon, ChevronRightIcon, PlayIcon, ChevronDownIcon, ChevronLeftIcon } from '@heroicons/react/24/outline';
import { useParams, useRouter, useSearchParams } from 'next/navigation';
import {
getScenarios,
createScenario,
updateScenario,
deleteScenario,
getRuns,
getRun,
getRunResults,
createRun,
createRunResult,
updateRunStatus,
createAggregateResult,
deleteRun,
} from '../../../actions/simulation_actions';
import { type WithStringId } from '../../../lib/types/types';
import { Scenario, SimulationRun, SimulationResult } from "../../../lib/types/testing_types";
import { Workflow } from "../../../lib/types/workflow_types";
import { z } from 'zod';
import { SimulationResultCard, ScenarioResultCard } from './components/RunComponents';
import { ScenarioList, ScenarioViewer } from './components/ScenarioComponents';
import { fetchWorkflow } from '../../../actions/workflow_actions';
import { StructuredPanel, ActionButton } from "../../../lib/components/structured-panel";
import {
ResizableHandle,
ResizablePanel,
ResizablePanelGroup,
} from "../../../../components/ui/resizable"
import { Pagination } from "../../../lib/components/pagination";
type ScenarioType = WithStringId<z.infer<typeof Scenario>>;
type SimulationRunType = WithStringId<z.infer<typeof SimulationRun>>;
type SimulationResultType = WithStringId<z.infer<typeof SimulationResult>>;
type SimulationReport = {
totalScenarios: number;
passedScenarios: number;
failedScenarios: number;
results: z.infer<typeof SimulationResult>[];
timestamp: Date;
};
const dummySimulator = async (scenario: ScenarioType, runId: string, projectId: string): Promise<z.infer<typeof SimulationResult>> => {
await new Promise(resolve => setTimeout(resolve, 500));
const passed = Math.random() > 0.5;
const result: z.infer<typeof SimulationResult> = {
projectId: projectId,
runId: runId,
scenarioId: scenario._id,
result: passed ? 'pass' : 'fail' as const,
details: passed
? "The bot successfully completed the conversation"
: "The bot could not handle the conversation",
};
await createRunResult(
projectId,
runId,
scenario._id,
result.result,
result.details
);
return result;
};
export default function SimulationApp() {
const { projectId } = useParams();
const router = useRouter();
const searchParams = useSearchParams();
const [scenarios, setScenarios] = useState<ScenarioType[]>([]);
const [selectedScenario, setSelectedScenario] = useState<ScenarioType | null>(null);
const [isEditing, setIsEditing] = useState(false);
const [menuOpenScenarioId, setMenuOpenScenarioId] = useState<string | null>(null);
const [isRunning, setIsRunning] = useState(false);
const [simulationReport, setSimulationReport] = useState<SimulationReport | null>(null);
const [expandedResults, setExpandedResults] = useState<Set<string>>(new Set());
const [runs, setRuns] = useState<SimulationRunType[]>([]);
const [activeRun, setActiveRun] = useState<SimulationRunType | null>(null);
const [runResults, setRunResults] = useState<SimulationResultType[]>([]);
const [isLoadingRuns, setIsLoadingRuns] = useState(true);
const [allRunResults, setAllRunResults] = useState<Record<string, SimulationResultType[]>>({});
const [workflowVersions, setWorkflowVersions] = useState<Record<string, WithStringId<z.infer<typeof Workflow>>>>({});
const [menuOpenId, setMenuOpenIdState] = useState<string | null>(null);
const runsPerPage = 10;
const currentPage = Number(searchParams.get('page')) || 1;
const setMenuOpenId = useCallback((id: string | null) => {
setMenuOpenIdState(id);
}, []);
// Load scenarios on mount
useEffect(() => {
if (!projectId) return;
getScenarios(projectId as string).then(setScenarios);
}, [projectId]);
useEffect(() => {
if (menuOpenScenarioId) {
const closeMenu = () => setMenuOpenScenarioId(null);
window.addEventListener('click', closeMenu);
return () => window.removeEventListener('click', closeMenu);
}
}, [menuOpenScenarioId]);
// Modify the fetchRuns function to also fetch results
const fetchRuns = useCallback(async () => {
if (!projectId) return;
setIsLoadingRuns(true);
try {
const runsData = await getRuns(projectId as string);
setRuns(runsData);
// Fetch results for all runs
const resultsPromises = runsData.map(run =>
getRunResults(projectId as string, run._id)
);
const allResults = await Promise.all(resultsPromises);
// Create a map of run ID to results
const resultsMap = runsData.reduce((acc, run, index) => ({
...acc,
[run._id]: allResults[index]
}), {});
setAllRunResults(resultsMap);
} catch (error) {
console.error('Error fetching runs:', error);
} finally {
setIsLoadingRuns(false);
}
}, [projectId]);
// Update the useEffect hooks to include fetchRuns
useEffect(() => {
if (!projectId) return;
fetchRuns();
}, [projectId, fetchRuns]);
useEffect(() => {
if (!projectId || !activeRun || activeRun.status === 'completed' || activeRun.status === 'cancelled') return;
const interval = setInterval(async () => {
try {
const updatedRun = await getRun(projectId as string, activeRun._id);
setActiveRun(updatedRun);
if (updatedRun.status === 'completed') {
const results = await getRunResults(projectId as string, activeRun._id);
setRunResults(results);
fetchRuns(); // Refresh the runs list
}
} catch (error) {
console.error('Error polling run status:', error);
}
}, 2000);
return () => clearInterval(interval);
}, [activeRun, projectId, fetchRuns]);
const createNewScenario = async () => {
if (!projectId) return;
const newScenarioId = await createScenario(
projectId as string,
'New Scenario',
''
);
// Refresh scenarios list
const updatedScenarios = await getScenarios(projectId as string);
setScenarios(updatedScenarios);
const newScenario = updatedScenarios.find(s => s._id === newScenarioId);
if (newScenario) {
setSelectedScenario(newScenario);
setIsEditing(true);
}
};
const handleUpdateScenario = async (updatedScenario: ScenarioType) => {
if (!projectId) return;
// First verify the scenario exists and get its current state
const currentScenarios = await getScenarios(projectId as string);
const existingScenario = currentScenarios.find(s => s._id === updatedScenario._id);
if (!existingScenario) {
console.error('Scenario not found');
return;
}
// Only update the specific fields that have changed
await updateScenario(
projectId as string,
updatedScenario._id,
{
name: updatedScenario.name,
description: updatedScenario.description,
criteria: updatedScenario.criteria,
context: updatedScenario.context,
}
);
// Just refresh the scenarios list without setting selected scenario
const updatedScenarios = await getScenarios(projectId as string);
setScenarios(updatedScenarios);
setIsEditing(false);
};
const handleCloseScenario = () => {
setSelectedScenario(null);
setIsEditing(false);
};
const handleDeleteScenario = async (scenarioId: string) => {
if (!projectId) return;
await deleteScenario(projectId as string, scenarioId);
const updatedScenarios = await getScenarios(projectId as string);
setScenarios(updatedScenarios);
if (selectedScenario?._id === scenarioId) {
setSelectedScenario(null);
setIsEditing(false);
}
setMenuOpenScenarioId(null);
};
const runAllScenarios = async () => {
if (!projectId) return;
setIsRunning(true);
setSimulationReport(null);
try {
// Get workflowId from localStorage
const workflowId = localStorage.getItem(`lastWorkflowId_${projectId}`);
if (!workflowId) {
throw new Error('No workflow selected. Please select a workflow first.');
}
// First verify the workflow exists before creating the run
let workflow;
try {
workflow = await fetchWorkflow(projectId as string, workflowId);
} catch (error) {
// If workflow doesn't exist, clear localStorage and throw error
localStorage.removeItem(`lastWorkflowId_${projectId}`);
throw new Error('Selected workflow no longer exists. Please select a new workflow.');
}
const newRun = await createRun(
projectId as string,
scenarios.map(s => s._id),
workflowId
);
setActiveRun(newRun);
// Store workflow version
setWorkflowVersions(prev => ({
...prev,
[workflowId]: workflow
}));
const shouldMock = process.env.NEXT_PUBLIC_MOCK_SIMULATION_RESULTS === 'true';
if (shouldMock) {
console.log('Using mock simulation...');
await updateRunStatus(projectId as string, newRun._id, 'running');
// Run all scenarios and collect results
const mockResults = await Promise.all(
scenarios.map(scenario =>
dummySimulator(scenario, newRun._id, projectId as string)
)
);
// Calculate and store aggregate results before marking as complete
const total = scenarios.length;
const pass = mockResults.filter(r => r.result === 'pass').length;
const fail = mockResults.filter(r => r.result === 'fail').length;
await createAggregateResult(
projectId as string,
newRun._id,
total,
pass,
fail
);
await updateRunStatus(
projectId as string,
newRun._id,
'completed',
new Date().toISOString()
);
const results = await getRunResults(projectId as string, newRun._id);
setRunResults(results);
const updatedRun = await getRun(projectId as string, newRun._id);
setActiveRun(updatedRun);
}
await fetchRuns();
} catch (error) {
console.error('Error starting scenarios:', error);
alert(error instanceof Error ? error.message : 'An error occurred while starting scenarios');
} finally {
setIsRunning(false);
}
};
const runSingleScenario = (scenario: ScenarioType) => {
// Store scenario ID in localStorage instead of URL parameter
localStorage.setItem('pendingScenarioId', scenario._id);
// Navigate to the playground without query parameter
router.push(`/projects/${projectId}/workflow`);
setMenuOpenScenarioId(null);
};
// Update the workflow versions fetching effect
useEffect(() => {
if (!projectId || !runs.length) return;
const fetchWorkflowVersions = async () => {
const workflowIds = Array.from(new Set(runs.map(run => run.workflowId)));
const versions: Record<string, WithStringId<z.infer<typeof Workflow>>> = {};
for (const workflowId of workflowIds) {
try {
const workflow = await fetchWorkflow(projectId as string, workflowId);
versions[workflowId] = workflow;
} catch (error) {
console.error(`Error fetching workflow ${workflowId}:`, error);
// Add a placeholder for deleted/invalid workflows
versions[workflowId] = {
_id: workflowId,
name: "Deleted/Invalid Workflow",
projectId: projectId as string,
agents: [],
prompts: [],
tools: [],
startAgent: "",
createdAt: new Date().toISOString(),
lastUpdatedAt: new Date().toISOString(),
};
}
}
setWorkflowVersions(versions);
};
fetchWorkflowVersions();
}, [projectId, runs]);
const handleCancelRun = async (runId: string) => {
if (!projectId) return;
try {
await updateRunStatus(projectId as string, runId, 'cancelled');
await fetchRuns(); // Refresh the runs list
} catch (error) {
console.error('Error cancelling run:', error);
}
};
const handleDeleteRun = async (runId: string) => {
if (!projectId) return;
try {
await deleteRun(projectId as string, runId);
await fetchRuns(); // Refresh the runs list
} catch (error) {
console.error('Error deleting run:', error);
}
};
const indexOfLastRun = currentPage * runsPerPage;
const indexOfFirstRun = indexOfLastRun - runsPerPage;
const currentRuns = runs.slice(indexOfFirstRun, indexOfLastRun);
const totalPages = Math.ceil(runs.length / runsPerPage);
return (
<ResizablePanelGroup direction="horizontal" className="h-screen gap-1">
<ResizablePanel minSize={10} defaultSize={15}>
<ScenarioList
scenarios={scenarios}
selectedId={selectedScenario?._id ?? null}
onSelect={(id) => setSelectedScenario(scenarios.find(s => s._id === id) ?? null)}
onAdd={createNewScenario}
onRunScenario={(id) => {
const scenario = scenarios.find(s => s._id === id);
if (scenario) runSingleScenario(scenario);
}}
onDeleteScenario={(id) => handleDeleteScenario(id)}
/>
</ResizablePanel>
<ResizableHandle />
<ResizablePanel minSize={20} defaultSize={85} className="overflow-auto">
{selectedScenario ? (
<ScenarioViewer
scenario={selectedScenario}
onSave={handleUpdateScenario}
onClose={handleCloseScenario}
/>
) : (
<StructuredPanel
title="SIMULATION RUNS"
tooltip="Run and view simulations"
actions={[
<ActionButton
key="run-all"
onClick={() => void runAllScenarios()}
disabled={isRunning}
icon={<PlayIcon className="w-4 h-4" />}
primary
>
Run All Scenarios
</ActionButton>
]}
>
<div className="p-6">
{/* Runs list */}
{isLoadingRuns ? (
<div>Loading runs...</div>
) : (
<div className="space-y-4">
{currentRuns.map((run) => (
<SimulationResultCard
key={run._id}
run={run}
results={allRunResults[run._id] || []}
scenarios={scenarios}
workflow={workflowVersions[run.workflowId]}
onCancelRun={handleCancelRun}
onDeleteRun={handleDeleteRun}
menuOpenId={menuOpenId}
setMenuOpenId={setMenuOpenId}
/>
))}
</div>
)}
{/* Pagination */}
{runs.length > runsPerPage && (
<div className="flex justify-center mt-4">
<Pagination
total={totalPages}
page={currentPage}
/>
</div>
)}
</div>
</StructuredPanel>
)}
</ResizablePanel>
</ResizablePanelGroup>
);
}

399
apps/rowboat/app/projects/[projectId]/simulation/components/RunComponents.tsx
View file

@ -1,399 +0,0 @@
'use client';
import { useState, useEffect } from 'react';
import { ChevronDownIcon, ChevronRightIcon, NoSymbolIcon, EllipsisVerticalIcon, ArrowDownTrayIcon, TrashIcon } from '@heroicons/react/24/outline';
import { WithStringId } from '../../../../lib/types/types';
import { Scenario, SimulationRun, SimulationResult, SimulationAggregateResult } from "../../../../lib/types/testing_types";
import { z } from 'zod';
import { Workflow } from "../../../../lib/types/workflow_types";
import { clsx } from 'clsx';
type ScenarioType = WithStringId<z.infer<typeof Scenario>>;
type SimulationRunType = WithStringId<z.infer<typeof SimulationRun>>;
type SimulationResultType = WithStringId<z.infer<typeof SimulationResult>>;
interface SimulationResultCardProps {
run: SimulationRunType;
results: SimulationResultType[];
scenarios: ScenarioType[];
workflow?: WithStringId<z.infer<typeof Workflow>>;
onCancelRun?: (runId: string) => void;
onDeleteRun?: (runId: string) => Promise<void>;
menuOpenId: string | null;
setMenuOpenId: (id: string | null) => void;
}
export const SimulationResultCard = ({ run, results, scenarios, workflow, onCancelRun, onDeleteRun, menuOpenId, setMenuOpenId }: SimulationResultCardProps) => {
const [isExpanded, setIsExpanded] = useState(false);
const [expandedScenarios, setExpandedScenarios] = useState<Set<string>>(new Set());
const [showDeleteConfirm, setShowDeleteConfirm] = useState(false);
const totalScenarios = run.aggregateResults?.total ?? run.scenarioIds.length;
const passedScenarios = run.aggregateResults?.pass ?? 0;
const failedScenarios = run.aggregateResults?.fail ?? 0;
const getStatusClass = (status: string) => {
const baseClass = "w-[110px] px-3 py-1 rounded text-xs text-center uppercase font-semibold inline-block";
switch (status) {
case 'completed':
case 'pass':
return `${baseClass} bg-green-50 dark:bg-green-900/20 text-green-800 dark:text-green-400`;
case 'failed':
case 'fail':
return `${baseClass} bg-red-50 dark:bg-red-900/20 text-red-800 dark:text-red-400`;
case 'error':
return `${baseClass} bg-orange-50 dark:bg-orange-900/20 text-orange-800 dark:text-orange-400`;
case 'cancelled':
return `${baseClass} bg-gray-50 dark:bg-neutral-800 text-gray-800 dark:text-neutral-400`;
case 'running':
case 'pending':
default:
return `${baseClass} bg-yellow-50 dark:bg-yellow-900/20 text-yellow-800 dark:text-yellow-400`;
}
};
const formatMainTitle = (date: string) => {
return `Run from ${new Date(date).toLocaleString('en-US', {
month: 'numeric',
day: 'numeric',
year: 'numeric',
hour: 'numeric',
minute: '2-digit',
second: '2-digit',
hour12: true
})}`;
};
const formatDateTime = (date: string) => {
return new Date(date).toLocaleString('en-US', {
month: 'numeric',
day: 'numeric',
year: 'numeric',
hour: 'numeric',
minute: '2-digit',
second: '2-digit',
hour12: true
});
};
const getDuration = () => {
if (!run.completedAt) return 'In Progress';
const start = new Date(run.startedAt);
const end = new Date(run.completedAt);
const diff = end.getTime() - start.getTime();
return `${(diff / 1000).toFixed(1)}s`;
};
const toggleScenario = (scenarioId: string, e: React.MouseEvent) => {
e.stopPropagation(); // Prevent triggering parent's onClick
setExpandedScenarios(prev => {
const newSet = new Set(prev);
if (newSet.has(scenarioId)) {
newSet.delete(scenarioId);
} else {
newSet.add(scenarioId);
}
return newSet;
});
};
useEffect(() => {
if (menuOpenId) {
const closeMenu = () => setMenuOpenId(null);
window.addEventListener('click', closeMenu);
return () => window.removeEventListener('click', closeMenu);
}
}, [menuOpenId, setMenuOpenId]);
return (
<div className="border dark:border-neutral-800 rounded-lg mb-4 shadow-sm">
<div
className={clsx(
"p-4 flex items-center justify-between cursor-pointer",
"transition-colors duration-200",
"hover:bg-neutral-100 dark:hover:bg-neutral-800",
"border-b border-transparent",
isExpanded && "border-b-neutral-200 dark:border-b-neutral-800"
)}
onClick={() => setIsExpanded(!isExpanded)}
>
<div className="flex items-center space-x-2">
{isExpanded ? (
<ChevronDownIcon className="h-5 w-5 text-gray-400 dark:text-neutral-500" />
) : (
<ChevronRightIcon className="h-5 w-5 text-gray-400 dark:text-neutral-500" />
)}
<div className="text-sm truncate dark:text-neutral-200">
{formatMainTitle(run.startedAt)}
</div>
</div>
<div className="flex items-center gap-2">
<span className={getStatusClass(run.status)}>
{run.status}
</span>
<div className="relative">
<button
onClick={(e) => {
e.stopPropagation();
setMenuOpenId(menuOpenId === run._id ? null : run._id);
}}
className="p-1 rounded-full hover:bg-gray-100 dark:hover:bg-neutral-700"
>
<EllipsisVerticalIcon className="h-5 w-5 text-gray-600 dark:text-neutral-400" />
</button>
{menuOpenId === run._id && (
<div className="absolute right-0 mt-1 w-48 rounded-md shadow-lg bg-white dark:bg-neutral-900 ring-1 ring-black ring-opacity-5 dark:ring-neutral-700 z-10">
<div className="py-1">
{(run.status === 'running' || run.status === 'pending') && onCancelRun && (
<button
onClick={(e) => {
e.stopPropagation();
onCancelRun(run._id);
setMenuOpenId(null);
}}
className="flex items-center px-4 py-2 text-sm text-red-600 dark:text-red-400 hover:bg-gray-100 dark:hover:bg-neutral-800 w-full"
>
<NoSymbolIcon className="h-4 w-4 mr-2" />
Cancel Run
</button>
)}
<button
disabled
className="flex items-center px-4 py-2 text-sm text-gray-400 dark:text-neutral-500 w-full cursor-not-allowed whitespace-nowrap"
>
<ArrowDownTrayIcon className="h-4 w-4 mr-2" />
Download transcripts
</button>
<button
onClick={(e) => {
e.stopPropagation();
setShowDeleteConfirm(true);
setMenuOpenId(null);
}}
className="flex items-center px-4 py-2 text-sm text-red-600 dark:text-red-400 hover:bg-gray-100 dark:hover:bg-neutral-800 w-full"
>
<TrashIcon className="h-4 w-4 mr-2" />
Delete run
</button>
</div>
</div>
)}
</div>
</div>
</div>
{isExpanded && (
<div className="p-4 border-t dark:border-neutral-800">
{run.status === 'error' ? (
<div className="text-orange-800 bg-orange-50 p-4 rounded-lg">
Your simulation could not be completed. Please run a new simulation again.
</div>
) : (
<>
{/* Workflow and timing information in a grid */}
<div className="grid grid-cols-3 gap-4 mb-6">
{workflow && (
<div className="bg-gray-50 dark:bg-neutral-800 p-4 rounded-lg">
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Workflow Version</div>
<div className="font-medium dark:text-neutral-200">{workflow.name}</div>
</div>
)}
<div className="bg-gray-50 dark:bg-neutral-800 p-4 rounded-lg">
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Completed</div>
<div className="text-sm dark:text-neutral-300">
{run.completedAt ? formatDateTime(run.completedAt) : 'Not completed'}
</div>
</div>
<div className="bg-gray-50 dark:bg-neutral-800 p-4 rounded-lg">
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Duration</div>
<div className="text-sm dark:text-neutral-300">{getDuration()}</div>
</div>
</div>
{/* Results statistics */}
<div className="grid grid-cols-3 gap-4 mb-6">
<div className="p-4 rounded-lg bg-gray-50 dark:bg-neutral-800">
<div className="text-sm text-gray-600 dark:text-neutral-400">Total Scenarios</div>
<div className="text-2xl font-semibold dark:text-neutral-200">{totalScenarios}</div>
</div>
<div className="p-4 rounded-lg bg-green-50 dark:bg-green-900/20">
<div className="text-sm text-green-600 dark:text-green-400">Passed</div>
<div className="text-2xl font-semibold text-green-700 dark:text-green-400">{passedScenarios}</div>
</div>
<div className="p-4 rounded-lg bg-red-50 dark:bg-red-900/20">
<div className="text-sm text-red-600 dark:text-red-400">Failed</div>
<div className="text-2xl font-semibold text-red-700 dark:text-red-400">{failedScenarios}</div>
</div>
</div>
<div className="space-y-2">
{run.scenarioIds.map(scenarioId => {
const scenario = scenarios.find(s => s._id === scenarioId);
const result = results.find(r => r.scenarioId === scenarioId);
const isScenarioExpanded = expandedScenarios.has(scenarioId);
return scenario && (
<div
key={scenarioId}
className={clsx(
"border dark:border-neutral-800 rounded-lg overflow-hidden",
"transition-colors duration-200",
result?.result === 'pass'
? 'bg-green-50/50 dark:bg-green-900/10 border-green-200 dark:border-green-900/50'
: result?.result === 'fail'
? 'bg-red-50/50 dark:bg-red-900/10 border-red-200 dark:border-red-900/50'
: 'bg-gray-50/50 dark:bg-neutral-900/50 border-gray-200 dark:border-neutral-800'
)}
>
<div
className={clsx(
"p-3 flex items-center justify-between cursor-pointer",
"hover:bg-white/50 dark:hover:bg-neutral-800/50"
)}
onClick={(e) => toggleScenario(scenarioId, e)}
>
<div className="flex items-center space-x-2">
{isScenarioExpanded ? (
<ChevronDownIcon className="h-4 w-4 text-gray-600 dark:text-neutral-400" />
) : (
<ChevronRightIcon className="h-4 w-4 text-gray-600 dark:text-neutral-400" />
)}
<span className="font-medium text-gray-900 dark:text-neutral-200">{scenario.name}</span>
</div>
{result && (
<span className={getStatusClass(result.result)}>
{result.result}
</span>
)}
</div>
{isScenarioExpanded && (
<div className="p-3 border-t border-opacity-50 dark:border-neutral-800 space-y-4">
<div>
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Description</div>
<div className="text-sm text-gray-700 dark:text-neutral-300">
{scenario.description}
</div>
</div>
<div>
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Criteria</div>
<div className="text-sm text-gray-700 dark:text-neutral-300">
{scenario.criteria || 'No criteria specified'}
</div>
</div>
<div>
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Context</div>
<div className="text-sm text-gray-700 dark:text-neutral-300">
{scenario.context || 'No context provided'}
</div>
</div>
{result && (
<div>
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Result Details</div>
<div className="text-sm text-gray-700 dark:text-neutral-300">
{result.details}
</div>
</div>
)}
</div>
)}
</div>
);
})}
</div>
</>
)}
</div>
)}
{showDeleteConfirm && (
<div className="fixed inset-0 bg-gray-600 bg-opacity-50 overflow-y-auto h-full w-full z-50">
<div className="relative top-20 mx-auto p-5 border w-96 shadow-lg rounded-md bg-white">
<div className="mt-3 text-center">
<h3 className="text-lg leading-6 font-medium text-gray-900 whitespace-nowrap">
Are you sure you want to delete this run?
</h3>
<div className="mt-6 flex justify-center space-x-4">
<button
onClick={() => setShowDeleteConfirm(false)}
className="px-4 py-2 bg-white text-gray-600 text-sm font-medium border rounded-md hover:bg-gray-50"
>
Retain
</button>
<button
onClick={async () => {
if (onDeleteRun) {
await onDeleteRun(run._id);
setShowDeleteConfirm(false);
}
}}
className="px-4 py-2 bg-red-600 text-white text-sm font-medium rounded-md hover:bg-red-700"
>
Delete
</button>
</div>
</div>
</div>
</div>
)}
</div>
);
};
interface ScenarioResultCardProps {
scenario: ScenarioType;
result?: SimulationResultType;
}
export const ScenarioResultCard = ({ scenario, result }: ScenarioResultCardProps) => {
const [isExpanded, setIsExpanded] = useState(false);
return (
<div className="border rounded-lg mb-2 last:mb-0">
<div
className="p-3 flex items-center justify-between cursor-pointer hover:bg-gray-50"
onClick={() => setIsExpanded(!isExpanded)}
>
<div className="flex items-center space-x-2">
{isExpanded ? (
<ChevronDownIcon className="h-4 w-4 text-gray-400" />
) : (
<ChevronRightIcon className="h-4 w-4 text-gray-400" />
)}
<span className="font-medium">{scenario.name}</span>
</div>
{result && (
<span className={`px-2 py-1 rounded-full text-xs ${
result.result === 'pass' ? 'bg-green-100 text-green-800' : 'bg-red-100 text-red-800'
}`}>
{result.result}
</span>
)}
</div>
{isExpanded && (
<div className="p-3 border-t space-y-2 bg-gray-50">
<div>
<div className="text-sm font-medium text-gray-600">Description</div>
<div className="text-sm">{scenario.description}</div>
</div>
<div>
<div className="text-sm font-medium text-gray-600">Criteria</div>
<div className="text-sm">{scenario.criteria}</div>
</div>
<div>
<div className="text-sm font-medium text-gray-600">Context</div>
<div className="text-sm">{scenario.context}</div>
</div>
{result && (
<div>
<div className="text-sm font-medium text-gray-600">Result Details</div>
<div className="text-sm">{result.details}</div>
</div>
)}
</div>
)}
</div>
);
};

204
apps/rowboat/app/projects/[projectId]/simulation/components/ScenarioComponents.tsx
View file

@ -1,204 +0,0 @@
'use client';
import { useState, useEffect, useCallback } from 'react';
import { Save, EllipsisVerticalIcon, PlayIcon, TrashIcon, X } from "lucide-react";
import { WithStringId } from '../../../../lib/types/types';
import { Scenario } from "../../../../lib/types/testing_types";
import { z } from 'zod';
import { EditableField } from '../../../../lib/components/editable-field';
import { FormSection } from '../../../../lib/components/form-section';
import { StructuredPanel, ActionButton } from "../../../../lib/components/structured-panel";
import clsx from "clsx";
import { Dropdown, DropdownItem, DropdownTrigger, DropdownMenu } from "@nextui-org/react";
import { SectionHeader, ListItem } from "../../../../lib/components/structured-list";
type ScenarioType = WithStringId<z.infer<typeof Scenario>>;
interface ScenarioViewerProps {
scenario: ScenarioType;
onSave: (scenario: ScenarioType) => void;
onClose: () => void;
}
export function ScenarioViewer({ scenario, onSave, onClose }: ScenarioViewerProps) {
const [editedScenario, setEditedScenario] = useState<ScenarioType>(scenario);
const [isDirty, setIsDirty] = useState(false);
const [nameError, setNameError] = useState<string | null>(null);
useEffect(() => {
setEditedScenario(scenario);
setIsDirty(false);
}, [scenario]);
const handleChange = useCallback((field: keyof ScenarioType, value: string) => {
if (field === 'name') {
setNameError(value.trim() ? null : 'Name is required');
}
setEditedScenario(prev => ({
...prev,
[field]: value,
}));
setIsDirty(true);
}, []);
const handleSave = useCallback(() => {
if (!editedScenario.name.trim()) {
setNameError('Name is required');
return;
}
onSave(editedScenario);
onClose();
}, [editedScenario, onSave, onClose]);
return (
<StructuredPanel
title="SCENARIO DETAILS"
actions={[
isDirty && (
<ActionButton
key="save"
onClick={handleSave}
icon={<Save className="w-4 h-4" />}
primary
>
Save
</ActionButton>
),
<ActionButton
key="close"
onClick={onClose}
icon={<X className="w-4 h-4" />}
>
Close
</ActionButton>
].filter(Boolean)}
>
<div className="flex flex-col gap-4">
<FormSection label="Name" showDivider>
<EditableField
value={editedScenario.name}
onChange={(value) => handleChange('name', value)}
multiline={false}
className="w-full"
showSaveButton={false}
placeholder="Enter an identifiable scenario name"
error={nameError}
/>
</FormSection>
<FormSection label="Description" showDivider>
<EditableField
value={editedScenario.description}
onChange={(value) => handleChange('description', value)}
multiline={true}
className="w-full"
showSaveButton={false}
placeholder="Describe the user scenario to be simulated"
/>
</FormSection>
<FormSection label="Criteria" showDivider>
<EditableField
value={editedScenario.criteria}
onChange={(value) => handleChange('criteria', value)}
multiline={true}
className="w-full"
showSaveButton={false}
placeholder="Enter success criteria for this scenario to pass in a simulation"
/>
</FormSection>
<FormSection label="Context">
<EditableField
value={editedScenario.context}
onChange={(value) => handleChange('context', value)}
multiline={true}
className="w-full"
showSaveButton={false}
placeholder="Provide context about the user to the assistant at the start of chat"
/>
</FormSection>
</div>
</StructuredPanel>
);
}
function ScenarioDropdown({
name,
onRun,
onDelete,
}: {
name: string;
onRun: () => void;
onDelete: () => void;
}) {
return (
<Dropdown>
<DropdownTrigger>
<EllipsisVerticalIcon size={16} />
</DropdownTrigger>
<DropdownMenu
onAction={(key) => {
if (key === 'run') onRun();
if (key === 'delete') onDelete();
}}
>
<DropdownItem
key="run"
startContent={<PlayIcon className="w-4 h-4" />}
>
Run scenario
</DropdownItem>
<DropdownItem
key="delete"
className="text-danger"
startContent={<TrashIcon className="w-4 h-4" />}
>
Delete
</DropdownItem>
</DropdownMenu>
</Dropdown>
);
}
export function ScenarioList({
scenarios,
selectedId,
onSelect,
onAdd,
onRunScenario,
onDeleteScenario,
}: {
scenarios: ScenarioType[];
selectedId: string | null;
onSelect: (id: string) => void;
onAdd: () => void;
onRunScenario: (id: string) => void;
onDeleteScenario: (id: string) => void;
}) {
return (
<StructuredPanel
title="TESTS"
tooltip="Browse and manage your test scenarios"
>
<div className="overflow-auto flex flex-col gap-1 justify-start">
<SectionHeader title="Scenarios" onAdd={onAdd} />
{scenarios.map((scenario) => (
<ListItem
key={scenario._id}
name={scenario.name}
isSelected={selectedId === scenario._id}
onClick={() => onSelect(scenario._id)}
rightElement={
<ScenarioDropdown
name={scenario.name}
onRun={() => onRunScenario(scenario._id)}
onDelete={() => onDeleteScenario(scenario._id)}
/>
}
/>
))}
</div>
</StructuredPanel>
);
}

10
apps/rowboat/app/projects/[projectId]/simulation/page.tsx
View file

@ -1,10 +0,0 @@
import { Metadata } from "next";
import SimulationApp from "./app";
export const metadata: Metadata = {
title: "Project simulation",
};
export default function SimulationPage() {
return <SimulationApp />;
}

64
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/app.tsx Normal file
View file

@ -0,0 +1,64 @@
"use client";
import Link from "next/link";
import { useRouter } from "next/navigation";
import { ScenariosApp } from "./scenarios_app";
import { ProfilesApp } from "./profiles_app";
import { SimulationsApp } from "./simulations_app";
import { usePathname } from "next/navigation";
import { RunsApp } from "./runs_app";
export function App({
projectId,
slug
}: {
projectId: string,
slug?: string[]
}) {
const router = useRouter();
const pathname = usePathname();
let selection: "scenarios" | "profiles" | "criteria" | "simulations" | "runs" = "runs";
if (!slug || slug.length === 0) {
router.push(`/projects/${projectId}/test/runs`);
} else if (slug[0] === "scenarios") {
selection = "scenarios";
} else if (slug[0] === "profiles") {
selection = "profiles";
} else if (slug[0] === "criteria") {
selection = "criteria";
} else if (slug[0] === "simulations") {
selection = "simulations";
} else if (slug[0] === "runs") {
selection = "runs";
}
let innerSlug: string[] = [];
if (slug && slug.length > 1) {
innerSlug = slug.slice(1);
}
const menuItems = [
{ label: "Scenarios", href: `/projects/${projectId}/test/scenarios` },
{ label: "Profiles", href: `/projects/${projectId}/test/profiles` },
{ label: "Simulations", href: `/projects/${projectId}/test/simulations` },
{ label: "Test Runs", href: `/projects/${projectId}/test/runs` },
];
return <div className="flex h-full">
<div className="w-40 shrink-0 p-2">
<ul>
{menuItems.map((item) => (
<li key={item.label}>
<Link
className={`block p-2 rounded-md text-sm ${pathname.startsWith(item.href) ? "bg-gray-100" : "hover:bg-gray-100"}`}
href={item.href}>{item.label}</Link>
</li>
))}
</ul>
</div>
<div className="grow border-l border-gray-200 p-2">
{selection === "scenarios" && <ScenariosApp projectId={projectId} slug={innerSlug} />}
{selection === "profiles" && <ProfilesApp projectId={projectId} slug={innerSlug} />}
{selection === "simulations" && <SimulationsApp projectId={projectId} slug={innerSlug} />}
{selection === "runs" && <RunsApp projectId={projectId} slug={innerSlug} />}
</div>
</div>;
}

8
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/page.tsx Normal file
View file

@ -0,0 +1,8 @@
import { App } from "./app";
export default function Page({ params }: { params: { projectId: string, slug?: string[] } }) {
return <App
projectId={params.projectId}
slug={params.slug}
/>;
}

535
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/profiles_app.tsx Normal file
View file

@ -0,0 +1,535 @@
import Link from "next/link";
import { WithStringId } from "@/app/lib/types/types";
import { TestProfile } from "@/app/lib/types/testing_types";
import { useEffect, useState, useRef } from "react";
import { createProfile, getProfile, listProfiles, updateProfile, deleteProfile, setDefaultProfile } from "@/app/actions/testing_actions";
import { Button, Input, Pagination, Spinner, Switch, Textarea, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter, Tooltip } from "@nextui-org/react";
import { useRouter, useSearchParams } from "next/navigation";
import { z } from "zod";
import { PlusIcon, ArrowLeftIcon, StarIcon } from "lucide-react";
import { FormStatusButton } from "@/app/lib/components/form-status-button";
import { RelativeTime } from "@primer/react"
import { getProjectConfig } from "@/app/actions/project_actions";
function EditProfile({
projectId,
profileId,
}: {
projectId: string,
profileId: string,
}) {
const router = useRouter();
const [profile, setProfile] = useState<WithStringId<z.infer<typeof TestProfile>> | null>(null);
const [loading, setLoading] = useState(true);
const [mockTools, setMockTools] = useState(false);
const [error, setError] = useState<string | null>(null);
const formRef = useRef<HTMLFormElement>(null);
useEffect(() => {
async function fetchProfile() {
setError(null);
try {
const profile = await getProfile(projectId, profileId);
setProfile(profile);
setMockTools(profile?.mockTools || false);
} catch (error) {
setError(`Unable to fetch profile: ${error}`);
} finally {
setLoading(false);
}
}
fetchProfile();
}, [profileId, projectId]);
async function handleSubmit(formData: FormData) {
setError(null);
try {
const name = formData.get("name") as string;
const context = formData.get("context") as string;
const mockPrompt = formData.get("mockPrompt") as string;
await updateProfile(projectId, profileId, {
name,
context,
mockTools,
mockPrompt: mockPrompt || undefined
});
router.push(`/projects/${projectId}/test/profiles/${profileId}`);
} catch (error) {
setError(`Unable to update profile: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">Edit Profile</h1>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => formRef.current?.requestSubmit()}>Retry</Button>
</div>}
{!loading && profile && (
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<Input
type="text"
name="name"
label="Name"
placeholder="Enter a name for the profile"
defaultValue={profile.name}
required
/>
<Textarea
name="context"
label="Context"
placeholder="Enter the context for this profile"
defaultValue={profile.context}
required
/>
<Switch
name="mockTools"
isSelected={mockTools}
onValueChange={(value) => {
setMockTools(value);
}}
className="self-start"
>
Mock Tools
</Switch>
{mockTools && <Textarea
name="mockPrompt"
label="Mock Prompt (Optional)"
placeholder="Enter a mock prompt"
defaultValue={profile.mockPrompt}
/>}
<div className="flex gap-2 items-center">
<FormStatusButton
props={{
className: "self-start",
children: "Update",
size: "sm",
type: "submit",
}}
/>
<Button
size="sm"
variant="flat"
as={Link}
href={`/projects/${projectId}/test/profiles/${profileId}`}
>
Cancel
</Button>
</div>
</form>
)}
</div>;
}
function ViewProfile({
projectId,
profileId,
}: {
projectId: string,
profileId: string,
}) {
const router = useRouter();
const [profile, setProfile] = useState<WithStringId<z.infer<typeof TestProfile>> | null>(null);
const [defaultProfileId, setDefaultProfileId] = useState<string | null>(null);
const [loading, setLoading] = useState(true);
const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
const [deleteError, setDeleteError] = useState<string | null>(null);
useEffect(() => {
async function fetchProfile() {
const profile = await getProfile(projectId, profileId);
const projectConfig = await getProjectConfig(projectId);
setProfile(profile);
setDefaultProfileId(projectConfig.defaultTestProfileId || null);
setLoading(false);
}
fetchProfile();
}, [projectId, profileId]);
async function handleDelete() {
try {
await deleteProfile(projectId, profileId);
router.push(`/projects/${projectId}/test/profiles`);
} catch (error) {
setDeleteError(`Failed to delete profile: ${error}`);
}
}
async function handleSetDefault() {
await setDefaultProfile(projectId, profileId);
router.push(`/projects/${projectId}/test/profiles`);
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">View Profile</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/profiles`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Profiles
</Button>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{!loading && !profile && <div className="text-gray-600 text-center">Profile not found</div>}
{!loading && profile && (
<>
<div className="flex flex-col gap-1 text-sm">
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Name</div>
<div className="flex-[2]">
<div className="flex flex-col gap-2">
<div>{profile.name}</div>
{defaultProfileId === profile._id && <div className="flex items-center gap-2">
<StarIcon className="w-4 h-4" />
<div className="text-gray-600">This is the default profile</div>
</div>}
</div>
</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Context</div>
<div className="flex-[2] whitespace-pre-wrap">{profile.context}</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Mock Tools</div>
<div className="flex-[2]">{profile.mockTools ? "Yes" : "No"}</div>
</div>
{profile.mockPrompt && <div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Mock Prompt</div>
<div className="flex-[2] whitespace-pre-wrap">{profile.mockPrompt}</div>
</div>}
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Created</div>
<div className="flex-[2]"><RelativeTime date={new Date(profile.createdAt)} /></div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Last Updated</div>
<div className="flex-[2]"><RelativeTime date={new Date(profile.lastUpdatedAt)} /></div>
</div>
</div>
<div className="flex gap-2 mt-4">
<Button
size="sm"
as={Link}
href={`/projects/${projectId}/test/profiles/${profileId}/edit`}
>
Edit
</Button>
{defaultProfileId !== profile._id && <Button
size="sm"
color="danger"
variant="flat"
onClick={() => setIsDeleteModalOpen(true)}
>
Delete
</Button>}
{defaultProfileId !== profile._id && <Button
size="sm"
onClick={() => handleSetDefault()}
startContent={<StarIcon className="w-4 h-4" />}
>
Set as default profile
</Button>}
</div>
<Modal
isOpen={isDeleteModalOpen}
onOpenChange={setIsDeleteModalOpen}
size="sm"
>
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Confirm Deletion</ModalHeader>
<ModalBody>
Are you sure you want to delete this profile?
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Cancel
</Button>
<Button
size="sm"
color="danger"
onPress={() => {
handleDelete();
onClose();
}}
>
Delete
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
<Modal
isOpen={deleteError !== null}
onOpenChange={() => setDeleteError(null)}
size="sm"
>
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Error</ModalHeader>
<ModalBody>
{deleteError}
</ModalBody>
<ModalFooter>
<Button
size="sm"
color="primary"
onPress={onClose}
>
Close
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
</>
)}
</div>;
}
function NewProfile({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const [error, setError] = useState<string | null>(null);
const [mockTools, setMockTools] = useState(false);
const formRef = useRef<HTMLFormElement>(null);
async function handleSubmit(formData: FormData) {
setError(null);
try {
const name = formData.get("name") as string;
const context = formData.get("context") as string;
const mockPrompt = formData.get("mockPrompt") as string;
const profile = await createProfile(projectId, {
name,
context,
mockTools,
mockPrompt: mockPrompt || undefined
});
router.push(`/projects/${projectId}/test/profiles/${profile._id}`);
} catch (error) {
setError(`Unable to create profile: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">New Profile</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/profiles`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Profiles
</Button>
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => formRef.current?.requestSubmit()}>Retry</Button>
</div>}
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<Input
type="text"
name="name"
label="Name"
placeholder="Enter a name for the profile"
required
/>
<Textarea
name="context"
label="Context"
placeholder="Enter the context for this profile"
required
/>
<Switch
name="mockTools"
isSelected={mockTools}
onValueChange={(value) => {
setMockTools(value);
}}
className="self-start"
>
Mock Tools
</Switch>
{mockTools && <Textarea
name="mockPrompt"
label="Mock Prompt (Optional)"
placeholder="Enter a mock prompt"
/>}
<FormStatusButton
props={{
className: "self-start",
children: "Create",
size: "sm",
type: "submit",
}}
/>
</form>
</div>;
}
function ProfileList({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const searchParams = useSearchParams();
const page = parseInt(searchParams.get("page") || "1");
const pageSize = 10;
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [profiles, setProfiles] = useState<WithStringId<z.infer<typeof TestProfile>>[]>([]);
const [defaultProfileId, setDefaultProfileId] = useState<string | null>(null);
const [total, setTotal] = useState(0);
useEffect(() => {
let ignore = false;
async function fetchProfiles() {
setLoading(true);
setError(null);
try {
const profiles = await listProfiles(projectId, page, pageSize);
const projectConfig = await getProjectConfig(projectId);
if (!ignore) {
setProfiles(profiles.profiles);
setTotal(Math.ceil(profiles.total / pageSize));
setDefaultProfileId(projectConfig.defaultTestProfileId || null);
}
} catch (error) {
if (!ignore) {
setError(`Unable to fetch profiles: ${error}`);
}
} finally {
if (!ignore) {
setLoading(false);
}
}
}
if (error == null) {
fetchProfiles();
}
return () => {
ignore = true;
};
}, [page, pageSize, error, projectId]);
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">Profiles</h1>
<Button
size="sm"
onClick={() => router.push(`/projects/${projectId}/test/profiles/new`)}
className="self-end"
startContent={<PlusIcon className="w-4 h-4" />}
>
New Profile
</Button>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => setError(null)}>Retry</Button>
</div>}
{!loading && !error && <>
{profiles.length === 0 && <div className="text-gray-600 text-center">No profiles found</div>}
{profiles.length > 0 && <div className="flex flex-col w-full">
{/* Header */}
<div className="grid grid-cols-8 py-2 bg-gray-100 font-semibold text-sm">
<div className="col-span-2 px-4">Name</div>
<div className="col-span-3 px-4">Context</div>
<div className="col-span-1 px-4">Mock Tools</div>
<div className="col-span-1 px-4">Created</div>
<div className="col-span-1 px-4">Updated</div>
</div>
{/* Rows */}
{profiles.map((profile) => (
<div key={profile._id} className="grid grid-cols-8 py-2 border-b hover:bg-gray-50 text-sm">
<div className="col-span-2 px-4 truncate">
<div className="flex items-center gap-2">
<Link
href={`/projects/${projectId}/test/profiles/${profile._id}`}
className="text-blue-600 hover:underline"
>
{profile.name}
</Link>
{defaultProfileId === profile._id && <Tooltip content="Default Profile">
<StarIcon className="w-4 h-4" />
</Tooltip>}
</div>
</div>
<div className="col-span-3 px-4 truncate">{profile.context}</div>
<div className="col-span-1 px-4">{profile.mockTools ? "Yes" : "No"}</div>
<div className="col-span-1 px-4 text-gray-600 truncate">
<RelativeTime date={new Date(profile.createdAt)} />
</div>
<div className="col-span-1 px-4 text-gray-600 truncate">
<RelativeTime date={new Date(profile.lastUpdatedAt)} />
</div>
</div>
))}
</div>}
{total > 1 && <Pagination
total={total}
page={page}
onChange={(page) => {
router.push(`/projects/${projectId}/test/profiles?page=${page}`);
}}
className="self-center"
/>}
</>}
</div>;
}
export function ProfilesApp({
projectId,
slug
}: {
projectId: string,
slug: string[]
}) {
let selection: "list" | "view" | "new" | "edit" = "list";
let profileId: string | null = null;
if (slug.length > 0) {
if (slug[0] === "new") {
selection = "new";
} else if (slug[slug.length - 1] === "edit") {
selection = "edit";
profileId = slug[0];
} else {
selection = "view";
profileId = slug[0];
}
}
return <>
{selection === "list" && <ProfileList projectId={projectId} />}
{selection === "new" && <NewProfile projectId={projectId} />}
{selection === "view" && profileId && <ViewProfile projectId={projectId} profileId={profileId} />}
{selection === "edit" && profileId && <EditProfile projectId={projectId} profileId={profileId} />}
</>;
}

456
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/runs_app.tsx Normal file
View file

@ -0,0 +1,456 @@
import Link from "next/link";
import { WithStringId } from "@/app/lib/types/types";
import { TestSimulation, TestRun } from "@/app/lib/types/testing_types";
import { useEffect, useState, useRef } from "react";
import { createRun, getRun, getSimulation, listRuns } from "@/app/actions/testing_actions";
import { Button, Input, Pagination, Spinner, Chip } from "@nextui-org/react";
import { useRouter, useSearchParams } from "next/navigation";
import { z } from "zod";
import { ArrowLeftIcon, PlusIcon, WorkflowIcon } from "lucide-react";
import { FormStatusButton } from "@/app/lib/components/form-status-button";
import { RelativeTime } from "@primer/react"
import { SimulationSelector } from "@/app/lib/components/selectors/simulation-selector";
import { WorkflowSelector } from "@/app/lib/components/selectors/workflow-selector";
import { Workflow } from "@/app/lib/types/workflow_types";
import { fetchWorkflow } from "@/app/actions/workflow_actions";
function NewRun({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const [error, setError] = useState<string | null>(null);
const formRef = useRef<HTMLFormElement>(null);
const [selectedSimulations, setSelectedSimulations] = useState<WithStringId<z.infer<typeof TestSimulation>>[]>([]);
const [isSimulationSelectorOpen, setIsSimulationSelectorOpen] = useState(false);
const [selectedWorkflow, setSelectedWorkflow] = useState<WithStringId<z.infer<typeof Workflow>> | null>(null);
const [isWorkflowSelectorOpen, setIsWorkflowSelectorOpen] = useState(false);
async function handleSubmit(formData: FormData) {
setError(null);
const simulationIds = selectedSimulations.map(sim => sim._id);
if (!selectedWorkflow) {
setError("Please select a workflow");
return;
}
if (simulationIds.length === 0) {
setError("Please select at least one simulation");
return;
}
try {
const run = await createRun(projectId, {
workflowId: selectedWorkflow._id,
simulationIds
});
router.push(`/projects/${projectId}/test/runs/${run._id}`);
} catch (error) {
setError(`Unable to create run: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">New Run</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/runs`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Runs
</Button>
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button
size="sm"
color="danger"
onClick={() => {
formRef.current?.requestSubmit();
}}
>
Retry
</Button>
</div>}
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<div className="flex flex-col gap-2">
<label className="text-sm font-medium">Workflow</label>
<div className="flex items-center gap-2">
{selectedWorkflow ? (
<div className="text-sm text-blue-600">{selectedWorkflow.name}</div>
) : (
<div className="text-sm text-gray-500">No workflow selected</div>
)}
<Button
size="sm"
onClick={() => setIsWorkflowSelectorOpen(true)}
type="button"
>
{selectedWorkflow ? "Change" : "Select"} Workflow
</Button>
</div>
</div>
<div className="flex flex-col gap-2">
<Button
size="sm"
onClick={() => setIsSimulationSelectorOpen(true)}
type="button"
className="self-start"
>
Select Simulations
</Button>
{selectedSimulations.length > 0 && (
<div className="flex flex-wrap gap-2">
{selectedSimulations.map((sim) => (
<Chip
key={sim._id}
onClose={() => setSelectedSimulations(prev => prev.filter(s => s._id !== sim._id))}
variant="flat"
className="py-1"
>
{sim.name}
</Chip>
))}
</div>
)}
</div>
<FormStatusButton
props={{
className: "self-start",
children: "Create Run",
size: "sm",
type: "submit",
isDisabled: !selectedWorkflow || selectedSimulations.length === 0,
}}
/>
</form>
<SimulationSelector
projectId={projectId}
isOpen={isSimulationSelectorOpen}
onOpenChange={setIsSimulationSelectorOpen}
onSelect={setSelectedSimulations}
initialSelected={selectedSimulations}
/>
<WorkflowSelector
projectId={projectId}
isOpen={isWorkflowSelectorOpen}
onOpenChange={setIsWorkflowSelectorOpen}
onSelect={setSelectedWorkflow}
/>
</div>;
}
function ViewRun({
projectId,
runId,
}: {
projectId: string,
runId: string,
}) {
const [run, setRun] = useState<WithStringId<z.infer<typeof TestRun>> | null>(null);
const [loading, setLoading] = useState(true);
const [workflow, setWorkflow] = useState<WithStringId<z.infer<typeof Workflow>> | null>(null);
const [simulations, setSimulations] = useState<WithStringId<z.infer<typeof TestSimulation>>[]>([]);
useEffect(() => {
async function fetchRun() {
const run = await getRun(projectId, runId);
setRun(run);
if (run) {
// Fetch workflow and simulations in parallel
const [workflowResult, simulationsResult] = await Promise.all([
fetchWorkflow(projectId, run.workflowId),
Promise.all(run.simulationIds.map(id => getSimulation(projectId, id)))
]);
setWorkflow(workflowResult);
setSimulations(simulationsResult.filter(s => s !== null));
}
setLoading(false);
}
fetchRun();
}, [runId, projectId]);
return <div className="h-full flex flex-col gap-4">
<div className="flex items-center justify-between">
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/runs`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Runs
</Button>
</div>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{!loading && !run && <div className="text-gray-600 text-center">Run not found</div>}
{!loading && run && (
<>
{/* Workflow and timing information in a grid */}
<div className="grid grid-cols-3 gap-4">
{workflow && (
<div className="bg-gray-50 dark:bg-neutral-800 p-4 rounded-lg">
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Workflow Version</div>
<div className="font-medium dark:text-neutral-200">{workflow.name}</div>
</div>
)}
<div className="bg-gray-50 dark:bg-neutral-800 p-4 rounded-lg">
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Completed</div>
<div className="text-sm dark:text-neutral-300">
{run.completedAt ? <RelativeTime date={new Date(run.completedAt)} /> : 'Not completed'}
</div>
</div>
<div className="bg-gray-50 dark:bg-neutral-800 p-4 rounded-lg">
<div className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-1">Duration</div>
<div className="text-sm dark:text-neutral-300">
{run.completedAt ?
`${((new Date(run.completedAt).getTime() - new Date(run.startedAt).getTime()) / 1000).toFixed(1)}s` :
'In Progress'}
</div>
</div>
</div>
{/* Results statistics */}
<div className="grid grid-cols-3 gap-4">
<div className="p-4 rounded-lg bg-gray-50 dark:bg-neutral-800">
<div className="text-sm text-gray-600 dark:text-neutral-400">Total Tests</div>
<div className="text-2xl font-semibold dark:text-neutral-200">{run.aggregateResults?.total || 0}</div>
</div>
<div className="p-4 rounded-lg bg-green-50 dark:bg-green-900/20">
<div className="text-sm text-green-600 dark:text-green-400">Passed</div>
<div className="text-2xl font-semibold text-green-700 dark:text-green-400">{run.aggregateResults?.passCount || 0}</div>
</div>
<div className="p-4 rounded-lg bg-red-50 dark:bg-red-900/20">
<div className="text-sm text-red-600 dark:text-red-400">Failed</div>
<div className="text-2xl font-semibold text-red-700 dark:text-red-400">{run.aggregateResults?.failCount || 0}</div>
</div>
</div>
{/* Simulations List */}
<div className="mt-4">
<h2 className="text-sm font-medium text-gray-600 dark:text-neutral-400 mb-2">Simulations</h2>
<div className="space-y-2">
{simulations.map(sim => (
<div key={sim._id} className="border dark:border-neutral-800 rounded-lg p-3">
<Link
href={`/projects/${projectId}/test/simulations/${sim._id}`}
className="text-blue-600 hover:underline"
>
{sim.name}
</Link>
</div>
))}
</div>
</div>
</>
)}
</div>;
}
function RunList({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const searchParams = useSearchParams();
const page = parseInt(searchParams.get("page") || "1");
const pageSize = 10;
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [runs, setRuns] = useState<WithStringId<z.infer<typeof TestRun>>[]>([]);
const [workflowMap, setWorkflowMap] = useState<Record<string, WithStringId<z.infer<typeof Workflow>>>>({});
const [total, setTotal] = useState(0);
useEffect(() => {
let ignore = false;
async function fetchRuns() {
setLoading(true);
setError(null);
try {
const result = await listRuns(projectId, page, pageSize);
if (!ignore) {
setRuns(result.runs);
setTotal(Math.ceil(result.total / pageSize));
}
} catch (error) {
if (!ignore) {
setError(`Unable to fetch runs: ${error}`);
}
} finally {
if (!ignore) {
setLoading(false);
}
}
}
if (error == null) {
fetchRuns();
}
return () => {
ignore = true;
};
}, [page, pageSize, error, projectId]);
useEffect(() => {
let ignore = false;
async function resolveWorkflows() {
const workflowIds = runs.reduce((acc, run) => {
if (!acc.includes(run.workflowId)) {
acc.push(run.workflowId);
}
return acc;
}, [] as string[]);
const workflows = await Promise.all(workflowIds.map((workflowId) => fetchWorkflow(projectId, workflowId)));
if (ignore) {
return;
}
setWorkflowMap(workflows.filter((workflow) => workflow !== null).reduce((acc, workflow) => {
acc[workflow._id] = workflow;
return acc;
}, {} as Record<string, WithStringId<z.infer<typeof Workflow>>>));
}
if (error == null) {
resolveWorkflows();
}
return () => {
ignore = true;
};
}, [runs, error, projectId]);
return <div className="h-full flex flex-col gap-4">
<div className="flex items-center justify-between">
<h1 className="text-xl font-semibold text-gray-800 dark:text-neutral-200">Test Runs</h1>
<Button
size="sm"
onClick={() => router.push(`/projects/${projectId}/test/runs/new`)}
startContent={<PlusIcon className="w-4 h-4" />}
>
New Run
</Button>
</div>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => setError(null)}>Retry</Button>
</div>}
{!loading && !error && <>
{runs.length === 0 && <div className="text-gray-600 text-center">No test runs found</div>}
{runs.length > 0 && <div className="space-y-4">
{runs.map((run) => (
<div key={run._id} className="border dark:border-neutral-800 rounded-lg shadow-sm">
<div className="p-4 flex items-center justify-between hover:bg-neutral-100 dark:hover:bg-neutral-800">
<div className="flex items-center space-x-4">
<Link
href={`/projects/${projectId}/test/runs/${run._id}`}
className="text-blue-600 hover:underline"
>
{run.name}
</Link>
{workflowMap[run.workflowId] && (
<div className="flex items-center gap-2 text-sm text-gray-600 dark:text-neutral-400">
<WorkflowIcon className="w-4 h-4 shrink-0" />
{workflowMap[run.workflowId].name}
</div>
)}
</div>
<div className="flex items-center gap-4">
<span className={getStatusClass(run.status)}>
{run.status}
</span>
<div className="text-sm text-gray-600 dark:text-neutral-400">
<RelativeTime date={new Date(run.startedAt)} />
</div>
</div>
</div>
{run.aggregateResults && (
<div className="border-t dark:border-neutral-800 px-4 py-2 bg-gray-50 dark:bg-neutral-900/50">
<div className="grid grid-cols-3 gap-4 text-sm">
<div className="text-gray-600 dark:text-neutral-400">
Total: {run.aggregateResults.total}
</div>
<div className="text-green-600 dark:text-green-400">
Passed: {run.aggregateResults.passCount}
</div>
<div className="text-red-600 dark:text-red-400">
Failed: {run.aggregateResults.failCount}
</div>
</div>
</div>
)}
</div>
))}
</div>}
{total > 1 && <Pagination
total={total}
page={page}
onChange={(page) => {
router.push(`/projects/${projectId}/test/runs?page=${page}`);
}}
className="self-center"
/>}
</>}
</div>;
}
// Helper function for status styling
function getStatusClass(status: string) {
const baseClass = "px-2 py-1 rounded text-xs uppercase font-medium";
switch (status) {
case 'completed':
return `${baseClass} bg-green-100 text-green-800`;
case 'failed':
case 'error':
return `${baseClass} bg-red-100 text-red-800`;
case 'cancelled':
return `${baseClass} bg-gray-100 text-gray-800`;
case 'running':
case 'pending':
default:
return `${baseClass} bg-yellow-100 text-yellow-800`;
}
}
export function RunsApp({
projectId,
slug
}: {
projectId: string,
slug: string[]
}) {
let selection: "list" | "view" | "new" = "list";
let runId: string | null = null;
if (slug.length > 0) {
if (slug[0] === "new") {
selection = "new";
} else {
selection = "view";
runId = slug[0];
}
}
return <>
{selection === "list" && <RunList projectId={projectId} />}
{selection === "new" && <NewRun projectId={projectId} />}
{selection === "view" && runId && <ViewRun projectId={projectId} runId={runId} />}
</>;
}

463
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/scenarios_app.tsx Normal file
View file

@ -0,0 +1,463 @@
import Link from "next/link";
import { WithStringId } from "@/app/lib/types/types";
import { TestScenario } from "@/app/lib/types/testing_types";
import { useEffect, useState, useRef } from "react";
import { createScenario, getScenario, listScenarios, updateScenario, deleteScenario } from "@/app/actions/testing_actions";
import { Button, Input, Pagination, Spinner, Textarea, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react";
import { useRouter, useSearchParams } from "next/navigation";
import { z } from "zod";
import { ArrowLeftIcon, PlusIcon } from "lucide-react";
import { FormStatusButton } from "@/app/lib/components/form-status-button";
import { RelativeTime } from "@primer/react"
function EditScenario({
projectId,
scenarioId,
}: {
projectId: string,
scenarioId: string,
}) {
const router = useRouter();
const [scenario, setScenario] = useState<WithStringId<z.infer<typeof TestScenario>> | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const formRef = useRef<HTMLFormElement>(null);
useEffect(() => {
async function fetchScenario() {
setError(null);
try {
const scenario = await getScenario(projectId, scenarioId);
setScenario(scenario);
} catch (error) {
setError(`Unable to fetch scenario: ${error}`);
} finally {
setLoading(false);
}
}
fetchScenario();
}, [scenarioId, projectId]);
async function handleSubmit(formData: FormData) {
setError(null);
try {
const name = formData.get("name") as string;
const description = formData.get("description") as string;
await updateScenario(projectId, scenarioId, { name, description });
router.push(`/projects/${projectId}/test/scenarios/${scenarioId}`);
} catch (error) {
setError(`Unable to update scenario: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">Edit Scenario</h1>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button
size="sm"
color="danger"
onClick={() => {
formRef.current?.requestSubmit();
}}
>
Retry
</Button>
</div>}
{!loading && scenario && (
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<Input
type="text"
name="name"
label="Name"
placeholder="Enter a name for the scenario"
defaultValue={scenario.name}
required
/>
<Textarea
name="description"
label="Description"
placeholder="Enter a description for the scenario"
defaultValue={scenario.description}
required
/>
<div className="flex gap-2 items-center">
<FormStatusButton
props={{
className: "self-start",
children: "Update",
size: "sm",
type: "submit",
}}
/>
<Button
size="sm"
variant="flat"
as={Link}
href={`/projects/${projectId}/test/scenarios/${scenarioId}`}
>
Cancel
</Button>
</div>
</form>
)}
</div>;
}
function ViewScenario({
projectId,
scenarioId,
}: {
projectId: string,
scenarioId: string,
}) {
const router = useRouter();
const [scenario, setScenario] = useState<WithStringId<z.infer<typeof TestScenario>> | null>(null);
const [loading, setLoading] = useState(true);
const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
const [deleteError, setDeleteError] = useState<string | null>(null);
useEffect(() => {
async function fetchScenario() {
const scenario = await getScenario(projectId, scenarioId);
setScenario(scenario);
setLoading(false);
}
fetchScenario();
}, [scenarioId, projectId]);
async function handleDelete() {
try {
await deleteScenario(projectId, scenarioId);
router.push(`/projects/${projectId}/test/scenarios`);
} catch (error) {
setDeleteError(`Failed to delete scenario: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">View Scenario</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/scenarios`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Scenarios
</Button>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{!loading && !scenario && <div className="text-gray-600 text-center">Scenario not found</div>}
{!loading && scenario && (
<>
<div className="flex flex-col gap-1 text-sm">
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Name</div>
<div className="flex-[2]">{scenario.name}</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Description</div>
<div className="flex-[2]">{scenario.description}</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Created</div>
<div className="flex-[2]"><RelativeTime date={new Date(scenario.createdAt)} /></div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Last Updated</div>
<div className="flex-[2]"><RelativeTime date={new Date(scenario.lastUpdatedAt)} /></div>
</div>
</div>
<div className="flex gap-2 mt-4">
<Button
size="sm"
as={Link}
href={`/projects/${projectId}/test/scenarios/${scenarioId}/edit`}
>
Edit
</Button>
<Button
size="sm"
color="danger"
variant="flat"
onClick={() => setIsDeleteModalOpen(true)}
>
Delete
</Button>
</div>
<Modal
isOpen={isDeleteModalOpen}
onOpenChange={setIsDeleteModalOpen}
size="sm"
>
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Confirm Deletion</ModalHeader>
<ModalBody>
Are you sure you want to delete this scenario?
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Cancel
</Button>
<Button
size="sm"
color="danger"
onPress={() => {
handleDelete();
onClose();
}}
>
Delete
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
<Modal
isOpen={deleteError !== null}
onOpenChange={() => setDeleteError(null)}
size="sm"
>
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Error</ModalHeader>
<ModalBody>
{deleteError}
</ModalBody>
<ModalFooter>
<Button
size="sm"
color="primary"
onPress={onClose}
>
Close
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
</>
)}
</div>;
}
function NewScenario({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const [error, setError] = useState<string | null>(null);
const formRef = useRef<HTMLFormElement>(null);
async function handleSubmit(formData: FormData) {
setError(null);
const name = formData.get("name") as string;
const description = formData.get("description") as string;
try {
const scenario = await createScenario(projectId, { name, description });
router.push(`/projects/${projectId}/test/scenarios/${scenario._id}`);
} catch (error) {
setError(`Unable to create scenario: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">New Scenario</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/scenarios`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Scenarios
</Button>
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button
size="sm"
color="danger"
onClick={() => {
formRef.current?.requestSubmit();
}}
>
Retry
</Button>
</div>}
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<Input
type="text"
name="name"
label="Name"
placeholder="Enter a name for the scenario"
required
/>
<Textarea
name="description"
label="Description"
placeholder="Enter a description for the scenario"
required
/>
<FormStatusButton
props={{
className: "self-start",
children: "Create",
size: "sm",
type: "submit",
}}
/>
</form>
</div>;
}
function ScenarioList({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const searchParams = useSearchParams();
const page = parseInt(searchParams.get("page") || "1");
const pageSize = 10;
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [scenarios, setScenarios] = useState<WithStringId<z.infer<typeof TestScenario>>[]>([]);
const [total, setTotal] = useState(0);
useEffect(() => {
let ignore = false;
async function fetchScenarios() {
setLoading(true);
setError(null);
try {
const scenarios = await listScenarios(projectId, page, pageSize);
if (!ignore) {
setScenarios(scenarios.scenarios);
setTotal(Math.ceil(scenarios.total / pageSize));
}
} catch (error) {
if (!ignore) {
setError(`Unable to fetch scenarios: ${error}`);
}
} finally {
if (!ignore) {
setLoading(false);
}
}
}
if (error == null) {
fetchScenarios();
}
return () => {
ignore = true;
};
}, [page, pageSize, error, projectId]);
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">Scenarios</h1>
<Button
size="sm"
onClick={() => router.push(`/projects/${projectId}/test/scenarios/new`)}
className="self-end"
startContent={<PlusIcon className="w-4 h-4" />}
>
New Scenario
</Button>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => setError(null)}>Retry</Button>
</div>}
{!loading && !error && <>
{scenarios.length === 0 && <div className="text-gray-600 text-center">No scenarios found</div>}
{scenarios.length > 0 && <div className="flex flex-col w-full">
{/* Header */}
<div className="grid grid-cols-7 py-2 bg-gray-100 font-semibold text-sm">
<div className="col-span-2 px-4">Name</div>
<div className="col-span-3 px-4">Description</div>
<div className="col-span-1 px-4">Created</div>
<div className="col-span-1 px-4">Updated</div>
</div>
{/* Rows */}
{scenarios.map((scenario) => (
<div key={scenario._id} className="grid grid-cols-7 py-2 border-b hover:bg-gray-50 text-sm">
<div className="col-span-2 px-4 truncate">
<Link
href={`/projects/${projectId}/test/scenarios/${scenario._id}`}
className="text-blue-600 hover:underline"
>
{scenario.name}
</Link>
</div>
<div className="col-span-3 px-4 truncate">{scenario.description}</div>
<div className="col-span-1 px-4 text-gray-600 truncate">
<RelativeTime date={new Date(scenario.createdAt)} />
</div>
<div className="col-span-1 px-4 text-gray-600 truncate">
<RelativeTime date={new Date(scenario.lastUpdatedAt)} />
</div>
</div>
))}
</div>}
{total > 1 && <Pagination
total={total}
page={page}
onChange={(page) => {
router.push(`/projects/${projectId}/test/scenarios?page=${page}`);
}}
className="self-center"
/>}
</>}
</div>;
}
export function ScenariosApp({
projectId,
slug
}: {
projectId: string,
slug: string[]
}) {
let selection: "list" | "view" | "new" | "edit" = "list";
let scenarioId: string | null = null;
if (slug.length > 0) {
if (slug[0] === "new") {
selection = "new";
} else if (slug[slug.length - 1] === "edit") {
selection = "edit";
scenarioId = slug[0];
} else {
selection = "view";
scenarioId = slug[0];
}
}
return <>
{selection === "list" && <ScenarioList projectId={projectId} />}
{selection === "new" && <NewScenario projectId={projectId} />}
{selection === "view" && scenarioId && <ViewScenario projectId={projectId} scenarioId={scenarioId} />}
{selection === "edit" && scenarioId && <EditScenario projectId={projectId} scenarioId={scenarioId} />}
</>;
}

690
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/simulations_app.tsx Normal file
View file

@ -0,0 +1,690 @@
import Link from "next/link";
import { WithStringId } from "@/app/lib/types/types";
import { TestProfile, TestScenario, TestSimulation } from "@/app/lib/types/testing_types";
import { useEffect, useState, useRef } from "react";
import { createSimulation, getSimulation, listSimulations, updateSimulation, deleteSimulation, listScenarios, getScenario, getProfile } from "@/app/actions/testing_actions";
import { Button, Input, Pagination, Spinner, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@nextui-org/react";
import { useRouter, useSearchParams } from "next/navigation";
import { z } from "zod";
import { PlusIcon, ArrowLeftIcon } from "lucide-react";
import { FormStatusButton } from "@/app/lib/components/form-status-button";
import { RelativeTime } from "@primer/react"
import { ScenarioSelector } from "@/app/lib/components/selectors/scenario-selector";
import { ProfileSelector } from "@/app/lib/components/selectors/profile-selector";
function EditSimulation({
projectId,
simulationId,
}: {
projectId: string,
simulationId: string,
}) {
const router = useRouter();
const [simulation, setSimulation] = useState<WithStringId<z.infer<typeof TestSimulation>> | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const formRef = useRef<HTMLFormElement>(null);
const [scenario, setScenario] = useState<WithStringId<z.infer<typeof TestScenario>> | null>(null);
const [profile, setProfile] = useState<WithStringId<z.infer<typeof TestProfile>> | null>(null);
const [isScenarioModalOpen, setIsScenarioModalOpen] = useState(false);
const [isProfileModalOpen, setIsProfileModalOpen] = useState(false);
useEffect(() => {
async function fetchSimulation() {
setError(null);
try {
const simulation = await getSimulation(projectId, simulationId);
setSimulation(simulation);
if (simulation) {
const [scenarioResult, profileResult] = await Promise.all([
getScenario(projectId, simulation.scenarioId),
getProfile(projectId, simulation.profileId),
]);
setScenario(scenarioResult);
setProfile(profileResult);
}
} catch (error) {
setError(`Unable to fetch simulation: ${error}`);
} finally {
setLoading(false);
}
}
fetchSimulation();
}, [simulationId, projectId]);
async function handleSubmit(formData: FormData) {
setError(null);
try {
const name = formData.get("name") as string;
const passCriteria = formData.get("passCriteria") as string;
if (!name || !passCriteria) {
throw new Error("Name and Pass Criteria are required");
}
if (!scenario || !profile) {
throw new Error("Please select all required fields");
}
await updateSimulation(projectId, simulationId, {
name,
scenarioId: scenario._id,
profileId: profile._id,
passCriteria
});
router.push(`/projects/${projectId}/test/simulations/${simulationId}`);
} catch (error) {
setError(`Unable to update simulation: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">Edit Simulation</h1>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => formRef.current?.requestSubmit()}>Retry</Button>
</div>}
{!loading && simulation && (
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<Input
type="text"
name="name"
label="Name"
placeholder="Enter a name for the simulation"
defaultValue={simulation.name}
required
/>
<Input
type="text"
name="passCriteria"
label="Pass Criteria"
placeholder="Enter the criteria for passing this simulation"
defaultValue={simulation.passCriteria}
required
/>
<div className="flex flex-col gap-2">
<label className="text-sm font-medium">Scenario</label>
<div className="flex items-center gap-2">
{scenario ? (
<div className="text-sm text-blue-600">{scenario.name}</div>
) : (
<div className="text-sm text-gray-500">No scenario selected</div>
)}
<Button
size="sm"
onClick={() => setIsScenarioModalOpen(true)}
type="button"
>
{scenario ? "Change" : "Select"} Scenario
</Button>
</div>
</div>
<div className="flex flex-col gap-2">
<label className="text-sm font-medium">Profile</label>
<div className="flex items-center gap-2">
{profile ? (
<div className="text-sm text-blue-600">{profile.name}</div>
) : (
<div className="text-sm text-gray-500">No profile selected</div>
)}
<Button
size="sm"
onClick={() => setIsProfileModalOpen(true)}
type="button"
>
{profile ? "Change" : "Select"} Profile
</Button>
</div>
</div>
<div className="flex gap-2 items-center">
<FormStatusButton
props={{
className: "self-start",
children: "Update",
size: "sm",
type: "submit",
isDisabled: !scenario || !profile,
}}
/>
<Button
size="sm"
variant="flat"
as={Link}
href={`/projects/${projectId}/test/simulations/${simulationId}`}
>
Cancel
</Button>
</div>
<ScenarioSelector
projectId={projectId}
isOpen={isScenarioModalOpen}
onOpenChange={setIsScenarioModalOpen}
onSelect={setScenario}
/>
<ProfileSelector
projectId={projectId}
isOpen={isProfileModalOpen}
onOpenChange={setIsProfileModalOpen}
onSelect={setProfile}
/>
</form>
)}
</div>;
}
function ViewSimulation({
projectId,
simulationId,
}: {
projectId: string,
simulationId: string,
}) {
const router = useRouter();
const [simulation, setSimulation] = useState<WithStringId<z.infer<typeof TestSimulation>> | null>(null);
const [loading, setLoading] = useState(true);
const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
const [deleteError, setDeleteError] = useState<string | null>(null);
const [scenario, setScenario] = useState<WithStringId<z.infer<typeof TestScenario>> | null>(null);
const [profile, setProfile] = useState<WithStringId<z.infer<typeof TestProfile>> | null>(null);
useEffect(() => {
async function fetchSimulation() {
const simulation = await getSimulation(projectId, simulationId);
setSimulation(simulation);
if (simulation) {
const [scenarioResult, profileResult] = await Promise.all([
getScenario(projectId, simulation.scenarioId),
getProfile(projectId, simulation.profileId),
]);
setScenario(scenarioResult);
setProfile(profileResult);
}
setLoading(false);
}
fetchSimulation();
}, [simulationId, projectId]);
async function handleDelete() {
try {
await deleteSimulation(projectId, simulationId);
router.push(`/projects/${projectId}/test/simulations`);
} catch (error) {
setDeleteError(`Failed to delete simulation: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">View Simulation</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/simulations`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Simulations
</Button>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{!loading && !simulation && <div className="text-gray-600 text-center">Simulation not found</div>}
{!loading && simulation && (
<>
<div className="flex flex-col gap-1 text-sm">
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Name</div>
<div className="flex-[2]">{simulation.name}</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Scenario</div>
<div className="flex-[2]">
{scenario ? (
<Link href={`/projects/${projectId}/test/scenarios/${scenario._id}`} className="text-blue-600 hover:underline">
{scenario.name}
</Link>
) : (
<div className="text-sm text-gray-500">No scenario selected</div>
)}
</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Profile</div>
<div className="flex-[2]">
{profile ? (
<Link href={`/projects/${projectId}/test/profiles/${profile._id}`} className="text-blue-600 hover:underline">
{profile.name}
</Link>
) : (
<div className="text-sm text-gray-500">No profile selected</div>
)}
</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Pass Criteria</div>
<div className="flex-[2]">{simulation.passCriteria}</div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Created</div>
<div className="flex-[2]"><RelativeTime date={new Date(simulation.createdAt)} /></div>
</div>
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Last Updated</div>
<div className="flex-[2]"><RelativeTime date={new Date(simulation.lastUpdatedAt)} /></div>
</div>
</div>
<div className="flex gap-2 mt-4">
<Button
size="sm"
as={Link}
href={`/projects/${projectId}/test/simulations/${simulationId}/edit`}
>
Edit
</Button>
<Button
size="sm"
color="danger"
variant="flat"
onClick={() => setIsDeleteModalOpen(true)}
>
Delete
</Button>
</div>
<Modal
isOpen={isDeleteModalOpen}
onOpenChange={setIsDeleteModalOpen}
size="sm"
>
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Confirm Deletion</ModalHeader>
<ModalBody>
Are you sure you want to delete this simulation?
</ModalBody>
<ModalFooter>
<Button size="sm" variant="flat" onPress={onClose}>
Cancel
</Button>
<Button
size="sm"
color="danger"
onPress={() => {
handleDelete();
onClose();
}}
>
Delete
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
<Modal
isOpen={deleteError !== null}
onOpenChange={() => setDeleteError(null)}
size="sm"
>
<ModalContent>
{(onClose) => (
<>
<ModalHeader>Error</ModalHeader>
<ModalBody>
{deleteError}
</ModalBody>
<ModalFooter>
<Button
size="sm"
color="primary"
onPress={onClose}
>
Close
</Button>
</ModalFooter>
</>
)}
</ModalContent>
</Modal>
</>
)}
</div>;
}
function NewSimulation({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const [error, setError] = useState<string | null>(null);
const [scenario, setScenario] = useState<WithStringId<z.infer<typeof TestScenario>> | null>(null);
const [profile, setProfile] = useState<WithStringId<z.infer<typeof TestProfile>> | null>(null);
const formRef = useRef<HTMLFormElement>(null);
const [isScenarioModalOpen, setIsScenarioModalOpen] = useState(false);
const [isProfileModalOpen, setIsProfileModalOpen] = useState(false);
async function handleSubmit(formData: FormData) {
setError(null);
try {
const name = formData.get("name") as string;
const passCriteria = formData.get("passCriteria") as string;
if (!name || !passCriteria) {
throw new Error("Name and Pass Criteria are required");
}
if (!scenario) {
throw new Error("Please select a scenario");
}
if (!profile) {
throw new Error("Please select a profile");
}
const result = await createSimulation(projectId, {
name,
scenarioId: scenario._id,
profileId: profile._id,
passCriteria,
});
router.push(`/projects/${projectId}/test/simulations/${result._id}`);
} catch (error) {
setError(`Unable to create simulation: ${error}`);
}
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">New Simulation</h1>
<Button
size="sm"
className="self-start"
as={Link}
href={`/projects/${projectId}/test/simulations`}
startContent={<ArrowLeftIcon className="w-4 h-4" />}
>
All Simulations
</Button>
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => formRef.current?.requestSubmit()}>Retry</Button>
</div>}
<form ref={formRef} action={handleSubmit} className="flex flex-col gap-2">
<Input
type="text"
name="name"
label="Name"
placeholder="Enter a name for the simulation"
required
/>
<Input
type="text"
name="passCriteria"
label="Pass Criteria"
placeholder="Enter the criteria for passing this simulation"
required
/>
<div className="flex flex-col gap-2">
<label className="text-sm font-medium">Scenario</label>
<div className="flex items-center gap-2">
{scenario ? (
<div className="text-sm text-blue-600">{scenario.name}</div>
) : (
<div className="text-sm text-gray-500">No scenario selected</div>
)}
<Button
size="sm"
onClick={() => setIsScenarioModalOpen(true)}
type="button"
>
{scenario ? "Change" : "Select"} Scenario
</Button>
</div>
</div>
<div className="flex flex-col gap-2">
<label className="text-sm font-medium">Profile</label>
<div className="flex items-center gap-2">
{profile ? (
<div className="text-sm text-blue-600">{profile.name}</div>
) : (
<div className="text-sm text-gray-500">No profile selected</div>
)}
<Button
size="sm"
onClick={() => setIsProfileModalOpen(true)}
type="button"
>
{profile ? "Change" : "Select"} Profile
</Button>
</div>
</div>
<FormStatusButton
props={{
className: "self-start",
children: "Create",
size: "sm",
type: "submit",
isDisabled: !scenario || !profile,
}}
/>
</form>
<ScenarioSelector
projectId={projectId}
isOpen={isScenarioModalOpen}
onOpenChange={setIsScenarioModalOpen}
onSelect={setScenario}
/>
<ProfileSelector
projectId={projectId}
isOpen={isProfileModalOpen}
onOpenChange={setIsProfileModalOpen}
onSelect={setProfile}
/>
</div>;
}
function SimulationList({
projectId,
}: {
projectId: string,
}) {
const router = useRouter();
const searchParams = useSearchParams();
const page = parseInt(searchParams.get("page") || "1");
const pageSize = 10;
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [simulationList, setSimulationList] = useState<WithStringId<z.infer<typeof TestSimulation>>[]>([]);
const [scenarioMap, setScenarioMap] = useState<Record<string, WithStringId<z.infer<typeof TestScenario>>>>({});
const [profileMap, setProfileMap] = useState<Record<string, WithStringId<z.infer<typeof TestProfile>>>>({});
const [total, setTotal] = useState(0);
useEffect(() => {
let ignore = false;
async function fetchSimulation() {
setLoading(true);
setError(null);
try {
const result = await listSimulations(projectId, page, pageSize);
if (!ignore) {
setSimulationList(result.simulations);
setTotal(Math.ceil(result.total / pageSize));
}
} catch (error) {
if (!ignore) {
setError(`Unable to fetch simulation: ${error}`);
}
} finally {
if (!ignore) {
setLoading(false);
}
}
}
if (error == null) {
fetchSimulation();
}
return () => {
ignore = true;
};
}, [page, pageSize, error, projectId]);
useEffect(() => {
let ignore = false;
async function resolveScenarios() {
const scenarioIds = simulationList.reduce((acc, simulation) => {
if (!acc.includes(simulation.scenarioId)) {
acc.push(simulation.scenarioId);
}
return acc;
}, [] as string[]);
const scenarios = await Promise.all(scenarioIds.map((scenarioId) => getScenario(projectId, scenarioId)));
if (ignore) {
return;
}
setScenarioMap(scenarios.filter((scenario) => scenario !== null).reduce((acc, scenario) => {
acc[scenario._id] = scenario;
return acc;
}, {} as Record<string, WithStringId<z.infer<typeof TestScenario>>>));
}
async function resolveProfiles() {
const profileIds = simulationList.reduce((acc, simulation) => {
if (!acc.includes(simulation.profileId)) {
acc.push(simulation.profileId);
}
return acc;
}, [] as string[]);
const profiles = await Promise.all(profileIds.map((profileId) => getProfile(projectId, profileId)));
if (ignore) {
return;
}
setProfileMap(profiles.filter((profile) => profile !== null).reduce((acc, profile) => {
acc[profile._id] = profile;
return acc;
}, {} as Record<string, WithStringId<z.infer<typeof TestProfile>>>));
}
if (error == null) {
resolveScenarios();
resolveProfiles();
}
return () => {
ignore = true;
};
}, [simulationList, error, projectId]);
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">Simulations</h1>
<Button
size="sm"
onClick={() => router.push(`/projects/${projectId}/test/simulations/new`)}
className="self-end"
startContent={<PlusIcon className="w-4 h-4" />}
>
New Simulation
</Button>
{loading && <div className="flex gap-2 items-center">
<Spinner size="sm" />
Loading...
</div>}
{error && <div className="bg-red-100 p-2 rounded-md text-red-800 flex items-center gap-2 text-sm">
{error}
<Button size="sm" color="danger" onClick={() => setError(null)}>Retry</Button>
</div>}
{!loading && !error && <>
{simulationList.length === 0 && <div className="text-gray-600 text-center">No simulation found</div>}
{simulationList.length > 0 && <div className="flex flex-col w-full">
{/* Header */}
<div className="grid grid-cols-9 py-2 bg-gray-100 font-semibold text-sm">
<div className="col-span-2 px-4">Name</div>
<div className="col-span-3 px-4">Scenario</div>
<div className="col-span-1 px-4">Profile</div>
<div className="col-span-1 px-4">Criteria</div>
<div className="col-span-1 px-4">Created</div>
<div className="col-span-1 px-4">Updated</div>
</div>
{/* Rows */}
{simulationList.map((simulation) => (
<div key={simulation._id} className="grid grid-cols-9 py-2 border-b hover:bg-gray-50 text-sm">
<div className="col-span-2 px-4 truncate">
<Link
href={`/projects/${projectId}/test/simulations/${simulation._id}`}
className="text-blue-600 hover:underline"
>
{simulation.name}
</Link>
</div>
<div className="col-span-3 px-4 truncate">
{scenarioMap[simulation.scenarioId]?.name || (
<span className="text-gray-500 font-mono text-xs">{simulation.scenarioId}</span>
)}
</div>
<div className="col-span-1 px-4 truncate">
{profileMap[simulation.profileId]?.name || (
<span className="text-gray-500 font-mono text-xs">{simulation.profileId}</span>
)}
</div>
<div className="col-span-1 px-4 truncate">
{simulation.passCriteria}
</div>
<div className="col-span-1 px-4 text-gray-600 truncate">
<RelativeTime date={new Date(simulation.createdAt)} />
</div>
<div className="col-span-1 px-4 text-gray-600 truncate">
<RelativeTime date={new Date(simulation.lastUpdatedAt)} />
</div>
</div>
))}
</div>}
{total > 1 && <Pagination
total={total}
page={page}
onChange={(page) => {
router.push(`/projects/${projectId}/test/simulations?page=${page}`);
}}
className="self-center"
/>}
</>}
</div>;
}
export function SimulationsApp({
projectId,
slug
}: {
projectId: string,
slug: string[]
}) {
let selection: "list" | "view" | "new" | "edit" = "list";
let simulationId: string | null = null;
if (slug.length > 0) {
if (slug[0] === "new") {
selection = "new";
} else if (slug[slug.length - 1] === "edit") {
selection = "edit";
simulationId = slug[0];
} else {
selection = "view";
simulationId = slug[0];
}
}
return <>
{selection === "list" && <SimulationList projectId={projectId} />}
{selection === "new" && <NewSimulation projectId={projectId} />}
{selection === "view" && simulationId && <ViewSimulation projectId={projectId} simulationId={simulationId} />}
{selection === "edit" && simulationId && <EditSimulation projectId={projectId} simulationId={simulationId} />}
</>;
}

12
apps/rowboat/app/projects/[projectId]/workflow/app.tsx
View file

@ -9,6 +9,8 @@ import { WorkflowSelector } from "./workflow_selector";
import { Spinner } from "@nextui-org/react"; import { Spinner } from "@nextui-org/react";
import { cloneWorkflow, createWorkflow, fetchPublishedWorkflowId, fetchWorkflow } from "../../../actions/workflow_actions"; import { cloneWorkflow, createWorkflow, fetchPublishedWorkflowId, fetchWorkflow } from "../../../actions/workflow_actions";
import { listDataSources } from "../../../actions/datasource_actions"; import { listDataSources } from "../../../actions/datasource_actions";
import { TestProfile } from "@/app/lib/types/testing_types";
import { getDefaultProfile } from "../../../actions/testing_actions";
export function App({ export function App({
projectId, projectId,
@ -19,6 +21,7 @@ export function App({
const [workflow, setWorkflow] = useState<WithStringId<z.infer<typeof Workflow>> | null>(null); const [workflow, setWorkflow] = useState<WithStringId<z.infer<typeof Workflow>> | null>(null);
const [publishedWorkflowId, setPublishedWorkflowId] = useState<string | null>(null); const [publishedWorkflowId, setPublishedWorkflowId] = useState<string | null>(null);
const [dataSources, setDataSources] = useState<WithStringId<z.infer<typeof DataSource>>[] | null>(null); const [dataSources, setDataSources] = useState<WithStringId<z.infer<typeof DataSource>>[] | null>(null);
const [defaultTestProfile, setDefaultTestProfile] = useState<z.infer<typeof TestProfile> | null>(null);
const [loading, setLoading] = useState(false); const [loading, setLoading] = useState(false);
const [autoSelectIfOnlyOneWorkflow, setAutoSelectIfOnlyOneWorkflow] = useState(true); const [autoSelectIfOnlyOneWorkflow, setAutoSelectIfOnlyOneWorkflow] = useState(true);
@ -27,11 +30,13 @@ export function App({
const workflow = await fetchWorkflow(projectId, workflowId); const workflow = await fetchWorkflow(projectId, workflowId);
const publishedWorkflowId = await fetchPublishedWorkflowId(projectId); const publishedWorkflowId = await fetchPublishedWorkflowId(projectId);
const dataSources = await listDataSources(projectId); const dataSources = await listDataSources(projectId);
const defaultTestProfile = await getDefaultProfile(projectId);
// Store the selected workflow ID in local storage // Store the selected workflow ID in local storage
localStorage.setItem(`lastWorkflowId_${projectId}`, workflowId); localStorage.setItem(`lastWorkflowId_${projectId}`, workflowId);
setWorkflow(workflow); setWorkflow(workflow);
setPublishedWorkflowId(publishedWorkflowId); setPublishedWorkflowId(publishedWorkflowId);
setDataSources(dataSources); setDataSources(dataSources);
setDefaultTestProfile(defaultTestProfile);
setLoading(false); setLoading(false);
}, [projectId]); }, [projectId]);
@ -47,11 +52,13 @@ export function App({
const workflow = await createWorkflow(projectId); const workflow = await createWorkflow(projectId);
const publishedWorkflowId = await fetchPublishedWorkflowId(projectId); const publishedWorkflowId = await fetchPublishedWorkflowId(projectId);
const dataSources = await listDataSources(projectId); const dataSources = await listDataSources(projectId);
const testProfile = await getDefaultProfile(projectId);
// Store the selected workflow ID in local storage // Store the selected workflow ID in local storage
localStorage.setItem(`lastWorkflowId_${projectId}`, workflow._id); localStorage.setItem(`lastWorkflowId_${projectId}`, workflow._id);
setWorkflow(workflow); setWorkflow(workflow);
setPublishedWorkflowId(publishedWorkflowId); setPublishedWorkflowId(publishedWorkflowId);
setDataSources(dataSources); setDataSources(dataSources);
setDefaultTestProfile(testProfile);
setLoading(false); setLoading(false);
} }
@ -60,11 +67,13 @@ export function App({
const workflow = await cloneWorkflow(projectId, workflowId); const workflow = await cloneWorkflow(projectId, workflowId);
const publishedWorkflowId = await fetchPublishedWorkflowId(projectId); const publishedWorkflowId = await fetchPublishedWorkflowId(projectId);
const dataSources = await listDataSources(projectId); const dataSources = await listDataSources(projectId);
const testProfile = await getDefaultProfile(projectId);
// Store the selected workflow ID in local storage // Store the selected workflow ID in local storage
localStorage.setItem(`lastWorkflowId_${projectId}`, workflow._id); localStorage.setItem(`lastWorkflowId_${projectId}`, workflow._id);
setWorkflow(workflow); setWorkflow(workflow);
setPublishedWorkflowId(publishedWorkflowId); setPublishedWorkflowId(publishedWorkflowId);
setDataSources(dataSources); setDataSources(dataSources);
setDefaultTestProfile(testProfile);
setLoading(false); setLoading(false);
} }
@ -98,10 +107,11 @@ export function App({
handleCreateNewVersion={handleCreateNewVersion} handleCreateNewVersion={handleCreateNewVersion}
autoSelectIfOnlyOneWorkflow={autoSelectIfOnlyOneWorkflow} autoSelectIfOnlyOneWorkflow={autoSelectIfOnlyOneWorkflow}
/>} />}
{!loading && workflow && (dataSources !== null) && <WorkflowEditor {!loading && workflow && (dataSources !== null) && (defaultTestProfile !== null) && <WorkflowEditor
key={workflow._id} key={workflow._id}
workflow={workflow} workflow={workflow}
dataSources={dataSources} dataSources={dataSources}
initialTestProfile={defaultTestProfile}
publishedWorkflowId={publishedWorkflowId} publishedWorkflowId={publishedWorkflowId}
handleShowSelector={handleShowSelector} handleShowSelector={handleShowSelector}
handleCloneVersion={handleCloneVersion} handleCloneVersion={handleCloneVersion}

4
apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx
View file

@ -29,6 +29,7 @@ import { BackIcon, HamburgerIcon, WorkflowIcon } from "../../../lib/components/i
import { CopyIcon, Layers2Icon, RadioIcon, RedoIcon, Sparkles, UndoIcon } from "lucide-react"; import { CopyIcon, Layers2Icon, RadioIcon, RedoIcon, Sparkles, UndoIcon } from "lucide-react";
import { EntityList } from "./entity_list"; import { EntityList } from "./entity_list";
import { CopilotMessage } from "../../../lib/types/copilot_types"; import { CopilotMessage } from "../../../lib/types/copilot_types";
import { TestProfile } from "@/app/lib/types/testing_types";
enablePatches(); enablePatches();
@ -533,12 +534,14 @@ export function WorkflowEditor({
publishedWorkflowId, publishedWorkflowId,
handleShowSelector, handleShowSelector,
handleCloneVersion, handleCloneVersion,
initialTestProfile,
}: { }: {
dataSources: WithStringId<z.infer<typeof DataSource>>[]; dataSources: WithStringId<z.infer<typeof DataSource>>[];
workflow: WithStringId<z.infer<typeof Workflow>>; workflow: WithStringId<z.infer<typeof Workflow>>;
publishedWorkflowId: string | null; publishedWorkflowId: string | null;
handleShowSelector: () => void; handleShowSelector: () => void;
handleCloneVersion: (workflowId: string) => void; handleCloneVersion: (workflowId: string) => void;
initialTestProfile: z.infer<typeof TestProfile>;
}) { }) {
const [state, dispatch] = useReducer<Reducer<State, Action>>(reducer, { const [state, dispatch] = useReducer<Reducer<State, Action>>(reducer, {
patches: [], patches: [],
@ -859,6 +862,7 @@ export function WorkflowEditor({
projectId={state.present.workflow.projectId} projectId={state.present.workflow.projectId}
workflow={state.present.workflow} workflow={state.present.workflow}
messageSubscriber={updateChatMessages} messageSubscriber={updateChatMessages}
initialTestProfile={initialTestProfile}
/> />
{state.present.selection?.type === "agent" && <AgentConfig {state.present.selection?.type === "agent" && <AgentConfig
key={state.present.selection.name} key={state.present.selection.name}

154
apps/simulation_runner/db.py
View file

@ -2,15 +2,25 @@ from pymongo import MongoClient
from bson import ObjectId from bson import ObjectId
import os import os
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from scenario_types import SimulationRun, Scenario, SimulationResult, SimulationAggregateResult from typing import Optional
from scenario_types import (
TestRun,
TestScenario,
TestProfile,
TestSimulation,
TestResult,
AggregateResults
)
MONGO_URI = os.environ.get("MONGODB_URI", "mongodb://localhost:27017/rowboat").strip() MONGO_URI = os.environ.get("MONGODB_URI", "mongodb://localhost:27017/rowboat").strip()
SCENARIOS_COLLECTION_NAME = "scenarios" # New collection names
API_KEYS_COLLECTION = "api_keys" TEST_SCENARIOS_COLLECTION = "test_scenarios"
SIMULATIONS_COLLECTION_NAME = "simulation_runs" TEST_PROFILES_COLLECTION = "test_profiles"
SIMULATION_RESULT_COLLECTION_NAME = "simulation_result" TEST_SIMULATIONS_COLLECTION = "test_simulations"
SIMULATION_AGGREGATE_RESULT_COLLECTION_NAME = "simulation_aggregate_result" TEST_RUNS_COLLECTION = "test_runs"
TEST_RESULTS_COLLECTION = "test_results"
API_KEYS_COLLECTION = "api_keys" # If still needed
def get_db(): def get_db():
client = MongoClient(MONGO_URI) client = MongoClient(MONGO_URI)
@ -21,6 +31,9 @@ def get_collection(collection_name: str):
return db[collection_name] return db[collection_name]
def get_api_key(project_id: str): def get_api_key(project_id: str):
"""
If you still use an API key pattern, adapt as needed.
"""
collection = get_collection(API_KEYS_COLLECTION) collection = get_collection(API_KEYS_COLLECTION)
doc = collection.find_one({"projectId": project_id}) doc = collection.find_one({"projectId": project_id})
if doc: if doc:
@ -28,71 +41,68 @@ def get_api_key(project_id: str):
else: else:
return None return None
def get_pending_simulation_run(): #
collection = get_collection(SIMULATIONS_COLLECTION_NAME) # TestRun helpers
#
def get_pending_run() -> Optional[TestRun]:
"""
Finds a run with 'pending' status, marks it 'running', and returns it.
"""
collection = get_collection(TEST_RUNS_COLLECTION)
doc = collection.find_one_and_update( doc = collection.find_one_and_update(
{"status": "pending"}, {"status": "pending"},
{"$set": {"status": "running"}}, {"$set": {"status": "running"}},
return_document=True return_document=True
) )
if doc: if doc:
return SimulationRun( return TestRun(
id=str(doc["_id"]), id=str(doc["_id"]),
projectId=doc["projectId"], projectId=doc["projectId"],
status="running", name=doc["name"],
scenarioIds=doc["scenarioIds"], simulationIds=doc["simulationIds"],
workflowId=doc["workflowId"], workflowId=doc["workflowId"],
status="running",
startedAt=doc["startedAt"], startedAt=doc["startedAt"],
completedAt=doc.get("completedAt") completedAt=doc.get("completedAt"),
aggregateResults=doc.get("aggregateResults"),
lastHeartbeat=doc.get("lastHeartbeat")
) )
return None return None
def set_simulation_run_to_completed(simulation_run: SimulationRun, aggregate_result: SimulationAggregateResult): def set_run_to_completed(test_run: TestRun, aggregate: AggregateResults):
collection = get_collection(SIMULATIONS_COLLECTION_NAME)
collection.update_one({"_id": ObjectId(simulation_run.id)}, {"$set": {"status": "completed", "aggregateResults": aggregate_result.model_dump(by_alias=True)}})
def get_scenarios_for_run(simulation_run: SimulationRun):
if simulation_run is None:
return []
collection = get_collection(SCENARIOS_COLLECTION_NAME)
scenarios = []
for doc in collection.find():
if doc["_id"] in [ObjectId(sid) for sid in simulation_run.scenarioIds]:
scenarios.append(Scenario(
id=str(doc["_id"]),
projectId=doc["projectId"],
name=doc["name"],
description=doc["description"],
criteria=doc["criteria"],
context=doc["context"],
createdAt=doc["createdAt"],
lastUpdatedAt=doc["lastUpdatedAt"]
))
return scenarios
def write_simulation_result(result: SimulationResult):
collection = get_collection(SIMULATION_RESULT_COLLECTION_NAME)
collection.insert_one(result.model_dump())
def update_simulation_run_heartbeat(simulation_run_id: str):
""" """
Updates the 'last_heartbeat' timestamp for a SimulationRun. Marks a test run 'completed' and sets the aggregate results.
""" """
collection = get_collection(TEST_RUNS_COLLECTION)
collection = get_collection(SIMULATIONS_COLLECTION_NAME)
collection.update_one( collection.update_one(
{"_id": ObjectId(simulation_run_id)}, {"_id": ObjectId(test_run.id)},
{
"$set": {
"status": "completed",
"aggregateResults": aggregate.model_dump(by_alias=True),
"completedAt": datetime.now(timezone.utc)
}
}
)
def update_run_heartbeat(run_id: str):
"""
Updates the 'lastHeartbeat' timestamp for a TestRun.
"""
collection = get_collection(TEST_RUNS_COLLECTION)
collection.update_one(
{"_id": ObjectId(run_id)},
{"$set": {"lastHeartbeat": datetime.now(timezone.utc)}} {"$set": {"lastHeartbeat": datetime.now(timezone.utc)}}
) )
def mark_stale_jobs_as_failed(): def mark_stale_jobs_as_failed(threshold_minutes: int = 20) -> int:
""" """
Finds any job in 'running' status whose last_heartbeat is older than 5 minutes, Finds any run in 'running' status whose lastHeartbeat is older than
and sets it to 'failed'. `threshold_minutes`, and sets it to 'failed'. Returns the count.
""" """
collection = get_collection(TEST_RUNS_COLLECTION)
collection = get_collection(SIMULATIONS_COLLECTION_NAME) stale_threshold = datetime.now(timezone.utc) - timedelta(minutes=threshold_minutes)
stale_threshold = datetime.now(timezone.utc) - timedelta(minutes=20)
result = collection.update_many( result = collection.update_many(
{ {
"status": "running", "status": "running",
@ -102,4 +112,46 @@ def mark_stale_jobs_as_failed():
"$set": {"status": "failed"} "$set": {"status": "failed"}
} }
) )
return result.modified_count # Number of jobs marked failed return result.modified_count
#
# TestSimulation helpers
#
def get_simulations_for_run(test_run: TestRun) -> list[TestSimulation]:
"""
Returns all simulations specified by a particular run.
"""
if test_run is None:
return []
collection = get_collection(TEST_SIMULATIONS_COLLECTION)
simulation_docs = collection.find({
"_id": {"$in": [ObjectId(sim_id) for sim_id in test_run.simulationIds]}
})
simulations = []
for doc in simulation_docs:
simulations.append(
TestSimulation(
id=str(doc["_id"]),
projectId=doc["projectId"],
name=doc["name"],
scenarioId=doc["scenarioId"],
profileId=doc["profileId"],
passCriteria=doc["passCriteria"],
createdAt=doc["createdAt"],
lastUpdatedAt=doc["lastUpdatedAt"]
)
)
return simulations
#
# TestResult helpers
#
def write_test_result(result: TestResult):
"""
Writes a test result into the `test_results` collection.
"""
collection = get_collection(TEST_RESULTS_COLLECTION)
collection.insert_one(result.model_dump())

61
apps/simulation_runner/scenario_types.py
View file

@ -2,39 +2,62 @@ from datetime import datetime
from typing import Optional, List, Literal from typing import Optional, List, Literal
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
run_status = Literal["pending", "running", "completed", "cancelled", "failed"] # Define run statuses to include the new "error" status
RunStatus = Literal["pending", "running", "completed", "cancelled", "failed", "error"]
class Scenario(BaseModel): class TestScenario(BaseModel):
# `_id` in Mongo will be stored as ObjectId; we return it as a string
id: str id: str
projectId: str projectId: str
name: str = "" name: str
description: str = "" description: str
criteria: str = ""
context: str = ""
createdAt: datetime createdAt: datetime
lastUpdatedAt: datetime lastUpdatedAt: datetime
class SimulationRun(BaseModel): class TestProfile(BaseModel):
id: str id: str
projectId: str projectId: str
status: Literal["pending", "running", "completed", "cancelled", "failed"] name: str
scenarioIds: List[str] context: str
createdAt: datetime
lastUpdatedAt: datetime
mockTools: bool
mockPrompt: Optional[str] = None
class TestSimulation(BaseModel):
id: str
projectId: str
name: str
scenarioId: str
profileId: str
passCriteria: str
createdAt: datetime
lastUpdatedAt: datetime
class AggregateResults(BaseModel):
total: int
passCount: int
failCount: int
class TestRun(BaseModel):
id: str
projectId: str
name: str
simulationIds: List[str]
workflowId: str workflowId: str
status: RunStatus
startedAt: datetime startedAt: datetime
lastHeartbeat: Optional[datetime] = None
completedAt: Optional[datetime] = None completedAt: Optional[datetime] = None
aggregateResults: Optional[dict] = None # By default, store aggregate results as a dict or the typed AggregateResults
aggregateResults: Optional[AggregateResults] = None
# The new schema does not mention lastHeartbeat,
# but you can keep it if you still want to track stale runs
lastHeartbeat: Optional[datetime] = None
class SimulationResult(BaseModel): class TestResult(BaseModel):
projectId: str projectId: str
runId: str runId: str
scenarioId: str simulationId: str
result: Literal["pass", "fail"] result: Literal["pass", "fail"]
details: str details: str
transcript: str
class SimulationAggregateResult(BaseModel):
total: int
pass_: int = Field(..., alias='pass')
fail: int

86
apps/simulation_runner/service.py
View file

@ -1,84 +1,104 @@
import asyncio import asyncio
import logging import logging
from typing import List from typing import List, Optional
from db import get_pending_simulation_run, get_scenarios_for_run, set_simulation_run_to_completed, get_api_key, mark_stale_jobs_as_failed, update_simulation_run_heartbeat
from scenario_types import SimulationRun, Scenario # Updated imports from your new db module and scenario_types
from simulation import simulate_scenarios from db import (
get_pending_run,
get_simulations_for_run,
set_run_to_completed,
get_api_key,
mark_stale_jobs_as_failed,
update_run_heartbeat
)
from scenario_types import TestRun, TestSimulation
# If you have a new simulation function, import it here.
# Otherwise, adapt the name as needed:
from simulation import simulate_simulations # or simulate_scenarios, if unchanged
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
class JobService: class JobService:
def __init__(self): def __init__(self):
self.poll_interval = 5 # seconds self.poll_interval = 5 # seconds
# Control concurrency of run processing
self.semaphore = asyncio.Semaphore(5) self.semaphore = asyncio.Semaphore(5)
async def poll_and_process_jobs(self, max_iterations: int = None): async def poll_and_process_jobs(self, max_iterations: Optional[int] = None):
""" """
Periodically checks for new jobs in MongoDB and processes them. Periodically checks for new runs in MongoDB and processes them.
""" """
# Start the stale-run check in the background
# Start the stale-job check in the background asyncio.create_task(self.fail_stale_runs_loop())
asyncio.create_task(self.fail_stale_jobs_loop())
iterations = 0 iterations = 0
while True: while True:
job = get_pending_simulation_run() run = get_pending_run() # <--- changed to match new DB function
if job: if run:
logging.info(f"Found new job: {job}. Processing...") logging.info(f"Found new run: {run}. Processing...")
asyncio.create_task(self.process_job(job)) asyncio.create_task(self.process_run(run))
iterations += 1 iterations += 1
if max_iterations is not None and iterations >= max_iterations: if max_iterations is not None and iterations >= max_iterations:
break break
# Sleep for the polling interval # Sleep for the polling interval
await asyncio.sleep(self.poll_interval) await asyncio.sleep(self.poll_interval)
async def process_job(self, job: SimulationRun): async def process_run(self, run: TestRun):
""" """
Calls the simulation function and updates job status upon completion. Calls the simulation function and updates run status upon completion.
""" """
async with self.semaphore: async with self.semaphore:
# Start heartbeat in background # Start heartbeat in background
stop_heartbeat_event = asyncio.Event() stop_heartbeat_event = asyncio.Event()
heartbeat_task = asyncio.create_task(self.heartbeat_loop(job.id, stop_heartbeat_event)) heartbeat_task = asyncio.create_task(self.heartbeat_loop(run.id, stop_heartbeat_event))
try: try:
scenarios = get_scenarios_for_run(job) # Fetch the simulations associated with this run
if not scenarios or len(scenarios) == 0: simulations = get_simulations_for_run(run)
logging.info(f"No scenarios found for job {job.id}") if not simulations:
logging.info(f"No simulations found for run {run.id}")
return return
api_key = get_api_key(job.projectId) # Fetch API key if needed
result = await simulate_scenarios(scenarios, job.id, job.workflowId, api_key) api_key = get_api_key(run.projectId)
# Perform your simulation logic
# adapt this call to your actual simulation function’s signature
aggregate_result = await simulate_simulations(
simulations=simulations,
run_id=run.id,
workflow_id=run.workflowId,
api_key=api_key
)
set_simulation_run_to_completed(job, result) # Mark run as completed with the aggregated result
logging.info(f"Job {job.id} completed.") set_run_to_completed(run, aggregate_result)
logging.info(f"Run {run.id} completed.")
except Exception as exc: except Exception as exc:
logging.error(f"Job {job.id} failed: {exc}") logging.error(f"Run {run.id} failed: {exc}")
finally: finally:
stop_heartbeat_event.set() stop_heartbeat_event.set()
await heartbeat_task await heartbeat_task
async def fail_stale_jobs_loop(self): async def fail_stale_runs_loop(self):
""" """
Periodically checks for stale jobs that haven't received a heartbeat in over 5 minutes, Periodically checks for stale runs (no heartbeat) and marks them as 'failed'.
and marks them as 'failed'.
""" """
while True: while True:
count = mark_stale_jobs_as_failed() count = mark_stale_jobs_as_failed()
if count > 0: if count > 0:
logging.warning(f"Marked {count} stale jobs as failed.") logging.warning(f"Marked {count} stale runs as failed.")
await asyncio.sleep(60) # Check every 60 seconds await asyncio.sleep(60) # Check every 60 seconds
async def heartbeat_loop(self, job_id: str, stop_event: asyncio.Event): async def heartbeat_loop(self, run_id: str, stop_event: asyncio.Event):
""" """
Periodically updates 'last_heartbeat' for the given job until 'stop_event' is set. Periodically updates 'lastHeartbeat' for the given run until 'stop_event' is set.
""" """
try: try:
while not stop_event.is_set(): while not stop_event.is_set():
update_simulation_run_heartbeat(job_id) update_run_heartbeat(run_id)
await asyncio.sleep(10) # Heartbeat interval in seconds await asyncio.sleep(10) # Heartbeat interval in seconds
except asyncio.CancelledError: except asyncio.CancelledError:
pass pass

149
apps/simulation_runner/simulation.py
View file

@ -1,46 +1,67 @@
from rowboat import Client, StatefulChat import asyncio
import logging
from typing import List from typing import List
import json import json
import os import os
import asyncio
from openai import OpenAI from openai import OpenAI
from scenario_types import Scenario, SimulationResult, SimulationAggregateResult
from db import write_simulation_result # Updated imports from your new schema/types
from scenario_types import TestSimulation, TestResult, AggregateResults
# If your DB functions changed names, adapt here:
from db import write_test_result # replaced write_simulation_result
from rowboat import Client, StatefulChat
openai_client = OpenAI() openai_client = OpenAI()
MODEL_NAME = "gpt-4o" MODEL_NAME = "gpt-4o"
ROWBOAT_API_HOST = os.environ.get("ROWBOAT_API_HOST", "http://127.0.0.1:3000").strip() ROWBOAT_API_HOST = os.environ.get("ROWBOAT_API_HOST", "http://127.0.0.1:3000").strip()
async def simulate_scenario(scenario: Scenario, rowboat_client: Client, workflow_id: str, max_iterations: int = 5) -> tuple[str, str, str]: async def simulate_simulation(
simulation: TestSimulation,
rowboat_client: Client,
workflow_id: str,
max_iterations: int = 5
) -> tuple[str, str, str]:
""" """
Runs a mock simulation for a given scenario asynchronously. Runs a mock simulation for a given TestSimulation asynchronously.
After simulating several turns of conversation, it evaluates the conversation. After simulating several turns of conversation, it evaluates the conversation.
Returns a tuple of (evaluation_result, details, transcript_str). Returns a tuple of (evaluation_result, details, transcript_str).
""" """
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
# Optionally embed passCriteria in the system prompt, if it’s relevant to context:
pass_criteria = simulation.passCriteria or ""
# Or place it separately below if you prefer.
# Prepare a Rowboat chat
support_chat = StatefulChat( support_chat = StatefulChat(
rowboat_client, rowboat_client,
system_prompt=f"{f'Context: {scenario.context}' if scenario.context else ''}", system_prompt=f"Context: {pass_criteria}" if pass_criteria else "",
workflow_id=workflow_id workflow_id=workflow_id
) )
# You might want to describe the simulation or scenario more thoroughly.
# Here, we just embed simulation.name in the system message:
messages = [ messages = [
{ {
"role": "system", "role": "system",
"content": f"Simulate the user based on the scenario: \n {scenario.description}" "content": (
f"Simulate the user based on this simulation:\n{simulation.name}"
)
} }
] ]
# ------------------------- # -------------------------
# 1) MAIN SIMULATION LOOP # (1) MAIN SIMULATION LOOP
# ------------------------- # -------------------------
for i in range(max_iterations): for _ in range(max_iterations):
openai_input = messages openai_input = messages
# Run OpenAI API call in a separate thread # Run OpenAI API call in a separate thread (non-blocking)
simulated_user_response = await loop.run_in_executor( simulated_user_response = await loop.run_in_executor(
None, # Use default thread pool None, # default ThreadPool
lambda: openai_client.chat.completions.create( lambda: openai_client.chat.completions.create(
model=MODEL_NAME, model=MODEL_NAME,
messages=openai_input, messages=openai_input,
@ -48,9 +69,9 @@ async def simulate_scenario(scenario: Scenario, rowboat_client: Client, workflow
) )
) )
simulated_content = simulated_user_response.choices[0].message.content simulated_content = simulated_user_response.choices[0].message.content.strip()
# Run support_chat.run in a thread if it's synchronous # Run Rowboat chat in a thread if it's synchronous
rowboat_response = await loop.run_in_executor( rowboat_response = await loop.run_in_executor(
None, None,
lambda: support_chat.run(simulated_content) lambda: support_chat.run(simulated_content)
@ -59,7 +80,7 @@ async def simulate_scenario(scenario: Scenario, rowboat_client: Client, workflow
messages.append({"role": "assistant", "content": rowboat_response}) messages.append({"role": "assistant", "content": rowboat_response})
# ------------------------- # -------------------------
# 2) EVALUATION STEP # (2) EVALUATION STEP
# ------------------------- # -------------------------
transcript_str = "" transcript_str = ""
for m in messages: for m in messages:
@ -67,19 +88,24 @@ async def simulate_scenario(scenario: Scenario, rowboat_client: Client, workflow
content = m.get("content", "") content = m.get("content", "")
transcript_str += f"{role.upper()}: {content}\n" transcript_str += f"{role.upper()}: {content}\n"
# We use passCriteria as the evaluation “criteria.”
evaluation_prompt = [ evaluation_prompt = [
{ {
"role": "system", "role": "system",
"content": ( "content": (
f"You are a neutral evaluator. Evaluate based on these criteria:\n{scenario.criteria}\n\nReturn ONLY a JSON object with format: " f"You are a neutral evaluator. Evaluate based on these criteria:\n"
'{"verdict": "pass", "details": <the reason for pass in 2 sentences>} if the support bot answered correctly, or {"verdict": "fail", "details": <the reason for fail in 2 sentences>} if not.' f"{simulation.passCriteria}\n\n"
"Return ONLY a JSON object in this format:\n"
'{"verdict": "pass", "details": <reason>} or '
'{"verdict": "fail", "details": <reason>}.'
) )
}, },
{ {
"role": "user", "role": "user",
"content": ( "content": (
f"Here is the conversation transcript:\n\n{transcript_str}\n\n" f"Here is the conversation transcript:\n\n{transcript_str}\n\n"
"Did the support bot answer correctly or not? Return only 'pass' or 'fail' for verdict, and a brief 2 sentence explanation for details." "Did the support bot answer correctly or not? "
"Return only 'pass' or 'fail' for verdict, and a brief explanation for details."
) )
} }
] ]
@ -91,51 +117,82 @@ async def simulate_scenario(scenario: Scenario, rowboat_client: Client, workflow
model=MODEL_NAME, model=MODEL_NAME,
messages=evaluation_prompt, messages=evaluation_prompt,
temperature=0.0, temperature=0.0,
# If your LLM supports a structured response format, you can specify it.
# Otherwise, remove or adapt 'response_format':
response_format={"type": "json_object"} response_format={"type": "json_object"}
) )
) )
if not eval_response.choices: if not eval_response.choices:
raise Exception("No evaluation response received from model") raise Exception("No evaluation response received from model")
else:
response_json = json.loads(eval_response.choices[0].message.content) response_json_str = eval_response.choices[0].message.content
evaluation_result = response_json.get("verdict") # Attempt to parse the JSON
details = response_json.get("details") response_json = json.loads(response_json_str)
if evaluation_result is None: evaluation_result = response_json.get("verdict")
raise Exception("No verdict field found in evaluation response") details = response_json.get("details")
if evaluation_result is None:
raise Exception("No 'verdict' field found in evaluation response")
return (evaluation_result, details, transcript_str) return (evaluation_result, details, transcript_str)
async def simulate_scenarios(scenarios: List[Scenario], runId: str, workflow_id: str, api_key: str, max_iterations: int = 5): async def simulate_simulations(
simulations: List[TestSimulation],
run_id: str,
workflow_id: str,
api_key: str,
max_iterations: int = 5
) -> AggregateResults:
""" """
Simulates a list of scenarios asynchronously and aggregates the results. Simulates a list of TestSimulations asynchronously and aggregates the results.
""" """
project_id = scenarios[0].projectId if not simulations:
# Return an empty result if there's nothing to simulate
return AggregateResults(total=0, pass_=0, fail=0)
# We assume all simulations belong to the same project
project_id = simulations[0].projectId
# Create a Rowboat client instance
client = Client( client = Client(
host=ROWBOAT_API_HOST, host=ROWBOAT_API_HOST,
project_id=project_id, project_id=project_id,
api_key=api_key api_key=api_key
) )
results = []
for scenario in scenarios: # Store results here
# Await the asynchronous simulate_scenario results: List[TestResult] = []
result, details, transcript = await simulate_scenario(scenario, client, workflow_id, max_iterations)
simulation_result = SimulationResult( for simulation in simulations:
projectId=project_id, # Run each simulation
runId=runId, verdict, details, transcript = await simulate_simulation(
scenarioId=scenario.id, simulation=simulation,
result=result, rowboat_client=client,
details=details, workflow_id=workflow_id,
transcript=transcript max_iterations=max_iterations
) )
results.append(simulation_result)
write_simulation_result(simulation_result)
aggregate_result = SimulationAggregateResult(**{ # Create a new TestResult
"total": len(scenarios), test_result = TestResult(
"pass": sum(1 for result in results if result.result == "pass"), projectId=project_id,
"fail": sum(1 for result in results if result.result == "fail") runId=run_id,
}) simulationId=simulation.id,
return aggregate_result result=verdict,
details=details
)
results.append(test_result)
# Persist the test result
write_test_result(test_result)
# Aggregate pass/fail
total_count = len(results)
pass_count = sum(1 for r in results if r.result == "pass")
fail_count = sum(1 for r in results if r.result == "fail")
return AggregateResults(
total=total_count,
passCount=pass_count,
failCount=fail_count
)