apunkt/rowboat
1
0
Fork 0
mirror of https://github.com/rowboatlabs/rowboat.git synced 2026-06-06 19:35:44 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix test profile implementation

Browse source
This commit is contained in:
ramnique 2025-03-03 19:39:51 +05:30
parent 4bd72ba245
commit 645d0c27f9
18 changed files with 128 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

4
apps/rowboat/app/actions/actions.ts
View file

@ -99,13 +99,13 @@ export async function getAssistantResponse(
};
}
export async function suggestToolResponse(toolId: string, projectId: string, messages: z.infer<typeof apiV1.ChatMessage>[], testProfile: z.infer<typeof TestProfile>): Promise<string> {
export async function suggestToolResponse(toolId: string, projectId: string, messages: z.infer<typeof apiV1.ChatMessage>[], mockInstructions: string): Promise<string> {
await projectAuthCheck(projectId);
if (!await check_query_limit(projectId)) {
throw new QueryLimitError();
}
return await mockToolResponse(toolId, messages, testProfile);
return await mockToolResponse(toolId, messages, mockInstructions);
}
export async function getInformationTool(

13
apps/rowboat/app/actions/project_actions.ts
View file

@ -41,7 +41,6 @@ export async function createProject(formData: FormData) {
const projectId = crypto.randomUUID();
const chatClientId = crypto.randomBytes(16).toString('base64url');
const secret = crypto.randomBytes(32).toString('hex');
const defaultTestProfileId = new ObjectId();
// create project
await projectsCollection.insertOne({
@ -54,7 +53,6 @@ export async function createProject(formData: FormData) {
secret,
nextWorkflowNumber: 1,
testRunCounter: 0,
defaultTestProfileId: defaultTestProfileId.toString(),
});
// add first workflow version
@ -70,17 +68,6 @@ export async function createProject(formData: FormData) {
name: `Version 1`,
});
// add default test profile
await testProfilesCollection.insertOne({
_id: defaultTestProfileId,
projectId,
name: "Default",
context: "",
mockTools: false,
createdAt: (new Date()).toISOString(),
lastUpdatedAt: (new Date()).toISOString(),
});
// add user to project
await projectMembersCollection.insertOne({
userId: user.sub,

34
apps/rowboat/app/actions/testing_actions.ts
View file

@ -171,13 +171,13 @@ export async function createSimulation(
data: {
name: string;
scenarioId: string;
profileId: string;
profileId: string | null;
passCriteria: string;
}
): Promise<WithStringId<z.infer<typeof TestSimulation>>> {
await projectAuthCheck(projectId);
const doc = {
const doc: z.infer<typeof TestSimulation> = {
...data,
projectId,
createdAt: new Date().toISOString(),
@ -196,7 +196,7 @@ export async function updateSimulation(
updates: {
name?: string;
scenarioId?: string;
profileId?: string;
profileId?: string | null;
passCriteria?: string;
}
): Promise<void> {
@ -245,34 +245,6 @@ export async function listProfiles(
};
}
export async function getDefaultProfile(projectId: string): Promise<WithStringId<z.infer<typeof TestProfile>> | null> {
await projectAuthCheck(projectId);
const project = await projectsCollection.findOne({ _id: projectId });
if (!project) {
return null;
}
if (!project.defaultTestProfileId) {
// create a default profile
const profile = await createProfile(projectId, {
name: 'Default',
context: '',
mockTools: false,
mockPrompt: '',
});
await setDefaultProfile(projectId, profile._id);
return profile;
}
return getProfile(projectId, project.defaultTestProfileId);
}
export async function setDefaultProfile(projectId: string, profileId: string): Promise<void> {
await projectAuthCheck(projectId);
await projectsCollection.updateOne(
{ _id: projectId },
{ $set: { defaultTestProfileId: profileId } }
);
}
export async function getProfile(projectId: string, profileId: string): Promise<WithStringId<z.infer<typeof TestProfile>> | null> {
await projectAuthCheck(projectId);

30
apps/rowboat/app/api/v1/[projectId]/chat/route.ts
View file

@ -72,9 +72,9 @@ export async function POST(
}
// if test profile is provided in the request, use it
let profile: z.infer<typeof TestProfile> | null = null;
let testProfile: z.infer<typeof TestProfile> | null = null;
if (result.data.testProfileId) {
const testProfile = await testProfilesCollection.findOne({
testProfile = await testProfilesCollection.findOne({
projectId: projectId,
_id: new ObjectId(result.data.testProfileId),
});
@ -82,34 +82,18 @@ export async function POST(
logger.log(`Test profile ${result.data.testProfileId} not found for project ${projectId}`);
return Response.json({ error: "Test profile not found" }, { status: 404 });
}
profile = testProfile;
} else {
// if no test profile is provided, use the default profile
const defaultProfile = await testProfilesCollection.findOne({
projectId: projectId,
_id: new ObjectId(project.defaultTestProfileId),
});
if (!defaultProfile) {
logger.log(`Default test profile not found for project ${projectId}`);
return Response.json({ error: "Default test profile not found" }, { status: 404 });
}
profile = defaultProfile;
}
if (!profile) {
logger.log(`No test profile found for project ${projectId}`);
return Response.json({ error: "No test profile found" }, { status: 404 });
}
// if profile has a context available, overwrite the system message in the request (if there is one)
let currentMessages = reqMessages;
if (profile.context) {
if (testProfile?.context) {
// if there is a system message, overwrite it
const systemMessageIndex = reqMessages.findIndex(m => m.role === "system");
if (systemMessageIndex !== -1) {
currentMessages[systemMessageIndex].content = profile.context;
currentMessages[systemMessageIndex].content = testProfile.context;
} else {
// if there is no system message, add one
currentMessages.unshift({ role: "system", content: profile.context });
currentMessages.unshift({ role: "system", content: testProfile.context });
}
}
@ -183,9 +167,9 @@ export async function POST(
try {
// if tool is supposed to be mocked, mock it
const workflowTool = workflow.tools.find(t => t.name === toolCall.function.name);
if (profile.mockTools) {
if (testProfile?.mockTools || workflowTool?.mockTool) {
logger.log(`Mocking tool call ${toolCall.function.name}`);
result = await mockToolResponse(toolCall.id, currentMessages, profile);
result = await mockToolResponse(toolCall.id, currentMessages, testProfile?.mockPrompt || workflowTool?.mockInstructions || '');
} else {
// else run the tool call by calling the client tool webhook
logger.log(`Running client tool webhook for tool ${toolCall.function.name}`);

10
apps/rowboat/app/lib/project_templates.ts
View file

@ -147,7 +147,7 @@ You are an helpful customer support assistant
"user_id"
]
},
"mockInPlayground": true,
"mockTool": true,
"autoSubmitMockedResponse": true
},
{
@ -165,7 +165,7 @@ You are an helpful customer support assistant
"param1"
]
},
"mockInPlayground": true,
"mockTool": true,
"autoSubmitMockedResponse": true
}
],
@ -289,13 +289,13 @@ You are an helpful customer support assistant
"orderId"
]
},
"mockInPlayground": true,
"mockTool": true,
"autoSubmitMockedResponse": true
},
{
"name": "retrieve_snippet",
"description": "This is a mock RAG service. Always return 2 paragraphs about a fictional scooter rental product, based on the query. Be verbose.",
"mockInPlayground": true,
"mockTool": true,
"autoSubmitMockedResponse": true,
"parameters": {
"type": "object",
@ -325,7 +325,7 @@ You are an helpful customer support assistant
"orderId"
]
},
"mockInPlayground": true,
"mockTool": true,
"autoSubmitMockedResponse": true
}
],

4
apps/rowboat/app/lib/types/agents_api_types.ts
View file

@ -44,7 +44,7 @@ export const AgenticAPIAgent = WorkflowAgent
export const AgenticAPIPrompt = WorkflowPrompt;
export const AgenticAPITool = WorkflowTool.omit({
mockInPlayground: true,
mockTool: true,
autoSubmitMockedResponse: true,
});
@ -91,7 +91,7 @@ export function convertWorkflowToAgenticAPI(workflow: z.infer<typeof Workflow>):
return agenticAgent;
}),
tools: workflow.tools.map(tool => {
const { mockInPlayground, autoSubmitMockedResponse, ...rest } = tool;
const { mockTool, autoSubmitMockedResponse, ...rest } = tool;
return {
...rest,
};

1
apps/rowboat/app/lib/types/project_types.ts
View file

@ -11,7 +11,6 @@ export const Project = z.object({
publishedWorkflowId: z.string().optional(),
nextWorkflowNumber: z.number().optional(),
testRunCounter: z.number().default(0),
defaultTestProfileId: z.string().optional(),
});export const ProjectMember = z.object({
userId: z.string(),
projectId: z.string(),

2
apps/rowboat/app/lib/types/testing_types.ts
View file

@ -22,7 +22,7 @@ export const TestSimulation = z.object({
projectId: z.string(),
name: z.string().min(1, "Name cannot be empty"),
scenarioId: z.string(),
profileId: z.string(),
profileId: z.string().nullable(),
passCriteria: z.string(),
createdAt: z.string().datetime(),
lastUpdatedAt: z.string().datetime(),

3
apps/rowboat/app/lib/types/workflow_types.ts
View file

@ -33,8 +33,9 @@ export const WorkflowPrompt = z.object({
export const WorkflowTool = z.object({
name: z.string(),
description: z.string(),
mockInPlayground: z.boolean().default(false).optional(),
mockTool: z.boolean().default(false).optional(),
autoSubmitMockedResponse: z.boolean().default(false).optional(),
mockInstructions: z.string().optional(),
parameters: z.object({
type: z.literal('object'),
properties: z.record(z.object({

9
apps/rowboat/app/lib/utils.ts
View file

@ -221,7 +221,7 @@ export class PrefixLogger {
}
}
export async function mockToolResponse(toolId: string, messages: z.infer<typeof ApiMessage>[], testProfile: z.infer<typeof TestProfile>): Promise<string> {
export async function mockToolResponse(toolId: string, messages: z.infer<typeof ApiMessage>[], mockInstructions: string): Promise<string> {
const prompt = `Given below is a chat between a user and a customer support assistant.
The assistant has requested a tool call with ID {{toolID}}.
@ -234,10 +234,6 @@ and also some instructions on how to mock the tool call.
{{messages}}
<<<END_OF_CHAT_HISTORY
>>>CONTEXT
{{context}}
<<<END_OF_CONTEXT
>>>MOCK_INSTRUCTIONS
{{mockInstructions}}
<<<END_OF_MOCK_INSTRUCTIONS
@ -246,8 +242,7 @@ The current date is {{date}}.
`
.replace('{{toolID}}', toolId)
.replace(`{{date}}`, new Date().toISOString())
.replace('{{context}}', testProfile.context)
.replace('{{mockInstructions}}', testProfile.mockPrompt || '')
.replace('{{mockInstructions}}', mockInstructions)
.replace('{{messages}}', JSON.stringify(messages.map((m) => {
let tool_calls;
if ('tool_calls' in m && m.role == 'assistant') {

6
apps/rowboat/app/projects/[projectId]/playground/app.tsx
View file

@ -24,16 +24,14 @@ export function App({
projectId,
workflow,
messageSubscriber,
initialTestProfile,
}: {
hidden?: boolean;
projectId: string;
workflow: z.infer<typeof Workflow>;
messageSubscriber?: (messages: z.infer<typeof apiV1.ChatMessage>[]) => void;
initialTestProfile: z.infer<typeof TestProfile>;
}) {
const [counter, setCounter] = useState<number>(0);
const [testProfile, setTestProfile] = useState<z.infer<typeof TestProfile>>(initialTestProfile);
const [testProfile, setTestProfile] = useState<z.infer<typeof TestProfile> | null>(null);
const [chat, setChat] = useState<z.infer<typeof PlaygroundChat>>({
projectId,
createdAt: new Date().toISOString(),
@ -42,7 +40,7 @@ export function App({
systemMessage: defaultSystemMessage,
});
function handleTestProfileChange(profile: WithStringId<z.infer<typeof TestProfile>>) {
function handleTestProfileChange(profile: WithStringId<z.infer<typeof TestProfile>> | null) {
setTestProfile(profile);
setCounter(counter + 1);
}

18
apps/rowboat/app/projects/[projectId]/playground/chat.tsx
View file

@ -15,21 +15,22 @@ import { CopyAsJsonButton } from "./copy-as-json-button";
import { TestProfile } from "@/app/lib/types/testing_types";
import { ProfileSelector } from "@/app/lib/components/selectors/profile-selector";
import { WithStringId } from "@/app/lib/types/types";
import { XCircleIcon, XIcon } from "lucide-react";
export function Chat({
chat,
projectId,
workflow,
messageSubscriber,
testProfile,
testProfile=null,
onTestProfileChange,
}: {
chat: z.infer<typeof PlaygroundChat>;
projectId: string;
workflow: z.infer<typeof Workflow>;
messageSubscriber?: (messages: z.infer<typeof apiV1.ChatMessage>[]) => void;
testProfile: z.infer<typeof TestProfile>;
onTestProfileChange: (profile: WithStringId<z.infer<typeof TestProfile>>) => void;
testProfile?: z.infer<typeof TestProfile> | null;
onTestProfileChange: (profile: WithStringId<z.infer<typeof TestProfile>> | null) => void;
}) {
const [messages, setMessages] = useState<z.infer<typeof apiV1.ChatMessage>[]>(chat.messages);
const [loadingAssistantResponse, setLoadingAssistantResponse] = useState<boolean>(false);
@ -41,7 +42,7 @@ export function Chat({
const [fetchResponseError, setFetchResponseError] = useState<string | null>(null);
const [lastAgenticRequest, setLastAgenticRequest] = useState<unknown | null>(null);
const [lastAgenticResponse, setLastAgenticResponse] = useState<unknown | null>(null);
const [systemMessage, setSystemMessage] = useState<string | undefined>(testProfile.context);
const [systemMessage, setSystemMessage] = useState<string | undefined>(testProfile?.context);
const [isProfileSelectorOpen, setIsProfileSelectorOpen] = useState(false);
// collect published tool call results
@ -279,15 +280,20 @@ export function Chat({
return <div className="relative h-full flex flex-col gap-8 pt-8 overflow-auto">
<CopyAsJsonButton onCopy={handleCopyChat} />
<div className="absolute top-0 left-0">
<div className="absolute top-0 left-0 flex items-center gap-1">
<Tooltip content={"Change profile"} placement="right">
<button
className="border border-gray-200 dark:border-gray-800 p-2 rounded-lg text-xs hover:bg-gray-100 dark:hover:bg-gray-800"
onClick={() => setIsProfileSelectorOpen(true)}
>
{`Test profile: ${testProfile.name}`}
{`${testProfile?.name || 'Select test profile'}`}
</button>
</Tooltip>
{testProfile && <Tooltip content={"Remove profile"} placement="right">
<button className="text-gray-500 dark:text-gray-400 hover:text-gray-700 dark:hover:text-gray-300" onClick={() => onTestProfileChange(null)}>
<XIcon className="w-4 h-4" />
</button>
</Tooltip>}
</div>
<ProfileSelector
projectId={projectId}

61
apps/rowboat/app/projects/[projectId]/playground/messages.tsx
View file

@ -94,7 +94,8 @@ function ToolCalls({
messages,
sender,
workflow,
testProfile,
testProfile=null,
systemMessage,
}: {
toolCalls: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'];
results: Record<string, z.infer<typeof apiV1.ToolMessage>>;
@ -103,7 +104,8 @@ function ToolCalls({
messages: z.infer<typeof apiV1.ChatMessage>[];
sender: string | null | undefined;
workflow: z.infer<typeof Workflow>;
testProfile: z.infer<typeof TestProfile>;
testProfile: z.infer<typeof TestProfile> | null;
systemMessage: string | undefined;
}) {
const resultsMap: Record<string, z.infer<typeof apiV1.ToolMessage>> = {};
@ -127,6 +129,7 @@ function ToolCalls({
sender={sender}
workflow={workflow}
testProfile={testProfile}
systemMessage={systemMessage}
/>
})}
</div>;
@ -140,7 +143,8 @@ function ToolCall({
messages,
sender,
workflow,
testProfile,
testProfile=null,
systemMessage,
}: {
toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number];
result: z.infer<typeof apiV1.ToolMessage> | undefined;
@ -149,7 +153,8 @@ function ToolCall({
messages: z.infer<typeof apiV1.ChatMessage>[];
sender: string | null | undefined;
workflow: z.infer<typeof Workflow>;
testProfile: z.infer<typeof TestProfile>;
testProfile: z.infer<typeof TestProfile> | null;
systemMessage: string | undefined;
}) {
let matchingWorkflowTool: z.infer<typeof WorkflowTool> | undefined;
for (const tool of workflow.tools) {
@ -181,7 +186,7 @@ function ToolCall({
sender={sender}
/>;
}
if (matchingWorkflowTool && !testProfile.mockTools) {
if (matchingWorkflowTool && testProfile && !testProfile.mockTools) {
return <ClientToolCall
toolCall={toolCall}
result={result}
@ -198,8 +203,9 @@ function ToolCall({
projectId={projectId}
messages={messages}
sender={sender}
autoSubmit={matchingWorkflowTool?.autoSubmitMockedResponse}
testProfile={testProfile}
workflowTool={matchingWorkflowTool}
systemMessage={systemMessage}
/>;
}
}
@ -413,8 +419,9 @@ function MockToolCall({
projectId,
messages,
sender,
autoSubmit = false,
testProfile,
testProfile=null,
workflowTool,
systemMessage,
}: {
toolCall: z.infer<typeof apiV1.AssistantMessageWithToolCalls>['tool_calls'][number];
result: z.infer<typeof apiV1.ToolMessage> | undefined;
@ -422,8 +429,9 @@ function MockToolCall({
projectId: string;
messages: z.infer<typeof apiV1.ChatMessage>[];
sender: string | null | undefined;
autoSubmit?: boolean;
testProfile: z.infer<typeof TestProfile>;
testProfile: z.infer<typeof TestProfile> | null;
workflowTool: z.infer<typeof WorkflowTool> | undefined;
systemMessage: string | undefined;
}) {
const [result, setResult] = useState<z.infer<typeof apiV1.ToolMessage> | undefined>(availableResult);
const [response, setResponse] = useState('');
@ -459,7 +467,18 @@ function MockToolCall({
async function process() {
setGeneratingResponse(true);
const response = await suggestToolResponse(toolCall.id, projectId, messages, testProfile);
const response = await suggestToolResponse(
toolCall.id,
projectId,
[{
role: 'system',
content: systemMessage || '',
createdAt: new Date().toISOString(),
version: 'v1',
chatId: '',
}, ...messages],
testProfile?.mockPrompt || workflowTool?.mockInstructions || '',
);
if (ignore) {
return;
}
@ -471,11 +490,11 @@ function MockToolCall({
return () => {
ignore = true;
};
}, [result, response, toolCall.id, projectId, messages, testProfile]);
}, [result, response, toolCall.id, projectId, messages, testProfile, systemMessage, workflowTool?.mockInstructions]);
// auto submit if autoSubmitMockedResponse is true
useEffect(() => {
if (!autoSubmit) {
if (!workflowTool?.autoSubmitMockedResponse) {
return;
}
if (result) {
@ -484,13 +503,14 @@ function MockToolCall({
if (response) {
handleSubmit();
}
}, [autoSubmit, response, handleSubmit, result]);
}, [workflowTool?.autoSubmitMockedResponse, response, handleSubmit, result]);
return <div className="flex flex-col gap-1">
{sender && <div className='text-gray-500 dark:text-gray-400 text-xs ml-3'>{sender}</div>}
<div className='border border-gray-300 dark:border-gray-700 p-2 pt-2 rounded-lg rounded-bl-none flex flex-col gap-2 mr-[30%] bg-white dark:bg-gray-900'>
<div className="flex items-center gap-2">
<CircleCheckIcon size={16} className="text-gray-500 dark:text-gray-400" />
{!result && <Spinner size="sm" />}
{result && <CircleCheckIcon size={16} className="text-gray-500 dark:text-gray-400" />}
<span className="text-sm text-gray-700 dark:text-gray-300">
Function Call: <code className='bg-gray-100 dark:bg-neutral-800 px-2 py-0.5 rounded font-mono'>{toolCall.function.name}</code>
</span>
@ -501,7 +521,7 @@ function MockToolCall({
{result && <ExpandableContent label='Result' content={result.content} expanded={false} />}
</div>
{!result && !autoSubmit && <div className='flex flex-col gap-2 mt-2'>
{!result && !workflowTool?.autoSubmitMockedResponse && <div className='flex flex-col gap-2 mt-2'>
<div>Response:</div>
<Textarea
maxRows={10}
@ -603,7 +623,7 @@ export function Messages({
loadingAssistantResponse,
loadingUserResponse,
workflow,
testProfile,
testProfile=null,
onSystemMessageChange,
}: {
projectId: string;
@ -614,7 +634,7 @@ export function Messages({
loadingAssistantResponse: boolean;
loadingUserResponse: boolean;
workflow: z.infer<typeof Workflow>;
testProfile: z.infer<typeof TestProfile>;
testProfile: z.infer<typeof TestProfile> | null;
onSystemMessageChange: (message: string) => void;
}) {
const messagesEndRef = useRef<HTMLDivElement>(null);
@ -628,9 +648,9 @@ export function Messages({
return <div className="grow pt-4 overflow-auto">
<div className="max-w-[768px] mx-auto flex flex-col gap-8">
<SystemMessage
content={testProfile.context}
content={testProfile?.context || systemMessage || ''}
onChange={onSystemMessageChange}
locked={true}
locked={testProfile !== null || messages.length > 0}
/>
{messages.map((message, index) => {
if (message.role === 'assistant') {
@ -645,6 +665,7 @@ export function Messages({
sender={message.agenticSender}
workflow={workflow}
testProfile={testProfile}
systemMessage={systemMessage}
/>;
} else {
// the assistant message createdAt is an ISO string timestamp

49
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/profiles_app.tsx
View file

@ -2,7 +2,7 @@ import Link from "next/link";
import { WithStringId } from "@/app/lib/types/types";
import { TestProfile } from "@/app/lib/types/testing_types";
import { useEffect, useState, useRef } from "react";
import { createProfile, getProfile, listProfiles, updateProfile, deleteProfile, setDefaultProfile } from "@/app/actions/testing_actions";
import { createProfile, getProfile, listProfiles, updateProfile, deleteProfile } from "@/app/actions/testing_actions";
import { Button, Input, Pagination, Spinner, Switch, Textarea, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter, Tooltip } from "@heroui/react";
import { useRouter, useSearchParams } from "next/navigation";
import { z } from "zod";
@ -134,7 +134,6 @@ function ViewProfile({
}) {
const router = useRouter();
const [profile, setProfile] = useState<WithStringId<z.infer<typeof TestProfile>> | null>(null);
const [defaultProfileId, setDefaultProfileId] = useState<string | null>(null);
const [loading, setLoading] = useState(true);
const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
const [deleteError, setDeleteError] = useState<string | null>(null);
@ -142,9 +141,7 @@ function ViewProfile({
useEffect(() => {
async function fetchProfile() {
const profile = await getProfile(projectId, profileId);
const projectConfig = await getProjectConfig(projectId);
setProfile(profile);
setDefaultProfileId(projectConfig.defaultTestProfileId || null);
setLoading(false);
}
fetchProfile();
@ -159,11 +156,6 @@ function ViewProfile({
}
}
async function handleSetDefault() {
await setDefaultProfile(projectId, profileId);
router.push(`/projects/${projectId}/test/profiles`);
}
return <div className="h-full flex flex-col gap-2">
<h1 className="text-medium font-bold text-gray-800 pb-2 border-b border-gray-200">View Profile</h1>
<Button
@ -186,13 +178,7 @@ function ViewProfile({
<div className="flex border-b py-2">
<div className="flex-[1] font-medium text-gray-600">Name</div>
<div className="flex-[2]">
<div className="flex flex-col gap-2">
<div>{profile.name}</div>
{defaultProfileId === profile._id && <div className="flex items-center gap-2">
<StarIcon className="w-4 h-4" />
<div className="text-gray-600">This is the default profile</div>
</div>}
</div>
<div className="flex-[2] whitespace-pre-wrap">{profile.name}</div>
</div>
</div>
<div className="flex border-b py-2">
@ -224,21 +210,14 @@ function ViewProfile({
>
Edit
</Button>
{defaultProfileId !== profile._id && <Button
<Button
size="sm"
color="danger"
variant="flat"
onClick={() => setIsDeleteModalOpen(true)}
>
Delete
</Button>}
{defaultProfileId !== profile._id && <Button
size="sm"
onClick={() => handleSetDefault()}
startContent={<StarIcon className="w-4 h-4" />}
>
Set as default profile
</Button>}
</Button>
</div>
<Modal
@ -399,7 +378,6 @@ function ProfileList({
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [profiles, setProfiles] = useState<WithStringId<z.infer<typeof TestProfile>>[]>([]);
const [defaultProfileId, setDefaultProfileId] = useState<string | null>(null);
const [total, setTotal] = useState(0);
useEffect(() => {
@ -410,11 +388,9 @@ function ProfileList({
setError(null);
try {
const profiles = await listProfiles(projectId, page, pageSize);
const projectConfig = await getProjectConfig(projectId);
if (!ignore) {
setProfiles(profiles.profiles);
setTotal(Math.ceil(profiles.total / pageSize));
setDefaultProfileId(projectConfig.defaultTestProfileId || null);
}
} catch (error) {
if (!ignore) {
@ -470,17 +446,12 @@ function ProfileList({
{profiles.map((profile) => (
<div key={profile._id} className="grid grid-cols-8 py-2 border-b hover:bg-gray-50 text-sm">
<div className="col-span-2 px-4 truncate">
<div className="flex items-center gap-2">
<Link
href={`/projects/${projectId}/test/profiles/${profile._id}`}
className="text-blue-600 hover:underline"
>
{profile.name}
</Link>
{defaultProfileId === profile._id && <Tooltip content="Default Profile">
<StarIcon className="w-4 h-4" />
</Tooltip>}
</div>
<Link
href={`/projects/${projectId}/test/profiles/${profile._id}`}
className="text-blue-600 hover:underline"
>
{profile.name}
</Link>
</div>
<div className="col-span-3 px-4 truncate">{profile.context}</div>
<div className="col-span-1 px-4">{profile.mockTools ? "Yes" : "No"}</div>

35
apps/rowboat/app/projects/[projectId]/test/[[...slug]]/simulations_app.tsx
View file

@ -38,7 +38,7 @@ function EditSimulation({
if (simulation) {
const [scenarioResult, profileResult] = await Promise.all([
getScenario(projectId, simulation.scenarioId),
getProfile(projectId, simulation.profileId),
simulation.profileId ? getProfile(projectId, simulation.profileId) : Promise.resolve(null),
]);
setScenario(scenarioResult);
setProfile(profileResult);
@ -62,14 +62,14 @@ function EditSimulation({
throw new Error("Name and Pass Criteria are required");
}
if (!scenario || !profile) {
throw new Error("Please select all required fields");
if (!scenario) {
throw new Error("Please select a scenario");
}
await updateSimulation(projectId, simulationId, {
name,
scenarioId: scenario._id,
profileId: profile._id,
profileId: profile?._id || null,
passCriteria
});
router.push(`/projects/${projectId}/test/simulations/${simulationId}`);
@ -124,13 +124,14 @@ function EditSimulation({
</div>
</div>
<div className="flex flex-col gap-2">
<label className="text-sm font-medium">Profile</label>
<label className="text-sm font-medium">Profile (optional)</label>
<div className="flex items-center gap-2">
{profile ? (
<div className="text-sm text-blue-600">{profile.name}</div>
) : (
<div className="text-sm text-gray-500">No profile selected</div>
)}
{profile && <Button size="sm" variant="bordered" onClick={() => setProfile(null)}>Remove</Button>}
<Button
size="sm"
onClick={() => setIsProfileModalOpen(true)}
@ -147,7 +148,7 @@ function EditSimulation({
children: "Update",
size: "sm",
type: "submit",
isDisabled: !scenario || !profile,
isDisabled: !scenario,
}}
/>
<Button
@ -199,7 +200,7 @@ function ViewSimulation({
if (simulation) {
const [scenarioResult, profileResult] = await Promise.all([
getScenario(projectId, simulation.scenarioId),
getProfile(projectId, simulation.profileId),
simulation.profileId ? getProfile(projectId, simulation.profileId) : Promise.resolve(null),
]);
setScenario(scenarioResult);
setProfile(profileResult);
@ -384,14 +385,11 @@ function NewSimulation({
if (!scenario) {
throw new Error("Please select a scenario");
}
if (!profile) {
throw new Error("Please select a profile");
}
const result = await createSimulation(projectId, {
name,
scenarioId: scenario._id,
profileId: profile._id,
profileId: profile?._id || null,
passCriteria,
});
router.push(`/projects/${projectId}/test/simulations/${result._id}`);
@ -448,13 +446,14 @@ function NewSimulation({
</div>
</div>
<div className="flex flex-col gap-2">
<label className="text-sm font-medium">Profile</label>
<label className="text-sm font-medium">Profile (optional)</label>
<div className="flex items-center gap-2">
{profile ? (
<div className="text-sm text-blue-600">{profile.name}</div>
) : (
<div className="text-sm text-gray-500">No profile selected</div>
)}
{profile && <Button size="sm" variant="bordered" onClick={() => setProfile(null)}>Remove</Button>}
<Button
size="sm"
onClick={() => setIsProfileModalOpen(true)}
@ -470,7 +469,7 @@ function NewSimulation({
children: "Create",
size: "sm",
type: "submit",
isDisabled: !scenario || !profile,
isDisabled: !scenario,
}}
/>
</form>
@ -559,7 +558,7 @@ function SimulationList({
}
async function resolveProfiles() {
const profileIds = simulationList.reduce((acc, simulation) => {
if (!acc.includes(simulation.profileId)) {
if (simulation.profileId && !acc.includes(simulation.profileId)) {
acc.push(simulation.profileId);
}
return acc;
@ -632,8 +631,12 @@ function SimulationList({
)}
</div>
<div className="col-span-1 px-4 truncate">
{profileMap[simulation.profileId]?.name || (
<span className="text-gray-500 font-mono text-xs">{simulation.profileId}</span>
{simulation.profileId ? (
profileMap[simulation.profileId]?.name || (
<span className="text-gray-500 font-mono text-xs">{simulation.profileId}</span>
)
) : (
<span className="text-gray-500 font-mono text-xs">None</span>
)}
</div>
<div className="col-span-1 px-4 truncate">

11
apps/rowboat/app/projects/[projectId]/workflow/app.tsx
View file

@ -10,7 +10,6 @@ import { Spinner } from "@heroui/react";
import { cloneWorkflow, createWorkflow, fetchPublishedWorkflowId, fetchWorkflow } from "../../../actions/workflow_actions";
import { listDataSources } from "../../../actions/datasource_actions";
import { TestProfile } from "@/app/lib/types/testing_types";
import { getDefaultProfile } from "../../../actions/testing_actions";
export function App({
projectId,
@ -21,7 +20,6 @@ export function App({
const [workflow, setWorkflow] = useState<WithStringId<z.infer<typeof Workflow>> | null>(null);
const [publishedWorkflowId, setPublishedWorkflowId] = useState<string | null>(null);
const [dataSources, setDataSources] = useState<WithStringId<z.infer<typeof DataSource>>[] | null>(null);
const [defaultTestProfile, setDefaultTestProfile] = useState<z.infer<typeof TestProfile> | null>(null);
const [loading, setLoading] = useState(false);
const [autoSelectIfOnlyOneWorkflow, setAutoSelectIfOnlyOneWorkflow] = useState(true);
@ -30,13 +28,11 @@ export function App({
const workflow = await fetchWorkflow(projectId, workflowId);
const publishedWorkflowId = await fetchPublishedWorkflowId(projectId);
const dataSources = await listDataSources(projectId);
const defaultTestProfile = await getDefaultProfile(projectId);
// Store the selected workflow ID in local storage
localStorage.setItem(`lastWorkflowId_${projectId}`, workflowId);
setWorkflow(workflow);
setPublishedWorkflowId(publishedWorkflowId);
setDataSources(dataSources);
setDefaultTestProfile(defaultTestProfile);
setLoading(false);
}, [projectId]);
@ -52,13 +48,11 @@ export function App({
const workflow = await createWorkflow(projectId);
const publishedWorkflowId = await fetchPublishedWorkflowId(projectId);
const dataSources = await listDataSources(projectId);
const testProfile = await getDefaultProfile(projectId);
// Store the selected workflow ID in local storage
localStorage.setItem(`lastWorkflowId_${projectId}`, workflow._id);
setWorkflow(workflow);
setPublishedWorkflowId(publishedWorkflowId);
setDataSources(dataSources);
setDefaultTestProfile(testProfile);
setLoading(false);
}
@ -67,13 +61,11 @@ export function App({
const workflow = await cloneWorkflow(projectId, workflowId);
const publishedWorkflowId = await fetchPublishedWorkflowId(projectId);
const dataSources = await listDataSources(projectId);
const testProfile = await getDefaultProfile(projectId);
// Store the selected workflow ID in local storage
localStorage.setItem(`lastWorkflowId_${projectId}`, workflow._id);
setWorkflow(workflow);
setPublishedWorkflowId(publishedWorkflowId);
setDataSources(dataSources);
setDefaultTestProfile(testProfile);
setLoading(false);
}
@ -107,11 +99,10 @@ export function App({
handleCreateNewVersion={handleCreateNewVersion}
autoSelectIfOnlyOneWorkflow={autoSelectIfOnlyOneWorkflow}
/>}
{!loading && workflow && (dataSources !== null) && (defaultTestProfile !== null) && <WorkflowEditor
{!loading && workflow && (dataSources !== null) && <WorkflowEditor
key={workflow._id}
workflow={workflow}
dataSources={dataSources}
initialTestProfile={defaultTestProfile}
publishedWorkflowId={publishedWorkflowId}
handleShowSelector={handleShowSelector}
handleCloneVersion={handleCloneVersion}

23
apps/rowboat/app/projects/[projectId]/workflow/tool_config.tsx
View file

@ -230,10 +230,10 @@ export function ToolConfig({
<div className="ml-4 flex flex-col gap-2">
<RadioGroup
defaultValue="mock"
value={tool.mockInPlayground ? "mock" : "api"}
value={tool.mockTool ? "mock" : "api"}
onValueChange={(value) => handleUpdate({
...tool,
mockInPlayground: value === "mock",
mockTool: value === "mock",
autoSubmitMockedResponse: value === "mock" ? true : undefined
})}
orientation="horizontal"
@ -264,7 +264,7 @@ export function ToolConfig({
</Radio>
</RadioGroup>
{tool.mockInPlayground && (
{tool.mockTool && <>
<div className="ml-0">
<Checkbox
key="autoSubmitMockedResponse"
@ -281,9 +281,22 @@ export function ToolConfig({
Auto-submit mocked response in playground
</Checkbox>
</div>
)}
{!tool.mockInPlayground && (
<Divider />
<EditableField
label="Mock instructions"
value={tool.mockInstructions || ''}
onChange={(value) => handleUpdate({
...tool,
mockInstructions: value
})}
placeholder="Enter mock instructions..."
multiline
/>
</>}
{!tool.mockTool && (
<div className="ml-0 text-danger text-xs">
Please configure your webhook in the <strong>Integrate</strong> page if you haven&apos;t already.
</div>

5
apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx
View file

@ -285,7 +285,7 @@ function reducer(state: State, action: Action): State {
type: "object",
properties: {},
},
mockInPlayground: true,
mockTool: true,
autoSubmitMockedResponse: true,
...action.tool
});
@ -534,14 +534,12 @@ export function WorkflowEditor({
publishedWorkflowId,
handleShowSelector,
handleCloneVersion,
initialTestProfile,
}: {
dataSources: WithStringId<z.infer<typeof DataSource>>[];
workflow: WithStringId<z.infer<typeof Workflow>>;
publishedWorkflowId: string | null;
handleShowSelector: () => void;
handleCloneVersion: (workflowId: string) => void;
initialTestProfile: z.infer<typeof TestProfile>;
}) {
const [state, dispatch] = useReducer<Reducer<State, Action>>(reducer, {
patches: [],
@ -862,7 +860,6 @@ export function WorkflowEditor({
projectId={state.present.workflow.projectId}
workflow={state.present.workflow}
messageSubscriber={updateChatMessages}
initialTestProfile={initialTestProfile}
/>
{state.present.selection?.type === "agent" && <AgentConfig
key={state.present.selection.name}