diff --git a/apps/rowboat/app/actions/simulation_actions.ts b/apps/rowboat/app/actions/simulation_actions.ts index 4dcd1de8..2b1e939d 100644 --- a/apps/rowboat/app/actions/simulation_actions.ts +++ b/apps/rowboat/app/actions/simulation_actions.ts @@ -1,11 +1,11 @@ 'use server'; import { ObjectId } from "mongodb"; -import { scenariosCollection } from "../lib/mongodb"; +import { scenariosCollection, simulationRunsCollection, simulationResultsCollection } from "../lib/mongodb"; import { z } from 'zod'; import { projectAuthCheck } from "./project_actions"; import { type WithStringId } from "../lib/types/types"; -import { Scenario } from "../lib/types/testing_types"; +import { Scenario, SimulationRun, SimulationResult } from "../lib/types/testing_types"; import { SimulationScenarioData } from "../lib/types/testing_types"; export async function getScenarios(projectId: string): Promise>[]> { @@ -83,4 +83,124 @@ export async function deleteScenario(projectId: string, scenarioId: string): Pro _id: new ObjectId(scenarioId), projectId, }); +} + +export async function getRuns(projectId: string): Promise>[]> { + await projectAuthCheck(projectId); + + const runs = await simulationRunsCollection + .find({ projectId }) + .sort({ startedAt: -1 }) // Most recent first + .toArray(); + + return runs.map(run => ({ + ...run, + _id: run._id.toString(), + })); +} + +export async function getRun(projectId: string, runId: string): Promise>> { + await projectAuthCheck(projectId); + + const run = await simulationRunsCollection.findOne({ + _id: new ObjectId(runId), + projectId, + }); + + if (!run) { + throw new Error('Run not found'); + } + + return { + ...run, + _id: run._id.toString(), + }; +} + +export async function createRun( + projectId: string, + scenarioIds: string[] +): Promise>> { + await projectAuthCheck(projectId); + + const run = { + projectId, + status: 'pending' as const, + scenarioIds, + startedAt: new Date().toISOString(), + }; + + const result = await simulationRunsCollection.insertOne(run); + + return { + ...run, + _id: result.insertedId.toString(), + }; +} + +export async function updateRunStatus( + projectId: string, + runId: string, + status: z.infer['status'], + completedAt?: string +): Promise { + await projectAuthCheck(projectId); + + const updateData: Partial> = { + status, + }; + + if (completedAt) { + updateData.completedAt = completedAt; + } + + await simulationRunsCollection.updateOne( + { + _id: new ObjectId(runId), + projectId, + }, + { + $set: updateData, + } + ); +} + +export async function getRunResults( + projectId: string, + runId: string +): Promise>[]> { + await projectAuthCheck(projectId); + + const results = await simulationResultsCollection + .find({ + runId, + projectId, + }) + .toArray(); + + return results.map(result => ({ + ...result, + _id: result._id.toString(), + })); +} + +export async function createRunResult( + projectId: string, + runId: string, + scenarioId: string, + result: z.infer['result'], + details: string +): Promise { + await projectAuthCheck(projectId); + + const resultDoc = { + projectId, + runId, + scenarioId, + result, + details, + }; + + const insertResult = await simulationResultsCollection.insertOne(resultDoc); + return insertResult.insertedId.toString(); } \ No newline at end of file diff --git a/apps/rowboat/app/lib/types/testing_types.ts b/apps/rowboat/app/lib/types/testing_types.ts index 16e6b129..cf5d20ad 100644 --- a/apps/rowboat/app/lib/types/testing_types.ts +++ b/apps/rowboat/app/lib/types/testing_types.ts @@ -35,6 +35,8 @@ export const SimulationRun = z.object({ }); export const SimulationResult = z.object({ + projectId: z.string(), + runId: z.string(), scenarioId: z.string(), result: z.union([z.literal('pass'), z.literal('fail')]), details: z.string() diff --git a/apps/rowboat/app/projects/[projectId]/simulation/app.tsx b/apps/rowboat/app/projects/[projectId]/simulation/app.tsx index a9384503..96b8ea28 100644 --- a/apps/rowboat/app/projects/[projectId]/simulation/app.tsx +++ b/apps/rowboat/app/projects/[projectId]/simulation/app.tsx @@ -1,49 +1,301 @@ 'use client'; import { useState, useEffect, useCallback } from 'react'; -import { PlusIcon, PencilIcon, XMarkIcon, DocumentDuplicateIcon, EllipsisVerticalIcon, TrashIcon, ChevronRightIcon, PlayIcon } from '@heroicons/react/24/outline'; +import { PlusIcon, PencilIcon, XMarkIcon, DocumentDuplicateIcon, EllipsisVerticalIcon, TrashIcon, ChevronRightIcon, PlayIcon, ChevronDownIcon } from '@heroicons/react/24/outline'; import { useParams, useRouter } from 'next/navigation'; import { getScenarios, createScenario, updateScenario, deleteScenario, + getRuns, + getRun, + getRunResults, + createRun, + createRunResult, + updateRunStatus, } from '../../../actions/simulation_actions'; import { type WithStringId } from '../../../lib/types/types'; -import { Scenario } from "../../../lib/types/testing_types"; +import { Scenario, SimulationRun, SimulationResult } from "../../../lib/types/testing_types"; import { z } from 'zod'; type ScenarioType = WithStringId>; - -type SimulationResult = { - scenarioId: string; - scenarioName: string; - passed: boolean; - details: string; - scenario: ScenarioType; -}; +type SimulationRunType = WithStringId>; +type SimulationResultType = WithStringId>; type SimulationReport = { totalScenarios: number; passedScenarios: number; failedScenarios: number; - results: SimulationResult[]; + results: z.infer[]; timestamp: Date; }; -const dummySimulator = async (scenario: ScenarioType): Promise => { +const dummySimulator = async (scenario: ScenarioType, runId: string, projectId: string): Promise> => { await new Promise(resolve => setTimeout(resolve, 500)); const passed = Math.random() > 0.5; - return { + // Create the result object with explicitly typed result + const result: z.infer = { + projectId: projectId, + runId: runId, scenarioId: scenario._id, - scenarioName: scenario.name, - passed, + result: passed ? 'pass' : 'fail' as const, // explicitly type as literal details: passed ? "The bot successfully completed the conversation" : "The bot could not handle the conversation", - scenario: scenario, }; + + // Write to DB using server action + await createRunResult( + projectId, + runId, + scenario._id, + result.result, + result.details + ); + + return result; +}; + +interface SimulationResultCardProps { + run: SimulationRunType; + results: SimulationResultType[]; + scenarios: ScenarioType[]; +} + +interface ScenarioResultCardProps { + scenario: ScenarioType; + result?: SimulationResultType; +} + +const SimulationResultCard = ({ run, results, scenarios }: SimulationResultCardProps) => { + const [isExpanded, setIsExpanded] = useState(false); + const [expandedScenarios, setExpandedScenarios] = useState>(new Set()); + + const statusLabelClass = "px-3 py-1 rounded text-xs min-w-[60px] text-center uppercase font-semibold"; + + const formatMainTitle = (date: string) => { + return `Run from ${new Date(date).toLocaleString('en-US', { + month: 'numeric', + day: 'numeric', + year: 'numeric', + hour: 'numeric', + minute: '2-digit', + second: '2-digit', + hour12: true + })}`; + }; + + const formatDateTime = (date: string) => { + return new Date(date).toLocaleString('en-US', { + month: 'numeric', + day: 'numeric', + year: 'numeric', + hour: 'numeric', + minute: '2-digit', + second: '2-digit', + hour12: true + }); + }; + + // Calculate statistics and duration + const totalScenarios = run.scenarioIds.length; + const passedScenarios = results.filter(r => r.result === 'pass').length; + const failedScenarios = results.filter(r => r.result === 'fail').length; + + const getDuration = () => { + if (!run.completedAt) return 'In Progress'; + const start = new Date(run.startedAt); + const end = new Date(run.completedAt); + const diff = end.getTime() - start.getTime(); + return `${(diff / 1000).toFixed(1)}s`; + }; + + const toggleScenario = (scenarioId: string, e: React.MouseEvent) => { + e.stopPropagation(); // Prevent triggering parent's onClick + setExpandedScenarios(prev => { + const newSet = new Set(prev); + if (newSet.has(scenarioId)) { + newSet.delete(scenarioId); + } else { + newSet.add(scenarioId); + } + return newSet; + }); + }; + + return ( +
+
setIsExpanded(!isExpanded)} + > +
+ {isExpanded ? ( + + ) : ( + + )} +
+ {formatMainTitle(run.startedAt)} +
+
+ + {run.status} + +
+ + {isExpanded && ( +
+ {/* Simplified timing information */} +
+
+ Completed: + {run.completedAt ? formatDateTime(run.completedAt) : 'Not completed'} +
+
+ Duration: + {getDuration()} +
+
+ + {/* Results statistics */} +
+
+
Total Scenarios
+
{totalScenarios}
+
+
+
Passed
+
{passedScenarios}
+
+
+
Failed
+
{failedScenarios}
+
+
+ +
+ {run.scenarioIds.map(scenarioId => { + const scenario = scenarios.find(s => s._id === scenarioId); + const result = results.find(r => r.scenarioId === scenarioId); + const isScenarioExpanded = expandedScenarios.has(scenarioId); + + return scenario && ( +
+
toggleScenario(scenarioId, e)} + > +
+ {isScenarioExpanded ? ( + + ) : ( + + )} + {scenario.name} +
+ {result && ( + + {result.result} + + )} +
+ + {isScenarioExpanded && ( +
+
+
Description
+
+ {scenario.description} +
+
+
+
Context
+
+ {scenario.context || 'No context provided'} +
+
+ {result && ( +
+
Result Details
+
+ {result.details} +
+
+ )} +
+ )} +
+ ); + })} +
+
+ )} +
+ ); +}; + +const ScenarioResultCard = ({ scenario, result }: ScenarioResultCardProps) => { + const [isExpanded, setIsExpanded] = useState(false); + + return ( +
+
setIsExpanded(!isExpanded)} + > +
+ {isExpanded ? ( + + ) : ( + + )} + {scenario.name} +
+ {result && ( + + {result.result} + + )} +
+ + {isExpanded && ( +
+
+
Description
+
{scenario.description}
+
+
+
Context
+
{scenario.context || 'No context provided'}
+
+ {result && ( +
+
Result Details
+
{result.details}
+
+ )} +
+ )} +
+ ); }; export default function SimulationApp() { @@ -56,6 +308,11 @@ export default function SimulationApp() { const [isRunning, setIsRunning] = useState(false); const [simulationReport, setSimulationReport] = useState(null); const [expandedResults, setExpandedResults] = useState>(new Set()); + const [runs, setRuns] = useState([]); + const [activeRun, setActiveRun] = useState(null); + const [runResults, setRunResults] = useState([]); + const [isLoadingRuns, setIsLoadingRuns] = useState(true); + const [allRunResults, setAllRunResults] = useState>({}); // Load scenarios on mount useEffect(() => { @@ -71,6 +328,61 @@ export default function SimulationApp() { } }, [menuOpenScenarioId]); + // Modify the fetchRuns function to also fetch results + const fetchRuns = useCallback(async () => { + if (!projectId) return; + setIsLoadingRuns(true); + try { + const runsData = await getRuns(projectId as string); + setRuns(runsData); + + // Fetch results for all runs + const resultsPromises = runsData.map(run => + getRunResults(projectId as string, run._id) + ); + const allResults = await Promise.all(resultsPromises); + + // Create a map of run ID to results + const resultsMap = runsData.reduce((acc, run, index) => ({ + ...acc, + [run._id]: allResults[index] + }), {}); + + setAllRunResults(resultsMap); + } catch (error) { + console.error('Error fetching runs:', error); + } finally { + setIsLoadingRuns(false); + } + }, [projectId]); + + // Update the useEffect hooks to include fetchRuns + useEffect(() => { + if (!projectId) return; + fetchRuns(); + }, [projectId, fetchRuns]); + + useEffect(() => { + if (!projectId || !activeRun || activeRun.status === 'completed' || activeRun.status === 'cancelled') return; + + const interval = setInterval(async () => { + try { + const updatedRun = await getRun(projectId as string, activeRun._id); + setActiveRun(updatedRun); + + if (updatedRun.status === 'completed') { + const results = await getRunResults(projectId as string, activeRun._id); + setRunResults(results); + fetchRuns(); // Refresh the runs list + } + } catch (error) { + console.error('Error polling run status:', error); + } + }, 2000); + + return () => clearInterval(interval); + }, [activeRun, projectId, fetchRuns]); + const createNewScenario = async () => { if (!projectId) return; const newScenarioId = await createScenario( @@ -127,31 +439,52 @@ export default function SimulationApp() { }; const runAllScenarios = async () => { + if (!projectId) return; setIsRunning(true); setSimulationReport(null); try { - const results: SimulationResult[] = []; - - // Run each scenario through the simulator - for (const scenario of scenarios) { - const result = await dummySimulator(scenario); - results.push(result); + // Create a new run using server action + const newRun = await createRun( + projectId as string, + scenarios.map(s => s._id) + ); + setActiveRun(newRun); + + // Check for the NEXT_PUBLIC_ prefixed env variable + if (process.env.NEXT_PUBLIC_MOCK_SIMULATION_RESULTS === 'true') { + console.log('Using mock simulation...'); // Debug log + + // First update run to 'running' status + await updateRunStatus(projectId as string, newRun._id, 'running'); + + // Generate and save all mock results + await Promise.all( + scenarios.map(scenario => + dummySimulator(scenario, newRun._id, projectId as string) + ) + ); + + // Update run status to completed + await updateRunStatus( + projectId as string, + newRun._id, + 'completed', + new Date().toISOString() + ); + + // Fetch the results back from DB to ensure consistency + const results = await getRunResults(projectId as string, newRun._id); + setRunResults(results); + + // Refresh the run to get its updated state + const updatedRun = await getRun(projectId as string, newRun._id); + setActiveRun(updatedRun); } - - // Generate report - const passedScenarios = results.filter(r => r.passed).length; - const report: SimulationReport = { - totalScenarios: scenarios.length, - passedScenarios, - failedScenarios: scenarios.length - passedScenarios, - results, - timestamp: new Date(), - }; - - setSimulationReport(report); + + await fetchRuns(); } catch (error) { - console.error('Error running scenarios:', error); + console.error('Error starting scenarios:', error); } finally { setIsRunning(false); } @@ -258,139 +591,36 @@ export default function SimulationApp() { ) ) : (
- {simulationReport ? ( - <> -
-
-

Simulation Results

-
- Run on {simulationReport.timestamp.toLocaleString()} -
-
- -
+
+

Simulation Runs

+ +
-
-
-
Total Scenarios
-
{simulationReport.totalScenarios}
-
-
-
Passed
-
- {simulationReport.passedScenarios} -
-
-
-
Failed
-
- {simulationReport.failedScenarios} -
-
-
- -
-

Detailed Results

-
- {simulationReport.results.map((result) => ( -
-
{ - const newExpandedResults = new Set(expandedResults); - if (expandedResults.has(result.scenarioId)) { - newExpandedResults.delete(result.scenarioId); - } else { - newExpandedResults.add(result.scenarioId); - } - setExpandedResults(newExpandedResults); - }} - > -
- - {result.scenarioName} -
-
- {result.passed ? 'Passed' : 'Failed'} -
-
- - {expandedResults.has(result.scenarioId) && ( -
-
-
-
- Name -
-
{result.scenario.name}
-
- -
-
- Description -
-
{result.scenario.description}
-
-
- -
-
- Details -
-
{result.details}
-
-
- )} -
- ))} -
-
- + {isLoadingRuns ? ( +
Loading runs...
+ ) : runs.length === 0 ? ( +
No simulation runs yet
) : ( - <> -
-

Scenarios

- +
+ {runs.map((run) => ( + + ))}
-
- Select a scenario or run all scenarios -
- )}
)} diff --git a/apps/rowboat/package-lock.json b/apps/rowboat/package-lock.json index 2f4b843e..32b815d2 100644 --- a/apps/rowboat/package-lock.json +++ b/apps/rowboat/package-lock.json @@ -13,7 +13,7 @@ "@aws-sdk/client-s3": "^3.743.0", "@aws-sdk/s3-request-presigner": "^3.743.0", "@google/generative-ai": "^0.21.0", - "@heroicons/react": "^2.1.1", + "@heroicons/react": "^2.2.0", "@langchain/core": "^0.3.7", "@langchain/textsplitters": "^0.1.0", "@mendable/firecrawl-js": "^1.0.3", diff --git a/apps/rowboat/package.json b/apps/rowboat/package.json index 9ddba6df..87f720f7 100644 --- a/apps/rowboat/package.json +++ b/apps/rowboat/package.json @@ -18,7 +18,7 @@ "@aws-sdk/client-s3": "^3.743.0", "@aws-sdk/s3-request-presigner": "^3.743.0", "@google/generative-ai": "^0.21.0", - "@heroicons/react": "^2.1.1", + "@heroicons/react": "^2.2.0", "@langchain/core": "^0.3.7", "@langchain/textsplitters": "^0.1.0", "@mendable/firecrawl-js": "^1.0.3",