Add aggregate result to db

apps/rowboat/app/actions/simulation_actions.ts

@@ -1,11 +1,11 @@
 'use server';
 
 import { ObjectId } from "mongodb";
-import { scenariosCollection, simulationRunsCollection, simulationResultsCollection } from "../lib/mongodb";
+import { scenariosCollection, simulationRunsCollection, simulationResultsCollection, simulationAggregateResultsCollection } from "../lib/mongodb";
 import { z } from 'zod';
 import { projectAuthCheck } from "./project_actions";
 import { type WithStringId } from "../lib/types/types";
-import { Scenario, SimulationRun, SimulationResult } from "../lib/types/testing_types";
+import { Scenario, SimulationRun, SimulationResult, SimulationAggregateResult } from "../lib/types/testing_types";
 import { SimulationScenarioData } from "../lib/types/testing_types";
 
 export async function getScenarios(projectId: string): Promise<WithStringId<z.infer<typeof Scenario>>[]> {
@@ -209,4 +209,45 @@ export async function createRunResult(
 
     const insertResult = await simulationResultsCollection.insertOne(resultDoc);
     return insertResult.insertedId.toString();
+}
+
+export async function createAggregateResult(
+    projectId: string,
+    runId: string,
+    total: number,
+    pass: number,
+    fail: number
+): Promise<void> {
+    await projectAuthCheck(projectId);
+
+    await simulationAggregateResultsCollection.insertOne({
+        projectId,
+        runId,
+        total,
+        pass,
+        fail,
+    });
+}
+
+export async function getAggregateResult(
+    projectId: string,
+    runId: string
+): Promise<z.infer<typeof SimulationAggregateResult> | null> {
+    await projectAuthCheck(projectId);
+
+    const result = await simulationAggregateResultsCollection.findOne({
+        projectId,
+        runId,
+    });
+
+    if (!result) return null;
+
+    // Only include the fields defined in the schema
+    return {
+        projectId: result.projectId,
+        runId: result.runId,
+        total: result.total,
+        pass: result.pass,
+        fail: result.fail
+    };
 } 

apps/rowboat/app/lib/mongodb.ts

@@ -7,7 +7,7 @@ import { Project } from "./types/project_types";
 import { EmbeddingDoc } from "./types/datasource_types";
 import { DataSourceDoc } from "./types/datasource_types";
 import { DataSource } from "./types/datasource_types";
-import { Scenario, SimulationResult, SimulationRun } from "./types/testing_types";
+import { Scenario, SimulationResult, SimulationRun, SimulationAggregateResult } from "./types/testing_types";
 import { z } from 'zod';
 
 const client = new MongoClient(process.env["MONGODB_CONNECTION_STRING"] || "mongodb://localhost:27017");
@@ -23,4 +23,5 @@ export const agentWorkflowsCollection = db.collection<z.infer<typeof Workflow>>(
 export const scenariosCollection = db.collection<z.infer<typeof Scenario>>("scenarios");
 export const apiKeysCollection = db.collection<z.infer<typeof ApiKey>>("api_keys");
 export const simulationRunsCollection = db.collection<z.infer<typeof SimulationRun>>("simulation_runs");
-export const simulationResultsCollection = db.collection<z.infer<typeof SimulationResult>>("simulation_results");
+export const simulationResultsCollection = db.collection<z.infer<typeof SimulationResult>>("simulation_results");
+export const simulationAggregateResultsCollection = db.collection<z.infer<typeof SimulationAggregateResult>>("simulation_aggregate_results");

apps/rowboat/app/lib/types/testing_types.ts

@@ -43,3 +43,10 @@ export const SimulationResult = z.object({
     details: z.string()
 });
 
+export const SimulationAggregateResult = z.object({
+    projectId: z.string(),
+    runId: z.string(),
+    total: z.number(),
+    pass: z.number(),
+    fail: z.number(),
+});

apps/rowboat/app/projects/[projectId]/simulation/app.tsx

@@ -14,6 +14,8 @@ import {
     createRun,
     createRunResult,
     updateRunStatus,
+    createAggregateResult,
+    getAggregateResult,
 } from '../../../actions/simulation_actions';
 import { type WithStringId } from '../../../lib/types/types';
 import { Scenario, SimulationRun, SimulationResult } from "../../../lib/types/testing_types";
@@ -37,18 +39,16 @@ const dummySimulator = async (scenario: ScenarioType, runId: string, projectId:
   await new Promise(resolve => setTimeout(resolve, 500));
   const passed = Math.random() > 0.5;
   
-  // Create the result object with explicitly typed result
   const result: z.infer<typeof SimulationResult> = {
     projectId: projectId,
     runId: runId,
     scenarioId: scenario._id,
-    result: passed ? 'pass' : 'fail' as const,  // explicitly type as literal
+    result: passed ? 'pass' : 'fail' as const,
     details: passed 
       ? "The bot successfully completed the conversation"
       : "The bot could not handle the conversation",
   };
 
-  // Write to DB using server action
   await createRunResult(
     projectId,
     runId,
@@ -207,30 +207,40 @@ export default function SimulationApp() {
     setSimulationReport(null);
 
     try {
-      // Create a new run using server action
       const newRun = await createRun(
         projectId as string,
         scenarios.map(s => s._id)
       );
       setActiveRun(newRun);
 
-      // Safely check for mock simulation flag
       const shouldMock = process.env.NEXT_PUBLIC_MOCK_SIMULATION_RESULTS === 'true';
       
       if (shouldMock) {
-        console.log('Using mock simulation...');  // Debug log
+        console.log('Using mock simulation...');
         
-        // First update run to 'running' status
         await updateRunStatus(projectId as string, newRun._id, 'running');
         
-        // Generate and save all mock results
-        await Promise.all(
+        // Run all scenarios and collect results
+        const mockResults = await Promise.all(
           scenarios.map(scenario => 
             dummySimulator(scenario, newRun._id, projectId as string)
           )
         );
 
-        // Update run status to completed
+        // Calculate aggregate results
+        const total = scenarios.length;
+        const pass = mockResults.filter(r => r.result === 'pass').length;
+        const fail = mockResults.filter(r => r.result === 'fail').length;
+
+        // Store aggregate results
+        await createAggregateResult(
+          projectId as string,
+          newRun._id,
+          total,
+          pass,
+          fail
+        );
+
         await updateRunStatus(
           projectId as string, 
           newRun._id, 
@@ -238,11 +248,9 @@ export default function SimulationApp() {
           new Date().toISOString()
         );
 
-        // Fetch the results back from DB to ensure consistency
         const results = await getRunResults(projectId as string, newRun._id);
         setRunResults(results);
         
-        // Refresh the run to get its updated state
         const updatedRun = await getRun(projectId as string, newRun._id);
         setActiveRun(updatedRun);
       }

apps/rowboat/app/projects/[projectId]/simulation/components/RunComponents.tsx

@@ -1,10 +1,11 @@
 'use client';
 
-import { useState } from 'react';
+import { useState, useEffect } from 'react';
 import { ChevronDownIcon, ChevronRightIcon } from '@heroicons/react/24/outline';
 import { WithStringId } from '../../../../lib/types/types';
-import { Scenario, SimulationRun, SimulationResult } from "../../../../lib/types/testing_types";
+import { Scenario, SimulationRun, SimulationResult, SimulationAggregateResult } from "../../../../lib/types/testing_types";
 import { z } from 'zod';
+import { getAggregateResult } from '../../../../actions/simulation_actions';
 
 type ScenarioType = WithStringId<z.infer<typeof Scenario>>;
 type SimulationRunType = WithStringId<z.infer<typeof SimulationRun>>;
@@ -19,6 +20,15 @@ interface SimulationResultCardProps {
 export const SimulationResultCard = ({ run, results, scenarios }: SimulationResultCardProps) => {
   const [isExpanded, setIsExpanded] = useState(false);
   const [expandedScenarios, setExpandedScenarios] = useState<Set<string>>(new Set());
+  const [aggregateResult, setAggregateResult] = useState<z.infer<typeof SimulationAggregateResult> | null>(null);
+
+  useEffect(() => {
+    if (run.projectId && run._id) {
+      getAggregateResult(run.projectId, run._id)
+        .then(setAggregateResult)
+        .catch(console.error);
+    }
+  }, [run.projectId, run._id]);
 
   const statusLabelClass = "px-3 py-1 rounded text-xs min-w-[60px] text-center uppercase font-semibold";
 
@@ -46,10 +56,10 @@ export const SimulationResultCard = ({ run, results, scenarios }: SimulationResu
     });
   };
 
-  // Calculate statistics and duration
-  const totalScenarios = run.scenarioIds.length;
-  const passedScenarios = results.filter(r => r.result === 'pass').length;
-  const failedScenarios = results.filter(r => r.result === 'fail').length;
+  // Replace the manual calculations with aggregate results
+  const totalScenarios = aggregateResult?.total ?? run.scenarioIds.length;
+  const passedScenarios = aggregateResult?.pass ?? 0;
+  const failedScenarios = aggregateResult?.fail ?? 0;
 
   const getDuration = () => {
     if (!run.completedAt) return 'In Progress';

apps/rowboat/app/projects/[projectId]/simulation/components/ScenarioComponents.tsx

@@ -1,7 +1,7 @@
 'use client';
 
-import { useState, useEffect } from 'react';
-import { PencilIcon, XMarkIcon, DocumentDuplicateIcon } from '@heroicons/react/24/outline';
+import { useState, useEffect, useCallback } from 'react';
+import { XMarkIcon } from '@heroicons/react/24/outline';
 import { WithStringId } from '../../../../lib/types/types';
 import { Scenario } from "../../../../lib/types/testing_types";
 import { z } from 'zod';
@@ -15,31 +15,44 @@ interface ScenarioViewerProps {
 }
 
 export function ScenarioViewer({ scenario, onSave, onClose }: ScenarioViewerProps) {
-  const [name, setName] = useState(scenario.name);
-  const [description, setDescription] = useState(scenario.description);
-  const [criteria, setCriteria] = useState(scenario.criteria || '');
-  const [context, setContext] = useState(scenario.context || '');
+  const [editedScenario, setEditedScenario] = useState<ScenarioType>(scenario);
+  const [saveTimeout, setSaveTimeout] = useState<NodeJS.Timeout | null>(null);
 
-  // Replace the existing useEffect with this debounced version
+  // Reset state when scenario changes
   useEffect(() => {
-    const timeoutId = setTimeout(() => {
-      // Only save if any value has actually changed
-      if (name !== scenario.name ||
-          description !== scenario.description ||
-          criteria !== scenario.criteria ||
-          context !== scenario.context) {
-        onSave({
-          ...scenario,
-          name,
-          description,
-          criteria,
-          context,
-        });
-      }
-    }, 500); // 500ms debounce
+    setEditedScenario(scenario);
+  }, [scenario]);
 
-    return () => clearTimeout(timeoutId);
-  }, [name, description, criteria, context, onSave, scenario]);
+  const handleChange = useCallback((field: keyof ScenarioType, value: string) => {
+    setEditedScenario(prev => ({
+      ...prev,
+      [field]: value,
+    }));
+
+    // Clear existing timeout
+    if (saveTimeout) {
+      clearTimeout(saveTimeout);
+    }
+
+    // Set new timeout
+    const timeoutId = setTimeout(() => {
+      onSave({
+        ...editedScenario,
+        [field]: value,
+      });
+    }, 500);
+
+    setSaveTimeout(timeoutId);
+  }, [editedScenario, onSave, saveTimeout]);
+
+  // Cleanup timeout on unmount
+  useEffect(() => {
+    return () => {
+      if (saveTimeout) {
+        clearTimeout(saveTimeout);
+      }
+    };
+  }, [saveTimeout]);
 
   return (
     <div>
@@ -58,8 +71,8 @@ export function ScenarioViewer({ scenario, onSave, onClose }: ScenarioViewerProp
           <div className="text-sm font-medium text-gray-500 uppercase tracking-wider mb-4">NAME</div>
           <input
             type="text"
-            value={name}
-            onChange={(e) => setName(e.target.value)}
+            value={editedScenario.name}
+            onChange={(e) => handleChange('name', e.target.value)}
             className="text-base border border-gray-200 rounded px-2 py-1 hover:border-gray-300 focus:border-blue-500 focus:ring-1 focus:ring-blue-500"
           />
         </div>
@@ -69,8 +82,8 @@ export function ScenarioViewer({ scenario, onSave, onClose }: ScenarioViewerProp
         <div className="flex flex-col">
           <div className="text-sm font-medium text-gray-500 uppercase tracking-wider mb-4">DESCRIPTION</div>
           <textarea
-            value={description}
-            onChange={(e) => setDescription(e.target.value)}
+            value={editedScenario.description}
+            onChange={(e) => handleChange('description', e.target.value)}
             className="text-base border border-gray-200 rounded px-2 py-1 hover:border-gray-300 focus:border-blue-500 focus:ring-1 focus:ring-blue-500 min-h-[24px] resize-none"
             style={{ height: 'auto', minHeight: '24px' }}
             onInput={(e) => {
@@ -86,8 +99,8 @@ export function ScenarioViewer({ scenario, onSave, onClose }: ScenarioViewerProp
         <div className="flex flex-col">
           <div className="text-sm font-medium text-gray-500 uppercase tracking-wider mb-4">CRITERIA</div>
           <textarea
-            value={criteria}
-            onChange={(e) => setCriteria(e.target.value)}
+            value={editedScenario.criteria}
+            onChange={(e) => handleChange('criteria', e.target.value)}
             className="text-base border border-gray-200 rounded px-2 py-1 hover:border-gray-300 focus:border-blue-500 focus:ring-1 focus:ring-blue-500 min-h-[24px] resize-none"
             style={{ height: 'auto', minHeight: '24px' }}
             onInput={(e) => {
@@ -103,8 +116,8 @@ export function ScenarioViewer({ scenario, onSave, onClose }: ScenarioViewerProp
         <div className="flex flex-col">
           <div className="text-sm font-medium text-gray-500 uppercase tracking-wider mb-4">CONTEXT</div>
           <textarea
-            value={context}
-            onChange={(e) => setContext(e.target.value)}
+            value={editedScenario.context}
+            onChange={(e) => handleChange('context', e.target.value)}
             className="text-base border border-gray-200 rounded px-2 py-1 hover:border-gray-300 focus:border-blue-500 focus:ring-1 focus:ring-blue-500 min-h-[24px] resize-none"
             style={{ height: 'auto', minHeight: '24px' }}
             onInput={(e) => {