From bc8fa905252a5ee8cac64368306c8f48b5dd1cec Mon Sep 17 00:00:00 2001 From: akhisud3195 Date: Fri, 14 Feb 2025 17:25:22 +0530 Subject: [PATCH] Add aggregate result to db --- .../rowboat/app/actions/simulation_actions.ts | 45 ++++++++++- apps/rowboat/app/lib/mongodb.ts | 5 +- apps/rowboat/app/lib/types/testing_types.ts | 7 ++ .../projects/[projectId]/simulation/app.tsx | 32 +++++--- .../simulation/components/RunComponents.tsx | 22 ++++-- .../components/ScenarioComponents.tsx | 77 +++++++++++-------- 6 files changed, 134 insertions(+), 54 deletions(-) diff --git a/apps/rowboat/app/actions/simulation_actions.ts b/apps/rowboat/app/actions/simulation_actions.ts index afdd3153..e28d24fd 100644 --- a/apps/rowboat/app/actions/simulation_actions.ts +++ b/apps/rowboat/app/actions/simulation_actions.ts @@ -1,11 +1,11 @@ 'use server'; import { ObjectId } from "mongodb"; -import { scenariosCollection, simulationRunsCollection, simulationResultsCollection } from "../lib/mongodb"; +import { scenariosCollection, simulationRunsCollection, simulationResultsCollection, simulationAggregateResultsCollection } from "../lib/mongodb"; import { z } from 'zod'; import { projectAuthCheck } from "./project_actions"; import { type WithStringId } from "../lib/types/types"; -import { Scenario, SimulationRun, SimulationResult } from "../lib/types/testing_types"; +import { Scenario, SimulationRun, SimulationResult, SimulationAggregateResult } from "../lib/types/testing_types"; import { SimulationScenarioData } from "../lib/types/testing_types"; export async function getScenarios(projectId: string): Promise>[]> { @@ -209,4 +209,45 @@ export async function createRunResult( const insertResult = await simulationResultsCollection.insertOne(resultDoc); return insertResult.insertedId.toString(); +} + +export async function createAggregateResult( + projectId: string, + runId: string, + total: number, + pass: number, + fail: number +): Promise { + await projectAuthCheck(projectId); + + await simulationAggregateResultsCollection.insertOne({ + projectId, + runId, + total, + pass, + fail, + }); +} + +export async function getAggregateResult( + projectId: string, + runId: string +): Promise | null> { + await projectAuthCheck(projectId); + + const result = await simulationAggregateResultsCollection.findOne({ + projectId, + runId, + }); + + if (!result) return null; + + // Only include the fields defined in the schema + return { + projectId: result.projectId, + runId: result.runId, + total: result.total, + pass: result.pass, + fail: result.fail + }; } \ No newline at end of file diff --git a/apps/rowboat/app/lib/mongodb.ts b/apps/rowboat/app/lib/mongodb.ts index fb6d6ca9..542166a1 100644 --- a/apps/rowboat/app/lib/mongodb.ts +++ b/apps/rowboat/app/lib/mongodb.ts @@ -7,7 +7,7 @@ import { Project } from "./types/project_types"; import { EmbeddingDoc } from "./types/datasource_types"; import { DataSourceDoc } from "./types/datasource_types"; import { DataSource } from "./types/datasource_types"; -import { Scenario, SimulationResult, SimulationRun } from "./types/testing_types"; +import { Scenario, SimulationResult, SimulationRun, SimulationAggregateResult } from "./types/testing_types"; import { z } from 'zod'; const client = new MongoClient(process.env["MONGODB_CONNECTION_STRING"] || "mongodb://localhost:27017"); @@ -23,4 +23,5 @@ export const agentWorkflowsCollection = db.collection>( export const scenariosCollection = db.collection>("scenarios"); export const apiKeysCollection = db.collection>("api_keys"); export const simulationRunsCollection = db.collection>("simulation_runs"); -export const simulationResultsCollection = db.collection>("simulation_results"); \ No newline at end of file +export const simulationResultsCollection = db.collection>("simulation_results"); +export const simulationAggregateResultsCollection = db.collection>("simulation_aggregate_results"); \ No newline at end of file diff --git a/apps/rowboat/app/lib/types/testing_types.ts b/apps/rowboat/app/lib/types/testing_types.ts index 289c2447..72b590fb 100644 --- a/apps/rowboat/app/lib/types/testing_types.ts +++ b/apps/rowboat/app/lib/types/testing_types.ts @@ -43,3 +43,10 @@ export const SimulationResult = z.object({ details: z.string() }); +export const SimulationAggregateResult = z.object({ + projectId: z.string(), + runId: z.string(), + total: z.number(), + pass: z.number(), + fail: z.number(), +}); \ No newline at end of file diff --git a/apps/rowboat/app/projects/[projectId]/simulation/app.tsx b/apps/rowboat/app/projects/[projectId]/simulation/app.tsx index b8e7d8ac..ffa55fd0 100644 --- a/apps/rowboat/app/projects/[projectId]/simulation/app.tsx +++ b/apps/rowboat/app/projects/[projectId]/simulation/app.tsx @@ -14,6 +14,8 @@ import { createRun, createRunResult, updateRunStatus, + createAggregateResult, + getAggregateResult, } from '../../../actions/simulation_actions'; import { type WithStringId } from '../../../lib/types/types'; import { Scenario, SimulationRun, SimulationResult } from "../../../lib/types/testing_types"; @@ -37,18 +39,16 @@ const dummySimulator = async (scenario: ScenarioType, runId: string, projectId: await new Promise(resolve => setTimeout(resolve, 500)); const passed = Math.random() > 0.5; - // Create the result object with explicitly typed result const result: z.infer = { projectId: projectId, runId: runId, scenarioId: scenario._id, - result: passed ? 'pass' : 'fail' as const, // explicitly type as literal + result: passed ? 'pass' : 'fail' as const, details: passed ? "The bot successfully completed the conversation" : "The bot could not handle the conversation", }; - // Write to DB using server action await createRunResult( projectId, runId, @@ -207,30 +207,40 @@ export default function SimulationApp() { setSimulationReport(null); try { - // Create a new run using server action const newRun = await createRun( projectId as string, scenarios.map(s => s._id) ); setActiveRun(newRun); - // Safely check for mock simulation flag const shouldMock = process.env.NEXT_PUBLIC_MOCK_SIMULATION_RESULTS === 'true'; if (shouldMock) { - console.log('Using mock simulation...'); // Debug log + console.log('Using mock simulation...'); - // First update run to 'running' status await updateRunStatus(projectId as string, newRun._id, 'running'); - // Generate and save all mock results - await Promise.all( + // Run all scenarios and collect results + const mockResults = await Promise.all( scenarios.map(scenario => dummySimulator(scenario, newRun._id, projectId as string) ) ); - // Update run status to completed + // Calculate aggregate results + const total = scenarios.length; + const pass = mockResults.filter(r => r.result === 'pass').length; + const fail = mockResults.filter(r => r.result === 'fail').length; + + // Store aggregate results + await createAggregateResult( + projectId as string, + newRun._id, + total, + pass, + fail + ); + await updateRunStatus( projectId as string, newRun._id, @@ -238,11 +248,9 @@ export default function SimulationApp() { new Date().toISOString() ); - // Fetch the results back from DB to ensure consistency const results = await getRunResults(projectId as string, newRun._id); setRunResults(results); - // Refresh the run to get its updated state const updatedRun = await getRun(projectId as string, newRun._id); setActiveRun(updatedRun); } diff --git a/apps/rowboat/app/projects/[projectId]/simulation/components/RunComponents.tsx b/apps/rowboat/app/projects/[projectId]/simulation/components/RunComponents.tsx index f5362038..8881d1c1 100644 --- a/apps/rowboat/app/projects/[projectId]/simulation/components/RunComponents.tsx +++ b/apps/rowboat/app/projects/[projectId]/simulation/components/RunComponents.tsx @@ -1,10 +1,11 @@ 'use client'; -import { useState } from 'react'; +import { useState, useEffect } from 'react'; import { ChevronDownIcon, ChevronRightIcon } from '@heroicons/react/24/outline'; import { WithStringId } from '../../../../lib/types/types'; -import { Scenario, SimulationRun, SimulationResult } from "../../../../lib/types/testing_types"; +import { Scenario, SimulationRun, SimulationResult, SimulationAggregateResult } from "../../../../lib/types/testing_types"; import { z } from 'zod'; +import { getAggregateResult } from '../../../../actions/simulation_actions'; type ScenarioType = WithStringId>; type SimulationRunType = WithStringId>; @@ -19,6 +20,15 @@ interface SimulationResultCardProps { export const SimulationResultCard = ({ run, results, scenarios }: SimulationResultCardProps) => { const [isExpanded, setIsExpanded] = useState(false); const [expandedScenarios, setExpandedScenarios] = useState>(new Set()); + const [aggregateResult, setAggregateResult] = useState | null>(null); + + useEffect(() => { + if (run.projectId && run._id) { + getAggregateResult(run.projectId, run._id) + .then(setAggregateResult) + .catch(console.error); + } + }, [run.projectId, run._id]); const statusLabelClass = "px-3 py-1 rounded text-xs min-w-[60px] text-center uppercase font-semibold"; @@ -46,10 +56,10 @@ export const SimulationResultCard = ({ run, results, scenarios }: SimulationResu }); }; - // Calculate statistics and duration - const totalScenarios = run.scenarioIds.length; - const passedScenarios = results.filter(r => r.result === 'pass').length; - const failedScenarios = results.filter(r => r.result === 'fail').length; + // Replace the manual calculations with aggregate results + const totalScenarios = aggregateResult?.total ?? run.scenarioIds.length; + const passedScenarios = aggregateResult?.pass ?? 0; + const failedScenarios = aggregateResult?.fail ?? 0; const getDuration = () => { if (!run.completedAt) return 'In Progress'; diff --git a/apps/rowboat/app/projects/[projectId]/simulation/components/ScenarioComponents.tsx b/apps/rowboat/app/projects/[projectId]/simulation/components/ScenarioComponents.tsx index 75ee547a..bd6978b9 100644 --- a/apps/rowboat/app/projects/[projectId]/simulation/components/ScenarioComponents.tsx +++ b/apps/rowboat/app/projects/[projectId]/simulation/components/ScenarioComponents.tsx @@ -1,7 +1,7 @@ 'use client'; -import { useState, useEffect } from 'react'; -import { PencilIcon, XMarkIcon, DocumentDuplicateIcon } from '@heroicons/react/24/outline'; +import { useState, useEffect, useCallback } from 'react'; +import { XMarkIcon } from '@heroicons/react/24/outline'; import { WithStringId } from '../../../../lib/types/types'; import { Scenario } from "../../../../lib/types/testing_types"; import { z } from 'zod'; @@ -15,31 +15,44 @@ interface ScenarioViewerProps { } export function ScenarioViewer({ scenario, onSave, onClose }: ScenarioViewerProps) { - const [name, setName] = useState(scenario.name); - const [description, setDescription] = useState(scenario.description); - const [criteria, setCriteria] = useState(scenario.criteria || ''); - const [context, setContext] = useState(scenario.context || ''); + const [editedScenario, setEditedScenario] = useState(scenario); + const [saveTimeout, setSaveTimeout] = useState(null); - // Replace the existing useEffect with this debounced version + // Reset state when scenario changes useEffect(() => { - const timeoutId = setTimeout(() => { - // Only save if any value has actually changed - if (name !== scenario.name || - description !== scenario.description || - criteria !== scenario.criteria || - context !== scenario.context) { - onSave({ - ...scenario, - name, - description, - criteria, - context, - }); - } - }, 500); // 500ms debounce + setEditedScenario(scenario); + }, [scenario]); - return () => clearTimeout(timeoutId); - }, [name, description, criteria, context, onSave, scenario]); + const handleChange = useCallback((field: keyof ScenarioType, value: string) => { + setEditedScenario(prev => ({ + ...prev, + [field]: value, + })); + + // Clear existing timeout + if (saveTimeout) { + clearTimeout(saveTimeout); + } + + // Set new timeout + const timeoutId = setTimeout(() => { + onSave({ + ...editedScenario, + [field]: value, + }); + }, 500); + + setSaveTimeout(timeoutId); + }, [editedScenario, onSave, saveTimeout]); + + // Cleanup timeout on unmount + useEffect(() => { + return () => { + if (saveTimeout) { + clearTimeout(saveTimeout); + } + }; + }, [saveTimeout]); return (
@@ -58,8 +71,8 @@ export function ScenarioViewer({ scenario, onSave, onClose }: ScenarioViewerProp
NAME
setName(e.target.value)} + value={editedScenario.name} + onChange={(e) => handleChange('name', e.target.value)} className="text-base border border-gray-200 rounded px-2 py-1 hover:border-gray-300 focus:border-blue-500 focus:ring-1 focus:ring-blue-500" />
@@ -69,8 +82,8 @@ export function ScenarioViewer({ scenario, onSave, onClose }: ScenarioViewerProp
DESCRIPTION