Reapply: Add run all scenarios detailed view

This commit is contained in:
akhisud3195 2025-02-14 15:44:48 +05:30
parent 5ef8321303
commit 43e7a9ea00
5 changed files with 521 additions and 169 deletions

View file

@ -1,11 +1,11 @@
'use server';
import { ObjectId } from "mongodb";
import { scenariosCollection } from "../lib/mongodb";
import { scenariosCollection, simulationRunsCollection, simulationResultsCollection } from "../lib/mongodb";
import { z } from 'zod';
import { projectAuthCheck } from "./project_actions";
import { type WithStringId } from "../lib/types/types";
import { Scenario } from "../lib/types/testing_types";
import { Scenario, SimulationRun, SimulationResult } from "../lib/types/testing_types";
import { SimulationScenarioData } from "../lib/types/testing_types";
export async function getScenarios(projectId: string): Promise<WithStringId<z.infer<typeof Scenario>>[]> {
@ -83,4 +83,124 @@ export async function deleteScenario(projectId: string, scenarioId: string): Pro
_id: new ObjectId(scenarioId),
projectId,
});
}
export async function getRuns(projectId: string): Promise<WithStringId<z.infer<typeof SimulationRun>>[]> {
await projectAuthCheck(projectId);
const runs = await simulationRunsCollection
.find({ projectId })
.sort({ startedAt: -1 }) // Most recent first
.toArray();
return runs.map(run => ({
...run,
_id: run._id.toString(),
}));
}
export async function getRun(projectId: string, runId: string): Promise<WithStringId<z.infer<typeof SimulationRun>>> {
await projectAuthCheck(projectId);
const run = await simulationRunsCollection.findOne({
_id: new ObjectId(runId),
projectId,
});
if (!run) {
throw new Error('Run not found');
}
return {
...run,
_id: run._id.toString(),
};
}
export async function createRun(
projectId: string,
scenarioIds: string[]
): Promise<WithStringId<z.infer<typeof SimulationRun>>> {
await projectAuthCheck(projectId);
const run = {
projectId,
status: 'pending' as const,
scenarioIds,
startedAt: new Date().toISOString(),
};
const result = await simulationRunsCollection.insertOne(run);
return {
...run,
_id: result.insertedId.toString(),
};
}
export async function updateRunStatus(
projectId: string,
runId: string,
status: z.infer<typeof SimulationRun>['status'],
completedAt?: string
): Promise<void> {
await projectAuthCheck(projectId);
const updateData: Partial<z.infer<typeof SimulationRun>> = {
status,
};
if (completedAt) {
updateData.completedAt = completedAt;
}
await simulationRunsCollection.updateOne(
{
_id: new ObjectId(runId),
projectId,
},
{
$set: updateData,
}
);
}
export async function getRunResults(
projectId: string,
runId: string
): Promise<WithStringId<z.infer<typeof SimulationResult>>[]> {
await projectAuthCheck(projectId);
const results = await simulationResultsCollection
.find({
runId,
projectId,
})
.toArray();
return results.map(result => ({
...result,
_id: result._id.toString(),
}));
}
export async function createRunResult(
projectId: string,
runId: string,
scenarioId: string,
result: z.infer<typeof SimulationResult>['result'],
details: string
): Promise<string> {
await projectAuthCheck(projectId);
const resultDoc = {
projectId,
runId,
scenarioId,
result,
details,
};
const insertResult = await simulationResultsCollection.insertOne(resultDoc);
return insertResult.insertedId.toString();
}

View file

@ -35,6 +35,8 @@ export const SimulationRun = z.object({
});
export const SimulationResult = z.object({
projectId: z.string(),
runId: z.string(),
scenarioId: z.string(),
result: z.union([z.literal('pass'), z.literal('fail')]),
details: z.string()

View file

@ -1,49 +1,301 @@
'use client';
import { useState, useEffect, useCallback } from 'react';
import { PlusIcon, PencilIcon, XMarkIcon, DocumentDuplicateIcon, EllipsisVerticalIcon, TrashIcon, ChevronRightIcon, PlayIcon } from '@heroicons/react/24/outline';
import { PlusIcon, PencilIcon, XMarkIcon, DocumentDuplicateIcon, EllipsisVerticalIcon, TrashIcon, ChevronRightIcon, PlayIcon, ChevronDownIcon } from '@heroicons/react/24/outline';
import { useParams, useRouter } from 'next/navigation';
import {
getScenarios,
createScenario,
updateScenario,
deleteScenario,
getRuns,
getRun,
getRunResults,
createRun,
createRunResult,
updateRunStatus,
} from '../../../actions/simulation_actions';
import { type WithStringId } from '../../../lib/types/types';
import { Scenario } from "../../../lib/types/testing_types";
import { Scenario, SimulationRun, SimulationResult } from "../../../lib/types/testing_types";
import { z } from 'zod';
type ScenarioType = WithStringId<z.infer<typeof Scenario>>;
type SimulationResult = {
scenarioId: string;
scenarioName: string;
passed: boolean;
details: string;
scenario: ScenarioType;
};
type SimulationRunType = WithStringId<z.infer<typeof SimulationRun>>;
type SimulationResultType = WithStringId<z.infer<typeof SimulationResult>>;
type SimulationReport = {
totalScenarios: number;
passedScenarios: number;
failedScenarios: number;
results: SimulationResult[];
results: z.infer<typeof SimulationResult>[];
timestamp: Date;
};
const dummySimulator = async (scenario: ScenarioType): Promise<SimulationResult> => {
const dummySimulator = async (scenario: ScenarioType, runId: string, projectId: string): Promise<z.infer<typeof SimulationResult>> => {
await new Promise(resolve => setTimeout(resolve, 500));
const passed = Math.random() > 0.5;
return {
// Create the result object with explicitly typed result
const result: z.infer<typeof SimulationResult> = {
projectId: projectId,
runId: runId,
scenarioId: scenario._id,
scenarioName: scenario.name,
passed,
result: passed ? 'pass' : 'fail' as const, // explicitly type as literal
details: passed
? "The bot successfully completed the conversation"
: "The bot could not handle the conversation",
scenario: scenario,
};
// Write to DB using server action
await createRunResult(
projectId,
runId,
scenario._id,
result.result,
result.details
);
return result;
};
interface SimulationResultCardProps {
run: SimulationRunType;
results: SimulationResultType[];
scenarios: ScenarioType[];
}
interface ScenarioResultCardProps {
scenario: ScenarioType;
result?: SimulationResultType;
}
const SimulationResultCard = ({ run, results, scenarios }: SimulationResultCardProps) => {
const [isExpanded, setIsExpanded] = useState(false);
const [expandedScenarios, setExpandedScenarios] = useState<Set<string>>(new Set());
const statusLabelClass = "px-3 py-1 rounded text-xs min-w-[60px] text-center uppercase font-semibold";
const formatMainTitle = (date: string) => {
return `Run from ${new Date(date).toLocaleString('en-US', {
month: 'numeric',
day: 'numeric',
year: 'numeric',
hour: 'numeric',
minute: '2-digit',
second: '2-digit',
hour12: true
})}`;
};
const formatDateTime = (date: string) => {
return new Date(date).toLocaleString('en-US', {
month: 'numeric',
day: 'numeric',
year: 'numeric',
hour: 'numeric',
minute: '2-digit',
second: '2-digit',
hour12: true
});
};
// Calculate statistics and duration
const totalScenarios = run.scenarioIds.length;
const passedScenarios = results.filter(r => r.result === 'pass').length;
const failedScenarios = results.filter(r => r.result === 'fail').length;
const getDuration = () => {
if (!run.completedAt) return 'In Progress';
const start = new Date(run.startedAt);
const end = new Date(run.completedAt);
const diff = end.getTime() - start.getTime();
return `${(diff / 1000).toFixed(1)}s`;
};
const toggleScenario = (scenarioId: string, e: React.MouseEvent) => {
e.stopPropagation(); // Prevent triggering parent's onClick
setExpandedScenarios(prev => {
const newSet = new Set(prev);
if (newSet.has(scenarioId)) {
newSet.delete(scenarioId);
} else {
newSet.add(scenarioId);
}
return newSet;
});
};
return (
<div className="border rounded-lg mb-4 shadow-sm">
<div
className="p-4 flex items-center justify-between cursor-pointer hover:bg-gray-50"
onClick={() => setIsExpanded(!isExpanded)}
>
<div className="flex items-center space-x-2">
{isExpanded ? (
<ChevronDownIcon className="h-5 w-5 text-gray-400" />
) : (
<ChevronRightIcon className="h-5 w-5 text-gray-400" />
)}
<div className="text-lg font-semibold">
{formatMainTitle(run.startedAt)}
</div>
</div>
<span className={`${statusLabelClass} ${
run.status === 'completed' ? 'bg-green-100 text-green-800' :
run.status === 'failed' ? 'bg-red-100 text-red-800' :
'bg-yellow-100 text-yellow-800'
}`}>
{run.status}
</span>
</div>
{isExpanded && (
<div className="p-4 border-t">
{/* Simplified timing information */}
<div className="mb-6 text-sm text-gray-500 space-y-1">
<div className="flex items-center">
<span className="w-24 text-gray-600">Completed:</span>
<span>{run.completedAt ? formatDateTime(run.completedAt) : 'Not completed'}</span>
</div>
<div className="flex items-center">
<span className="w-24 text-gray-600">Duration:</span>
<span>{getDuration()}</span>
</div>
</div>
{/* Results statistics */}
<div className="grid grid-cols-3 gap-4 mb-6">
<div className="p-4 rounded-lg bg-gray-50">
<div className="text-sm text-gray-600">Total Scenarios</div>
<div className="text-2xl font-semibold">{totalScenarios}</div>
</div>
<div className="p-4 rounded-lg bg-green-50">
<div className="text-sm text-green-600">Passed</div>
<div className="text-2xl font-semibold text-green-700">{passedScenarios}</div>
</div>
<div className="p-4 rounded-lg bg-red-50">
<div className="text-sm text-red-600">Failed</div>
<div className="text-2xl font-semibold text-red-700">{failedScenarios}</div>
</div>
</div>
<div className="space-y-2">
{run.scenarioIds.map(scenarioId => {
const scenario = scenarios.find(s => s._id === scenarioId);
const result = results.find(r => r.scenarioId === scenarioId);
const isScenarioExpanded = expandedScenarios.has(scenarioId);
return scenario && (
<div
key={scenarioId}
className={`border rounded-lg overflow-hidden ${
result?.result === 'pass' ? 'bg-green-50 border-green-200' :
result?.result === 'fail' ? 'bg-red-50 border-red-200' :
'bg-gray-50 border-gray-200'
}`}
>
<div
className="p-3 flex items-center justify-between cursor-pointer hover:bg-opacity-80"
onClick={(e) => toggleScenario(scenarioId, e)}
>
<div className="flex items-center space-x-2">
{isScenarioExpanded ? (
<ChevronDownIcon className="h-4 w-4 text-gray-600" />
) : (
<ChevronRightIcon className="h-4 w-4 text-gray-600" />
)}
<span className="font-medium text-gray-900">{scenario.name}</span>
</div>
{result && (
<span className={`${statusLabelClass} ${
result.result === 'pass' ? 'bg-green-200 text-green-900' : 'bg-red-200 text-red-900'
}`}>
{result.result}
</span>
)}
</div>
{isScenarioExpanded && (
<div className="p-3 border-t border-opacity-50 space-y-4">
<div>
<div className="text-sm font-medium mb-1">Description</div>
<div className="text-sm text-gray-700">
{scenario.description}
</div>
</div>
<div>
<div className="text-sm font-medium mb-1">Context</div>
<div className="text-sm text-gray-700">
{scenario.context || 'No context provided'}
</div>
</div>
{result && (
<div>
<div className="text-sm font-medium mb-1">Result Details</div>
<div className="text-sm text-gray-700">
{result.details}
</div>
</div>
)}
</div>
)}
</div>
);
})}
</div>
</div>
)}
</div>
);
};
const ScenarioResultCard = ({ scenario, result }: ScenarioResultCardProps) => {
const [isExpanded, setIsExpanded] = useState(false);
return (
<div className="border rounded-lg mb-2 last:mb-0">
<div
className="p-3 flex items-center justify-between cursor-pointer hover:bg-gray-50"
onClick={() => setIsExpanded(!isExpanded)}
>
<div className="flex items-center space-x-2">
{isExpanded ? (
<ChevronDownIcon className="h-4 w-4 text-gray-400" />
) : (
<ChevronRightIcon className="h-4 w-4 text-gray-400" />
)}
<span className="font-medium">{scenario.name}</span>
</div>
{result && (
<span className={`px-2 py-1 rounded-full text-xs ${
result.result === 'pass' ? 'bg-green-100 text-green-800' : 'bg-red-100 text-red-800'
}`}>
{result.result}
</span>
)}
</div>
{isExpanded && (
<div className="p-3 border-t space-y-2 bg-gray-50">
<div>
<div className="text-sm font-medium text-gray-600">Description</div>
<div className="text-sm">{scenario.description}</div>
</div>
<div>
<div className="text-sm font-medium text-gray-600">Context</div>
<div className="text-sm">{scenario.context || 'No context provided'}</div>
</div>
{result && (
<div>
<div className="text-sm font-medium text-gray-600">Result Details</div>
<div className="text-sm">{result.details}</div>
</div>
)}
</div>
)}
</div>
);
};
export default function SimulationApp() {
@ -56,6 +308,11 @@ export default function SimulationApp() {
const [isRunning, setIsRunning] = useState(false);
const [simulationReport, setSimulationReport] = useState<SimulationReport | null>(null);
const [expandedResults, setExpandedResults] = useState<Set<string>>(new Set());
const [runs, setRuns] = useState<SimulationRunType[]>([]);
const [activeRun, setActiveRun] = useState<SimulationRunType | null>(null);
const [runResults, setRunResults] = useState<SimulationResultType[]>([]);
const [isLoadingRuns, setIsLoadingRuns] = useState(true);
const [allRunResults, setAllRunResults] = useState<Record<string, SimulationResultType[]>>({});
// Load scenarios on mount
useEffect(() => {
@ -71,6 +328,61 @@ export default function SimulationApp() {
}
}, [menuOpenScenarioId]);
// Modify the fetchRuns function to also fetch results
const fetchRuns = useCallback(async () => {
if (!projectId) return;
setIsLoadingRuns(true);
try {
const runsData = await getRuns(projectId as string);
setRuns(runsData);
// Fetch results for all runs
const resultsPromises = runsData.map(run =>
getRunResults(projectId as string, run._id)
);
const allResults = await Promise.all(resultsPromises);
// Create a map of run ID to results
const resultsMap = runsData.reduce((acc, run, index) => ({
...acc,
[run._id]: allResults[index]
}), {});
setAllRunResults(resultsMap);
} catch (error) {
console.error('Error fetching runs:', error);
} finally {
setIsLoadingRuns(false);
}
}, [projectId]);
// Update the useEffect hooks to include fetchRuns
useEffect(() => {
if (!projectId) return;
fetchRuns();
}, [projectId, fetchRuns]);
useEffect(() => {
if (!projectId || !activeRun || activeRun.status === 'completed' || activeRun.status === 'cancelled') return;
const interval = setInterval(async () => {
try {
const updatedRun = await getRun(projectId as string, activeRun._id);
setActiveRun(updatedRun);
if (updatedRun.status === 'completed') {
const results = await getRunResults(projectId as string, activeRun._id);
setRunResults(results);
fetchRuns(); // Refresh the runs list
}
} catch (error) {
console.error('Error polling run status:', error);
}
}, 2000);
return () => clearInterval(interval);
}, [activeRun, projectId, fetchRuns]);
const createNewScenario = async () => {
if (!projectId) return;
const newScenarioId = await createScenario(
@ -127,31 +439,52 @@ export default function SimulationApp() {
};
const runAllScenarios = async () => {
if (!projectId) return;
setIsRunning(true);
setSimulationReport(null);
try {
const results: SimulationResult[] = [];
// Run each scenario through the simulator
for (const scenario of scenarios) {
const result = await dummySimulator(scenario);
results.push(result);
// Create a new run using server action
const newRun = await createRun(
projectId as string,
scenarios.map(s => s._id)
);
setActiveRun(newRun);
// Check for the NEXT_PUBLIC_ prefixed env variable
if (process.env.NEXT_PUBLIC_MOCK_SIMULATION_RESULTS === 'true') {
console.log('Using mock simulation...'); // Debug log
// First update run to 'running' status
await updateRunStatus(projectId as string, newRun._id, 'running');
// Generate and save all mock results
await Promise.all(
scenarios.map(scenario =>
dummySimulator(scenario, newRun._id, projectId as string)
)
);
// Update run status to completed
await updateRunStatus(
projectId as string,
newRun._id,
'completed',
new Date().toISOString()
);
// Fetch the results back from DB to ensure consistency
const results = await getRunResults(projectId as string, newRun._id);
setRunResults(results);
// Refresh the run to get its updated state
const updatedRun = await getRun(projectId as string, newRun._id);
setActiveRun(updatedRun);
}
// Generate report
const passedScenarios = results.filter(r => r.passed).length;
const report: SimulationReport = {
totalScenarios: scenarios.length,
passedScenarios,
failedScenarios: scenarios.length - passedScenarios,
results,
timestamp: new Date(),
};
setSimulationReport(report);
await fetchRuns();
} catch (error) {
console.error('Error running scenarios:', error);
console.error('Error starting scenarios:', error);
} finally {
setIsRunning(false);
}
@ -258,139 +591,36 @@ export default function SimulationApp() {
)
) : (
<div className="space-y-6">
{simulationReport ? (
<>
<div className="flex justify-between items-center mb-6">
<div>
<h1 className="text-2xl font-bold">Simulation Results</h1>
<div className="text-sm text-gray-500 mt-1">
Run on {simulationReport.timestamp.toLocaleString()}
</div>
</div>
<button
onClick={runAllScenarios}
disabled={isRunning || scenarios.length === 0}
className={`px-4 py-2 rounded-md text-white ${
isRunning || scenarios.length === 0
? 'bg-gray-400 cursor-not-allowed'
: 'bg-blue-600 hover:bg-blue-700'
}`}
>
{isRunning ? 'Running...' : 'Run All Scenarios'}
</button>
</div>
<div className="flex justify-between items-center mb-6">
<h1 className="text-2xl font-bold">Simulation Runs</h1>
<button
onClick={runAllScenarios}
disabled={isRunning || scenarios.length === 0}
className={`px-4 py-2 rounded-md text-white ${
isRunning || scenarios.length === 0
? 'bg-gray-400 cursor-not-allowed'
: 'bg-blue-600 hover:bg-blue-700'
}`}
>
{isRunning ? 'Running...' : 'Run All Scenarios'}
</button>
</div>
<div className="grid grid-cols-3 gap-4">
<div className="bg-gray-50 p-4 rounded-lg min-h-[100px] flex flex-col justify-center">
<div className="text-sm text-gray-500">Total Scenarios</div>
<div className="text-2xl font-bold">{simulationReport.totalScenarios}</div>
</div>
<div className="bg-green-50 p-4 rounded-lg min-h-[100px] flex flex-col justify-center">
<div className="text-sm text-green-600">Passed</div>
<div className="text-2xl font-bold text-green-600">
{simulationReport.passedScenarios}
</div>
</div>
<div className="bg-red-50 p-4 rounded-lg min-h-[100px] flex flex-col justify-center">
<div className="text-sm text-red-600">Failed</div>
<div className="text-2xl font-bold text-red-600">
{simulationReport.failedScenarios}
</div>
</div>
</div>
<div className="mt-8">
<h2 className="text-lg font-semibold mb-4">Detailed Results</h2>
<div className="space-y-2">
{simulationReport.results.map((result) => (
<div
key={result.scenarioId}
className={`p-4 rounded-lg border ${
result.passed ? 'border-green-200 bg-green-50' : 'border-red-200 bg-red-50'
}`}
>
<div
className="flex justify-between items-center cursor-pointer"
onClick={() => {
const newExpandedResults = new Set(expandedResults);
if (expandedResults.has(result.scenarioId)) {
newExpandedResults.delete(result.scenarioId);
} else {
newExpandedResults.add(result.scenarioId);
}
setExpandedResults(newExpandedResults);
}}
>
<div className="font-medium flex items-center gap-2">
<ChevronRightIcon
className={`h-5 w-5 transform transition-transform ${
expandedResults.has(result.scenarioId) ? 'rotate-90' : ''
}`}
/>
{result.scenarioName}
</div>
<div
className={`px-2 py-1 rounded text-sm w-16 text-center ${
result.passed
? 'bg-green-100 text-green-800'
: 'bg-red-100 text-red-800'
}`}
>
{result.passed ? 'Passed' : 'Failed'}
</div>
</div>
{expandedResults.has(result.scenarioId) && (
<div className="mt-4 pl-7 space-y-4">
<div className="grid grid-cols-2 gap-4">
<div>
<div className="text-sm font-medium text-gray-500 uppercase tracking-wider mb-1">
Name
</div>
<div className="text-sm">{result.scenario.name}</div>
</div>
<div>
<div className="text-sm font-medium text-gray-500 uppercase tracking-wider mb-1">
Description
</div>
<div className="text-sm">{result.scenario.description}</div>
</div>
</div>
<div>
<div className="text-sm font-medium text-gray-500 uppercase tracking-wider mb-1">
Details
</div>
<div className="text-sm">{result.details}</div>
</div>
</div>
)}
</div>
))}
</div>
</div>
</>
{isLoadingRuns ? (
<div className="text-center py-4">Loading runs...</div>
) : runs.length === 0 ? (
<div className="text-center py-4 text-gray-500">No simulation runs yet</div>
) : (
<>
<div className="flex justify-between items-center mb-6">
<h1 className="text-2xl font-bold">Scenarios</h1>
<button
onClick={runAllScenarios}
disabled={isRunning || scenarios.length === 0}
className={`px-4 py-2 rounded-md text-white ${
isRunning || scenarios.length === 0
? 'bg-gray-400 cursor-not-allowed'
: 'bg-blue-600 hover:bg-blue-700'
}`}
>
{isRunning ? 'Running...' : 'Run All Scenarios'}
</button>
<div className="space-y-4">
{runs.map((run) => (
<SimulationResultCard
key={run._id}
run={run}
results={allRunResults[run._id] || []}
scenarios={scenarios}
/>
))}
</div>
<div className="text-center text-gray-500 mt-10">
Select a scenario or run all scenarios
</div>
</>
)}
</div>
)}

View file

@ -13,7 +13,7 @@
"@aws-sdk/client-s3": "^3.743.0",
"@aws-sdk/s3-request-presigner": "^3.743.0",
"@google/generative-ai": "^0.21.0",
"@heroicons/react": "^2.1.1",
"@heroicons/react": "^2.2.0",
"@langchain/core": "^0.3.7",
"@langchain/textsplitters": "^0.1.0",
"@mendable/firecrawl-js": "^1.0.3",

View file

@ -18,7 +18,7 @@
"@aws-sdk/client-s3": "^3.743.0",
"@aws-sdk/s3-request-presigner": "^3.743.0",
"@google/generative-ai": "^0.21.0",
"@heroicons/react": "^2.1.1",
"@heroicons/react": "^2.2.0",
"@langchain/core": "^0.3.7",
"@langchain/textsplitters": "^0.1.0",
"@mendable/firecrawl-js": "^1.0.3",