feat: add qa node in workflow builder (#172)

* feat: add qa node in workflow builder

* feat: add qa analysis token usage in usage_info

* fix: mask the API key in QA node

* feat: add advanced configuration in QA node
This commit is contained in:
Abhishek 2026-02-25 13:53:30 +05:30 committed by GitHub
parent f1f4830012
commit a836825b83
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
30 changed files with 1619 additions and 265 deletions

View file

@ -0,0 +1,208 @@
"use client";
import { useEffect, useState } from "react";
import { getDefaultConfigurationsApiV1UserConfigurationsDefaultsGet } from "@/client/sdk.gen";
import { Checkbox } from "@/components/ui/checkbox";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
interface SchemaProperty {
type?: string;
default?: string | number | boolean;
enum?: string[];
examples?: string[];
$ref?: string;
}
interface ProviderSchema {
properties: Record<string, SchemaProperty>;
required?: string[];
$defs?: Record<string, SchemaProperty>;
}
interface LLMConfigSelectorProps {
provider: string;
onProviderChange: (provider: string) => void;
model: string;
onModelChange: (model: string) => void;
apiKey: string;
onApiKeyChange: (apiKey: string) => void;
}
export function LLMConfigSelector({
provider,
onProviderChange,
model,
onModelChange,
apiKey,
onApiKeyChange,
}: LLMConfigSelectorProps) {
const [schemas, setSchemas] = useState<Record<string, ProviderSchema>>({});
const [isManualModelInput, setIsManualModelInput] = useState(false);
useEffect(() => {
const fetchSchemas = async () => {
const response =
await getDefaultConfigurationsApiV1UserConfigurationsDefaultsGet();
if (response.data?.llm) {
setSchemas(response.data.llm as unknown as Record<string, ProviderSchema>);
}
};
fetchSchemas();
}, []);
const availableProviders = Object.keys(schemas);
const providerSchema = schemas[provider];
const getModelOptions = (): string[] => {
if (!providerSchema) return [];
const modelSchema = providerSchema.properties.model;
const actualSchema =
modelSchema?.$ref && providerSchema.$defs
? providerSchema.$defs[modelSchema.$ref.split("/").pop() || ""]
: modelSchema;
return actualSchema?.examples || [];
};
const modelOptions = getModelOptions();
// Check if current model is not in options (custom model)
useEffect(() => {
if (model && modelOptions.length > 0 && !modelOptions.includes(model)) {
setIsManualModelInput(true);
}
}, [model, modelOptions]);
const handleProviderChange = (newProvider: string) => {
if (!newProvider) return;
onProviderChange(newProvider);
const newSchema = schemas[newProvider];
if (newSchema?.properties?.model) {
const modelSchema = newSchema.properties.model;
const actualSchema =
modelSchema.$ref && newSchema.$defs
? newSchema.$defs[modelSchema.$ref.split("/").pop() || ""]
: modelSchema;
const defaultModel =
(actualSchema?.default as string) ||
actualSchema?.examples?.[0] ||
"";
onModelChange(defaultModel);
}
setIsManualModelInput(false);
};
return (
<div className="space-y-4 p-3 border rounded-md bg-muted/10">
<div className="grid grid-cols-2 gap-4">
<div className="space-y-2">
<Label>Provider</Label>
<Select value={provider} onValueChange={handleProviderChange}>
<SelectTrigger className="w-full">
<SelectValue placeholder="Select provider" />
</SelectTrigger>
<SelectContent>
{availableProviders.map((p) => (
<SelectItem key={p} value={p}>
{p}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div className="space-y-2">
<Label>Model</Label>
{isManualModelInput ? (
<div className="space-y-2">
<Input
type="text"
placeholder="Enter model name"
value={model}
onChange={(e) => onModelChange(e.target.value)}
/>
<div className="flex items-center space-x-2">
<Checkbox
id="qa-manual-model"
checked={isManualModelInput}
onCheckedChange={(checked) => {
setIsManualModelInput(checked as boolean);
if (!checked && modelOptions.length > 0) {
onModelChange(modelOptions[0]);
}
}}
/>
<Label
htmlFor="qa-manual-model"
className="text-sm font-normal cursor-pointer"
>
Add Model Manually
</Label>
</div>
</div>
) : modelOptions.length > 0 ? (
<div className="space-y-2">
<Select
value={model}
onValueChange={(v) => {
if (v) onModelChange(v);
}}
>
<SelectTrigger className="w-full">
<SelectValue placeholder="Select model" />
</SelectTrigger>
<SelectContent>
{modelOptions.map((m) => (
<SelectItem key={m} value={m}>
{m}
</SelectItem>
))}
</SelectContent>
</Select>
<div className="flex items-center space-x-2">
<Checkbox
id="qa-manual-model-dropdown"
checked={isManualModelInput}
onCheckedChange={(checked) =>
setIsManualModelInput(checked as boolean)
}
/>
<Label
htmlFor="qa-manual-model-dropdown"
className="text-sm font-normal cursor-pointer"
>
Add Model Manually
</Label>
</div>
</div>
) : (
<Input
type="text"
placeholder="Enter model name"
value={model}
onChange={(e) => onModelChange(e.target.value)}
/>
)}
</div>
</div>
<div className="space-y-2">
<Label>API Key</Label>
<Input
type="text"
placeholder="Enter API key"
value={apiKey}
onChange={(e) => onApiKeyChange(e.target.value)}
/>
</div>
</div>
);
}

View file

@ -1,4 +1,4 @@
import { ExternalLink, Globe, Headset, Link2, LucideIcon, OctagonX, Play, Webhook, X } from 'lucide-react';
import { ClipboardCheck, ExternalLink, Globe, Headset, Link2, LucideIcon, OctagonX, Play, Webhook, X } from 'lucide-react';
import { useEffect } from 'react';
import { Button } from '@/components/ui/button';
@ -57,12 +57,18 @@ const TRIGGER_NODE_TYPES: NodeTypeConfig[] = [
}
];
const WEBHOOK_NODE_TYPES: NodeTypeConfig[] = [
const INTEGRATION_NODE_TYPES: NodeTypeConfig[] = [
{
type: NodeType.WEBHOOK,
label: 'Webhook',
description: 'Send HTTP request after workflow completion',
icon: Link2
},
{
type: NodeType.QA,
label: 'QA Analysis',
description: 'Run LLM quality analysis after each call',
icon: ClipboardCheck
}
];
@ -163,7 +169,7 @@ export default function AddNodePanel({ isOpen, onNodeSelect, onClose }: AddNodeP
<NodeSection
title="Integrations"
nodes={WEBHOOK_NODE_TYPES}
nodes={INTEGRATION_NODE_TYPES}
onNodeSelect={onNodeSelect}
/>
</div>

View file

@ -0,0 +1,348 @@
import { NodeProps, NodeToolbar, Position } from "@xyflow/react";
import { ChevronDown, ChevronRight, Circle, ClipboardCheck, Edit, Trash2Icon } from "lucide-react";
import { memo, useEffect, useMemo, useState } from "react";
import { useWorkflow } from "@/app/workflow/[workflowId]/contexts/WorkflowContext";
import { FlowNodeData } from "@/components/flow/types";
import { LLMConfigSelector } from "@/components/LLMConfigSelector";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Switch } from "@/components/ui/switch";
import { Textarea } from "@/components/ui/textarea";
import { NodeContent } from "./common/NodeContent";
import { NodeEditDialog } from "./common/NodeEditDialog";
import { useNodeHandlers } from "./common/useNodeHandlers";
interface QANodeProps extends NodeProps {
data: FlowNodeData;
}
export const QANode = memo(({ data, selected, id }: QANodeProps) => {
const { open, setOpen, handleSaveNodeData, handleDeleteNode } = useNodeHandlers({ id });
const { saveWorkflow } = useWorkflow();
// Form state
const [name, setName] = useState(data.name || "QA Analysis");
const [qaEnabled, setQaEnabled] = useState(data.qa_enabled ?? true);
const [useWorkflowLlm, setUseWorkflowLlm] = useState(data.qa_use_workflow_llm ?? true);
const [qaProvider, setQaProvider] = useState(data.qa_provider || "openai");
const [qaModel, setQaModel] = useState(data.qa_model || "gpt-4.1");
const [qaApiKey, setQaApiKey] = useState(data.qa_api_key || "");
const [qaSystemPrompt, setQaSystemPrompt] = useState(data.qa_system_prompt || "");
const [minCallDuration, setMinCallDuration] = useState(data.qa_min_call_duration ?? 15);
const [qaVoicemailCalls, setQaVoicemailCalls] = useState(data.qa_voicemail_calls ?? false);
const [qaSampleRate, setQaSampleRate] = useState(data.qa_sample_rate ?? 100);
const isDirty = useMemo(() => {
return (
name !== (data.name || "QA Analysis") ||
qaEnabled !== (data.qa_enabled ?? true) ||
useWorkflowLlm !== (data.qa_use_workflow_llm ?? true) ||
qaProvider !== (data.qa_provider || "openai") ||
qaModel !== (data.qa_model || "gpt-4.1") ||
qaApiKey !== (data.qa_api_key || "") ||
qaSystemPrompt !== (data.qa_system_prompt || "") ||
minCallDuration !== (data.qa_min_call_duration ?? 15) ||
qaVoicemailCalls !== (data.qa_voicemail_calls ?? false) ||
qaSampleRate !== (data.qa_sample_rate ?? 100)
);
}, [name, qaEnabled, useWorkflowLlm, qaProvider, qaModel, qaApiKey, qaSystemPrompt, minCallDuration, qaVoicemailCalls, qaSampleRate, data]);
const handleSave = async () => {
handleSaveNodeData({
...data,
name,
qa_enabled: qaEnabled,
qa_use_workflow_llm: useWorkflowLlm,
qa_provider: qaProvider,
qa_model: qaModel,
qa_api_key: qaApiKey,
qa_system_prompt: qaSystemPrompt,
qa_min_call_duration: minCallDuration,
qa_voicemail_calls: qaVoicemailCalls,
qa_sample_rate: qaSampleRate,
});
setOpen(false);
setTimeout(async () => {
await saveWorkflow();
}, 100);
};
const resetFormState = () => {
setName(data.name || "QA Analysis");
setQaEnabled(data.qa_enabled ?? true);
setUseWorkflowLlm(data.qa_use_workflow_llm ?? true);
setQaProvider(data.qa_provider || "openai");
setQaModel(data.qa_model || "gpt-4.1");
setQaApiKey(data.qa_api_key || "");
setQaSystemPrompt(data.qa_system_prompt || "");
setMinCallDuration(data.qa_min_call_duration ?? 15);
setQaVoicemailCalls(data.qa_voicemail_calls ?? false);
setQaSampleRate(data.qa_sample_rate ?? 100);
};
const handleOpenChange = (newOpen: boolean) => {
if (newOpen) {
resetFormState();
}
setOpen(newOpen);
};
useEffect(() => {
if (open) {
resetFormState();
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [data, open]);
return (
<>
<NodeContent
selected={selected}
invalid={data.invalid}
selected_through_edge={data.selected_through_edge}
hovered_through_edge={data.hovered_through_edge}
title={data.name || "QA Analysis"}
icon={<ClipboardCheck />}
nodeType="qa"
onDoubleClick={() => handleOpenChange(true)}
nodeId={id}
>
<div className="space-y-2">
<div className="flex items-center gap-2">
<span className="text-xs font-mono bg-muted px-1.5 py-0.5 rounded">
{data.qa_use_workflow_llm !== false
? "Workflow LLM"
: `${data.qa_provider || "openai"}/${data.qa_model || "gpt-4.1"}`}
</span>
</div>
<div className="flex items-center gap-1.5">
<Circle
className={`h-2 w-2 ${data.qa_enabled !== false ? "fill-green-500 text-green-500" : "fill-gray-400 text-gray-400"}`}
/>
<span className="text-xs text-muted-foreground">
{data.qa_enabled !== false ? "Enabled" : "Disabled"}
</span>
</div>
</div>
</NodeContent>
<NodeToolbar isVisible={selected} position={Position.Right}>
<div className="flex flex-col gap-1">
<Button onClick={() => handleOpenChange(true)} variant="outline" size="icon">
<Edit />
</Button>
<Button onClick={handleDeleteNode} variant="outline" size="icon">
<Trash2Icon />
</Button>
</div>
</NodeToolbar>
<NodeEditDialog
open={open}
onOpenChange={handleOpenChange}
nodeData={data}
title="Edit QA Analysis"
onSave={handleSave}
isDirty={isDirty}
>
{open && (
<QANodeEditForm
name={name}
setName={setName}
qaEnabled={qaEnabled}
setQaEnabled={setQaEnabled}
useWorkflowLlm={useWorkflowLlm}
setUseWorkflowLlm={setUseWorkflowLlm}
qaProvider={qaProvider}
setQaProvider={setQaProvider}
qaModel={qaModel}
setQaModel={setQaModel}
qaApiKey={qaApiKey}
setQaApiKey={setQaApiKey}
qaSystemPrompt={qaSystemPrompt}
setQaSystemPrompt={setQaSystemPrompt}
minCallDuration={minCallDuration}
setMinCallDuration={setMinCallDuration}
qaVoicemailCalls={qaVoicemailCalls}
setQaVoicemailCalls={setQaVoicemailCalls}
qaSampleRate={qaSampleRate}
setQaSampleRate={setQaSampleRate}
/>
)}
</NodeEditDialog>
</>
);
});
interface QANodeEditFormProps {
name: string;
setName: (value: string) => void;
qaEnabled: boolean;
setQaEnabled: (value: boolean) => void;
useWorkflowLlm: boolean;
setUseWorkflowLlm: (value: boolean) => void;
qaProvider: string;
setQaProvider: (value: string) => void;
qaModel: string;
setQaModel: (value: string) => void;
qaApiKey: string;
setQaApiKey: (value: string) => void;
qaSystemPrompt: string;
setQaSystemPrompt: (value: string) => void;
minCallDuration: number;
setMinCallDuration: (value: number) => void;
qaVoicemailCalls: boolean;
setQaVoicemailCalls: (value: boolean) => void;
qaSampleRate: number;
setQaSampleRate: (value: number) => void;
}
const QANodeEditForm = ({
name,
setName,
qaEnabled,
setQaEnabled,
useWorkflowLlm,
setUseWorkflowLlm,
qaProvider,
setQaProvider,
qaModel,
setQaModel,
qaApiKey,
setQaApiKey,
qaSystemPrompt,
setQaSystemPrompt,
minCallDuration,
setMinCallDuration,
qaVoicemailCalls,
setQaVoicemailCalls,
qaSampleRate,
setQaSampleRate,
}: QANodeEditFormProps) => {
const [advancedOpen, setAdvancedOpen] = useState(false);
return (
<div className="space-y-4">
<div className="grid gap-2">
<Label>Name</Label>
<Label className="text-xs text-muted-foreground">
A display name for this QA analysis node.
</Label>
<Input value={name} onChange={(e) => setName(e.target.value)} />
</div>
<div className="flex items-center space-x-2 p-2 border rounded-md bg-muted/20">
<Switch id="qa-enabled" checked={qaEnabled} onCheckedChange={setQaEnabled} />
<Label htmlFor="qa-enabled">Enabled</Label>
<Label className="text-xs text-muted-foreground ml-2">
Whether this QA analysis runs after each call.
</Label>
</div>
<div className="flex items-center space-x-2 p-2 border rounded-md bg-muted/20">
<Switch
id="use-workflow-llm"
checked={useWorkflowLlm}
onCheckedChange={setUseWorkflowLlm}
/>
<Label htmlFor="use-workflow-llm">Use Workflow LLM</Label>
<Label className="text-xs text-muted-foreground ml-2">
Use the LLM configured in your account settings.
</Label>
</div>
{!useWorkflowLlm && (
<LLMConfigSelector
provider={qaProvider}
onProviderChange={setQaProvider}
model={qaModel}
onModelChange={setQaModel}
apiKey={qaApiKey}
onApiKeyChange={setQaApiKey}
/>
)}
<div className="grid gap-2">
<Label>System Prompt</Label>
<Label className="text-xs text-muted-foreground">
The prompt sent to the LLM for QA analysis. Use {'{metrics}'} placeholder for
call metrics.
</Label>
<Textarea
value={qaSystemPrompt}
onChange={(e) => setQaSystemPrompt(e.target.value)}
className="min-h-[300px] font-mono text-xs"
placeholder="Enter QA analysis system prompt..."
/>
</div>
{/* Advanced Configuration */}
<div className="border rounded-md">
<button
type="button"
className="flex items-center gap-2 w-full p-3 text-sm font-medium hover:bg-muted/50 transition-colors"
onClick={() => setAdvancedOpen(!advancedOpen)}
>
{advancedOpen ? (
<ChevronDown className="h-4 w-4" />
) : (
<ChevronRight className="h-4 w-4" />
)}
Advanced Configuration
</button>
{advancedOpen && (
<div className="px-3 pb-3 space-y-4 border-t pt-3">
<div className="grid gap-2">
<Label>Minimum Call Duration (seconds)</Label>
<Label className="text-xs text-muted-foreground">
Calls shorter than this duration will be skipped from QA analysis.
</Label>
<Input
type="number"
min={0}
value={minCallDuration}
onChange={(e) => setMinCallDuration(Number(e.target.value))}
/>
</div>
<div className="flex items-center space-x-2 p-2 border rounded-md bg-muted/20">
<Switch
id="qa-voicemail"
checked={qaVoicemailCalls}
onCheckedChange={setQaVoicemailCalls}
/>
<Label htmlFor="qa-voicemail">QA Voicemail Calls</Label>
<Label className="text-xs text-muted-foreground ml-2">
Run QA analysis on calls that reached voicemail.
</Label>
</div>
<div className="grid gap-2">
<Label>Sample Rate (%)</Label>
<Label className="text-xs text-muted-foreground">
Percentage of eligible calls to run QA analysis on.
</Label>
<Input
type="number"
min={1}
max={100}
value={qaSampleRate}
onChange={(e) =>
setQaSampleRate(
Math.min(100, Math.max(1, Number(e.target.value)))
)
}
/>
</div>
</div>
)}
</div>
</div>
);
};
QANode.displayName = "QANode";

View file

@ -12,7 +12,7 @@ interface NodeContentProps {
hovered_through_edge?: boolean;
title: string;
icon: ReactNode;
nodeType?: 'start' | 'agent' | 'end' | 'global' | 'trigger' | 'webhook';
nodeType?: 'start' | 'agent' | 'end' | 'global' | 'trigger' | 'webhook' | 'qa';
hasSourceHandle?: boolean;
hasTargetHandle?: boolean;
children?: ReactNode;
@ -36,6 +36,8 @@ const getNodeTypeBadge = (nodeType?: string) => {
return { label: 'API Trigger', className: 'bg-purple-500 text-white' };
case 'webhook':
return { label: 'Webhook', className: 'bg-indigo-500 text-white' };
case 'qa':
return { label: 'QA Analysis', className: 'bg-teal-500 text-white' };
default:
return { label: 'Node', className: 'bg-zinc-500 text-white' };
}

View file

@ -1,6 +1,7 @@
export * from './AgentNode';
export * from './EndCall';
export * from './GlobalNode';
export * from './QANode';
export * from './StartCall';
export * from './TriggerNode';
export * from './WebhookNode';

View file

@ -4,7 +4,8 @@ export enum NodeType {
END_CALL = 'endCall',
GLOBAL_NODE = 'globalNode',
TRIGGER = 'trigger',
WEBHOOK = 'webhook'
WEBHOOK = 'webhook',
QA = 'qa'
}
export type FlowNodeData = {
@ -40,6 +41,16 @@ export type FlowNodeData = {
max_retries: number;
retry_delay_seconds: number;
};
// QA node specific
qa_enabled?: boolean;
qa_system_prompt?: string;
qa_use_workflow_llm?: boolean;
qa_provider?: string;
qa_model?: string;
qa_api_key?: string;
qa_min_call_duration?: number;
qa_voicemail_calls?: boolean;
qa_sample_rate?: number;
// Tools - array of tool UUIDs that can be invoked by this node
tool_uuids?: string[];
// Documents - array of knowledge base document UUIDs that can be referenced by this node