mirror of
https://github.com/dograh-hq/dograh.git
synced 2026-06-28 08:49:42 +02:00
feat: add rtf log when user speaks when muted
This commit is contained in:
Abhishek Kumar
parent
93c45580e7
commit
1967a71935
13 changed files with 196 additions and 31 deletions
|
|
@ -17,6 +17,7 @@ interface RealtimeFeedbackEvent {
|
|||
result?: string;
|
||||
node_name?: string;
|
||||
previous_node?: string;
|
||||
allow_interrupt?: boolean;
|
||||
ttfb_seconds?: number;
|
||||
processor?: string;
|
||||
model?: string;
|
||||
|
|
@ -79,6 +80,9 @@ function convertLogEventsToTranscriptEvents(events: RealtimeFeedbackEvent[]): Tr
|
|||
case 'rtf-pipeline-error':
|
||||
type = 'pipeline-error';
|
||||
break;
|
||||
case 'rtf-interrupt-warning':
|
||||
type = 'interrupt-warning';
|
||||
break;
|
||||
default:
|
||||
type = 'bot-text';
|
||||
}
|
||||
|
|
@ -93,6 +97,7 @@ function convertLogEventsToTranscriptEvents(events: RealtimeFeedbackEvent[]): Tr
|
|||
status,
|
||||
nodeName: event.payload.node_name,
|
||||
previousNode: event.payload.previous_node,
|
||||
allowInterrupt: event.payload.allow_interrupt,
|
||||
ttfbSeconds: event.payload.ttfb_seconds,
|
||||
processor: event.payload.processor,
|
||||
model: event.payload.model,
|
||||
|
|
@ -114,6 +119,7 @@ function convertLiveMessagesToTranscriptEvents(messages: FeedbackMessage[]): Tra
|
|||
status: msg.status,
|
||||
nodeName: msg.nodeName,
|
||||
previousNode: msg.previousNode,
|
||||
allowInterrupt: msg.allowInterrupt,
|
||||
ttfbSeconds: msg.ttfbSeconds,
|
||||
processor: msg.processor,
|
||||
model: msg.model,
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ export const UnifiedTranscript = ({
|
|||
functionName: msg.functionName,
|
||||
status: msg.status,
|
||||
nodeName: msg.nodeName,
|
||||
allowInterrupt: msg.allowInterrupt,
|
||||
ttfbSeconds: msg.ttfbSeconds,
|
||||
fatal: msg.fatal,
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -1,16 +1,17 @@
|
|||
'use client';
|
||||
|
||||
import { AlertTriangle, Brain, GitBranch, Wrench } from 'lucide-react';
|
||||
import { AlertTriangle, Brain, ExternalLink, GitBranch, MicOff, Wrench } from 'lucide-react';
|
||||
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
export interface TranscriptMessageData {
|
||||
id: string;
|
||||
type: 'user-transcription' | 'bot-text' | 'function-call' | 'node-transition' | 'ttfb-metric' | 'pipeline-error';
|
||||
type: 'user-transcription' | 'bot-text' | 'function-call' | 'node-transition' | 'ttfb-metric' | 'pipeline-error' | 'interrupt-warning';
|
||||
text: string;
|
||||
final?: boolean;
|
||||
functionName?: string;
|
||||
nodeName?: string;
|
||||
allowInterrupt?: boolean;
|
||||
ttfbSeconds?: number;
|
||||
fatal?: boolean;
|
||||
}
|
||||
|
|
@ -37,6 +38,31 @@ export function TranscriptMessage({ message, nextMessage }: TranscriptMessagePro
|
|||
);
|
||||
}
|
||||
|
||||
// Interrupt warning - show as an amber alert (one-time)
|
||||
if (message.type === 'interrupt-warning') {
|
||||
return (
|
||||
<div className="flex items-start gap-2 px-3 py-2 rounded-lg bg-amber-500/10 border border-amber-500/20">
|
||||
<MicOff className="h-4 w-4 text-amber-500 mt-0.5 shrink-0" />
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="text-xs font-medium text-amber-700 dark:text-amber-400">
|
||||
Interruption Disabled
|
||||
</div>
|
||||
<div className="text-sm text-amber-600 dark:text-amber-300 mt-0.5">
|
||||
{message.text}
|
||||
</div>
|
||||
<a
|
||||
href="https://docs.dograh.com/configurations/interruption"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="inline-flex items-center gap-1 text-xs text-amber-600 dark:text-amber-400 hover:underline mt-1"
|
||||
>
|
||||
Learn more <ExternalLink className="h-3 w-3" />
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Pipeline error - show as a red alert
|
||||
if (message.type === 'pipeline-error') {
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ interface UseWebSocketRTCProps {
|
|||
|
||||
export interface FeedbackMessage {
|
||||
id: string;
|
||||
type: 'user-transcription' | 'bot-text' | 'function-call' | 'node-transition' | 'ttfb-metric' | 'pipeline-error';
|
||||
type: 'user-transcription' | 'bot-text' | 'function-call' | 'node-transition' | 'ttfb-metric' | 'pipeline-error' | 'interrupt-warning';
|
||||
text: string;
|
||||
final?: boolean;
|
||||
timestamp: string;
|
||||
|
|
@ -27,6 +27,7 @@ export interface FeedbackMessage {
|
|||
// Node transition fields
|
||||
nodeName?: string;
|
||||
previousNode?: string;
|
||||
allowInterrupt?: boolean;
|
||||
// TTFB metric fields
|
||||
ttfbSeconds?: number;
|
||||
processor?: string;
|
||||
|
|
@ -82,6 +83,12 @@ export const useWebSocketRTC = ({ workflowId, workflowRunId, accessToken, initia
|
|||
|
||||
const pc_id = useRef(generateSecureId());
|
||||
|
||||
// Mute/speaking state tracking refs (ephemeral signals, not rendered directly)
|
||||
const userMutedRef = useRef(false);
|
||||
const firstBotSpeechCompletedRef = useRef(false);
|
||||
const currentAllowInterruptRef = useRef<boolean | undefined>(undefined);
|
||||
const interruptWarningShownRef = useRef(false);
|
||||
|
||||
// Get WebSocket URL from client configuration
|
||||
const getWebSocketUrl = useCallback(() => {
|
||||
// Get base URL from client configuration
|
||||
|
|
@ -287,6 +294,24 @@ export const useWebSocketRTC = ({ workflowId, workflowRunId, accessToken, initia
|
|||
|
||||
case 'rtf-user-transcription': {
|
||||
const transcription = message.payload;
|
||||
|
||||
// Show one-time warning if user speaks while muted on a no-interrupt node
|
||||
// Skip during initial bot greeting (muted by MuteUntilFirstBotComplete strategy)
|
||||
if (
|
||||
!interruptWarningShownRef.current &&
|
||||
firstBotSpeechCompletedRef.current &&
|
||||
userMutedRef.current &&
|
||||
currentAllowInterruptRef.current === false
|
||||
) {
|
||||
interruptWarningShownRef.current = true;
|
||||
setFeedbackMessages(prev => [...prev, {
|
||||
id: `interrupt-warning-${Date.now()}`,
|
||||
type: 'interrupt-warning',
|
||||
text: 'Interruption is disabled for this step. The bot will finish speaking before processing your input. You can enable interruption in the workflow editor.',
|
||||
timestamp: new Date().toISOString(),
|
||||
}]);
|
||||
}
|
||||
|
||||
setFeedbackMessages(prev => {
|
||||
// Step 1: Finalize the last bot message (user started speaking)
|
||||
const messagesWithBotFinalized = prev.map((msg, idx) => {
|
||||
|
|
@ -322,7 +347,7 @@ export const useWebSocketRTC = ({ workflowId, workflowRunId, accessToken, initia
|
|||
// Append to existing bot message
|
||||
return [
|
||||
...prev.slice(0, -1),
|
||||
{ ...last, text: last.text + message.payload.text }
|
||||
{ ...last, text: last.text + ' ' + message.payload.text }
|
||||
];
|
||||
}
|
||||
// Start new bot message
|
||||
|
|
@ -368,13 +393,15 @@ export const useWebSocketRTC = ({ workflowId, workflowRunId, accessToken, initia
|
|||
}
|
||||
|
||||
case 'rtf-node-transition': {
|
||||
const { node_name, previous_node } = message.payload;
|
||||
const { node_name, previous_node_name, allow_interrupt } = message.payload;
|
||||
currentAllowInterruptRef.current = allow_interrupt;
|
||||
setFeedbackMessages(prev => [...prev, {
|
||||
id: `node-${Date.now()}`,
|
||||
type: 'node-transition',
|
||||
text: node_name,
|
||||
nodeName: node_name,
|
||||
previousNode: previous_node,
|
||||
previousNode: previous_node_name,
|
||||
allowInterrupt: allow_interrupt,
|
||||
timestamp: new Date().toISOString(),
|
||||
}]);
|
||||
break;
|
||||
|
|
@ -407,6 +434,24 @@ export const useWebSocketRTC = ({ workflowId, workflowRunId, accessToken, initia
|
|||
break;
|
||||
}
|
||||
|
||||
// Ephemeral state signals — update refs only, no UI messages
|
||||
case 'rtf-bot-started-speaking':
|
||||
break;
|
||||
|
||||
case 'rtf-bot-stopped-speaking':
|
||||
if (!firstBotSpeechCompletedRef.current) {
|
||||
firstBotSpeechCompletedRef.current = true;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'rtf-user-mute-started':
|
||||
userMutedRef.current = true;
|
||||
break;
|
||||
|
||||
case 'rtf-user-mute-stopped':
|
||||
userMutedRef.current = false;
|
||||
break;
|
||||
|
||||
default:
|
||||
logger.warn('Unknown message type:', message.type);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
*/
|
||||
|
||||
export interface TranscriptEvent {
|
||||
type: 'user-transcription' | 'bot-text' | 'function-call' | 'node-transition' | 'ttfb-metric' | 'pipeline-error';
|
||||
type: 'user-transcription' | 'bot-text' | 'function-call' | 'node-transition' | 'ttfb-metric' | 'pipeline-error' | 'interrupt-warning';
|
||||
text: string;
|
||||
final?: boolean;
|
||||
timestamp: string;
|
||||
|
|
@ -13,6 +13,7 @@ export interface TranscriptEvent {
|
|||
status?: 'running' | 'completed';
|
||||
nodeName?: string;
|
||||
previousNode?: string;
|
||||
allowInterrupt?: boolean;
|
||||
ttfbSeconds?: number;
|
||||
processor?: string;
|
||||
model?: string;
|
||||
|
|
@ -28,6 +29,7 @@ export interface ProcessedMessage {
|
|||
functionName?: string;
|
||||
status?: 'running' | 'completed';
|
||||
nodeName?: string;
|
||||
allowInterrupt?: boolean;
|
||||
ttfbSeconds?: number;
|
||||
fatal?: boolean;
|
||||
}
|
||||
|
|
@ -69,7 +71,7 @@ export function processTranscriptEvents(events: TranscriptEvent[]): ProcessedMes
|
|||
} else if (event.type === 'bot-text') {
|
||||
// Combine consecutive bot-text from the same turn
|
||||
if (currentBotText && currentBotText.event.turn === event.turn) {
|
||||
currentBotText.text = currentBotText.text + event.text;
|
||||
currentBotText.text = currentBotText.text + ' ' + event.text;
|
||||
} else {
|
||||
flushBotText();
|
||||
currentBotText = { event, text: event.text };
|
||||
|
|
@ -144,6 +146,7 @@ function convertToProcessedMessage(event: TranscriptEvent, overrideText?: string
|
|||
functionName: event.functionName,
|
||||
status: event.status,
|
||||
nodeName: event.nodeName,
|
||||
allowInterrupt: event.allowInterrupt,
|
||||
ttfbSeconds: event.ttfbSeconds,
|
||||
fatal: event.fatal,
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue