apunkt/dograh
1
0
Fork 0
mirror of https://github.com/dograh-hq/dograh.git synced 2026-06-07 07:55:16 +02:00
Code Issues Projects Releases Packages Wiki Activity

chore: unify the call logs and real time events

Browse source
This commit is contained in:
Abhishek Kumar 2026-01-15 16:16:32 +05:30
parent d25f898a8f
commit 58f0bbe184
16 changed files with 753 additions and 359 deletions
Download patch file Download diff file Expand all files Collapse all files

3
api/services/pipecat/event_handlers.py
View file

@ -47,7 +47,6 @@ def register_transport_event_handlers(
num_channels=num_channels,
)
in_memory_transcript_buffer = InMemoryTranscriptBuffer(workflow_run_id)
in_memory_logs_buffer = InMemoryLogsBuffer(workflow_run_id)
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, participant):
@ -71,7 +70,7 @@ def register_transport_event_handlers(
await task.cancel()
# Return the buffers so they can be passed to other handlers
return in_memory_audio_buffer, in_memory_transcript_buffer, in_memory_logs_buffer
return in_memory_audio_buffer, in_memory_transcript_buffer
def register_task_event_handler(

38
api/services/pipecat/realtime_feedback_observer.py
View file

@ -1,11 +1,16 @@
"""Real-time feedback observer for sending pipeline events to the frontend.
This observer watches pipeline frames and sends relevant events (transcriptions,
bot text) over WebSocket to provide real-time feedback in the UI.
bot text, function calls, TTFB metrics) over WebSocket to provide real-time
feedback in the UI.
For frames with presentation timestamps (pts), like TTSTextFrame, we respect
the timing by queuing them and sending at the appropriate time, similar to
how base_output.py handles timed frames.
Note: Node transition events are sent directly from PipecatEngine.set_node()
rather than being observed here, to ensure precise timing at the moment of
node changes.
"""
import asyncio
@ -24,20 +29,30 @@ from pipecat.frames.frames import (
FunctionCallResultFrame,
InterimTranscriptionFrame,
InterruptionFrame,
MetricsFrame,
StopFrame,
TranscriptionFrame,
TTSTextFrame,
)
from pipecat.metrics.metrics import TTFBMetricsData
from pipecat.observers.base_observer import BaseObserver, FramePushed
from pipecat.processors.frame_processor import FrameDirection
from pipecat.utils.time import nanoseconds_to_seconds
class RealtimeFeedbackObserver(BaseObserver):
"""Observer that sends real-time transcription and bot response events via WebSocket.
"""Observer that sends real-time transcription, bot response, and metrics via WebSocket.
Observes pipeline frames and sends events for:
- User transcriptions (interim and final)
- Bot TTS text (with pts-based timing)
- Function calls (start/end)
- TTFB metrics (LLM generation time only - filters to processors containing "LLM")
For frames with pts (presentation timestamp), we queue them and send at the
appropriate time to sync with audio playback.
Note: Node transitions are handled by PipecatEngine.set_node() callback.
"""
def __init__(
@ -132,6 +147,8 @@ class RealtimeFeedbackObserver(BaseObserver):
frame = data.frame
frame_direction = data.direction
logger.trace(f"{self} Received Frame: {frame} Direction: {frame_direction}")
# Handle pipeline termination - stop clock task
if isinstance(frame, (EndFrame, CancelFrame, StopFrame)):
await self._cancel_clock_task()
@ -226,6 +243,23 @@ class RealtimeFeedbackObserver(BaseObserver):
},
}
)
# Handle TTFB metrics - capture LLM generation time only
elif isinstance(frame, MetricsFrame):
# Check if this MetricsFrame contains TTFB data from an LLM processor
for metric_data in frame.data:
if isinstance(metric_data, TTFBMetricsData):
# Only send TTFB if it's from an LLM processor
if metric_data.processor and "LLM" in metric_data.processor:
await self._send_message(
{
"type": "rtf-ttfb-metric",
"payload": {
"ttfb_seconds": metric_data.value,
"processor": metric_data.processor,
"model": metric_data.model,
},
}
)
async def _send_message(self, message: dict):
"""Send message via WebSocket AND append to logs buffer, handling errors gracefully."""

42
api/services/pipecat/run_pipeline.py
View file

@ -14,6 +14,7 @@ from api.services.pipecat.event_handlers import (
register_transcript_handler,
register_transport_event_handlers,
)
from api.services.pipecat.in_memory_buffers import InMemoryLogsBuffer
from api.services.pipecat.pipeline_builder import (
build_pipeline,
create_pipeline_components,
@ -467,11 +468,45 @@ async def _run_pipeline(
ReactFlowDTO.model_validate(workflow.workflow_definition_with_fallback)
)
# Create in-memory logs buffer early so it can be used by engine callbacks
in_memory_logs_buffer = InMemoryLogsBuffer(workflow_run_id)
# Create node transition callback if WebSocket sender is available
node_transition_callback = None
ws_sender = get_ws_sender(workflow_run_id)
if ws_sender:
async def send_node_transition(
node_name: str, previous_node: Optional[str]
) -> None:
"""Send node transition event via WebSocket AND log to buffer."""
message = {
"type": "rtf-node-transition",
"payload": {
"node_name": node_name,
"previous_node": previous_node,
},
}
# Send via WebSocket
try:
await ws_sender(message)
except Exception as e:
logger.debug(f"Failed to send node transition via WebSocket: {e}")
# Log to in-memory buffer
try:
await in_memory_logs_buffer.append(message)
except Exception as e:
logger.error(f"Failed to append node transition to logs buffer: {e}")
node_transition_callback = send_node_transition
engine = PipecatEngine(
llm=llm,
workflow=workflow_graph,
call_context_vars=merged_call_context_vars,
workflow_run_id=workflow_run_id,
node_transition_callback=node_transition_callback,
)
# Create pipeline components with audio configuration and engine
@ -573,7 +608,7 @@ async def _run_pipeline(
await engine.initialize()
# Register event handlers
in_memory_audio_buffer, in_memory_transcript_buffer, in_memory_logs_buffer = (
in_memory_audio_buffer, in_memory_transcript_buffer = (
register_transport_event_handlers(
task,
transport,
@ -585,10 +620,11 @@ async def _run_pipeline(
)
# Add real-time feedback observer if WebSocket sender is available
ws_sender = get_ws_sender(workflow_run_id)
# Note: ws_sender was already fetched earlier for node_transition_callback
if ws_sender:
feedback_observer = RealtimeFeedbackObserver(
ws_sender=ws_sender, logs_buffer=in_memory_logs_buffer
ws_sender=ws_sender,
logs_buffer=in_memory_logs_buffer,
)
task.add_observer(feedback_observer)

17
api/services/workflow/pipecat_engine.py
View file

@ -62,6 +62,9 @@ class PipecatEngine:
call_context_vars: dict,
audio_buffer: Optional["AudioBuffer"] = None,
workflow_run_id: Optional[int] = None,
node_transition_callback: Optional[
Callable[[str, Optional[str]], Awaitable[None]]
] = None,
):
self.task = task
self.llm = llm
@ -71,6 +74,7 @@ class PipecatEngine:
self._call_context_vars = call_context_vars
self._audio_buffer = audio_buffer
self._workflow_run_id = workflow_run_id
self._node_transition_callback = node_transition_callback
self._initialized = False
self._client_disconnected = False
self._call_disposed = False
@ -359,9 +363,20 @@ class PipecatEngine:
f"Executing node: name: {node.name} is_static: {node.is_static} allow_interrupt: {node.allow_interrupt} is_end: {node.is_end}"
)
# Track previous node for transition event
previous_node_name = self._current_node.name if self._current_node else None
# Set current node for all nodes (including static ones) so STT mute filter works
self._current_node = node
# Send node transition event if callback is provided
if self._node_transition_callback:
try:
await self._node_transition_callback(node.name, previous_node_name)
except Exception as e:
# Log but don't fail - feedback is non-critical
logger.debug(f"Failed to send node transition event: {e}")
# Handle start nodes
if node.is_start:
await self._handle_start_node(node)
@ -693,5 +708,3 @@ class PipecatEngine:
and not self._user_response_timeout_task.done()
):
self._user_response_timeout_task.cancel()
# Note: Native VoicemailDetector cleanup is handled by the pipeline

6
ui/src/app/workflow/[workflowId]/run/[runId]/BrowserCall.tsx
View file

@ -9,7 +9,7 @@ import {
ApiKeyErrorDialog,
AudioControls,
ConnectionStatus,
RealtimeFeedbackPanel,
RealtimeFeedback,
WorkflowConfigErrorDialog
} from "./components";
import { useWebSocketRTC } from "./hooks";
@ -142,9 +142,9 @@ const BrowserCall = ({ workflowId, workflowRunId, accessToken, initialContextVar
{/* Show transcript panel */}
<div className="w-1/3 h-full shrink-0">
<RealtimeFeedbackPanel
<RealtimeFeedback
mode="live"
messages={feedbackMessages}
isVisible={true}
isCallActive={connectionActive}
isCallCompleted={isCompleted}
/>

161
ui/src/app/workflow/[workflowId]/run/[runId]/components/RealtimeFeedback.tsx Normal file
View file

@ -0,0 +1,161 @@
'use client';
import { FeedbackMessage } from '../hooks/useWebSocketRTC';
import { processLiveMessages, processTranscriptEvents, TranscriptEvent } from '../utils/processTranscriptEvents';
import { UnifiedTranscript } from './UnifiedTranscript';
// Historical log event format from the backend
interface RealtimeFeedbackEvent {
type: string;
payload: {
text?: string;
final?: boolean;
user_id?: string;
timestamp?: string;
function_name?: string;
tool_call_id?: string;
result?: string;
node_name?: string;
previous_node?: string;
ttfb_seconds?: number;
processor?: string;
model?: string;
};
timestamp: string;
turn: number;
}
export interface WorkflowRunLogs {
realtime_feedback_events?: RealtimeFeedbackEvent[];
}
// Props for live mode (WebSocket messages)
interface LiveModeProps {
mode: 'live';
messages: FeedbackMessage[];
isCallActive: boolean;
isCallCompleted: boolean;
}
// Props for historical mode (API logs)
interface HistoricalModeProps {
mode: 'historical';
logs: WorkflowRunLogs | null;
}
type RealtimeFeedbackProps = LiveModeProps | HistoricalModeProps;
/**
* Convert backend log events to unified TranscriptEvent format
*/
function convertLogEventsToTranscriptEvents(events: RealtimeFeedbackEvent[]): TranscriptEvent[] {
return events.map(event => {
let type: TranscriptEvent['type'];
let status: TranscriptEvent['status'];
switch (event.type) {
case 'rtf-user-transcription':
type = 'user-transcription';
break;
case 'rtf-bot-text':
type = 'bot-text';
break;
case 'rtf-function-call-start':
type = 'function-call';
status = 'running';
break;
case 'rtf-function-call-end':
type = 'function-call';
status = 'completed';
break;
case 'rtf-node-transition':
type = 'node-transition';
break;
case 'rtf-ttfb-metric':
type = 'ttfb-metric';
break;
default:
type = 'bot-text';
}
return {
type,
text: event.payload.text || event.payload.result || event.payload.function_name || event.payload.node_name || '',
final: event.payload.final,
timestamp: event.timestamp,
turn: event.turn,
functionName: event.payload.function_name,
status,
nodeName: event.payload.node_name,
previousNode: event.payload.previous_node,
ttfbSeconds: event.payload.ttfb_seconds,
processor: event.payload.processor,
model: event.payload.model,
};
});
}
/**
* Convert live WebSocket messages to unified TranscriptEvent format
*/
function convertLiveMessagesToTranscriptEvents(messages: FeedbackMessage[]): TranscriptEvent[] {
return messages.map(msg => ({
type: msg.type,
text: msg.text,
final: msg.final,
timestamp: msg.timestamp,
functionName: msg.functionName,
status: msg.status,
nodeName: msg.nodeName,
previousNode: msg.previousNode,
ttfbSeconds: msg.ttfbSeconds,
processor: msg.processor,
model: msg.model,
}));
}
/**
* Single unified component that handles both live WebSocket messages
* and historical logs from the API.
*/
export const RealtimeFeedback = (props: RealtimeFeedbackProps) => {
if (props.mode === 'historical') {
// Historical mode - process logs from API
const rawEvents = props.logs?.realtime_feedback_events;
const messages = rawEvents
? processTranscriptEvents(convertLogEventsToTranscriptEvents(rawEvents))
: [];
return (
<UnifiedTranscript
messages={messages}
status="ended"
title="Call Transcript"
emptyState={{
title: "No conversation recorded",
subtitle: "Real-time feedback events were not captured for this call"
}}
/>
);
}
// Live mode - process WebSocket messages (optimized - messages already accumulated)
const { messages, isCallActive, isCallCompleted } = props;
const status = isCallActive ? 'live' : isCallCompleted ? 'ended' : 'ready';
const processedMessages = processLiveMessages(convertLiveMessagesToTranscriptEvents(messages));
return (
<UnifiedTranscript
messages={processedMessages}
status={status}
title="Live Transcript"
autoScroll={true}
emptyState={{
title: "No messages yet",
subtitle: isCallActive
? "Start speaking to see the transcript"
: "Start the call to begin the conversation"
}}
/>
);
};

166
ui/src/app/workflow/[workflowId]/run/[runId]/components/RealtimeFeedbackLogs.tsx
View file

@ -1,166 +0,0 @@
'use client';
import { CheckCircle, MessageSquare, MicOff,Wrench } from 'lucide-react';
import { cn } from '@/lib/utils';
interface RealtimeFeedbackEvent {
type: string;
payload: {
text?: string;
final?: boolean;
user_id?: string;
timestamp?: string;
function_name?: string;
tool_call_id?: string;
result?: string;
};
timestamp: string;
turn: number;
}
export interface WorkflowRunLogs {
realtime_feedback_events?: RealtimeFeedbackEvent[];
}
interface RealtimeFeedbackLogsProps {
logs: WorkflowRunLogs | null;
}
const EventItem = ({ event }: { event: RealtimeFeedbackEvent }) => {
// Function call message - centered
if (event.type === 'rtf-function-call-start' || event.type === 'rtf-function-call-end') {
return (
<div className="flex justify-center">
<div className="px-3 py-1.5 rounded-full text-xs bg-amber-500/10 border border-amber-500/20 inline-flex items-center gap-2">
{event.type === 'rtf-function-call-start' ? (
<Wrench className="h-3 w-3 text-amber-500" />
) : (
<CheckCircle className="h-3 w-3 text-green-500" />
)}
<span className="font-mono text-amber-700 dark:text-amber-400">
{event.payload.function_name}()
</span>
{event.type === 'rtf-function-call-end' && (
<span className="text-muted-foreground">✓</span>
)}
</div>
</div>
);
}
const isUser = event.type === 'rtf-user-transcription';
// User messages on right, bot messages on left
return (
<div className={cn(
"flex",
isUser ? "justify-end" : "justify-start"
)}>
<div
className={cn(
"max-w-[85%] px-3 py-2 rounded-2xl text-sm",
isUser
? "bg-primary text-primary-foreground rounded-br-md"
: "bg-muted rounded-bl-md"
)}
>
<div className="whitespace-pre-wrap leading-relaxed">{event.payload.text}</div>
</div>
</div>
);
};
function processEvents(events: RealtimeFeedbackEvent[]): RealtimeFeedbackEvent[] {
// Filter out interim transcriptions
const filteredEvents = events.filter(event => {
if (event.type === 'rtf-user-transcription' && !event.payload.final) {
return false;
}
return true;
});
// Combine consecutive rtf-bot-text events by turn
const processed: RealtimeFeedbackEvent[] = [];
let currentBotText: RealtimeFeedbackEvent | null = null;
for (const event of filteredEvents) {
if (event.type === 'rtf-bot-text') {
if (currentBotText && currentBotText.turn === event.turn) {
// Same turn, combine the text
currentBotText.payload.text = (currentBotText.payload.text || '') + ' ' + (event.payload.text || '');
} else {
// Different turn or first bot text
if (currentBotText) {
processed.push(currentBotText);
}
// Deep copy to avoid mutating original event
currentBotText = {
...event,
payload: { ...event.payload }
};
}
} else {
// Not a bot text event
if (currentBotText) {
processed.push(currentBotText);
currentBotText = null;
}
processed.push(event);
}
}
// Don't forget the last bot text if there is one
if (currentBotText) {
processed.push(currentBotText);
}
return processed;
}
export function RealtimeFeedbackLogs({ logs }: RealtimeFeedbackLogsProps) {
const rawEvents = logs?.realtime_feedback_events;
const events = rawEvents ? processEvents(rawEvents) : undefined;
return (
<div className="w-full h-full flex flex-col bg-background border-l border-border">
{/* Header */}
<div className="px-4 py-3 border-b border-border shrink-0">
<div className="flex items-center justify-center gap-2">
<MessageSquare className="h-4 w-4 text-muted-foreground shrink-0" />
<span className="font-medium text-sm whitespace-nowrap">Call Transcript</span>
<div className="flex items-center gap-1 text-xs px-2 py-0.5 rounded-full shrink-0 bg-muted text-muted-foreground">
<MicOff className="h-3 w-3" />
<span>Ended</span>
</div>
</div>
</div>
{/* Messages */}
<div className="flex-1 overflow-y-auto">
{!events || events.length === 0 ? (
<div className="flex flex-col items-center justify-center h-full text-muted-foreground text-sm">
<MessageSquare className="h-10 w-10 mb-4 opacity-30" />
<p className="font-medium">No conversation recorded</p>
<p className="text-xs mt-1 text-center px-4">
Real-time feedback events were not captured for this call
</p>
</div>
) : (
<div className="space-y-3 p-4">
{events.map((event, index) => (
<EventItem key={index} event={event} />
))}
</div>
)}
</div>
{/* Footer with message count */}
{events && events.length > 0 && (
<div className="px-4 py-2 border-t border-border text-xs text-muted-foreground shrink-0">
{events.filter(e => e.type === 'rtf-user-transcription' || e.type === 'rtf-bot-text').length} messages
</div>
)}
</div>
);
}

152
ui/src/app/workflow/[workflowId]/run/[runId]/components/RealtimeFeedbackPanel.tsx
View file

@ -1,152 +0,0 @@
"use client";
import { Loader2, MessageSquare, Mic, MicOff, Wrench } from "lucide-react";
import { useEffect, useRef } from "react";
import { cn } from "@/lib/utils";
import { FeedbackMessage } from "../hooks/useWebSocketRTC";
interface RealtimeFeedbackPanelProps {
messages: FeedbackMessage[];
isVisible: boolean;
isCallActive: boolean;
isCallCompleted: boolean;
}
const MessageItem = ({ msg }: { msg: FeedbackMessage }) => {
// Function call message - centered
if (msg.type === 'function-call') {
return (
<div className="flex justify-center">
<div className="px-3 py-1.5 rounded-full text-xs bg-amber-500/10 border border-amber-500/20 inline-flex items-center gap-2">
{msg.status === 'running' ? (
<Loader2 className="h-3 w-3 animate-spin text-amber-500" />
) : (
<Wrench className="h-3 w-3 text-amber-500" />
)}
<span className="font-mono text-amber-700 dark:text-amber-400">
{msg.functionName}()
</span>
{msg.status === 'completed' && (
<span className="text-muted-foreground">✓</span>
)}
</div>
</div>
);
}
const isUser = msg.type === 'user-transcription';
// User messages on right, bot messages on left
return (
<div className={cn(
"flex",
isUser ? "justify-end" : "justify-start"
)}>
<div
className={cn(
"max-w-[85%] px-3 py-2 rounded-2xl text-sm",
isUser
? "bg-primary text-primary-foreground rounded-br-md"
: "bg-muted rounded-bl-md",
!msg.final && "opacity-70"
)}
>
<div className="whitespace-pre-wrap leading-relaxed">{msg.text}</div>
{!msg.final && (
<div className={cn(
"text-[10px] mt-1 italic",
isUser ? "text-primary-foreground/70" : "text-muted-foreground"
)}>
speaking...
</div>
)}
</div>
</div>
);
};
export const RealtimeFeedbackPanel = ({
messages,
isVisible,
isCallActive,
isCallCompleted
}: RealtimeFeedbackPanelProps) => {
const scrollRef = useRef<HTMLDivElement>(null);
// Auto-scroll to bottom when new messages arrive
useEffect(() => {
if (scrollRef.current) {
scrollRef.current.scrollTop = scrollRef.current.scrollHeight;
}
}, [messages]);
if (!isVisible) return null;
return (
<div className="w-full h-full flex flex-col bg-background border-l border-border">
{/* Header */}
<div className="px-4 py-3 border-b border-border shrink-0">
<div className="flex items-center justify-center gap-2">
<MessageSquare className="h-4 w-4 text-muted-foreground shrink-0" />
<span className="font-medium text-sm whitespace-nowrap">Live Transcript</span>
<div className={cn(
"flex items-center gap-1 text-xs px-2 py-0.5 rounded-full shrink-0",
isCallActive
? "bg-green-500/10 text-green-600 dark:text-green-400"
: isCallCompleted
? "bg-muted text-muted-foreground"
: "bg-muted text-muted-foreground"
)}>
{isCallActive ? (
<>
<Mic className="h-3 w-3" />
<span>Live</span>
</>
) : isCallCompleted ? (
<>
<MicOff className="h-3 w-3" />
<span>Ended</span>
</>
) : (
<>
<MicOff className="h-3 w-3" />
<span>Ready</span>
</>
)}
</div>
</div>
</div>
{/* Messages */}
<div ref={scrollRef} className="flex-1 overflow-y-auto">
{messages.length === 0 ? (
<div className="flex flex-col items-center justify-center h-full text-muted-foreground text-sm">
<MessageSquare className="h-10 w-10 mb-4 opacity-30" />
<p className="font-medium">No messages yet</p>
<p className="text-xs mt-1 text-center px-4">
{isCallActive
? "Start speaking to see the transcript"
: "Start the call to begin the conversation"
}
</p>
</div>
) : (
<div className="space-y-3 p-4">
{messages.map((msg) => (
<MessageItem key={msg.id} msg={msg} />
))}
</div>
)}
</div>
{/* Footer with message count */}
{messages.length > 0 && (
<div className="px-4 py-2 border-t border-border text-xs text-muted-foreground shrink-0">
{messages.filter(m => m.type !== 'function-call').length} messages
</div>
)}
</div>
);
};

96
ui/src/app/workflow/[workflowId]/run/[runId]/components/UnifiedTranscript.tsx Normal file
View file

@ -0,0 +1,96 @@
"use client";
import { useEffect, useRef } from "react";
import { ProcessedMessage } from "../utils/processTranscriptEvents";
import { TranscriptContainer } from "./shared/TranscriptContainer";
import { TranscriptEmptyState } from "./shared/TranscriptEmptyState";
import { TranscriptMessage, TranscriptMessageData } from "./shared/TranscriptMessage";
interface UnifiedTranscriptProps {
messages: ProcessedMessage[];
status: 'ready' | 'live' | 'ended';
title?: string;
autoScroll?: boolean;
emptyState?: {
title: string;
subtitle: string;
};
}
export const UnifiedTranscript = ({
messages,
status,
title,
autoScroll = false,
emptyState
}: UnifiedTranscriptProps) => {
const scrollRef = useRef<HTMLDivElement>(null);
// Auto-scroll to bottom when new messages arrive (for live mode)
useEffect(() => {
if (autoScroll && scrollRef.current) {
scrollRef.current.scrollTop = scrollRef.current.scrollHeight;
}
}, [messages, autoScroll]);
// Calculate message count (exclude system messages like function calls, node transitions, TTFB)
const messageCount = messages.filter(
m => m.type === 'user-transcription' || m.type === 'bot-text'
).length;
// Convert ProcessedMessage to TranscriptMessageData
const transcriptMessages: TranscriptMessageData[] = messages.map(msg => ({
id: msg.id,
type: msg.type,
text: msg.text,
final: msg.final,
functionName: msg.functionName,
status: msg.status,
nodeName: msg.nodeName,
ttfbSeconds: msg.ttfbSeconds,
}));
// Default empty state
const defaultEmptyState = {
title: status === 'live' ? "No messages yet" : "No conversation recorded",
subtitle: status === 'live'
? "Start speaking to see the transcript"
: "Real-time feedback events were not captured"
};
const emptyStateToShow = emptyState || defaultEmptyState;
return (
<TranscriptContainer
title={title || (status === 'live' ? 'Live Transcript' : 'Call Transcript')}
status={status}
messageCount={messageCount > 0 ? messageCount : undefined}
>
<div ref={scrollRef} className="flex-1 overflow-y-auto">
{messages.length === 0 ? (
<TranscriptEmptyState
title={emptyStateToShow.title}
subtitle={emptyStateToShow.subtitle}
/>
) : (
<div className="space-y-3 p-4">
{transcriptMessages.map((msg, index) => {
// Skip standalone TTFB metrics (they're rendered inline with bot text)
if (msg.type === 'ttfb-metric') {
return null;
}
return (
<TranscriptMessage
key={msg.id}
message={msg}
nextMessage={transcriptMessages[index + 1]}
/>
);
})}
</div>
)}
</div>
</TranscriptContainer>
);
};

3
ui/src/app/workflow/[workflowId]/run/[runId]/components/index.ts
View file

@ -2,6 +2,5 @@ export * from './ApiKeyErrorDialog';
export * from './AudioControls';
export * from './ConnectionStatus';
export * from './ContextDisplay';
export * from './RealtimeFeedbackLogs';
export * from './RealtimeFeedbackPanel';
export * from './RealtimeFeedback';
export * from './WorkflowConfigErrorDialog';

72
ui/src/app/workflow/[workflowId]/run/[runId]/components/shared/TranscriptContainer.tsx Normal file
View file

@ -0,0 +1,72 @@
'use client';
import { MessageSquare, Mic, MicOff } from 'lucide-react';
import { ReactNode } from 'react';
import { cn } from '@/lib/utils';
type CallStatus = 'ready' | 'live' | 'ended';
interface TranscriptContainerProps {
title: string;
status: CallStatus;
children: ReactNode;
messageCount?: number;
}
const STATUS_CONFIG = {
ready: {
icon: MicOff,
label: 'Ready',
className: 'bg-muted text-muted-foreground',
},
live: {
icon: Mic,
label: 'Live',
className: 'bg-green-500/10 text-green-600 dark:text-green-400',
},
ended: {
icon: MicOff,
label: 'Ended',
className: 'bg-muted text-muted-foreground',
},
};
export function TranscriptContainer({
title,
status,
children,
messageCount
}: TranscriptContainerProps) {
const statusConfig = STATUS_CONFIG[status];
const StatusIcon = statusConfig.icon;
return (
<div className="w-full h-full flex flex-col bg-background border-l border-border">
{/* Header */}
<div className="px-4 py-3 border-b border-border shrink-0">
<div className="flex items-center justify-center gap-2">
<MessageSquare className="h-4 w-4 text-muted-foreground shrink-0" />
<span className="font-medium text-sm whitespace-nowrap">{title}</span>
<div className={cn(
"flex items-center gap-1 text-xs px-2 py-0.5 rounded-full shrink-0",
statusConfig.className
)}>
<StatusIcon className="h-3 w-3" />
<span>{statusConfig.label}</span>
</div>
</div>
</div>
{/* Content */}
{children}
{/* Footer with message count */}
{messageCount !== undefined && messageCount > 0 && (
<div className="px-4 py-2 border-t border-border text-xs text-muted-foreground shrink-0">
{messageCount} messages
</div>
)}
</div>
);
}

20
ui/src/app/workflow/[workflowId]/run/[runId]/components/shared/TranscriptEmptyState.tsx Normal file
View file

@ -0,0 +1,20 @@
'use client';
import { MessageSquare } from 'lucide-react';
interface TranscriptEmptyStateProps {
title: string;
subtitle: string;
}
export function TranscriptEmptyState({ title, subtitle }: TranscriptEmptyStateProps) {
return (
<div className="flex flex-col items-center justify-center h-full text-muted-foreground text-sm">
<MessageSquare className="h-10 w-10 mb-4 opacity-30" />
<p className="font-medium">{title}</p>
<p className="text-xs mt-1 text-center px-4">
{subtitle}
</p>
</div>
);
}

110
ui/src/app/workflow/[workflowId]/run/[runId]/components/shared/TranscriptMessage.tsx Normal file
View file

@ -0,0 +1,110 @@
'use client';
import { Brain, GitBranch, Wrench } from 'lucide-react';
import { cn } from '@/lib/utils';
export interface TranscriptMessageData {
id: string;
type: 'user-transcription' | 'bot-text' | 'function-call' | 'node-transition' | 'ttfb-metric';
text: string;
final?: boolean;
functionName?: string;
nodeName?: string;
ttfbSeconds?: number;
}
interface TranscriptMessageProps {
message: TranscriptMessageData;
nextMessage?: TranscriptMessageData;
}
export function TranscriptMessage({ message, nextMessage }: TranscriptMessageProps) {
// Node transition - show as section divider
if (message.type === 'node-transition') {
return (
<div className="flex items-center gap-2 py-2">
<div className="flex-1 h-px bg-border"></div>
<div className="px-2 py-1 rounded-md text-xs bg-blue-500/10 border border-blue-500/20 inline-flex items-center gap-1.5">
<GitBranch className="h-3 w-3 text-blue-500" />
<span className="font-medium text-blue-700 dark:text-blue-400">
{message.nodeName}
</span>
</div>
<div className="flex-1 h-px bg-border"></div>
</div>
);
}
// TTFB metric - don't render standalone, it'll be shown with bot messages and function calls
if (message.type === 'ttfb-metric') {
return null;
}
// Function call message - centered with TTFB if present
if (message.type === 'function-call') {
const ttfbMetric = nextMessage?.type === 'ttfb-metric' ? nextMessage : null;
return (
<div className="flex flex-col items-center gap-1">
{/* Show TTFB metric above function call */}
{ttfbMetric && ttfbMetric.ttfbSeconds !== undefined && (
<div className="flex items-center gap-1.5 text-xs text-muted-foreground">
<Brain className="h-3 w-3" />
<span className="font-medium">Reasoning Delay:</span>
<span>{(ttfbMetric.ttfbSeconds * 1000).toFixed(0)}ms</span>
</div>
)}
<div className="px-3 py-1.5 rounded-full text-xs bg-amber-500/10 border border-amber-500/20 inline-flex items-center gap-2">
<Wrench className="h-3 w-3 text-amber-500" />
<span className="font-mono text-amber-700 dark:text-amber-400">
{message.functionName}()
</span>
</div>
</div>
);
}
const isUser = message.type === 'user-transcription';
const isBot = message.type === 'bot-text';
// Check if next message is a TTFB metric (for bot messages)
const ttfbMetric = isBot && nextMessage?.type === 'ttfb-metric' ? nextMessage : null;
// User messages on right, bot messages on left
return (
<div className={cn(
"flex",
isUser ? "justify-end" : "justify-start"
)}>
<div className="flex flex-col gap-1 max-w-[85%]">
{/* Show TTFB metric above bot messages */}
{ttfbMetric && ttfbMetric.ttfbSeconds !== undefined && (
<div className="flex items-center gap-1.5 text-xs text-muted-foreground px-1">
<Brain className="h-3 w-3" />
<span className="font-medium">Reasoning Delay:</span>
<span>{(ttfbMetric.ttfbSeconds * 1000).toFixed(0)}ms</span>
</div>
)}
<div
className={cn(
"px-3 py-2 rounded-2xl text-sm",
isUser
? "bg-primary text-primary-foreground rounded-br-md"
: "bg-muted rounded-bl-md",
!message.final && "opacity-70"
)}
>
<div className="whitespace-pre-wrap leading-relaxed">{message.text}</div>
{!message.final && (
<div className={cn(
"text-[10px] mt-1 italic",
isUser ? "text-primary-foreground/70" : "text-muted-foreground"
)}>
speaking...
</div>
)}
</div>
</div>
</div>
);
}

75
ui/src/app/workflow/[workflowId]/run/[runId]/hooks/useWebSocketRTC.tsx
View file

@ -17,12 +17,19 @@ interface UseWebSocketRTCProps {
export interface FeedbackMessage {
id: string;
type: 'user-transcription' | 'bot-text' | 'function-call';
type: 'user-transcription' | 'bot-text' | 'function-call' | 'node-transition' | 'ttfb-metric';
text: string;
final?: boolean;
timestamp: string;
functionName?: string;
status?: 'running' | 'completed';
// Node transition fields
nodeName?: string;
previousNode?: string;
// TTFB metric fields
ttfbSeconds?: number;
processor?: string;
model?: string;
}
export const useWebSocketRTC = ({ workflowId, workflowRunId, accessToken, initialContextVariables }: UseWebSocketRTCProps) => {
@ -285,35 +292,26 @@ export const useWebSocketRTC = ({ workflowId, workflowRunId, accessToken, initia
case 'rtf-user-transcription': {
const transcription = message.payload;
setFeedbackMessages(prev => {
// Mark last bot message as final (user started speaking)
const withBotFinalized = prev.map((m, i) =>
i === prev.length - 1 && m.type === 'bot-text' && !m.final
? { ...m, final: true }
: m
// Step 1: Finalize the last bot message (user started speaking)
const messagesWithBotFinalized = prev.map((msg, idx) => {
const isLastMessage = idx === prev.length - 1;
const isUnfinalizedBotMessage = msg.type === 'bot-text' && !msg.final;
return isLastMessage && isUnfinalizedBotMessage
? { ...msg, final: true }
: msg;
});
// Step 2: Remove any previous interim transcription
const messagesWithoutInterim = messagesWithBotFinalized.filter(
msg => !(msg.type === 'user-transcription' && !msg.final)
);
// For interim transcriptions, replace the last interim
if (!transcription.final) {
const withoutLastInterim = withBotFinalized.filter(
m => !(m.type === 'user-transcription' && !m.final)
);
return [...withoutLastInterim, {
id: `user-${Date.now()}`,
type: 'user-transcription',
text: transcription.text,
final: false,
timestamp: new Date().toISOString(),
}];
}
// For final transcriptions, replace interim with final
const withoutInterim = withBotFinalized.filter(
m => !(m.type === 'user-transcription' && !m.final)
);
return [...withoutInterim, {
// Step 3: Add new transcription (interim or final)
return [...messagesWithoutInterim, {
id: `user-${Date.now()}`,
type: 'user-transcription',
text: transcription.text,
final: true,
final: transcription.final,
timestamp: new Date().toISOString(),
}];
});
@ -381,6 +379,33 @@ export const useWebSocketRTC = ({ workflowId, workflowRunId, accessToken, initia
break;
}
case 'rtf-node-transition': {
const { node_name, previous_node } = message.payload;
setFeedbackMessages(prev => [...prev, {
id: `node-${Date.now()}`,
type: 'node-transition',
text: node_name,
nodeName: node_name,
previousNode: previous_node,
timestamp: new Date().toISOString(),
}]);
break;
}
case 'rtf-ttfb-metric': {
const { ttfb_seconds, processor, model } = message.payload;
setFeedbackMessages(prev => [...prev, {
id: `ttfb-${Date.now()}`,
type: 'ttfb-metric',
text: `${(ttfb_seconds * 1000).toFixed(0)}ms`,
ttfbSeconds: ttfb_seconds,
processor,
model,
timestamp: new Date().toISOString(),
}]);
break;
}
default:
logger.warn('Unknown message type:', message.type);
}

4
ui/src/app/workflow/[workflowId]/run/[runId]/page.tsx
View file

@ -6,7 +6,7 @@ import { useParams } from 'next/navigation';
import { useEffect, useRef, useState } from 'react';
import BrowserCall from '@/app/workflow/[workflowId]/run/[runId]/BrowserCall';
import { RealtimeFeedbackLogs, WorkflowRunLogs } from '@/app/workflow/[workflowId]/run/[runId]/components/RealtimeFeedbackLogs';
import { RealtimeFeedback, WorkflowRunLogs } from '@/app/workflow/[workflowId]/run/[runId]/components/RealtimeFeedback';
import WorkflowLayout from '@/app/workflow/WorkflowLayout';
import { getWorkflowRunApiV1WorkflowWorkflowIdRunsRunIdGet } from '@/client/sdk.gen';
import { MediaPreviewButtons, MediaPreviewDialog } from '@/components/MediaPreviewDialog';
@ -255,7 +255,7 @@ export default function WorkflowRunPage() {
{/* Transcript panel - 1/3 width */}
<div className="w-1/3 h-full shrink-0">
<RealtimeFeedbackLogs logs={workflowRun?.logs} />
<RealtimeFeedback mode="historical" logs={workflowRun?.logs} />
</div>
</div>
);

147
ui/src/app/workflow/[workflowId]/run/[runId]/utils/processTranscriptEvents.ts Normal file
View file

@ -0,0 +1,147 @@
/**
* Utility to process realtime feedback events into a unified transcript format.
* Used by both live WebSocket messages and post-call logs.
*/
export interface TranscriptEvent {
type: 'user-transcription' | 'bot-text' | 'function-call' | 'node-transition' | 'ttfb-metric';
text: string;
final?: boolean;
timestamp: string;
turn?: number;
functionName?: string;
status?: 'running' | 'completed';
nodeName?: string;
previousNode?: string;
ttfbSeconds?: number;
processor?: string;
model?: string;
}
export interface ProcessedMessage {
id: string;
type: TranscriptEvent['type'];
text: string;
final?: boolean;
timestamp: string;
functionName?: string;
status?: 'running' | 'completed';
nodeName?: string;
ttfbSeconds?: number;
}
/**
* Process transcript events (both live and historical).
* Combines consecutive bot-text by turn and associates TTFB metrics.
*/
export function processTranscriptEvents(events: TranscriptEvent[]): ProcessedMessage[] {
// Filter out interim transcriptions and function-call-start events
const filteredEvents = events.filter(event => {
if (event.type === 'user-transcription' && !event.final) return false;
if (event.type === 'function-call' && event.status === 'running') return false;
return true;
});
const processed: ProcessedMessage[] = [];
let currentBotText: { event: TranscriptEvent; text: string } | null = null;
let pendingTtfb: TranscriptEvent | null = null;
const flushBotText = () => {
if (!currentBotText) return;
processed.push(convertToProcessedMessage(currentBotText.event, currentBotText.text));
// Add the pending TTFB metric if it exists
if (pendingTtfb) {
processed.push(convertToProcessedMessage(pendingTtfb));
pendingTtfb = null;
}
currentBotText = null;
};
for (const event of filteredEvents) {
if (event.type === 'ttfb-metric') {
// Store TTFB to associate with the next bot-text or function-call
pendingTtfb = event;
} else if (event.type === 'bot-text') {
// Combine consecutive bot-text from the same turn
if (currentBotText && currentBotText.event.turn === event.turn) {
currentBotText.text = currentBotText.text + ' ' + event.text;
} else {
flushBotText();
currentBotText = { event, text: event.text };
}
} else {
// Handle other events (user-transcription, function-call, node-transition)
flushBotText();
processed.push(convertToProcessedMessage(event));
// Add pending TTFB after function calls
if (event.type === 'function-call' && pendingTtfb) {
processed.push(convertToProcessedMessage(pendingTtfb));
pendingTtfb = null;
}
}
}
// Flush any remaining bot text
flushBotText();
return processed;
}
/**
* Process live messages - optimized version.
*
* Optimizations rely on useWebSocketRTC.tsx already handling:
* - Bot text accumulation (consecutive chunks combined with spaces)
* - Interim transcription filtering (only final transcriptions kept)
* - Function call status (start events filtered, only completed kept)
*
* This function only needs to:
* - Associate TTFB metrics with the preceding bot-text or function-call
* - Convert to ProcessedMessage format
*/
export function processLiveMessages(messages: TranscriptEvent[]): ProcessedMessage[] {
const processed: ProcessedMessage[] = [];
let pendingTtfb: TranscriptEvent | null = null;
for (const msg of messages) {
if (msg.type === 'ttfb-metric') {
// Store TTFB to associate with next message
pendingTtfb = msg;
} else {
// Add the message
processed.push(convertToProcessedMessage(msg));
// Add pending TTFB after final bot-text or completed function calls
if ((msg.type === 'bot-text' && msg.final) ||
(msg.type === 'function-call' && msg.status === 'completed')) {
if (pendingTtfb) {
processed.push(convertToProcessedMessage(pendingTtfb));
pendingTtfb = null;
}
}
}
}
return processed;
}
// Alias for backward compatibility
export const processHistoricalEvents = processTranscriptEvents;
function convertToProcessedMessage(event: TranscriptEvent, overrideText?: string): ProcessedMessage {
return {
id: `${event.type}-${event.timestamp}`,
type: event.type,
text: overrideText ?? event.text,
final: event.final ?? true,
timestamp: event.timestamp,
functionName: event.functionName,
status: event.status,
nodeName: event.nodeName,
ttfbSeconds: event.ttfbSeconds,
};
}