From ac01f7775ecc47418e1c6d9b6f921ca4375029c4 Mon Sep 17 00:00:00 2001 From: Sabiha Khan <87858386+chewwbaka@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:47:43 +0530 Subject: [PATCH] Feat/enhanced timestamped transcript (#501) * feat: add additional timestamps in call transcript optionally * fi: timestamp precision to millisec instead of micro * fix: address enhanced transcript review issues * fix: non vad user turn timestamp --- api/services/pipecat/event_handlers.py | 5 +- api/services/pipecat/in_memory_buffers.py | 129 ++++++++++++++- .../pipecat/realtime_feedback_events.py | 6 + .../pipecat/realtime_feedback_observer.py | 33 ++++ api/services/pipecat/run_pipeline.py | 5 + api/tests/test_realtime_feedback_events.py | 37 +++++ api/tests/test_realtime_feedback_observer.py | 148 +++++++++++++++++- api/utils/transcript.py | 23 ++- .../components/ConfigurationsDialog.tsx | 1 + .../workflow/[workflowId]/settings/page.tsx | 53 ++++++- .../components/workflow/conversation/types.ts | 1 + ui/src/types/workflow-configurations.ts | 15 ++ 12 files changed, 441 insertions(+), 15 deletions(-) diff --git a/api/services/pipecat/event_handlers.py b/api/services/pipecat/event_handlers.py index dbd27998..c57ecea8 100644 --- a/api/services/pipecat/event_handlers.py +++ b/api/services/pipecat/event_handlers.py @@ -70,6 +70,7 @@ def register_event_handlers( pre_call_fetch_task: asyncio.Task | None = None, user_provider_id: str | None = None, integration_runtime_sessions: list[IntegrationRuntimeSession] | None = None, + include_transcript_end_timestamps: bool = False, ): """Register all event handlers for transport and task events. @@ -386,7 +387,9 @@ def register_event_handlers( else: logger.debug("Bot audio buffer is empty, skipping upload") - transcript_text = in_memory_logs_buffer.generate_transcript_text() + transcript_text = in_memory_logs_buffer.generate_transcript_text( + include_end_timestamps=include_transcript_end_timestamps + ) if not transcript_text: logger.debug("No transcript events in logs buffer, skipping upload") diff --git a/api/services/pipecat/in_memory_buffers.py b/api/services/pipecat/in_memory_buffers.py index 12d29018..349c1add 100644 --- a/api/services/pipecat/in_memory_buffers.py +++ b/api/services/pipecat/in_memory_buffers.py @@ -107,6 +107,12 @@ class InMemoryLogsBuffer: self._turn_counter = 0 self._current_node_id: Optional[str] = None self._current_node_name: Optional[str] = None + self._user_speech_start_timestamp: Optional[str] = None + self._user_speech_end_timestamp: Optional[str] = None + self._user_speech_start_from_vad = False + self._user_speech_end_from_vad = False + self._bot_speech_start_timestamp: Optional[str] = None + self._bot_speech_end_timestamp: Optional[str] = None def set_current_node(self, node_id: str, node_name: str): """Set the current node ID and name to be injected into subsequent events.""" @@ -123,11 +129,126 @@ class InMemoryLogsBuffer: """Get the current node name.""" return self._current_node_name + @staticmethod + def _now_iso() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds") + + def mark_user_started_speaking( + self, timestamp: Optional[str] = None, *, from_vad: bool = False + ): + """Record when the user started speaking for the current turn.""" + vad_interval_is_open = ( + self._user_speech_start_from_vad and self._user_speech_end_timestamp is None + ) + if vad_interval_is_open and not from_vad: + return + + self._user_speech_start_timestamp = timestamp or self._now_iso() + self._user_speech_end_timestamp = None + self._user_speech_start_from_vad = from_vad + self._user_speech_end_from_vad = False + self._update_latest_payload_start_timestamp( + RealtimeFeedbackType.USER_TRANSCRIPTION.value, + self._user_speech_start_timestamp, + require_final=True, + ) + + def mark_user_stopped_speaking( + self, timestamp: Optional[str] = None, *, from_vad: bool = False + ): + """Record when the user stopped speaking and update the latest user event.""" + if self._user_speech_end_from_vad and not from_vad: + return + + self._user_speech_end_timestamp = timestamp or self._now_iso() + self._user_speech_end_from_vad = from_vad + self._update_latest_payload_end_timestamp( + RealtimeFeedbackType.USER_TRANSCRIPTION.value, + self._user_speech_end_timestamp, + require_final=True, + ) + + def mark_bot_started_speaking(self, timestamp: Optional[str] = None): + """Record when the bot started speaking for the current assistant turn.""" + self._bot_speech_start_timestamp = timestamp or self._now_iso() + self._bot_speech_end_timestamp = None + self._update_latest_payload_start_timestamp( + RealtimeFeedbackType.BOT_TEXT.value, + self._bot_speech_start_timestamp, + ) + + def mark_bot_stopped_speaking(self, timestamp: Optional[str] = None): + """Record when the bot stopped speaking and update the latest bot event.""" + self._bot_speech_end_timestamp = timestamp or self._now_iso() + self._update_latest_payload_end_timestamp( + RealtimeFeedbackType.BOT_TEXT.value, + self._bot_speech_end_timestamp, + ) + + def _find_latest_open_payload( + self, event_type: str, *, require_final: bool = False + ) -> dict | None: + for event in reversed(self._events): + if event.get("type") != event_type: + continue + payload = event.get("payload") + if not isinstance(payload, dict): + continue + if require_final and payload.get("final") is not True: + continue + if payload.get("end_timestamp"): + continue + return payload + return None + + def _update_latest_payload_start_timestamp( + self, event_type: str, start_timestamp: str, *, require_final: bool = False + ): + payload = self._find_latest_open_payload( + event_type, require_final=require_final + ) + if payload is not None: + payload["timestamp"] = start_timestamp + + def _update_latest_payload_end_timestamp( + self, event_type: str, end_timestamp: str, *, require_final: bool = False + ): + payload = self._find_latest_open_payload( + event_type, require_final=require_final + ) + if payload is not None: + payload["end_timestamp"] = end_timestamp + + def _event_with_speech_timestamps(self, event: dict) -> dict: + event_type = event.get("type") + payload = event.get("payload") + if not isinstance(payload, dict): + return event + + payload_with_timestamps = dict(payload) + if ( + event_type == RealtimeFeedbackType.USER_TRANSCRIPTION.value + and payload.get("final") is True + ): + if self._user_speech_start_timestamp: + payload_with_timestamps["timestamp"] = self._user_speech_start_timestamp + if self._user_speech_end_timestamp: + payload_with_timestamps["end_timestamp"] = self._user_speech_end_timestamp + elif event_type == RealtimeFeedbackType.BOT_TEXT.value: + bot_interval_is_active = self._bot_speech_end_timestamp is None + if bot_interval_is_active and self._bot_speech_start_timestamp: + payload_with_timestamps["timestamp"] = self._bot_speech_start_timestamp + + if payload_with_timestamps == payload: + return event + return {**event, "payload": payload_with_timestamps} + async def append(self, event: dict): """Append a feedback event to the buffer with timestamp and current node.""" + event = self._event_with_speech_timestamps(event) timestamped_event = stamp_realtime_feedback_event( event, - timestamp=datetime.now(UTC).isoformat(), + timestamp=self._now_iso(), turn=self._turn_counter, node_id=self._current_node_id, node_name=self._current_node_name, @@ -166,13 +287,15 @@ class InMemoryLogsBuffer: return True return False - def generate_transcript_text(self) -> str: + def generate_transcript_text(self, *, include_end_timestamps: bool = False) -> str: """Generate transcript text from logged events. Filters for rtf-user-transcription (final) and rtf-bot-text events, formats them as '[timestamp] user/assistant: text\\n'. """ - return _generate_transcript_text(self._sorted_events()) + return _generate_transcript_text( + self._sorted_events(), include_end_timestamps=include_end_timestamps + ) @property def is_empty(self) -> bool: diff --git a/api/services/pipecat/realtime_feedback_events.py b/api/services/pipecat/realtime_feedback_events.py index e140fc63..c9e4321f 100644 --- a/api/services/pipecat/realtime_feedback_events.py +++ b/api/services/pipecat/realtime_feedback_events.py @@ -30,6 +30,7 @@ def build_user_transcription_event( text: str, final: bool, timestamp: str | None = None, + end_timestamp: str | None = None, user_id: str | None = None, ) -> dict[str, Any]: payload: dict[str, Any] = { @@ -38,6 +39,8 @@ def build_user_transcription_event( } if timestamp is not None: payload["timestamp"] = timestamp + if end_timestamp is not None: + payload["end_timestamp"] = end_timestamp if user_id is not None: payload["user_id"] = user_id return { @@ -50,10 +53,13 @@ def build_bot_text_event( *, text: str, timestamp: str | None = None, + end_timestamp: str | None = None, ) -> dict[str, Any]: payload: dict[str, Any] = {"text": text} if timestamp is not None: payload["timestamp"] = timestamp + if end_timestamp is not None: + payload["end_timestamp"] = end_timestamp return { "type": RealtimeFeedbackType.BOT_TEXT.value, "payload": payload, diff --git a/api/services/pipecat/realtime_feedback_observer.py b/api/services/pipecat/realtime_feedback_observer.py index 67d635d9..0644edbe 100644 --- a/api/services/pipecat/realtime_feedback_observer.py +++ b/api/services/pipecat/realtime_feedback_observer.py @@ -21,6 +21,7 @@ node changes. """ import json +from datetime import UTC, datetime from typing import TYPE_CHECKING, Awaitable, Callable, Optional, Set from loguru import logger @@ -52,8 +53,12 @@ from pipecat.frames.frames import ( TranscriptionFrame, TTSSpeakFrame, TTSTextFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, UserMuteStartedFrame, UserMuteStoppedFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, ) from pipecat.metrics.metrics import TTFBMetricsData from pipecat.observers.base_observer import BaseObserver, FramePushed @@ -62,6 +67,10 @@ from pipecat.transports.base_output import BaseOutputTransport from pipecat.utils.enums import RealtimeFeedbackType +def _epoch_seconds_to_utc_iso(timestamp: float) -> str: + return datetime.fromtimestamp(timestamp, UTC).isoformat(timespec="milliseconds") + + class RealtimeFeedbackObserver(BaseObserver): """Observer that sends real-time events via WebSocket and persists final transcripts. @@ -138,13 +147,35 @@ class RealtimeFeedbackObserver(BaseObserver): return # Bot speaking state - WS only (ephemeral state signals, not persisted) elif isinstance(frame, BotStartedSpeakingFrame): + if self._logs_buffer: + self._logs_buffer.mark_bot_started_speaking() await self._send_ws( {"type": RealtimeFeedbackType.BOT_STARTED_SPEAKING.value, "payload": {}} ) elif isinstance(frame, BotStoppedSpeakingFrame): + if self._logs_buffer: + self._logs_buffer.mark_bot_stopped_speaking() await self._send_ws( {"type": RealtimeFeedbackType.BOT_STOPPED_SPEAKING.value, "payload": {}} ) + elif isinstance(frame, UserStartedSpeakingFrame): + if self._logs_buffer: + self._logs_buffer.mark_user_started_speaking() + elif isinstance(frame, UserStoppedSpeakingFrame): + if self._logs_buffer: + self._logs_buffer.mark_user_stopped_speaking() + elif isinstance(frame, VADUserStartedSpeakingFrame): + if self._logs_buffer: + self._logs_buffer.mark_user_started_speaking( + _epoch_seconds_to_utc_iso(frame.timestamp - frame.start_secs), + from_vad=True, + ) + elif isinstance(frame, VADUserStoppedSpeakingFrame): + if self._logs_buffer: + self._logs_buffer.mark_user_stopped_speaking( + _epoch_seconds_to_utc_iso(frame.timestamp - frame.stop_secs), + from_vad=True, + ) # User mute state - WS only (ephemeral state signals, not persisted) elif isinstance(frame, UserMuteStartedFrame): await self._send_ws( @@ -314,6 +345,7 @@ def register_turn_log_handlers( text=message.content, final=True, timestamp=message.timestamp, + end_timestamp=getattr(message, "end_timestamp", None), ) ) except Exception as e: @@ -327,6 +359,7 @@ def register_turn_log_handlers( build_bot_text_event( text=message.content, timestamp=message.timestamp, + end_timestamp=getattr(message, "end_timestamp", None), ) ) except Exception as e: diff --git a/api/services/pipecat/run_pipeline.py b/api/services/pipecat/run_pipeline.py index 4bfedc7c..29ab79ca 100644 --- a/api/services/pipecat/run_pipeline.py +++ b/api/services/pipecat/run_pipeline.py @@ -545,6 +545,10 @@ async def _run_pipeline_impl( max_call_duration_seconds = DEFAULT_MAX_CALL_DURATION_SECONDS max_user_idle_timeout = DEFAULT_MAX_USER_IDLE_TIMEOUT_SECONDS keyterms = None # Dictionary words for STT boosting + transcript_config = run_configs.get("transcript_configuration") or {} + include_transcript_end_timestamps = bool( + transcript_config.get("include_end_timestamps", False) + ) if run_configs: if "max_call_duration" in run_configs: @@ -1045,6 +1049,7 @@ async def _run_pipeline_impl( pre_call_fetch_task=pre_call_fetch_task, user_provider_id=user_provider_id, integration_runtime_sessions=integration_runtime_sessions, + include_transcript_end_timestamps=include_transcript_end_timestamps, ) register_audio_data_handler(audio_buffer, workflow_run_id, in_memory_audio_buffer) diff --git a/api/tests/test_realtime_feedback_events.py b/api/tests/test_realtime_feedback_events.py index 95c0859b..1cfcaf89 100644 --- a/api/tests/test_realtime_feedback_events.py +++ b/api/tests/test_realtime_feedback_events.py @@ -1,10 +1,12 @@ from api.services.pipecat.realtime_feedback_events import ( build_bot_text_event, build_function_call_end_event, + build_user_transcription_event, build_node_transition_event, realtime_feedback_event_sort_key, stamp_realtime_feedback_event, ) +from api.utils.transcript import generate_transcript_text def test_build_function_call_end_event_serializes_results(): @@ -51,3 +53,38 @@ def test_stamp_and_sort_realtime_feedback_events(): assert events == [bot_text, node_transition] assert node_transition["node_id"] == "node-1" assert node_transition["node_name"] == "Greeting" + + +def test_transcript_can_include_end_timestamps_without_changing_default_format(): + events = [ + stamp_realtime_feedback_event( + build_bot_text_event( + text="Can you confirm your date of birth?", + timestamp="2026-01-01T00:00:01+00:00", + end_timestamp="2026-01-01T00:00:04+00:00", + ), + timestamp="2026-01-01T00:00:05+00:00", + turn=0, + ), + stamp_realtime_feedback_event( + build_user_transcription_event( + text="January fifth", + final=True, + timestamp="2026-01-01T00:00:06+00:00", + end_timestamp="2026-01-01T00:00:08+00:00", + ), + timestamp="2026-01-01T00:00:09+00:00", + turn=1, + ), + ] + + assert generate_transcript_text(events) == ( + "[2026-01-01T00:00:01+00:00] assistant: Can you confirm your date of birth?\n" + "[2026-01-01T00:00:06+00:00] user: January fifth\n" + ) + assert generate_transcript_text(events, include_end_timestamps=True) == ( + "[2026-01-01T00:00:01+00:00 -> 2026-01-01T00:00:04+00:00] " + "assistant: Can you confirm your date of birth?\n" + "[2026-01-01T00:00:06+00:00 -> 2026-01-01T00:00:08+00:00] " + "user: January fifth\n" + ) diff --git a/api/tests/test_realtime_feedback_observer.py b/api/tests/test_realtime_feedback_observer.py index 8057fd6f..781339b5 100644 --- a/api/tests/test_realtime_feedback_observer.py +++ b/api/tests/test_realtime_feedback_observer.py @@ -1,7 +1,17 @@ from types import SimpleNamespace +import re import pytest -from pipecat.frames.frames import TranscriptionFrame, TTSTextFrame +from pipecat.frames.frames import ( + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TranscriptionFrame, + TTSTextFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, +) from pipecat.observers.base_observer import FramePushed from pipecat.processors.frame_processor import FrameDirection from pipecat.transports.base_output import BaseOutputTransport @@ -144,3 +154,139 @@ async def test_turn_log_handlers_persist_user_message_added_events(): "timestamp": "2026-01-01T00:00:00+00:00", } assert events[0]["turn"] == 1 + + +@pytest.mark.asyncio +async def test_observer_attaches_backend_speaking_intervals_to_logged_transcript_events(): + async def ws_sender(_message): + pass + + logs_buffer = InMemoryLogsBuffer(workflow_run_id=123) + observer = RealtimeFeedbackObserver(ws_sender=ws_sender, logs_buffer=logs_buffer) + + user_aggregator = _FakeAggregator() + assistant_aggregator = _FakeAggregator() + register_turn_log_handlers(logs_buffer, user_aggregator, assistant_aggregator) + + await observer.on_push_frame( + _frame_pushed(UserStartedSpeakingFrame(), FrameDirection.DOWNSTREAM) + ) + await observer.on_push_frame( + _frame_pushed(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM) + ) + await user_aggregator.handlers["on_user_turn_message_added"]( + user_aggregator, + SimpleNamespace( + content="January fifth", + timestamp="aggregator-user-start", + ), + ) + + await observer.on_push_frame( + _frame_pushed(BotStartedSpeakingFrame(), FrameDirection.DOWNSTREAM) + ) + await assistant_aggregator.handlers["on_assistant_turn_stopped"]( + assistant_aggregator, + SimpleNamespace( + content="Thank you", + timestamp="aggregator-bot-start", + ), + ) + await observer.on_push_frame( + _frame_pushed(BotStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM) + ) + + user_event, bot_event = [ + event + for event in logs_buffer.get_events() + if event["type"] in {"rtf-user-transcription", "rtf-bot-text"} + ] + + assert user_event["payload"]["timestamp"] != "aggregator-user-start" + assert re.match( + r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+00:00$", + user_event["payload"]["timestamp"], + ) + assert user_event["payload"]["end_timestamp"] + assert bot_event["payload"]["timestamp"] != "aggregator-bot-start" + assert re.match( + r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+00:00$", + bot_event["payload"]["timestamp"], + ) + assert bot_event["payload"]["end_timestamp"] + + +@pytest.mark.asyncio +async def test_observer_uses_vad_user_speech_times_over_turn_closure_times(): + async def ws_sender(_message): + pass + + logs_buffer = InMemoryLogsBuffer(workflow_run_id=123) + observer = RealtimeFeedbackObserver(ws_sender=ws_sender, logs_buffer=logs_buffer) + user_aggregator = _FakeAggregator() + assistant_aggregator = _FakeAggregator() + register_turn_log_handlers(logs_buffer, user_aggregator, assistant_aggregator) + + await observer.on_push_frame( + _frame_pushed( + VADUserStartedSpeakingFrame(start_secs=0.2, timestamp=1000.2), + FrameDirection.DOWNSTREAM, + ) + ) + await observer.on_push_frame( + _frame_pushed( + VADUserStoppedSpeakingFrame(stop_secs=0.5, timestamp=1002.5), + FrameDirection.DOWNSTREAM, + ) + ) + await observer.on_push_frame( + _frame_pushed(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM) + ) + + await user_aggregator.handlers["on_user_turn_message_added"]( + user_aggregator, + SimpleNamespace(content="Hello", timestamp="aggregator-user-start"), + ) + + user_event = logs_buffer.get_events()[0] + assert user_event["payload"]["timestamp"] == "1970-01-01T00:16:40.000+00:00" + assert user_event["payload"]["end_timestamp"] == "1970-01-01T00:16:42.000+00:00" + + +@pytest.mark.asyncio +async def test_completed_bot_speech_interval_is_not_reused_for_next_pre_playback_text(): + logs_buffer = InMemoryLogsBuffer(workflow_run_id=123) + + logs_buffer.mark_bot_started_speaking("2026-01-01T00:00:01.000+00:00") + await logs_buffer.append({"type": "rtf-bot-text", "payload": {"text": "First"}}) + logs_buffer.mark_bot_stopped_speaking("2026-01-01T00:00:02.000+00:00") + + await logs_buffer.append({"type": "rtf-bot-text", "payload": {"text": "Second"}}) + second_event = logs_buffer.get_events()[-1] + + assert second_event["payload"] == {"text": "Second"} + + +@pytest.mark.asyncio +async def test_non_vad_user_turn_after_completed_vad_turn_gets_fresh_timestamps(): + logs_buffer = InMemoryLogsBuffer(workflow_run_id=123) + + logs_buffer.mark_user_started_speaking( + "2026-01-01T00:00:01.000+00:00", from_vad=True + ) + logs_buffer.mark_user_stopped_speaking( + "2026-01-01T00:00:02.000+00:00", from_vad=True + ) + await logs_buffer.append( + {"type": "rtf-user-transcription", "payload": {"text": "First", "final": True}} + ) + + logs_buffer.mark_user_started_speaking("2026-01-01T00:00:10.000+00:00") + logs_buffer.mark_user_stopped_speaking("2026-01-01T00:00:12.000+00:00") + await logs_buffer.append( + {"type": "rtf-user-transcription", "payload": {"text": "Second", "final": True}} + ) + + second_event = logs_buffer.get_events()[-1] + assert second_event["payload"]["timestamp"] == "2026-01-01T00:00:10.000+00:00" + assert second_event["payload"]["end_timestamp"] == "2026-01-01T00:00:12.000+00:00" diff --git a/api/utils/transcript.py b/api/utils/transcript.py index 4de11a41..6f8d32ee 100644 --- a/api/utils/transcript.py +++ b/api/utils/transcript.py @@ -3,11 +3,26 @@ from typing import List from pipecat.utils.enums import RealtimeFeedbackType -def generate_transcript_text(events: List[dict]) -> str: +def _format_timestamp_range(payload: dict, event: dict, include_end_timestamps: bool) -> str: + start_timestamp = payload.get("timestamp") or event.get("timestamp", "") + if not include_end_timestamps: + return start_timestamp + + end_timestamp = payload.get("end_timestamp") + if end_timestamp: + return f"{start_timestamp} -> {end_timestamp}" if start_timestamp else end_timestamp + return start_timestamp + + +def generate_transcript_text( + events: List[dict], *, include_end_timestamps: bool = False +) -> str: """Generate transcript text from realtime feedback events. Filters for rtf-user-transcription (final) and rtf-bot-text events, - formats them as '[timestamp] user/assistant: text\\n'. + formats them as '[timestamp] user/assistant: text\\n'. When + include_end_timestamps is True, formats as + '[start_timestamp -> end_timestamp] user/assistant: text\\n'. """ lines: List[str] = [] for event in events: @@ -18,11 +33,11 @@ def generate_transcript_text(events: List[dict]) -> str: event_type == RealtimeFeedbackType.USER_TRANSCRIPTION.value and payload.get("final") is True ): - timestamp = payload.get("timestamp") or event.get("timestamp", "") + timestamp = _format_timestamp_range(payload, event, include_end_timestamps) prefix = f"[{timestamp}] " if timestamp else "" lines.append(f"{prefix}user: {payload.get('text', '')}\n") elif event_type == RealtimeFeedbackType.BOT_TEXT.value: - timestamp = payload.get("timestamp") or event.get("timestamp", "") + timestamp = _format_timestamp_range(payload, event, include_end_timestamps) prefix = f"[{timestamp}] " if timestamp else "" lines.append(f"{prefix}assistant: {payload.get('text', '')}\n") diff --git a/ui/src/app/workflow/[workflowId]/components/ConfigurationsDialog.tsx b/ui/src/app/workflow/[workflowId]/components/ConfigurationsDialog.tsx index 2d288a61..86ce8385 100644 --- a/ui/src/app/workflow/[workflowId]/components/ConfigurationsDialog.tsx +++ b/ui/src/app/workflow/[workflowId]/components/ConfigurationsDialog.tsx @@ -78,6 +78,7 @@ export const ConfigurationsDialog = ({ turn_start_min_words: turnStartMinWords, provisional_vad_pause_secs: provisionalVadPauseSecs, turn_stop_strategy: turnStopStrategy, + transcript_configuration: resolvedWorkflowConfigurations.transcript_configuration, context_compaction_enabled: contextCompactionEnabled, }, name); onOpenChange(false); diff --git a/ui/src/app/workflow/[workflowId]/settings/page.tsx b/ui/src/app/workflow/[workflowId]/settings/page.tsx index 6a97baba..14ed3ce1 100644 --- a/ui/src/app/workflow/[workflowId]/settings/page.tsx +++ b/ui/src/app/workflow/[workflowId]/settings/page.tsx @@ -48,6 +48,7 @@ import { DEFAULT_PROVISIONAL_VAD_PAUSE_SECS, DEFAULT_TURN_START_MIN_WORDS, DEFAULT_VOICEMAIL_DETECTION_CONFIGURATION, + resolveWorkflowConfigurations, TURN_START_STRATEGY_OPTIONS, type TurnStartStrategy, type TurnStopStrategy, @@ -293,6 +294,9 @@ function GeneralSection({ const [contextCompactionEnabled, setContextCompactionEnabled] = useState( workflowConfigurations.context_compaction_enabled, ); + const [includeTranscriptEndTimestamps, setIncludeTranscriptEndTimestamps] = useState( + workflowConfigurations.transcript_configuration?.include_end_timestamps ?? false, + ); const [isSaving, setIsSaving] = useState(false); const [isUploadingAudio, setIsUploadingAudio] = useState(false); const [audioUploadError, setAudioUploadError] = useState(null); @@ -314,9 +318,11 @@ function GeneralSection({ turnStartMinWords !== workflowConfigurations.turn_start_min_words || provisionalVadPauseSecs !== workflowConfigurations.provisional_vad_pause_secs || turnStopStrategy !== workflowConfigurations.turn_stop_strategy || - contextCompactionEnabled !== workflowConfigurations.context_compaction_enabled + contextCompactionEnabled !== workflowConfigurations.context_compaction_enabled || + includeTranscriptEndTimestamps !== + (workflowConfigurations.transcript_configuration?.include_end_timestamps ?? false) ); - }, [name, workflowName, ambientNoiseConfig, maxCallDuration, maxUserIdleTimeout, smartTurnStopSecs, turnStartStrategy, turnStartMinWords, provisionalVadPauseSecs, turnStopStrategy, contextCompactionEnabled, workflowConfigurations]); + }, [name, workflowName, ambientNoiseConfig, maxCallDuration, maxUserIdleTimeout, smartTurnStopSecs, turnStartStrategy, turnStartMinWords, provisionalVadPauseSecs, turnStopStrategy, contextCompactionEnabled, includeTranscriptEndTimestamps, workflowConfigurations]); useUnsavedChanges("general", isDirty); @@ -393,6 +399,10 @@ function GeneralSection({ provisional_vad_pause_secs: provisionalVadPauseSecs, turn_stop_strategy: turnStopStrategy, context_compaction_enabled: contextCompactionEnabled, + transcript_configuration: { + ...(workflowConfigurations.transcript_configuration ?? {}), + include_end_timestamps: includeTranscriptEndTimestamps, + }, }, name, ); @@ -686,6 +696,34 @@ function GeneralSection({ + {/* Transcript */} +
+
+

Transcript

+

+ Include start and stop timestamps for each speaker in the uploaded transcript. +

+
+
+ + +
+
+ 
+                            {`[2026-07-06T10:00:00.000Z -> 2026-07-06T10:00:04.800Z] assistant: Can you confirm your date of birth?
+[2026-07-06T10:00:06.200Z -> 2026-07-06T10:00:08.700Z] user: January fifth, nineteen ninety.`}
+
+
+
+ + + {/* Context Compaction */}
@@ -1458,6 +1496,9 @@ function WorkflowSettingsInner({ initialWorkflowConfigurations, user, }); + const resolvedWorkflowConfigurationsForRender = workflowConfigurations + ? resolveWorkflowConfigurations(workflowConfigurations) + : null; useEffect(() => { if (hasFetchedModelConfiguration.current) return; @@ -1532,18 +1573,18 @@ function WorkflowSettingsInner({
{/* Sections */}
- {workflowConfigurations && ( + {resolvedWorkflowConfigurationsForRender && ( <> {/* General */} diff --git a/ui/src/components/workflow/conversation/types.ts b/ui/src/components/workflow/conversation/types.ts index 51e835e9..36d9b9b3 100644 --- a/ui/src/components/workflow/conversation/types.ts +++ b/ui/src/components/workflow/conversation/types.ts @@ -38,6 +38,7 @@ export interface RealtimeFeedbackEvent { final?: boolean; user_id?: string; timestamp?: string; + end_timestamp?: string; function_name?: string; tool_call_id?: string; arguments?: unknown; diff --git a/ui/src/types/workflow-configurations.ts b/ui/src/types/workflow-configurations.ts index 5c31ae41..dffc985b 100644 --- a/ui/src/types/workflow-configurations.ts +++ b/ui/src/types/workflow-configurations.ts @@ -60,6 +60,14 @@ export const DEFAULT_VOICEMAIL_DETECTION_CONFIGURATION: VoicemailDetectionConfig long_speech_timeout: 8.0, }; +export interface TranscriptConfiguration { + include_end_timestamps: boolean; +} + +export const DEFAULT_TRANSCRIPT_CONFIGURATION: TranscriptConfiguration = { + include_end_timestamps: false, +}; + export interface ModelOverrides { llm?: { provider?: string; @@ -115,6 +123,7 @@ export type WorkflowConfigurations = WorkflowConfigurationBase & { turn_stop_strategy: TurnStopStrategy; // Strategy for detecting end of user turn dictionary?: string; // Comma-separated words for voice agent to listen for voicemail_detection?: VoicemailDetectionConfiguration; + transcript_configuration: TranscriptConfiguration; context_compaction_enabled: boolean; // Summarize context on node transitions to remove stale tool calls model_overrides?: ModelOverrides; // Per-workflow model configuration overrides model_configuration_v2_override?: OrganizationAiModelConfigurationV2; // Full v2 model configuration override @@ -134,6 +143,7 @@ const FALLBACK_WORKFLOW_CONFIGURATIONS: WorkflowConfigurations = { provisional_vad_pause_secs: DEFAULT_PROVISIONAL_VAD_PAUSE_SECS, turn_stop_strategy: 'transcription', // Default to transcription-based detection dictionary: '', + transcript_configuration: DEFAULT_TRANSCRIPT_CONFIGURATION, context_compaction_enabled: false, }; @@ -186,5 +196,10 @@ export function resolveWorkflowConfigurations( configurations?.context_compaction_enabled ?? defaults?.context_compaction_enabled ?? FALLBACK_WORKFLOW_CONFIGURATIONS.context_compaction_enabled, + transcript_configuration: { + ...DEFAULT_TRANSCRIPT_CONFIGURATION, + ...(defaults?.transcript_configuration as Partial | undefined), + ...(configurations?.transcript_configuration as Partial | undefined), + }, }; }