Feat/enhanced timestamped transcript (#501)

* feat: add additional timestamps in call transcript optionally

* fi: timestamp precision to millisec instead of micro

* fix: address enhanced transcript review issues

* fix: non vad user turn timestamp

api/services/pipecat/event_handlers.py

@@ -70,6 +70,7 @@ def register_event_handlers(
     pre_call_fetch_task: asyncio.Task | None = None,
     user_provider_id: str | None = None,
     integration_runtime_sessions: list[IntegrationRuntimeSession] | None = None,
+    include_transcript_end_timestamps: bool = False,
 ):
     """Register all event handlers for transport and task events.
 
@@ -386,7 +387,9 @@ def register_event_handlers(
             else:
                 logger.debug("Bot audio buffer is empty, skipping upload")
 
-            transcript_text = in_memory_logs_buffer.generate_transcript_text()
+            transcript_text = in_memory_logs_buffer.generate_transcript_text(
+                include_end_timestamps=include_transcript_end_timestamps
+            )
             if not transcript_text:
                 logger.debug("No transcript events in logs buffer, skipping upload")
 

api/services/pipecat/in_memory_buffers.py

@@ -107,6 +107,12 @@ class InMemoryLogsBuffer:
         self._turn_counter = 0
         self._current_node_id: Optional[str] = None
         self._current_node_name: Optional[str] = None
+        self._user_speech_start_timestamp: Optional[str] = None
+        self._user_speech_end_timestamp: Optional[str] = None
+        self._user_speech_start_from_vad = False
+        self._user_speech_end_from_vad = False
+        self._bot_speech_start_timestamp: Optional[str] = None
+        self._bot_speech_end_timestamp: Optional[str] = None
 
     def set_current_node(self, node_id: str, node_name: str):
         """Set the current node ID and name to be injected into subsequent events."""
@@ -123,11 +129,126 @@ class InMemoryLogsBuffer:
         """Get the current node name."""
         return self._current_node_name
 
+    @staticmethod
+    def _now_iso() -> str:
+        return datetime.now(UTC).isoformat(timespec="milliseconds")
+
+    def mark_user_started_speaking(
+        self, timestamp: Optional[str] = None, *, from_vad: bool = False
+    ):
+        """Record when the user started speaking for the current turn."""
+        vad_interval_is_open = (
+            self._user_speech_start_from_vad and self._user_speech_end_timestamp is None
+        )
+        if vad_interval_is_open and not from_vad:
+            return
+
+        self._user_speech_start_timestamp = timestamp or self._now_iso()
+        self._user_speech_end_timestamp = None
+        self._user_speech_start_from_vad = from_vad
+        self._user_speech_end_from_vad = False
+        self._update_latest_payload_start_timestamp(
+            RealtimeFeedbackType.USER_TRANSCRIPTION.value,
+            self._user_speech_start_timestamp,
+            require_final=True,
+        )
+
+    def mark_user_stopped_speaking(
+        self, timestamp: Optional[str] = None, *, from_vad: bool = False
+    ):
+        """Record when the user stopped speaking and update the latest user event."""
+        if self._user_speech_end_from_vad and not from_vad:
+            return
+
+        self._user_speech_end_timestamp = timestamp or self._now_iso()
+        self._user_speech_end_from_vad = from_vad
+        self._update_latest_payload_end_timestamp(
+            RealtimeFeedbackType.USER_TRANSCRIPTION.value,
+            self._user_speech_end_timestamp,
+            require_final=True,
+        )
+
+    def mark_bot_started_speaking(self, timestamp: Optional[str] = None):
+        """Record when the bot started speaking for the current assistant turn."""
+        self._bot_speech_start_timestamp = timestamp or self._now_iso()
+        self._bot_speech_end_timestamp = None
+        self._update_latest_payload_start_timestamp(
+            RealtimeFeedbackType.BOT_TEXT.value,
+            self._bot_speech_start_timestamp,
+        )
+
+    def mark_bot_stopped_speaking(self, timestamp: Optional[str] = None):
+        """Record when the bot stopped speaking and update the latest bot event."""
+        self._bot_speech_end_timestamp = timestamp or self._now_iso()
+        self._update_latest_payload_end_timestamp(
+            RealtimeFeedbackType.BOT_TEXT.value,
+            self._bot_speech_end_timestamp,
+        )
+
+    def _find_latest_open_payload(
+        self, event_type: str, *, require_final: bool = False
+    ) -> dict | None:
+        for event in reversed(self._events):
+            if event.get("type") != event_type:
+                continue
+            payload = event.get("payload")
+            if not isinstance(payload, dict):
+                continue
+            if require_final and payload.get("final") is not True:
+                continue
+            if payload.get("end_timestamp"):
+                continue
+            return payload
+        return None
+
+    def _update_latest_payload_start_timestamp(
+        self, event_type: str, start_timestamp: str, *, require_final: bool = False
+    ):
+        payload = self._find_latest_open_payload(
+            event_type, require_final=require_final
+        )
+        if payload is not None:
+            payload["timestamp"] = start_timestamp
+
+    def _update_latest_payload_end_timestamp(
+        self, event_type: str, end_timestamp: str, *, require_final: bool = False
+    ):
+        payload = self._find_latest_open_payload(
+            event_type, require_final=require_final
+        )
+        if payload is not None:
+            payload["end_timestamp"] = end_timestamp
+
+    def _event_with_speech_timestamps(self, event: dict) -> dict:
+        event_type = event.get("type")
+        payload = event.get("payload")
+        if not isinstance(payload, dict):
+            return event
+
+        payload_with_timestamps = dict(payload)
+        if (
+            event_type == RealtimeFeedbackType.USER_TRANSCRIPTION.value
+            and payload.get("final") is True
+        ):
+            if self._user_speech_start_timestamp:
+                payload_with_timestamps["timestamp"] = self._user_speech_start_timestamp
+            if self._user_speech_end_timestamp:
+                payload_with_timestamps["end_timestamp"] = self._user_speech_end_timestamp
+        elif event_type == RealtimeFeedbackType.BOT_TEXT.value:
+            bot_interval_is_active = self._bot_speech_end_timestamp is None
+            if bot_interval_is_active and self._bot_speech_start_timestamp:
+                payload_with_timestamps["timestamp"] = self._bot_speech_start_timestamp
+
+        if payload_with_timestamps == payload:
+            return event
+        return {**event, "payload": payload_with_timestamps}
+
     async def append(self, event: dict):
         """Append a feedback event to the buffer with timestamp and current node."""
+        event = self._event_with_speech_timestamps(event)
         timestamped_event = stamp_realtime_feedback_event(
             event,
-            timestamp=datetime.now(UTC).isoformat(),
+            timestamp=self._now_iso(),
             turn=self._turn_counter,
             node_id=self._current_node_id,
             node_name=self._current_node_name,
@@ -166,13 +287,15 @@ class InMemoryLogsBuffer:
                 return True
         return False
 
-    def generate_transcript_text(self) -> str:
+    def generate_transcript_text(self, *, include_end_timestamps: bool = False) -> str:
         """Generate transcript text from logged events.
 
         Filters for rtf-user-transcription (final) and rtf-bot-text events,
         formats them as '[timestamp] user/assistant: text\\n'.
         """
-        return _generate_transcript_text(self._sorted_events())
+        return _generate_transcript_text(
+            self._sorted_events(), include_end_timestamps=include_end_timestamps
+        )
 
     @property
     def is_empty(self) -> bool:

api/services/pipecat/realtime_feedback_events.py

@@ -30,6 +30,7 @@ def build_user_transcription_event(
     text: str,
     final: bool,
     timestamp: str | None = None,
+    end_timestamp: str | None = None,
     user_id: str | None = None,
 ) -> dict[str, Any]:
     payload: dict[str, Any] = {
@@ -38,6 +39,8 @@ def build_user_transcription_event(
     }
     if timestamp is not None:
         payload["timestamp"] = timestamp
+    if end_timestamp is not None:
+        payload["end_timestamp"] = end_timestamp
     if user_id is not None:
         payload["user_id"] = user_id
     return {
@@ -50,10 +53,13 @@ def build_bot_text_event(
     *,
     text: str,
     timestamp: str | None = None,
+    end_timestamp: str | None = None,
 ) -> dict[str, Any]:
     payload: dict[str, Any] = {"text": text}
     if timestamp is not None:
         payload["timestamp"] = timestamp
+    if end_timestamp is not None:
+        payload["end_timestamp"] = end_timestamp
     return {
         "type": RealtimeFeedbackType.BOT_TEXT.value,
         "payload": payload,

api/services/pipecat/realtime_feedback_observer.py

@@ -21,6 +21,7 @@ node changes.
 """
 
 import json
+from datetime import UTC, datetime
 from typing import TYPE_CHECKING, Awaitable, Callable, Optional, Set
 
 from loguru import logger
@@ -52,8 +53,12 @@ from pipecat.frames.frames import (
     TranscriptionFrame,
     TTSSpeakFrame,
     TTSTextFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
     UserMuteStartedFrame,
     UserMuteStoppedFrame,
+    VADUserStartedSpeakingFrame,
+    VADUserStoppedSpeakingFrame,
 )
 from pipecat.metrics.metrics import TTFBMetricsData
 from pipecat.observers.base_observer import BaseObserver, FramePushed
@@ -62,6 +67,10 @@ from pipecat.transports.base_output import BaseOutputTransport
 from pipecat.utils.enums import RealtimeFeedbackType
 
 
+def _epoch_seconds_to_utc_iso(timestamp: float) -> str:
+    return datetime.fromtimestamp(timestamp, UTC).isoformat(timespec="milliseconds")
+
+
 class RealtimeFeedbackObserver(BaseObserver):
     """Observer that sends real-time events via WebSocket and persists final transcripts.
 
@@ -138,13 +147,35 @@ class RealtimeFeedbackObserver(BaseObserver):
             return
         # Bot speaking state - WS only (ephemeral state signals, not persisted)
         elif isinstance(frame, BotStartedSpeakingFrame):
+            if self._logs_buffer:
+                self._logs_buffer.mark_bot_started_speaking()
             await self._send_ws(
                 {"type": RealtimeFeedbackType.BOT_STARTED_SPEAKING.value, "payload": {}}
             )
         elif isinstance(frame, BotStoppedSpeakingFrame):
+            if self._logs_buffer:
+                self._logs_buffer.mark_bot_stopped_speaking()
             await self._send_ws(
                 {"type": RealtimeFeedbackType.BOT_STOPPED_SPEAKING.value, "payload": {}}
             )
+        elif isinstance(frame, UserStartedSpeakingFrame):
+            if self._logs_buffer:
+                self._logs_buffer.mark_user_started_speaking()
+        elif isinstance(frame, UserStoppedSpeakingFrame):
+            if self._logs_buffer:
+                self._logs_buffer.mark_user_stopped_speaking()
+        elif isinstance(frame, VADUserStartedSpeakingFrame):
+            if self._logs_buffer:
+                self._logs_buffer.mark_user_started_speaking(
+                    _epoch_seconds_to_utc_iso(frame.timestamp - frame.start_secs),
+                    from_vad=True,
+                )
+        elif isinstance(frame, VADUserStoppedSpeakingFrame):
+            if self._logs_buffer:
+                self._logs_buffer.mark_user_stopped_speaking(
+                    _epoch_seconds_to_utc_iso(frame.timestamp - frame.stop_secs),
+                    from_vad=True,
+                )
         # User mute state - WS only (ephemeral state signals, not persisted)
         elif isinstance(frame, UserMuteStartedFrame):
             await self._send_ws(
@@ -314,6 +345,7 @@ def register_turn_log_handlers(
                     text=message.content,
                     final=True,
                     timestamp=message.timestamp,
+                    end_timestamp=getattr(message, "end_timestamp", None),
                 )
             )
         except Exception as e:
@@ -327,6 +359,7 @@ def register_turn_log_handlers(
                     build_bot_text_event(
                         text=message.content,
                         timestamp=message.timestamp,
+                        end_timestamp=getattr(message, "end_timestamp", None),
                     )
                 )
             except Exception as e:

api/services/pipecat/run_pipeline.py

@@ -545,6 +545,10 @@ async def _run_pipeline_impl(
     max_call_duration_seconds = DEFAULT_MAX_CALL_DURATION_SECONDS
     max_user_idle_timeout = DEFAULT_MAX_USER_IDLE_TIMEOUT_SECONDS
     keyterms = None  # Dictionary words for STT boosting
+    transcript_config = run_configs.get("transcript_configuration") or {}
+    include_transcript_end_timestamps = bool(
+        transcript_config.get("include_end_timestamps", False)
+    )
 
     if run_configs:
         if "max_call_duration" in run_configs:
@@ -1045,6 +1049,7 @@ async def _run_pipeline_impl(
         pre_call_fetch_task=pre_call_fetch_task,
         user_provider_id=user_provider_id,
         integration_runtime_sessions=integration_runtime_sessions,
+        include_transcript_end_timestamps=include_transcript_end_timestamps,
     )
 
     register_audio_data_handler(audio_buffer, workflow_run_id, in_memory_audio_buffer)