Feat/enhanced timestamped transcript (#501)

* feat: add additional timestamps in call transcript optionally

* fi: timestamp precision to millisec instead of micro

* fix: address enhanced transcript review issues

* fix: non vad user turn timestamp

api/tests/test_realtime_feedback_events.py

@@ -1,10 +1,12 @@
 from api.services.pipecat.realtime_feedback_events import (
     build_bot_text_event,
     build_function_call_end_event,
+    build_user_transcription_event,
     build_node_transition_event,
     realtime_feedback_event_sort_key,
     stamp_realtime_feedback_event,
 )
+from api.utils.transcript import generate_transcript_text
 
 
 def test_build_function_call_end_event_serializes_results():
@@ -51,3 +53,38 @@ def test_stamp_and_sort_realtime_feedback_events():
     assert events == [bot_text, node_transition]
     assert node_transition["node_id"] == "node-1"
     assert node_transition["node_name"] == "Greeting"
+
+
+def test_transcript_can_include_end_timestamps_without_changing_default_format():
+    events = [
+        stamp_realtime_feedback_event(
+            build_bot_text_event(
+                text="Can you confirm your date of birth?",
+                timestamp="2026-01-01T00:00:01+00:00",
+                end_timestamp="2026-01-01T00:00:04+00:00",
+            ),
+            timestamp="2026-01-01T00:00:05+00:00",
+            turn=0,
+        ),
+        stamp_realtime_feedback_event(
+            build_user_transcription_event(
+                text="January fifth",
+                final=True,
+                timestamp="2026-01-01T00:00:06+00:00",
+                end_timestamp="2026-01-01T00:00:08+00:00",
+            ),
+            timestamp="2026-01-01T00:00:09+00:00",
+            turn=1,
+        ),
+    ]
+
+    assert generate_transcript_text(events) == (
+        "[2026-01-01T00:00:01+00:00] assistant: Can you confirm your date of birth?\n"
+        "[2026-01-01T00:00:06+00:00] user: January fifth\n"
+    )
+    assert generate_transcript_text(events, include_end_timestamps=True) == (
+        "[2026-01-01T00:00:01+00:00 -> 2026-01-01T00:00:04+00:00] "
+        "assistant: Can you confirm your date of birth?\n"
+        "[2026-01-01T00:00:06+00:00 -> 2026-01-01T00:00:08+00:00] "
+        "user: January fifth\n"
+    )

api/tests/test_realtime_feedback_observer.py

@@ -1,7 +1,17 @@
 from types import SimpleNamespace
+import re
 
 import pytest
-from pipecat.frames.frames import TranscriptionFrame, TTSTextFrame
+from pipecat.frames.frames import (
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
+    TranscriptionFrame,
+    TTSTextFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
+    VADUserStartedSpeakingFrame,
+    VADUserStoppedSpeakingFrame,
+)
 from pipecat.observers.base_observer import FramePushed
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.transports.base_output import BaseOutputTransport
@@ -144,3 +154,139 @@ async def test_turn_log_handlers_persist_user_message_added_events():
         "timestamp": "2026-01-01T00:00:00+00:00",
     }
     assert events[0]["turn"] == 1
+
+
+@pytest.mark.asyncio
+async def test_observer_attaches_backend_speaking_intervals_to_logged_transcript_events():
+    async def ws_sender(_message):
+        pass
+
+    logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
+    observer = RealtimeFeedbackObserver(ws_sender=ws_sender, logs_buffer=logs_buffer)
+
+    user_aggregator = _FakeAggregator()
+    assistant_aggregator = _FakeAggregator()
+    register_turn_log_handlers(logs_buffer, user_aggregator, assistant_aggregator)
+
+    await observer.on_push_frame(
+        _frame_pushed(UserStartedSpeakingFrame(), FrameDirection.DOWNSTREAM)
+    )
+    await observer.on_push_frame(
+        _frame_pushed(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM)
+    )
+    await user_aggregator.handlers["on_user_turn_message_added"](
+        user_aggregator,
+        SimpleNamespace(
+            content="January fifth",
+            timestamp="aggregator-user-start",
+        ),
+    )
+
+    await observer.on_push_frame(
+        _frame_pushed(BotStartedSpeakingFrame(), FrameDirection.DOWNSTREAM)
+    )
+    await assistant_aggregator.handlers["on_assistant_turn_stopped"](
+        assistant_aggregator,
+        SimpleNamespace(
+            content="Thank you",
+            timestamp="aggregator-bot-start",
+        ),
+    )
+    await observer.on_push_frame(
+        _frame_pushed(BotStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM)
+    )
+
+    user_event, bot_event = [
+        event
+        for event in logs_buffer.get_events()
+        if event["type"] in {"rtf-user-transcription", "rtf-bot-text"}
+    ]
+
+    assert user_event["payload"]["timestamp"] != "aggregator-user-start"
+    assert re.match(
+        r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+00:00$",
+        user_event["payload"]["timestamp"],
+    )
+    assert user_event["payload"]["end_timestamp"]
+    assert bot_event["payload"]["timestamp"] != "aggregator-bot-start"
+    assert re.match(
+        r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+00:00$",
+        bot_event["payload"]["timestamp"],
+    )
+    assert bot_event["payload"]["end_timestamp"]
+
+
+@pytest.mark.asyncio
+async def test_observer_uses_vad_user_speech_times_over_turn_closure_times():
+    async def ws_sender(_message):
+        pass
+
+    logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
+    observer = RealtimeFeedbackObserver(ws_sender=ws_sender, logs_buffer=logs_buffer)
+    user_aggregator = _FakeAggregator()
+    assistant_aggregator = _FakeAggregator()
+    register_turn_log_handlers(logs_buffer, user_aggregator, assistant_aggregator)
+
+    await observer.on_push_frame(
+        _frame_pushed(
+            VADUserStartedSpeakingFrame(start_secs=0.2, timestamp=1000.2),
+            FrameDirection.DOWNSTREAM,
+        )
+    )
+    await observer.on_push_frame(
+        _frame_pushed(
+            VADUserStoppedSpeakingFrame(stop_secs=0.5, timestamp=1002.5),
+            FrameDirection.DOWNSTREAM,
+        )
+    )
+    await observer.on_push_frame(
+        _frame_pushed(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM)
+    )
+
+    await user_aggregator.handlers["on_user_turn_message_added"](
+        user_aggregator,
+        SimpleNamespace(content="Hello", timestamp="aggregator-user-start"),
+    )
+
+    user_event = logs_buffer.get_events()[0]
+    assert user_event["payload"]["timestamp"] == "1970-01-01T00:16:40.000+00:00"
+    assert user_event["payload"]["end_timestamp"] == "1970-01-01T00:16:42.000+00:00"
+
+
+@pytest.mark.asyncio
+async def test_completed_bot_speech_interval_is_not_reused_for_next_pre_playback_text():
+    logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
+
+    logs_buffer.mark_bot_started_speaking("2026-01-01T00:00:01.000+00:00")
+    await logs_buffer.append({"type": "rtf-bot-text", "payload": {"text": "First"}})
+    logs_buffer.mark_bot_stopped_speaking("2026-01-01T00:00:02.000+00:00")
+
+    await logs_buffer.append({"type": "rtf-bot-text", "payload": {"text": "Second"}})
+    second_event = logs_buffer.get_events()[-1]
+
+    assert second_event["payload"] == {"text": "Second"}
+
+
+@pytest.mark.asyncio
+async def test_non_vad_user_turn_after_completed_vad_turn_gets_fresh_timestamps():
+    logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
+
+    logs_buffer.mark_user_started_speaking(
+        "2026-01-01T00:00:01.000+00:00", from_vad=True
+    )
+    logs_buffer.mark_user_stopped_speaking(
+        "2026-01-01T00:00:02.000+00:00", from_vad=True
+    )
+    await logs_buffer.append(
+        {"type": "rtf-user-transcription", "payload": {"text": "First", "final": True}}
+    )
+
+    logs_buffer.mark_user_started_speaking("2026-01-01T00:00:10.000+00:00")
+    logs_buffer.mark_user_stopped_speaking("2026-01-01T00:00:12.000+00:00")
+    await logs_buffer.append(
+        {"type": "rtf-user-transcription", "payload": {"text": "Second", "final": True}}
+    )
+
+    second_event = logs_buffer.get_events()[-1]
+    assert second_event["payload"]["timestamp"] == "2026-01-01T00:00:10.000+00:00"
+    assert second_event["payload"]["end_timestamp"] == "2026-01-01T00:00:12.000+00:00"