apunkt/dograh
1
0
Fork 0
mirror of https://github.com/dograh-hq/dograh.git synced 2026-07-10 11:12:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

Feat/enhanced timestamped transcript (#501)

Browse source 
* feat: add additional timestamps in call transcript optionally

* fi: timestamp precision to millisec instead of micro

* fix: address enhanced transcript review issues

* fix: non vad user turn timestamp
This commit is contained in:
Sabiha Khan 2026-07-07 14:47:43 +05:30 • committed by GitHub
parent d9b9a1efc8
commit ac01f7775e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 441 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

148
api/tests/test_realtime_feedback_observer.py
View file

@ -1,7 +1,17 @@
from types import SimpleNamespace
import re
import pytest
from pipecat.frames.frames import TranscriptionFrame, TTSTextFrame
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
TranscriptionFrame,
TTSTextFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
VADUserStartedSpeakingFrame,
VADUserStoppedSpeakingFrame,
)
from pipecat.observers.base_observer import FramePushed
from pipecat.processors.frame_processor import FrameDirection
from pipecat.transports.base_output import BaseOutputTransport
@ -144,3 +154,139 @@ async def test_turn_log_handlers_persist_user_message_added_events():
"timestamp": "2026-01-01T00:00:00+00:00",
}
assert events[0]["turn"] == 1
@pytest.mark.asyncio
async def test_observer_attaches_backend_speaking_intervals_to_logged_transcript_events():
async def ws_sender(_message):
pass
logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
observer = RealtimeFeedbackObserver(ws_sender=ws_sender, logs_buffer=logs_buffer)
user_aggregator = _FakeAggregator()
assistant_aggregator = _FakeAggregator()
register_turn_log_handlers(logs_buffer, user_aggregator, assistant_aggregator)
await observer.on_push_frame(
_frame_pushed(UserStartedSpeakingFrame(), FrameDirection.DOWNSTREAM)
)
await observer.on_push_frame(
_frame_pushed(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM)
)
await user_aggregator.handlers["on_user_turn_message_added"](
user_aggregator,
SimpleNamespace(
content="January fifth",
timestamp="aggregator-user-start",
),
)
await observer.on_push_frame(
_frame_pushed(BotStartedSpeakingFrame(), FrameDirection.DOWNSTREAM)
)
await assistant_aggregator.handlers["on_assistant_turn_stopped"](
assistant_aggregator,
SimpleNamespace(
content="Thank you",
timestamp="aggregator-bot-start",
),
)
await observer.on_push_frame(
_frame_pushed(BotStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM)
)
user_event, bot_event = [
event
for event in logs_buffer.get_events()
if event["type"] in {"rtf-user-transcription", "rtf-bot-text"}
]
assert user_event["payload"]["timestamp"] != "aggregator-user-start"
assert re.match(
r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+00:00$",
user_event["payload"]["timestamp"],
)
assert user_event["payload"]["end_timestamp"]
assert bot_event["payload"]["timestamp"] != "aggregator-bot-start"
assert re.match(
r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+00:00$",
bot_event["payload"]["timestamp"],
)
assert bot_event["payload"]["end_timestamp"]
@pytest.mark.asyncio
async def test_observer_uses_vad_user_speech_times_over_turn_closure_times():
async def ws_sender(_message):
pass
logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
observer = RealtimeFeedbackObserver(ws_sender=ws_sender, logs_buffer=logs_buffer)
user_aggregator = _FakeAggregator()
assistant_aggregator = _FakeAggregator()
register_turn_log_handlers(logs_buffer, user_aggregator, assistant_aggregator)
await observer.on_push_frame(
_frame_pushed(
VADUserStartedSpeakingFrame(start_secs=0.2, timestamp=1000.2),
FrameDirection.DOWNSTREAM,
)
)
await observer.on_push_frame(
_frame_pushed(
VADUserStoppedSpeakingFrame(stop_secs=0.5, timestamp=1002.5),
FrameDirection.DOWNSTREAM,
)
)
await observer.on_push_frame(
_frame_pushed(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM)
)
await user_aggregator.handlers["on_user_turn_message_added"](
user_aggregator,
SimpleNamespace(content="Hello", timestamp="aggregator-user-start"),
)
user_event = logs_buffer.get_events()[0]
assert user_event["payload"]["timestamp"] == "1970-01-01T00:16:40.000+00:00"
assert user_event["payload"]["end_timestamp"] == "1970-01-01T00:16:42.000+00:00"
@pytest.mark.asyncio
async def test_completed_bot_speech_interval_is_not_reused_for_next_pre_playback_text():
logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
logs_buffer.mark_bot_started_speaking("2026-01-01T00:00:01.000+00:00")
await logs_buffer.append({"type": "rtf-bot-text", "payload": {"text": "First"}})
logs_buffer.mark_bot_stopped_speaking("2026-01-01T00:00:02.000+00:00")
await logs_buffer.append({"type": "rtf-bot-text", "payload": {"text": "Second"}})
second_event = logs_buffer.get_events()[-1]
assert second_event["payload"] == {"text": "Second"}
@pytest.mark.asyncio
async def test_non_vad_user_turn_after_completed_vad_turn_gets_fresh_timestamps():
logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
logs_buffer.mark_user_started_speaking(
"2026-01-01T00:00:01.000+00:00", from_vad=True
)
logs_buffer.mark_user_stopped_speaking(
"2026-01-01T00:00:02.000+00:00", from_vad=True
)
await logs_buffer.append(
{"type": "rtf-user-transcription", "payload": {"text": "First", "final": True}}
)
logs_buffer.mark_user_started_speaking("2026-01-01T00:00:10.000+00:00")
logs_buffer.mark_user_stopped_speaking("2026-01-01T00:00:12.000+00:00")
await logs_buffer.append(
{"type": "rtf-user-transcription", "payload": {"text": "Second", "final": True}}
)
second_event = logs_buffer.get_events()[-1]
assert second_event["payload"]["timestamp"] == "2026-01-01T00:00:10.000+00:00"
assert second_event["payload"]["end_timestamp"] == "2026-01-01T00:00:12.000+00:00"