mirror of
https://github.com/dograh-hq/dograh.git
synced 2026-07-10 11:12:13 +02:00
Feat/enhanced timestamped transcript (#501)
* feat: add additional timestamps in call transcript optionally * fi: timestamp precision to millisec instead of micro * fix: address enhanced transcript review issues * fix: non vad user turn timestamp
This commit is contained in:
parent
d9b9a1efc8
commit
ac01f7775e
12 changed files with 441 additions and 15 deletions
|
|
@ -70,6 +70,7 @@ def register_event_handlers(
|
|||
pre_call_fetch_task: asyncio.Task | None = None,
|
||||
user_provider_id: str | None = None,
|
||||
integration_runtime_sessions: list[IntegrationRuntimeSession] | None = None,
|
||||
include_transcript_end_timestamps: bool = False,
|
||||
):
|
||||
"""Register all event handlers for transport and task events.
|
||||
|
||||
|
|
@ -386,7 +387,9 @@ def register_event_handlers(
|
|||
else:
|
||||
logger.debug("Bot audio buffer is empty, skipping upload")
|
||||
|
||||
transcript_text = in_memory_logs_buffer.generate_transcript_text()
|
||||
transcript_text = in_memory_logs_buffer.generate_transcript_text(
|
||||
include_end_timestamps=include_transcript_end_timestamps
|
||||
)
|
||||
if not transcript_text:
|
||||
logger.debug("No transcript events in logs buffer, skipping upload")
|
||||
|
||||
|
|
|
|||
|
|
@ -107,6 +107,12 @@ class InMemoryLogsBuffer:
|
|||
self._turn_counter = 0
|
||||
self._current_node_id: Optional[str] = None
|
||||
self._current_node_name: Optional[str] = None
|
||||
self._user_speech_start_timestamp: Optional[str] = None
|
||||
self._user_speech_end_timestamp: Optional[str] = None
|
||||
self._user_speech_start_from_vad = False
|
||||
self._user_speech_end_from_vad = False
|
||||
self._bot_speech_start_timestamp: Optional[str] = None
|
||||
self._bot_speech_end_timestamp: Optional[str] = None
|
||||
|
||||
def set_current_node(self, node_id: str, node_name: str):
|
||||
"""Set the current node ID and name to be injected into subsequent events."""
|
||||
|
|
@ -123,11 +129,126 @@ class InMemoryLogsBuffer:
|
|||
"""Get the current node name."""
|
||||
return self._current_node_name
|
||||
|
||||
@staticmethod
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="milliseconds")
|
||||
|
||||
def mark_user_started_speaking(
|
||||
self, timestamp: Optional[str] = None, *, from_vad: bool = False
|
||||
):
|
||||
"""Record when the user started speaking for the current turn."""
|
||||
vad_interval_is_open = (
|
||||
self._user_speech_start_from_vad and self._user_speech_end_timestamp is None
|
||||
)
|
||||
if vad_interval_is_open and not from_vad:
|
||||
return
|
||||
|
||||
self._user_speech_start_timestamp = timestamp or self._now_iso()
|
||||
self._user_speech_end_timestamp = None
|
||||
self._user_speech_start_from_vad = from_vad
|
||||
self._user_speech_end_from_vad = False
|
||||
self._update_latest_payload_start_timestamp(
|
||||
RealtimeFeedbackType.USER_TRANSCRIPTION.value,
|
||||
self._user_speech_start_timestamp,
|
||||
require_final=True,
|
||||
)
|
||||
|
||||
def mark_user_stopped_speaking(
|
||||
self, timestamp: Optional[str] = None, *, from_vad: bool = False
|
||||
):
|
||||
"""Record when the user stopped speaking and update the latest user event."""
|
||||
if self._user_speech_end_from_vad and not from_vad:
|
||||
return
|
||||
|
||||
self._user_speech_end_timestamp = timestamp or self._now_iso()
|
||||
self._user_speech_end_from_vad = from_vad
|
||||
self._update_latest_payload_end_timestamp(
|
||||
RealtimeFeedbackType.USER_TRANSCRIPTION.value,
|
||||
self._user_speech_end_timestamp,
|
||||
require_final=True,
|
||||
)
|
||||
|
||||
def mark_bot_started_speaking(self, timestamp: Optional[str] = None):
|
||||
"""Record when the bot started speaking for the current assistant turn."""
|
||||
self._bot_speech_start_timestamp = timestamp or self._now_iso()
|
||||
self._bot_speech_end_timestamp = None
|
||||
self._update_latest_payload_start_timestamp(
|
||||
RealtimeFeedbackType.BOT_TEXT.value,
|
||||
self._bot_speech_start_timestamp,
|
||||
)
|
||||
|
||||
def mark_bot_stopped_speaking(self, timestamp: Optional[str] = None):
|
||||
"""Record when the bot stopped speaking and update the latest bot event."""
|
||||
self._bot_speech_end_timestamp = timestamp or self._now_iso()
|
||||
self._update_latest_payload_end_timestamp(
|
||||
RealtimeFeedbackType.BOT_TEXT.value,
|
||||
self._bot_speech_end_timestamp,
|
||||
)
|
||||
|
||||
def _find_latest_open_payload(
|
||||
self, event_type: str, *, require_final: bool = False
|
||||
) -> dict | None:
|
||||
for event in reversed(self._events):
|
||||
if event.get("type") != event_type:
|
||||
continue
|
||||
payload = event.get("payload")
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
if require_final and payload.get("final") is not True:
|
||||
continue
|
||||
if payload.get("end_timestamp"):
|
||||
continue
|
||||
return payload
|
||||
return None
|
||||
|
||||
def _update_latest_payload_start_timestamp(
|
||||
self, event_type: str, start_timestamp: str, *, require_final: bool = False
|
||||
):
|
||||
payload = self._find_latest_open_payload(
|
||||
event_type, require_final=require_final
|
||||
)
|
||||
if payload is not None:
|
||||
payload["timestamp"] = start_timestamp
|
||||
|
||||
def _update_latest_payload_end_timestamp(
|
||||
self, event_type: str, end_timestamp: str, *, require_final: bool = False
|
||||
):
|
||||
payload = self._find_latest_open_payload(
|
||||
event_type, require_final=require_final
|
||||
)
|
||||
if payload is not None:
|
||||
payload["end_timestamp"] = end_timestamp
|
||||
|
||||
def _event_with_speech_timestamps(self, event: dict) -> dict:
|
||||
event_type = event.get("type")
|
||||
payload = event.get("payload")
|
||||
if not isinstance(payload, dict):
|
||||
return event
|
||||
|
||||
payload_with_timestamps = dict(payload)
|
||||
if (
|
||||
event_type == RealtimeFeedbackType.USER_TRANSCRIPTION.value
|
||||
and payload.get("final") is True
|
||||
):
|
||||
if self._user_speech_start_timestamp:
|
||||
payload_with_timestamps["timestamp"] = self._user_speech_start_timestamp
|
||||
if self._user_speech_end_timestamp:
|
||||
payload_with_timestamps["end_timestamp"] = self._user_speech_end_timestamp
|
||||
elif event_type == RealtimeFeedbackType.BOT_TEXT.value:
|
||||
bot_interval_is_active = self._bot_speech_end_timestamp is None
|
||||
if bot_interval_is_active and self._bot_speech_start_timestamp:
|
||||
payload_with_timestamps["timestamp"] = self._bot_speech_start_timestamp
|
||||
|
||||
if payload_with_timestamps == payload:
|
||||
return event
|
||||
return {**event, "payload": payload_with_timestamps}
|
||||
|
||||
async def append(self, event: dict):
|
||||
"""Append a feedback event to the buffer with timestamp and current node."""
|
||||
event = self._event_with_speech_timestamps(event)
|
||||
timestamped_event = stamp_realtime_feedback_event(
|
||||
event,
|
||||
timestamp=datetime.now(UTC).isoformat(),
|
||||
timestamp=self._now_iso(),
|
||||
turn=self._turn_counter,
|
||||
node_id=self._current_node_id,
|
||||
node_name=self._current_node_name,
|
||||
|
|
@ -166,13 +287,15 @@ class InMemoryLogsBuffer:
|
|||
return True
|
||||
return False
|
||||
|
||||
def generate_transcript_text(self) -> str:
|
||||
def generate_transcript_text(self, *, include_end_timestamps: bool = False) -> str:
|
||||
"""Generate transcript text from logged events.
|
||||
|
||||
Filters for rtf-user-transcription (final) and rtf-bot-text events,
|
||||
formats them as '[timestamp] user/assistant: text\\n'.
|
||||
"""
|
||||
return _generate_transcript_text(self._sorted_events())
|
||||
return _generate_transcript_text(
|
||||
self._sorted_events(), include_end_timestamps=include_end_timestamps
|
||||
)
|
||||
|
||||
@property
|
||||
def is_empty(self) -> bool:
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ def build_user_transcription_event(
|
|||
text: str,
|
||||
final: bool,
|
||||
timestamp: str | None = None,
|
||||
end_timestamp: str | None = None,
|
||||
user_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
|
|
@ -38,6 +39,8 @@ def build_user_transcription_event(
|
|||
}
|
||||
if timestamp is not None:
|
||||
payload["timestamp"] = timestamp
|
||||
if end_timestamp is not None:
|
||||
payload["end_timestamp"] = end_timestamp
|
||||
if user_id is not None:
|
||||
payload["user_id"] = user_id
|
||||
return {
|
||||
|
|
@ -50,10 +53,13 @@ def build_bot_text_event(
|
|||
*,
|
||||
text: str,
|
||||
timestamp: str | None = None,
|
||||
end_timestamp: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {"text": text}
|
||||
if timestamp is not None:
|
||||
payload["timestamp"] = timestamp
|
||||
if end_timestamp is not None:
|
||||
payload["end_timestamp"] = end_timestamp
|
||||
return {
|
||||
"type": RealtimeFeedbackType.BOT_TEXT.value,
|
||||
"payload": payload,
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ node changes.
|
|||
"""
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Awaitable, Callable, Optional, Set
|
||||
|
||||
from loguru import logger
|
||||
|
|
@ -52,8 +53,12 @@ from pipecat.frames.frames import (
|
|||
TranscriptionFrame,
|
||||
TTSSpeakFrame,
|
||||
TTSTextFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
UserMuteStartedFrame,
|
||||
UserMuteStoppedFrame,
|
||||
VADUserStartedSpeakingFrame,
|
||||
VADUserStoppedSpeakingFrame,
|
||||
)
|
||||
from pipecat.metrics.metrics import TTFBMetricsData
|
||||
from pipecat.observers.base_observer import BaseObserver, FramePushed
|
||||
|
|
@ -62,6 +67,10 @@ from pipecat.transports.base_output import BaseOutputTransport
|
|||
from pipecat.utils.enums import RealtimeFeedbackType
|
||||
|
||||
|
||||
def _epoch_seconds_to_utc_iso(timestamp: float) -> str:
|
||||
return datetime.fromtimestamp(timestamp, UTC).isoformat(timespec="milliseconds")
|
||||
|
||||
|
||||
class RealtimeFeedbackObserver(BaseObserver):
|
||||
"""Observer that sends real-time events via WebSocket and persists final transcripts.
|
||||
|
||||
|
|
@ -138,13 +147,35 @@ class RealtimeFeedbackObserver(BaseObserver):
|
|||
return
|
||||
# Bot speaking state - WS only (ephemeral state signals, not persisted)
|
||||
elif isinstance(frame, BotStartedSpeakingFrame):
|
||||
if self._logs_buffer:
|
||||
self._logs_buffer.mark_bot_started_speaking()
|
||||
await self._send_ws(
|
||||
{"type": RealtimeFeedbackType.BOT_STARTED_SPEAKING.value, "payload": {}}
|
||||
)
|
||||
elif isinstance(frame, BotStoppedSpeakingFrame):
|
||||
if self._logs_buffer:
|
||||
self._logs_buffer.mark_bot_stopped_speaking()
|
||||
await self._send_ws(
|
||||
{"type": RealtimeFeedbackType.BOT_STOPPED_SPEAKING.value, "payload": {}}
|
||||
)
|
||||
elif isinstance(frame, UserStartedSpeakingFrame):
|
||||
if self._logs_buffer:
|
||||
self._logs_buffer.mark_user_started_speaking()
|
||||
elif isinstance(frame, UserStoppedSpeakingFrame):
|
||||
if self._logs_buffer:
|
||||
self._logs_buffer.mark_user_stopped_speaking()
|
||||
elif isinstance(frame, VADUserStartedSpeakingFrame):
|
||||
if self._logs_buffer:
|
||||
self._logs_buffer.mark_user_started_speaking(
|
||||
_epoch_seconds_to_utc_iso(frame.timestamp - frame.start_secs),
|
||||
from_vad=True,
|
||||
)
|
||||
elif isinstance(frame, VADUserStoppedSpeakingFrame):
|
||||
if self._logs_buffer:
|
||||
self._logs_buffer.mark_user_stopped_speaking(
|
||||
_epoch_seconds_to_utc_iso(frame.timestamp - frame.stop_secs),
|
||||
from_vad=True,
|
||||
)
|
||||
# User mute state - WS only (ephemeral state signals, not persisted)
|
||||
elif isinstance(frame, UserMuteStartedFrame):
|
||||
await self._send_ws(
|
||||
|
|
@ -314,6 +345,7 @@ def register_turn_log_handlers(
|
|||
text=message.content,
|
||||
final=True,
|
||||
timestamp=message.timestamp,
|
||||
end_timestamp=getattr(message, "end_timestamp", None),
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
|
|
@ -327,6 +359,7 @@ def register_turn_log_handlers(
|
|||
build_bot_text_event(
|
||||
text=message.content,
|
||||
timestamp=message.timestamp,
|
||||
end_timestamp=getattr(message, "end_timestamp", None),
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -545,6 +545,10 @@ async def _run_pipeline_impl(
|
|||
max_call_duration_seconds = DEFAULT_MAX_CALL_DURATION_SECONDS
|
||||
max_user_idle_timeout = DEFAULT_MAX_USER_IDLE_TIMEOUT_SECONDS
|
||||
keyterms = None # Dictionary words for STT boosting
|
||||
transcript_config = run_configs.get("transcript_configuration") or {}
|
||||
include_transcript_end_timestamps = bool(
|
||||
transcript_config.get("include_end_timestamps", False)
|
||||
)
|
||||
|
||||
if run_configs:
|
||||
if "max_call_duration" in run_configs:
|
||||
|
|
@ -1045,6 +1049,7 @@ async def _run_pipeline_impl(
|
|||
pre_call_fetch_task=pre_call_fetch_task,
|
||||
user_provider_id=user_provider_id,
|
||||
integration_runtime_sessions=integration_runtime_sessions,
|
||||
include_transcript_end_timestamps=include_transcript_end_timestamps,
|
||||
)
|
||||
|
||||
register_audio_data_handler(audio_buffer, workflow_run_id, in_memory_audio_buffer)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
from api.services.pipecat.realtime_feedback_events import (
|
||||
build_bot_text_event,
|
||||
build_function_call_end_event,
|
||||
build_user_transcription_event,
|
||||
build_node_transition_event,
|
||||
realtime_feedback_event_sort_key,
|
||||
stamp_realtime_feedback_event,
|
||||
)
|
||||
from api.utils.transcript import generate_transcript_text
|
||||
|
||||
|
||||
def test_build_function_call_end_event_serializes_results():
|
||||
|
|
@ -51,3 +53,38 @@ def test_stamp_and_sort_realtime_feedback_events():
|
|||
assert events == [bot_text, node_transition]
|
||||
assert node_transition["node_id"] == "node-1"
|
||||
assert node_transition["node_name"] == "Greeting"
|
||||
|
||||
|
||||
def test_transcript_can_include_end_timestamps_without_changing_default_format():
|
||||
events = [
|
||||
stamp_realtime_feedback_event(
|
||||
build_bot_text_event(
|
||||
text="Can you confirm your date of birth?",
|
||||
timestamp="2026-01-01T00:00:01+00:00",
|
||||
end_timestamp="2026-01-01T00:00:04+00:00",
|
||||
),
|
||||
timestamp="2026-01-01T00:00:05+00:00",
|
||||
turn=0,
|
||||
),
|
||||
stamp_realtime_feedback_event(
|
||||
build_user_transcription_event(
|
||||
text="January fifth",
|
||||
final=True,
|
||||
timestamp="2026-01-01T00:00:06+00:00",
|
||||
end_timestamp="2026-01-01T00:00:08+00:00",
|
||||
),
|
||||
timestamp="2026-01-01T00:00:09+00:00",
|
||||
turn=1,
|
||||
),
|
||||
]
|
||||
|
||||
assert generate_transcript_text(events) == (
|
||||
"[2026-01-01T00:00:01+00:00] assistant: Can you confirm your date of birth?\n"
|
||||
"[2026-01-01T00:00:06+00:00] user: January fifth\n"
|
||||
)
|
||||
assert generate_transcript_text(events, include_end_timestamps=True) == (
|
||||
"[2026-01-01T00:00:01+00:00 -> 2026-01-01T00:00:04+00:00] "
|
||||
"assistant: Can you confirm your date of birth?\n"
|
||||
"[2026-01-01T00:00:06+00:00 -> 2026-01-01T00:00:08+00:00] "
|
||||
"user: January fifth\n"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,17 @@
|
|||
from types import SimpleNamespace
|
||||
import re
|
||||
|
||||
import pytest
|
||||
from pipecat.frames.frames import TranscriptionFrame, TTSTextFrame
|
||||
from pipecat.frames.frames import (
|
||||
BotStartedSpeakingFrame,
|
||||
BotStoppedSpeakingFrame,
|
||||
TranscriptionFrame,
|
||||
TTSTextFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
VADUserStartedSpeakingFrame,
|
||||
VADUserStoppedSpeakingFrame,
|
||||
)
|
||||
from pipecat.observers.base_observer import FramePushed
|
||||
from pipecat.processors.frame_processor import FrameDirection
|
||||
from pipecat.transports.base_output import BaseOutputTransport
|
||||
|
|
@ -144,3 +154,139 @@ async def test_turn_log_handlers_persist_user_message_added_events():
|
|||
"timestamp": "2026-01-01T00:00:00+00:00",
|
||||
}
|
||||
assert events[0]["turn"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observer_attaches_backend_speaking_intervals_to_logged_transcript_events():
|
||||
async def ws_sender(_message):
|
||||
pass
|
||||
|
||||
logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
|
||||
observer = RealtimeFeedbackObserver(ws_sender=ws_sender, logs_buffer=logs_buffer)
|
||||
|
||||
user_aggregator = _FakeAggregator()
|
||||
assistant_aggregator = _FakeAggregator()
|
||||
register_turn_log_handlers(logs_buffer, user_aggregator, assistant_aggregator)
|
||||
|
||||
await observer.on_push_frame(
|
||||
_frame_pushed(UserStartedSpeakingFrame(), FrameDirection.DOWNSTREAM)
|
||||
)
|
||||
await observer.on_push_frame(
|
||||
_frame_pushed(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM)
|
||||
)
|
||||
await user_aggregator.handlers["on_user_turn_message_added"](
|
||||
user_aggregator,
|
||||
SimpleNamespace(
|
||||
content="January fifth",
|
||||
timestamp="aggregator-user-start",
|
||||
),
|
||||
)
|
||||
|
||||
await observer.on_push_frame(
|
||||
_frame_pushed(BotStartedSpeakingFrame(), FrameDirection.DOWNSTREAM)
|
||||
)
|
||||
await assistant_aggregator.handlers["on_assistant_turn_stopped"](
|
||||
assistant_aggregator,
|
||||
SimpleNamespace(
|
||||
content="Thank you",
|
||||
timestamp="aggregator-bot-start",
|
||||
),
|
||||
)
|
||||
await observer.on_push_frame(
|
||||
_frame_pushed(BotStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM)
|
||||
)
|
||||
|
||||
user_event, bot_event = [
|
||||
event
|
||||
for event in logs_buffer.get_events()
|
||||
if event["type"] in {"rtf-user-transcription", "rtf-bot-text"}
|
||||
]
|
||||
|
||||
assert user_event["payload"]["timestamp"] != "aggregator-user-start"
|
||||
assert re.match(
|
||||
r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+00:00$",
|
||||
user_event["payload"]["timestamp"],
|
||||
)
|
||||
assert user_event["payload"]["end_timestamp"]
|
||||
assert bot_event["payload"]["timestamp"] != "aggregator-bot-start"
|
||||
assert re.match(
|
||||
r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+00:00$",
|
||||
bot_event["payload"]["timestamp"],
|
||||
)
|
||||
assert bot_event["payload"]["end_timestamp"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observer_uses_vad_user_speech_times_over_turn_closure_times():
|
||||
async def ws_sender(_message):
|
||||
pass
|
||||
|
||||
logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
|
||||
observer = RealtimeFeedbackObserver(ws_sender=ws_sender, logs_buffer=logs_buffer)
|
||||
user_aggregator = _FakeAggregator()
|
||||
assistant_aggregator = _FakeAggregator()
|
||||
register_turn_log_handlers(logs_buffer, user_aggregator, assistant_aggregator)
|
||||
|
||||
await observer.on_push_frame(
|
||||
_frame_pushed(
|
||||
VADUserStartedSpeakingFrame(start_secs=0.2, timestamp=1000.2),
|
||||
FrameDirection.DOWNSTREAM,
|
||||
)
|
||||
)
|
||||
await observer.on_push_frame(
|
||||
_frame_pushed(
|
||||
VADUserStoppedSpeakingFrame(stop_secs=0.5, timestamp=1002.5),
|
||||
FrameDirection.DOWNSTREAM,
|
||||
)
|
||||
)
|
||||
await observer.on_push_frame(
|
||||
_frame_pushed(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM)
|
||||
)
|
||||
|
||||
await user_aggregator.handlers["on_user_turn_message_added"](
|
||||
user_aggregator,
|
||||
SimpleNamespace(content="Hello", timestamp="aggregator-user-start"),
|
||||
)
|
||||
|
||||
user_event = logs_buffer.get_events()[0]
|
||||
assert user_event["payload"]["timestamp"] == "1970-01-01T00:16:40.000+00:00"
|
||||
assert user_event["payload"]["end_timestamp"] == "1970-01-01T00:16:42.000+00:00"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_bot_speech_interval_is_not_reused_for_next_pre_playback_text():
|
||||
logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
|
||||
|
||||
logs_buffer.mark_bot_started_speaking("2026-01-01T00:00:01.000+00:00")
|
||||
await logs_buffer.append({"type": "rtf-bot-text", "payload": {"text": "First"}})
|
||||
logs_buffer.mark_bot_stopped_speaking("2026-01-01T00:00:02.000+00:00")
|
||||
|
||||
await logs_buffer.append({"type": "rtf-bot-text", "payload": {"text": "Second"}})
|
||||
second_event = logs_buffer.get_events()[-1]
|
||||
|
||||
assert second_event["payload"] == {"text": "Second"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_vad_user_turn_after_completed_vad_turn_gets_fresh_timestamps():
|
||||
logs_buffer = InMemoryLogsBuffer(workflow_run_id=123)
|
||||
|
||||
logs_buffer.mark_user_started_speaking(
|
||||
"2026-01-01T00:00:01.000+00:00", from_vad=True
|
||||
)
|
||||
logs_buffer.mark_user_stopped_speaking(
|
||||
"2026-01-01T00:00:02.000+00:00", from_vad=True
|
||||
)
|
||||
await logs_buffer.append(
|
||||
{"type": "rtf-user-transcription", "payload": {"text": "First", "final": True}}
|
||||
)
|
||||
|
||||
logs_buffer.mark_user_started_speaking("2026-01-01T00:00:10.000+00:00")
|
||||
logs_buffer.mark_user_stopped_speaking("2026-01-01T00:00:12.000+00:00")
|
||||
await logs_buffer.append(
|
||||
{"type": "rtf-user-transcription", "payload": {"text": "Second", "final": True}}
|
||||
)
|
||||
|
||||
second_event = logs_buffer.get_events()[-1]
|
||||
assert second_event["payload"]["timestamp"] == "2026-01-01T00:00:10.000+00:00"
|
||||
assert second_event["payload"]["end_timestamp"] == "2026-01-01T00:00:12.000+00:00"
|
||||
|
|
|
|||
|
|
@ -3,11 +3,26 @@ from typing import List
|
|||
from pipecat.utils.enums import RealtimeFeedbackType
|
||||
|
||||
|
||||
def generate_transcript_text(events: List[dict]) -> str:
|
||||
def _format_timestamp_range(payload: dict, event: dict, include_end_timestamps: bool) -> str:
|
||||
start_timestamp = payload.get("timestamp") or event.get("timestamp", "")
|
||||
if not include_end_timestamps:
|
||||
return start_timestamp
|
||||
|
||||
end_timestamp = payload.get("end_timestamp")
|
||||
if end_timestamp:
|
||||
return f"{start_timestamp} -> {end_timestamp}" if start_timestamp else end_timestamp
|
||||
return start_timestamp
|
||||
|
||||
|
||||
def generate_transcript_text(
|
||||
events: List[dict], *, include_end_timestamps: bool = False
|
||||
) -> str:
|
||||
"""Generate transcript text from realtime feedback events.
|
||||
|
||||
Filters for rtf-user-transcription (final) and rtf-bot-text events,
|
||||
formats them as '[timestamp] user/assistant: text\\n'.
|
||||
formats them as '[timestamp] user/assistant: text\\n'. When
|
||||
include_end_timestamps is True, formats as
|
||||
'[start_timestamp -> end_timestamp] user/assistant: text\\n'.
|
||||
"""
|
||||
lines: List[str] = []
|
||||
for event in events:
|
||||
|
|
@ -18,11 +33,11 @@ def generate_transcript_text(events: List[dict]) -> str:
|
|||
event_type == RealtimeFeedbackType.USER_TRANSCRIPTION.value
|
||||
and payload.get("final") is True
|
||||
):
|
||||
timestamp = payload.get("timestamp") or event.get("timestamp", "")
|
||||
timestamp = _format_timestamp_range(payload, event, include_end_timestamps)
|
||||
prefix = f"[{timestamp}] " if timestamp else ""
|
||||
lines.append(f"{prefix}user: {payload.get('text', '')}\n")
|
||||
elif event_type == RealtimeFeedbackType.BOT_TEXT.value:
|
||||
timestamp = payload.get("timestamp") or event.get("timestamp", "")
|
||||
timestamp = _format_timestamp_range(payload, event, include_end_timestamps)
|
||||
prefix = f"[{timestamp}] " if timestamp else ""
|
||||
lines.append(f"{prefix}assistant: {payload.get('text', '')}\n")
|
||||
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ export const ConfigurationsDialog = ({
|
|||
turn_start_min_words: turnStartMinWords,
|
||||
provisional_vad_pause_secs: provisionalVadPauseSecs,
|
||||
turn_stop_strategy: turnStopStrategy,
|
||||
transcript_configuration: resolvedWorkflowConfigurations.transcript_configuration,
|
||||
context_compaction_enabled: contextCompactionEnabled,
|
||||
}, name);
|
||||
onOpenChange(false);
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ import {
|
|||
DEFAULT_PROVISIONAL_VAD_PAUSE_SECS,
|
||||
DEFAULT_TURN_START_MIN_WORDS,
|
||||
DEFAULT_VOICEMAIL_DETECTION_CONFIGURATION,
|
||||
resolveWorkflowConfigurations,
|
||||
TURN_START_STRATEGY_OPTIONS,
|
||||
type TurnStartStrategy,
|
||||
type TurnStopStrategy,
|
||||
|
|
@ -293,6 +294,9 @@ function GeneralSection({
|
|||
const [contextCompactionEnabled, setContextCompactionEnabled] = useState(
|
||||
workflowConfigurations.context_compaction_enabled,
|
||||
);
|
||||
const [includeTranscriptEndTimestamps, setIncludeTranscriptEndTimestamps] = useState(
|
||||
workflowConfigurations.transcript_configuration?.include_end_timestamps ?? false,
|
||||
);
|
||||
const [isSaving, setIsSaving] = useState(false);
|
||||
const [isUploadingAudio, setIsUploadingAudio] = useState(false);
|
||||
const [audioUploadError, setAudioUploadError] = useState<string | null>(null);
|
||||
|
|
@ -314,9 +318,11 @@ function GeneralSection({
|
|||
turnStartMinWords !== workflowConfigurations.turn_start_min_words ||
|
||||
provisionalVadPauseSecs !== workflowConfigurations.provisional_vad_pause_secs ||
|
||||
turnStopStrategy !== workflowConfigurations.turn_stop_strategy ||
|
||||
contextCompactionEnabled !== workflowConfigurations.context_compaction_enabled
|
||||
contextCompactionEnabled !== workflowConfigurations.context_compaction_enabled ||
|
||||
includeTranscriptEndTimestamps !==
|
||||
(workflowConfigurations.transcript_configuration?.include_end_timestamps ?? false)
|
||||
);
|
||||
}, [name, workflowName, ambientNoiseConfig, maxCallDuration, maxUserIdleTimeout, smartTurnStopSecs, turnStartStrategy, turnStartMinWords, provisionalVadPauseSecs, turnStopStrategy, contextCompactionEnabled, workflowConfigurations]);
|
||||
}, [name, workflowName, ambientNoiseConfig, maxCallDuration, maxUserIdleTimeout, smartTurnStopSecs, turnStartStrategy, turnStartMinWords, provisionalVadPauseSecs, turnStopStrategy, contextCompactionEnabled, includeTranscriptEndTimestamps, workflowConfigurations]);
|
||||
|
||||
useUnsavedChanges("general", isDirty);
|
||||
|
||||
|
|
@ -393,6 +399,10 @@ function GeneralSection({
|
|||
provisional_vad_pause_secs: provisionalVadPauseSecs,
|
||||
turn_stop_strategy: turnStopStrategy,
|
||||
context_compaction_enabled: contextCompactionEnabled,
|
||||
transcript_configuration: {
|
||||
...(workflowConfigurations.transcript_configuration ?? {}),
|
||||
include_end_timestamps: includeTranscriptEndTimestamps,
|
||||
},
|
||||
},
|
||||
name,
|
||||
);
|
||||
|
|
@ -686,6 +696,34 @@ function GeneralSection({
|
|||
|
||||
<Separator />
|
||||
|
||||
{/* Transcript */}
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<h3 className="text-sm font-medium">Transcript</h3>
|
||||
<p className="text-xs text-muted-foreground mt-0.5">
|
||||
Include start and stop timestamps for each speaker in the uploaded transcript.
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center justify-between">
|
||||
<Label htmlFor="transcript-end-timestamps-enabled" className="text-sm">
|
||||
Enhanced Timestamped Transcript
|
||||
</Label>
|
||||
<Switch
|
||||
id="transcript-end-timestamps-enabled"
|
||||
checked={includeTranscriptEndTimestamps}
|
||||
onCheckedChange={setIncludeTranscriptEndTimestamps}
|
||||
/>
|
||||
</div>
|
||||
<div className="rounded-md border bg-muted/20 p-3">
|
||||
<pre className="whitespace-pre-wrap text-xs leading-relaxed text-muted-foreground">
|
||||
{`[2026-07-06T10:00:00.000Z -> 2026-07-06T10:00:04.800Z] assistant: Can you confirm your date of birth?
|
||||
[2026-07-06T10:00:06.200Z -> 2026-07-06T10:00:08.700Z] user: January fifth, nineteen ninety.`}
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<Separator />
|
||||
|
||||
{/* Context Compaction */}
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
|
|
@ -1458,6 +1496,9 @@ function WorkflowSettingsInner({
|
|||
initialWorkflowConfigurations,
|
||||
user,
|
||||
});
|
||||
const resolvedWorkflowConfigurationsForRender = workflowConfigurations
|
||||
? resolveWorkflowConfigurations(workflowConfigurations)
|
||||
: null;
|
||||
|
||||
useEffect(() => {
|
||||
if (hasFetchedModelConfiguration.current) return;
|
||||
|
|
@ -1532,18 +1573,18 @@ function WorkflowSettingsInner({
|
|||
<div className="mx-auto flex max-w-5xl gap-8 px-6 py-8">
|
||||
{/* Sections */}
|
||||
<div className="min-w-0 flex-1 space-y-8">
|
||||
{workflowConfigurations && (
|
||||
{resolvedWorkflowConfigurationsForRender && (
|
||||
<>
|
||||
{/* General */}
|
||||
<GeneralSection
|
||||
workflowConfigurations={workflowConfigurations}
|
||||
workflowConfigurations={resolvedWorkflowConfigurationsForRender}
|
||||
workflowName={workflowName || workflow.name}
|
||||
workflowId={workflowId}
|
||||
onSave={saveWorkflowConfigurations}
|
||||
/>
|
||||
|
||||
<WorkflowModelOverridesSection
|
||||
workflowConfigurations={workflowConfigurations}
|
||||
workflowConfigurations={resolvedWorkflowConfigurationsForRender}
|
||||
workflowName={workflowName}
|
||||
onSave={saveWorkflowConfigurations}
|
||||
modelConfigurationDefaults={modelConfigurationDefaults}
|
||||
|
|
@ -1563,7 +1604,7 @@ function WorkflowSettingsInner({
|
|||
|
||||
{/* Voicemail Detection */}
|
||||
<VoicemailSection
|
||||
workflowConfigurations={workflowConfigurations}
|
||||
workflowConfigurations={resolvedWorkflowConfigurationsForRender}
|
||||
workflowName={workflowName}
|
||||
onSave={saveWorkflowConfigurations}
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ export interface RealtimeFeedbackEvent {
|
|||
final?: boolean;
|
||||
user_id?: string;
|
||||
timestamp?: string;
|
||||
end_timestamp?: string;
|
||||
function_name?: string;
|
||||
tool_call_id?: string;
|
||||
arguments?: unknown;
|
||||
|
|
|
|||
|
|
@ -60,6 +60,14 @@ export const DEFAULT_VOICEMAIL_DETECTION_CONFIGURATION: VoicemailDetectionConfig
|
|||
long_speech_timeout: 8.0,
|
||||
};
|
||||
|
||||
export interface TranscriptConfiguration {
|
||||
include_end_timestamps: boolean;
|
||||
}
|
||||
|
||||
export const DEFAULT_TRANSCRIPT_CONFIGURATION: TranscriptConfiguration = {
|
||||
include_end_timestamps: false,
|
||||
};
|
||||
|
||||
export interface ModelOverrides {
|
||||
llm?: {
|
||||
provider?: string;
|
||||
|
|
@ -115,6 +123,7 @@ export type WorkflowConfigurations = WorkflowConfigurationBase & {
|
|||
turn_stop_strategy: TurnStopStrategy; // Strategy for detecting end of user turn
|
||||
dictionary?: string; // Comma-separated words for voice agent to listen for
|
||||
voicemail_detection?: VoicemailDetectionConfiguration;
|
||||
transcript_configuration: TranscriptConfiguration;
|
||||
context_compaction_enabled: boolean; // Summarize context on node transitions to remove stale tool calls
|
||||
model_overrides?: ModelOverrides; // Per-workflow model configuration overrides
|
||||
model_configuration_v2_override?: OrganizationAiModelConfigurationV2; // Full v2 model configuration override
|
||||
|
|
@ -134,6 +143,7 @@ const FALLBACK_WORKFLOW_CONFIGURATIONS: WorkflowConfigurations = {
|
|||
provisional_vad_pause_secs: DEFAULT_PROVISIONAL_VAD_PAUSE_SECS,
|
||||
turn_stop_strategy: 'transcription', // Default to transcription-based detection
|
||||
dictionary: '',
|
||||
transcript_configuration: DEFAULT_TRANSCRIPT_CONFIGURATION,
|
||||
context_compaction_enabled: false,
|
||||
};
|
||||
|
||||
|
|
@ -186,5 +196,10 @@ export function resolveWorkflowConfigurations(
|
|||
configurations?.context_compaction_enabled
|
||||
?? defaults?.context_compaction_enabled
|
||||
?? FALLBACK_WORKFLOW_CONFIGURATIONS.context_compaction_enabled,
|
||||
transcript_configuration: {
|
||||
...DEFAULT_TRANSCRIPT_CONFIGURATION,
|
||||
...(defaults?.transcript_configuration as Partial<TranscriptConfiguration> | undefined),
|
||||
...(configurations?.transcript_configuration as Partial<TranscriptConfiguration> | undefined),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue