apunkt/dograh
1
0
Fork 0
mirror of https://github.com/dograh-hq/dograh.git synced 2026-06-19 08:28:10 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat: add chat based testing for voice agent (#308)

Browse source 
* feat: add backend foundations

* feat: add text chat UI

* chore: simplify the reload behaviour

* fix: fix upgrade banner to be triggered after package upload

* feat: simplify TesterPanel design

* chore: fix formatting and generate client

* chore: fix tracing for text chat mode

* fix: fix revert and edit CTA

* refactor: refactor TesterPanel into smaller components

* feat: enable runtime transition of nodes

* fix: fix review comments
This commit is contained in:
Abhishek 2026-05-21 15:20:02 +05:30 • committed by GitHub
parent 67479e98fd
commit d97d1d72cd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
96 changed files with 7630 additions and 1684 deletions
Download patch file Download diff file Expand all files Collapse all files

4
api/tests/test_is_private_ip_candidate.py
View file

@ -167,9 +167,7 @@ class TestIsLocalOrCgnatIp:
class TestKeepCandidate:
def test_private_relay_candidate_survives_private_policy(self):
candidate = (
"candidate:111 1 udp 41885439 192.168.1.50 50000 typ relay raddr 0.0.0.0 rport 0"
)
candidate = "candidate:111 1 udp 41885439 192.168.1.50 50000 typ relay raddr 0.0.0.0 rport 0"
assert _keep_candidate(candidate, NonRelayFilterPolicy.PRIVATE) is True
def test_private_host_candidate_drops_under_private_policy(self):

53
api/tests/test_realtime_feedback_events.py Normal file
View file

@ -0,0 +1,53 @@
from api.services.pipecat.realtime_feedback_events import (
build_bot_text_event,
build_function_call_end_event,
build_node_transition_event,
realtime_feedback_event_sort_key,
stamp_realtime_feedback_event,
)
def test_build_function_call_end_event_serializes_results():
event = build_function_call_end_event(
function_name="lookup_contact",
tool_call_id="tool-1",
result={"contact_id": 42},
)
assert event == {
"type": "rtf-function-call-end",
"payload": {
"function_name": "lookup_contact",
"tool_call_id": "tool-1",
"result": "{'contact_id': 42}",
},
}
def test_stamp_and_sort_realtime_feedback_events():
node_transition = stamp_realtime_feedback_event(
build_node_transition_event(
node_id="node-1",
node_name="Greeting",
previous_node_id=None,
previous_node_name=None,
),
timestamp="2026-01-01T00:00:03+00:00",
turn=0,
node_id="node-1",
node_name="Greeting",
)
bot_text = stamp_realtime_feedback_event(
build_bot_text_event(
text="Hello there",
timestamp="2026-01-01T00:00:01+00:00",
),
timestamp="2026-01-01T00:00:02+00:00",
turn=0,
)
events = sorted([node_transition, bot_text], key=realtime_feedback_event_sort_key)
assert events == [bot_text, node_transition]
assert node_transition["node_id"] == "node-1"
assert node_transition["node_name"] == "Greeting"

99
api/tests/test_text_and_audio_playback.py
View file

@ -382,6 +382,105 @@ class TestStartGreeting:
result = engine.get_start_greeting()
assert result == ("text", "Hello Alice!")
@pytest.mark.asyncio
async def test_queue_node_opening_queues_text_greeting(
self, text_workflow: WorkflowGraph
):
"""Fresh node entry with a greeting should queue TTS and skip LLM bootstrap."""
llm = Mock()
llm.queue_frame = AsyncMock()
task = Mock()
task.queue_frame = AsyncMock()
engine = PipecatEngine(
llm=llm,
context=LLMContext(),
workflow=text_workflow,
call_context_vars={},
workflow_run_id=1,
)
engine.set_task(task)
result = await engine.queue_node_opening(
node_id=text_workflow.start_node_id,
previous_node_id=None,
generate_if_no_greeting=True,
)
assert result == "greeting"
llm.queue_frame.assert_not_awaited()
queued_frame = task.queue_frame.await_args.args[0]
assert isinstance(queued_frame, TTSSpeakFrame)
assert queued_frame.text == TEXT_GREETING
assert queued_frame.append_to_context is True
@pytest.mark.asyncio
async def test_queue_node_opening_falls_back_to_llm_without_greeting(self):
"""When a node has no greeting, the engine should queue initial LLM generation."""
dto = ReactFlowDTO(
nodes=[
RFNodeDTO(
id="start",
type="startCall",
position=Position(x=0, y=0),
data=StartCallNodeData(
name="Start",
prompt="Prompt",
is_start=True,
add_global_prompt=False,
extraction_enabled=False,
),
),
RFNodeDTO(
id="end",
type="endCall",
position=Position(x=0, y=200),
data=EndCallNodeData(
name="End",
prompt="End",
is_end=True,
add_global_prompt=False,
extraction_enabled=False,
),
),
],
edges=[
RFEdgeDTO(
id="e",
source="start",
target="end",
data=EdgeDataDTO(label="End", condition="End"),
),
],
)
workflow = WorkflowGraph(dto)
context = LLMContext()
llm = Mock()
llm.queue_frame = AsyncMock()
task = Mock()
task.queue_frame = AsyncMock()
engine = PipecatEngine(
llm=llm,
context=context,
workflow=workflow,
call_context_vars={},
workflow_run_id=1,
)
engine.set_task(task)
result = await engine.queue_node_opening(
node_id=workflow.start_node_id,
previous_node_id=None,
generate_if_no_greeting=True,
)
assert result == "llm"
task.queue_frame.assert_not_awaited()
queued_frame = llm.queue_frame.await_args.args[0]
assert isinstance(queued_frame, LLMContextFrame)
assert queued_frame.context is context
# ─── Tests: Transition Speech (Pipeline) ────────────────────────

126
api/tests/test_text_chat_logs.py Normal file
View file

@ -0,0 +1,126 @@
from api.services.workflow.text_chat_logs import (
build_text_chat_realtime_feedback_events,
visible_text_chat_turns,
)
def test_visible_text_chat_turns_trims_to_cursor_branch():
session_data = {
"cursor_turn_id": "turn-2",
"turns": [
{"id": "turn-1"},
{"id": "turn-2"},
{"id": "turn-3"},
],
}
assert visible_text_chat_turns(session_data) == [
{"id": "turn-1"},
{"id": "turn-2"},
]
def test_build_text_chat_realtime_feedback_events_uses_visible_branch_and_dedupes_node_transitions():
session_data = {
"cursor_turn_id": "turn-2",
"turns": [
{
"id": "turn-1",
"created_at": "2026-01-01T00:00:00+00:00",
"events": [
{
"type": "node_transition",
"created_at": "2026-01-01T00:00:00+00:00",
"payload": {
"node_id": "node-start",
"node_name": "Start",
"previous_node_id": None,
"previous_node_name": None,
"allow_interrupt": False,
},
}
],
"user_message": None,
"assistant_message": {
"text": "Hello",
"created_at": "2026-01-01T00:00:01+00:00",
},
},
{
"id": "turn-2",
"created_at": "2026-01-01T00:00:02+00:00",
"events": [
{
"type": "node_transition",
"created_at": "2026-01-01T00:00:02+00:00",
"payload": {
"node_id": "node-start",
"node_name": "Start",
"previous_node_id": None,
"previous_node_name": None,
"allow_interrupt": False,
},
},
{
"type": "tool_call_started",
"created_at": "2026-01-01T00:00:03+00:00",
"payload": {
"function_name": "lookup_contact",
"tool_call_id": "tool-1",
},
},
{
"type": "tool_call_result",
"created_at": "2026-01-01T00:00:04+00:00",
"payload": {
"function_name": "lookup_contact",
"tool_call_id": "tool-1",
"result": {"contact_id": 42},
},
},
],
"user_message": {
"text": "Find Abhishek",
"created_at": "2026-01-01T00:00:02+00:00",
},
"assistant_message": {
"text": "I found one match.",
"created_at": "2026-01-01T00:00:05+00:00",
},
},
{
"id": "turn-3",
"created_at": "2026-01-01T00:00:06+00:00",
"events": [
{
"type": "execution_error",
"created_at": "2026-01-01T00:00:06+00:00",
"payload": {"message": "Should be hidden after rewind"},
}
],
"user_message": {
"text": "This turn is rewound away",
"created_at": "2026-01-01T00:00:06+00:00",
},
"assistant_message": None,
},
],
}
events = build_text_chat_realtime_feedback_events(session_data)
assert [event["type"] for event in events] == [
"rtf-node-transition",
"rtf-bot-text",
"rtf-user-transcription",
"rtf-function-call-start",
"rtf-function-call-end",
"rtf-bot-text",
]
assert events[0]["payload"]["node_name"] == "Start"
assert events[2]["payload"]["text"] == "Find Abhishek"
assert events[4]["payload"]["result"] == "{'contact_id': 42}"
assert all(
event.get("payload", {}).get("error") != "Should be hidden after rewind"
for event in events
)

91
api/tests/test_text_chat_session_service.py Normal file
View file

@ -0,0 +1,91 @@
from unittest.mock import AsyncMock
import pytest
import api.services.workflow.text_chat_session_service as text_chat_session_service
from api.db.models import WorkflowRunTextSessionModel
from api.services.workflow.text_chat_session_service import (
TextChatSessionExecutionError,
TextChatTurnNotFoundError,
_reload_text_chat_session,
build_pending_text_chat_turn,
truncate_text_chat_future_turns,
validate_text_chat_turn_cursor,
)
def test_build_pending_text_chat_turn_sets_pending_shape():
turn = build_pending_text_chat_turn(user_text="Hello")
assert turn["id"].startswith("turn_")
assert turn["status"] == "pending"
assert turn["user_message"]["text"] == "Hello"
assert turn["assistant_message"] is None
assert turn["events"] == []
assert turn["usage"] == {}
def test_truncate_text_chat_future_turns_moves_rewound_branch_to_discarded_future():
session_data = {
"cursor_turn_id": "turn-2",
"turns": [
{"id": "turn-1"},
{"id": "turn-2"},
{"id": "turn-3"},
],
"discarded_future": [],
}
active_turns, discarded_future = truncate_text_chat_future_turns(session_data)
assert active_turns == [{"id": "turn-1"}, {"id": "turn-2"}]
assert discarded_future[0]["rewound_from_turn_id"] == "turn-2"
assert discarded_future[0]["turns"] == [{"id": "turn-3"}]
def test_validate_text_chat_turn_cursor_raises_for_missing_turn():
with pytest.raises(TextChatTurnNotFoundError):
validate_text_chat_turn_cursor(
{"turns": [{"id": "turn-1"}]},
"turn-404",
)
@pytest.mark.asyncio
async def test_reload_text_chat_session_uses_run_id_to_resolve_organization(
monkeypatch,
):
reloaded_session = WorkflowRunTextSessionModel(workflow_run_id=123)
get_org_id = AsyncMock(return_value=77)
get_text_session = AsyncMock(return_value=reloaded_session)
monkeypatch.setattr(
text_chat_session_service.db_client,
"get_organization_id_by_workflow_run_id",
get_org_id,
)
monkeypatch.setattr(
text_chat_session_service.db_client,
"get_workflow_run_text_session",
get_text_session,
)
result = await _reload_text_chat_session(123)
assert result is reloaded_session
get_org_id.assert_awaited_once_with(123)
get_text_session.assert_awaited_once_with(123, organization_id=77)
@pytest.mark.asyncio
async def test_reload_text_chat_session_raises_when_run_organization_is_missing(
monkeypatch,
):
monkeypatch.setattr(
text_chat_session_service.db_client,
"get_organization_id_by_workflow_run_id",
AsyncMock(return_value=None),
)
with pytest.raises(TextChatSessionExecutionError, match="organization not found"):
await _reload_text_chat_session(123)

181
api/tests/test_workflow_run_cost.py Normal file
View file

@ -0,0 +1,181 @@
from datetime import UTC, datetime
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from api.services.pricing import workflow_run_cost as workflow_run_cost_mod
from api.services.pricing.workflow_run_cost import (
apply_usage_delta_to_organization,
build_workflow_run_cost_info,
calculate_workflow_run_cost,
)
def _make_workflow_run():
return SimpleNamespace(
id=123,
workflow_id=456,
mode="textchat",
created_at=datetime.now(UTC),
usage_info={
"llm": {},
"tts": {},
"stt": {},
"call_duration_seconds": 7,
},
cost_info={},
workflow=SimpleNamespace(
organization_id=42,
user=SimpleNamespace(selected_organization_id=42),
),
)
@pytest.mark.asyncio
async def test_build_workflow_run_cost_info_does_not_update_org_usage(monkeypatch):
workflow_run = _make_workflow_run()
get_org = AsyncMock(return_value=SimpleNamespace(id=42, price_per_second_usd=1.5))
update_usage = AsyncMock()
monkeypatch.setattr(
workflow_run_cost_mod.db_client, "get_organization_by_id", get_org
)
monkeypatch.setattr(
workflow_run_cost_mod.db_client, "update_usage_after_run", update_usage
)
cost_info = await build_workflow_run_cost_info(workflow_run)
assert cost_info is not None
assert cost_info["call_duration_seconds"] == 7
assert "cost_breakdown" in cost_info
assert "dograh_token_usage" in cost_info
assert cost_info["charge_usd"] == 10.5
update_usage.assert_not_called()
@pytest.mark.asyncio
async def test_calculate_workflow_run_cost_keeps_org_usage_side_effect_in_wrapper(
monkeypatch,
):
workflow_run = _make_workflow_run()
get_org = AsyncMock(return_value=SimpleNamespace(id=42, price_per_second_usd=None))
update_run = AsyncMock()
update_usage = AsyncMock()
monkeypatch.setattr(
workflow_run_cost_mod.db_client,
"get_workflow_run_by_id",
AsyncMock(return_value=workflow_run),
)
monkeypatch.setattr(
workflow_run_cost_mod.db_client, "get_organization_by_id", get_org
)
monkeypatch.setattr(
workflow_run_cost_mod.db_client, "update_workflow_run", update_run
)
monkeypatch.setattr(
workflow_run_cost_mod.db_client, "update_usage_after_run", update_usage
)
await calculate_workflow_run_cost(workflow_run.id)
update_run.assert_awaited_once()
saved_kwargs = update_run.await_args.kwargs
assert saved_kwargs["run_id"] == workflow_run.id
assert "cost_breakdown" in saved_kwargs["cost_info"]
update_usage.assert_awaited_once()
@pytest.mark.asyncio
async def test_apply_usage_delta_to_organization_uses_incremental_costs(
monkeypatch,
):
workflow_run = _make_workflow_run()
workflow_run.cost_info = {"call_id": "preserve-me"}
usage_delta_one = {
"llm": {
"OpenAILLMService#0|||gpt-4.1-mini": {
"prompt_tokens": 1_000,
"completion_tokens": 100,
"total_tokens": 1_100,
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
}
},
"tts": {},
"stt": {},
"call_duration_seconds": 3,
}
usage_delta_two = {
"llm": {
"OpenAILLMService#0|||gpt-4.1-mini": {
"prompt_tokens": 2_000,
"completion_tokens": 50,
"total_tokens": 2_050,
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
}
},
"tts": {},
"stt": {},
"call_duration_seconds": 4,
}
merged_usage = {
"llm": {
"OpenAILLMService#0|||gpt-4.1-mini": {
"prompt_tokens": 3_000,
"completion_tokens": 150,
"total_tokens": 3_150,
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
}
},
"tts": {},
"stt": {},
"call_duration_seconds": 7,
}
get_org = AsyncMock(return_value=SimpleNamespace(id=42, price_per_second_usd=1.5))
update_usage = AsyncMock()
monkeypatch.setattr(
workflow_run_cost_mod.db_client, "get_organization_by_id", get_org
)
monkeypatch.setattr(
workflow_run_cost_mod.db_client, "update_usage_after_run", update_usage
)
first_delta = await apply_usage_delta_to_organization(workflow_run, usage_delta_one)
second_delta = await apply_usage_delta_to_organization(
workflow_run, usage_delta_two
)
total_workflow_run = SimpleNamespace(**workflow_run.__dict__)
total_workflow_run.usage_info = merged_usage
total_cost = await build_workflow_run_cost_info(total_workflow_run)
assert first_delta is not None
assert second_delta is not None
assert total_cost is not None
assert update_usage.await_count == 2
assert update_usage.await_args_list[0].args == (
42,
first_delta["dograh_token_usage"],
3.0,
first_delta["charge_usd"],
)
assert update_usage.await_args_list[1].args == (
42,
second_delta["dograh_token_usage"],
4.0,
second_delta["charge_usd"],
)
assert (
first_delta["dograh_token_usage"] + second_delta["dograh_token_usage"]
) == pytest.approx(total_cost["dograh_token_usage"])
assert (
first_delta["charge_usd"] + second_delta["charge_usd"]
== total_cost["charge_usd"]
)

1194
api/tests/test_workflow_text_chat.py Normal file
View file

File diff suppressed because it is too large Load diff