mirror of
https://github.com/dograh-hq/dograh.git
synced 2026-06-22 08:38:13 +02:00
7d053320df
* fix: disable duplicate trigger nodes in workflow builder AddNodePanel: disable trigger buttons and show tooltip when a trigger already exists on the canvas, using bySpecName to identify trigger- category specs from the live node list. useWorkflowState: preflight in saveWorkflow rejects saves with multiple trigger nodes via a sonner toast before the network request is made. text_chat_session_service: include the original exception message in TextChatSessionExecutionError so the HTTP 500 detail surfaces the root cause without DB inspection. Closes #378 * style: format test_text_chat_session_service.py with ruff * chore: retrigger CI checks * fix(workflow): enforce node instance constraints --------- Co-authored-by: Abhishek Kumar <abhishek@a6k.me>
72 lines
2.8 KiB
Python
72 lines
2.8 KiB
Python
"""Regression test for QA analysis when the LLM returns a non-dict JSON value.
|
|
|
|
``parse_llm_json`` is explicitly designed to return a list when the model emits
|
|
a top-level JSON array (see ``test_json_parser.py``). The QA analyzers then call
|
|
``parsed.get(...)`` on the result. For a list that raises ``AttributeError``,
|
|
which is NOT caught by the surrounding ``except (json.JSONDecodeError, ValueError)``
|
|
— so a stray array response crashed the whole QA run instead of degrading to
|
|
empty results.
|
|
"""
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, Mock, patch
|
|
|
|
import pytest
|
|
|
|
from api.services.workflow.qa import analysis as qa_analysis
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_whole_call_qa_tolerates_array_llm_response():
|
|
"""A top-level JSON array from the QA LLM degrades to empty results."""
|
|
qa_data = SimpleNamespace(qa_system_prompt="Summarize: {transcript}")
|
|
workflow_run = SimpleNamespace(
|
|
logs={
|
|
"realtime_feedback_events": [
|
|
{"role": "user", "content": "hello"},
|
|
{"role": "assistant", "content": "hi there"},
|
|
]
|
|
},
|
|
usage_info={"call_duration_seconds": 12},
|
|
)
|
|
warning_mock = Mock()
|
|
|
|
with (
|
|
patch.object(
|
|
qa_analysis, "build_conversation_structure", return_value=[{"x": 1}]
|
|
),
|
|
patch.object(qa_analysis, "format_transcript", return_value="user: hello"),
|
|
patch.object(qa_analysis, "compute_call_metrics", return_value={}),
|
|
patch.object(
|
|
qa_analysis,
|
|
"resolve_llm_config",
|
|
new=AsyncMock(return_value=("openai", "gpt-4o", "sk-test", {})),
|
|
),
|
|
patch.object(
|
|
qa_analysis, "create_llm_service_from_provider", return_value=object()
|
|
),
|
|
patch.object(
|
|
qa_analysis,
|
|
"_run_llm_inference",
|
|
new=AsyncMock(return_value='["tag1", "tag2"]'),
|
|
),
|
|
patch.object(qa_analysis, "setup_langfuse_parent_context", return_value=None),
|
|
patch.object(qa_analysis, "add_qa_span_to_trace", return_value=None),
|
|
patch.object(qa_analysis.logger, "warning", warning_mock),
|
|
):
|
|
# Before the fix this raised AttributeError: 'list' object has no
|
|
# attribute 'get'.
|
|
result = await qa_analysis._run_whole_call_qa_analysis(
|
|
qa_data, workflow_run, workflow_run_id=99
|
|
)
|
|
|
|
node_result = result["node_results"]["whole_call"]
|
|
assert node_result["tags"] == []
|
|
assert node_result["summary"] == ""
|
|
assert node_result["score"] is None
|
|
warning_mock.assert_called_once()
|
|
warning_message = warning_mock.call_args.args[0]
|
|
assert "non-object JSON" in warning_message
|
|
assert "run 99" in warning_message
|
|
assert "list" in warning_message
|
|
assert "tag1" not in warning_message
|