feat: add workflow graph constraints fixtures

This commit is contained in:
Abhishek Kumar 2026-05-08 16:02:51 +05:30
parent 6d93be3ef6
commit 5a358d4d29
38 changed files with 447 additions and 49 deletions

View file

@ -27,7 +27,7 @@ from api.services.workflow.dto import (
StartCallRFNode,
VariableType,
)
from api.services.workflow.workflow import WorkflowGraph
from api.services.workflow.workflow_graph import WorkflowGraph
START_CALL_SYSTEM_PROMPT = "Start Call System Prompt"
AGENT_SYSTEM_PROMPT = "Agent Node System Prompt"

View file

@ -0,0 +1,24 @@
{
"nodes": [
{
"id": "s",
"type": "startCall",
"position": {"x": 0, "y": 0},
"data": {"name": "Start", "prompt": "Greet the caller."}
},
{
"id": "a",
"type": "agentNode",
"position": {"x": 200, "y": 0},
"data": {"name": "Agent", "prompt": "Continue the conversation."}
}
],
"edges": [
{
"id": "a-s",
"source": "a",
"target": "s",
"data": {"label": "loop back to start", "condition": "always"}
}
]
}

View file

@ -0,0 +1,30 @@
{
"nodes": [
{
"id": "s",
"type": "startCall",
"position": {"x": -200, "y": 0},
"data": {"name": "Start", "prompt": "Greet the caller."}
},
{
"id": "a",
"type": "agentNode",
"position": {"x": 0, "y": 0},
"data": {"name": "Agent", "prompt": "Talk to the caller."}
},
{
"id": "w",
"type": "webhook",
"position": {"x": 200, "y": 0},
"data": {"name": "Notify CRM"}
}
],
"edges": [
{
"id": "a-w",
"source": "a",
"target": "w",
"data": {"label": "post to webhook", "condition": "always"}
}
]
}

View file

@ -0,0 +1,30 @@
{
"nodes": [
{
"id": "s",
"type": "startCall",
"position": {"x": -200, "y": 0},
"data": {"name": "Start", "prompt": "Greet the caller."}
},
{
"id": "g",
"type": "globalNode",
"position": {"x": 0, "y": 0},
"data": {"name": "Global", "prompt": "Shared system prompt."}
},
{
"id": "a",
"type": "agentNode",
"position": {"x": 200, "y": 0},
"data": {"name": "Agent", "prompt": "Talk to the caller."}
}
],
"edges": [
{
"id": "g-a",
"source": "g",
"target": "a",
"data": {"label": "global to agent", "condition": "always"}
}
]
}

View file

@ -0,0 +1,30 @@
{
"nodes": [
{
"id": "s",
"type": "startCall",
"position": {"x": -200, "y": 0},
"data": {"name": "Start", "prompt": "Greet the caller."}
},
{
"id": "w",
"type": "webhook",
"position": {"x": 0, "y": 0},
"data": {"name": "Notify CRM"}
},
{
"id": "e",
"type": "endCall",
"position": {"x": 200, "y": 0},
"data": {"name": "End", "prompt": "Say goodbye."}
}
],
"edges": [
{
"id": "w-e",
"source": "w",
"target": "e",
"data": {"label": "after webhook", "condition": "always"}
}
]
}

View file

@ -0,0 +1,24 @@
{
"nodes": [
{
"id": "s",
"type": "startCall",
"position": {"x": -200, "y": 0},
"data": {"name": "Start", "prompt": "Greet the caller."}
},
{
"id": "e",
"type": "endCall",
"position": {"x": 0, "y": 0},
"data": {"name": "End", "prompt": "Say goodbye."}
}
],
"edges": [
{
"id": "ghost-e",
"source": "ghost-node",
"target": "e",
"data": {"label": "from nowhere", "condition": "always"}
}
]
}

View file

@ -0,0 +1,18 @@
{
"nodes": [
{
"id": "s",
"type": "startCall",
"position": {"x": 0, "y": 0},
"data": {"name": "Start", "prompt": "Greet the caller."}
}
],
"edges": [
{
"id": "s-ghost",
"source": "s",
"target": "ghost-node",
"data": {"label": "to nowhere", "condition": "always"}
}
]
}

View file

@ -0,0 +1,36 @@
{
"nodes": [
{
"id": "s",
"type": "startCall",
"position": {"x": 0, "y": 0},
"data": {"name": "Start", "prompt": "Greet the caller."}
},
{
"id": "a",
"type": "agentNode",
"position": {"x": 200, "y": 0},
"data": {"name": "Agent", "prompt": "Continue the conversation."}
},
{
"id": "e",
"type": "endCall",
"position": {"x": 400, "y": 0},
"data": {"name": "End", "prompt": "Say goodbye."}
}
],
"edges": [
{
"id": "s-a",
"source": "s",
"target": "a",
"data": {"label": "start to agent", "condition": "always"}
},
{
"id": "a-e",
"source": "a",
"target": "e",
"data": {"label": "agent to end", "condition": "always"}
}
]
}

View file

@ -0,0 +1,24 @@
{
"nodes": [
{
"id": "a",
"type": "agentNode",
"position": {"x": 0, "y": 0},
"data": {"name": "Agent", "prompt": "Talk to the caller."}
},
{
"id": "e",
"type": "endCall",
"position": {"x": 200, "y": 0},
"data": {"name": "End", "prompt": "Say goodbye."}
}
],
"edges": [
{
"id": "a-e",
"source": "a",
"target": "e",
"data": {"label": "agent to end", "condition": "always"}
}
]
}

View file

@ -0,0 +1,11 @@
{
"nodes": [
{
"id": "s",
"type": "startCall",
"position": {"x": 0, "y": 0},
"data": {"name": "Start", "prompt": "Greet the caller and hang up."}
}
],
"edges": []
}

View file

@ -4,14 +4,14 @@ import pytest
from api.services.workflow.dto import ReactFlowDTO, sanitize_workflow_definition
_FIXTURES_DIR = Path(__file__).parent / "definitions"
_FIXTURES_DIR = Path(__file__).parent / "dto_fixtures"
@pytest.mark.asyncio
async def test_dto():
# Path resolved relative to this test file so the test works regardless
# of the cwd pytest is invoked from.
with open(_FIXTURES_DIR / "rf-1.json", "r") as f:
with open(_FIXTURES_DIR / "sample_branching_workflow.json", "r") as f:
dto = ReactFlowDTO.model_validate_json(f.read())
assert dto is not None

View file

@ -31,7 +31,7 @@ from pipecat.tests.mock_transport import MockTransport
from pipecat.transports.base_transport import TransportParams
from api.services.workflow.pipecat_engine import PipecatEngine
from api.services.workflow.workflow import WorkflowGraph
from api.services.workflow.workflow_graph import WorkflowGraph
from api.tests.conftest import (
AGENT_SYSTEM_PROMPT,
END_CALL_SYSTEM_PROMPT,

View file

@ -57,7 +57,7 @@ from api.services.workflow.pipecat_engine_custom_tools import CustomToolManager
from api.services.workflow.pipecat_engine_variable_extractor import (
VariableExtractionManager,
)
from api.services.workflow.workflow import WorkflowGraph
from api.services.workflow.workflow_graph import WorkflowGraph
from api.tests.conftest import END_CALL_SYSTEM_PROMPT, START_CALL_SYSTEM_PROMPT
from pipecat.tests import MockLLMService, MockTTSService

View file

@ -49,7 +49,7 @@ from pipecat.turns.user_turn_strategies import UserTurnStrategies
from pipecat.utils.time import time_now_iso8601
from api.services.workflow.pipecat_engine import PipecatEngine
from api.services.workflow.workflow import WorkflowGraph
from api.services.workflow.workflow_graph import WorkflowGraph
from pipecat.tests import MockLLMService, MockTTSService

View file

@ -22,7 +22,7 @@ from pipecat.tests.mock_transport import MockTransport
from pipecat.transports.base_transport import TransportParams
from api.services.workflow.pipecat_engine import PipecatEngine
from api.services.workflow.workflow import WorkflowGraph
from api.services.workflow.workflow_graph import WorkflowGraph
from api.tests.conftest import END_CALL_SYSTEM_PROMPT
from pipecat.tests import MockLLMService, MockTTSService

View file

@ -35,7 +35,7 @@ from api.services.workflow.pipecat_engine import PipecatEngine
from api.services.workflow.pipecat_engine_variable_extractor import (
VariableExtractionManager,
)
from api.services.workflow.workflow import WorkflowGraph
from api.services.workflow.workflow_graph import WorkflowGraph
from pipecat.tests import MockLLMService, MockTTSService

View file

@ -32,7 +32,7 @@ from api.services.workflow.pipecat_engine import PipecatEngine
from api.services.workflow.pipecat_engine_variable_extractor import (
VariableExtractionManager,
)
from api.services.workflow.workflow import WorkflowGraph
from api.services.workflow.workflow_graph import WorkflowGraph
from pipecat.tests import MockLLMService, MockTTSService

View file

@ -43,7 +43,7 @@ from api.services.workflow.dto import (
)
from api.services.workflow.pipecat_engine import PipecatEngine
from api.services.workflow.pipecat_engine_custom_tools import CustomToolManager
from api.services.workflow.workflow import WorkflowGraph
from api.services.workflow.workflow_graph import WorkflowGraph
from pipecat.tests import MockLLMService, MockTTSService
# ─── Constants ──────────────────────────────────────────────────

View file

@ -54,7 +54,7 @@ from api.services.workflow.pipecat_engine import PipecatEngine
from api.services.workflow.pipecat_engine_variable_extractor import (
VariableExtractionManager,
)
from api.services.workflow.workflow import WorkflowGraph
from api.services.workflow.workflow_graph import WorkflowGraph
from pipecat.tests import MockLLMService, MockTTSService

View file

@ -44,7 +44,7 @@ from pipecat.turns.user_turn_strategies import UserTurnStrategies
from pipecat.utils.time import time_now_iso8601
from api.services.workflow.pipecat_engine import PipecatEngine
from api.services.workflow.workflow import WorkflowGraph
from api.services.workflow.workflow_graph import WorkflowGraph
from pipecat.tests import MockLLMService, MockTTSService

View file

@ -48,7 +48,7 @@ from api.services.workflow.pipecat_engine import PipecatEngine
from api.services.workflow.pipecat_engine_variable_extractor import (
VariableExtractionManager,
)
from api.services.workflow.workflow import WorkflowGraph
from api.services.workflow.workflow_graph import WorkflowGraph
from pipecat.tests import MockLLMService, MockTTSService

View file

@ -0,0 +1,117 @@
"""Regression tests for WorkflowGraph edge/graph constraints + the
admin audit script that mirrors them.
Each fixture in `dto_fixtures/` is either a clean workflow or a single
category of violation we found in production. We pin two layers:
1. WorkflowGraph semantic gate used by `/publish`, the SDK API, and
both MCP tools. Driven by `NodeSpec.graph_constraints`. If this
layer ever stops rejecting one of these fixtures, the production
write paths will quietly start accepting bad workflows again.
2. audit_definition (api.services.admin_utils.local_exec) read-only
sweep over persisted rows used to find legacy/imported breakage.
Pinned so refactors of the rule set don't silently change the
verdicts the migration relies on.
DTO-level shape validation is covered by `test_dto.py` and isn't
re-pinned here.
"""
import json
from pathlib import Path
import pytest
from api.services.admin_utils.local_exec import audit_definition
from api.services.workflow.dto import ReactFlowDTO
from api.services.workflow.workflow_graph import WorkflowGraph
_FIXTURES_DIR = Path(__file__).parent / "dto_fixtures"
def _load(name: str) -> tuple[str, dict]:
raw = (_FIXTURES_DIR / f"{name}.json").read_text()
return raw, json.loads(raw)
# (fixture_name, expected_audit_reasons, expected_graph_messages)
#
# expected_graph_messages semantics:
# None — DTO rejects upstream, WorkflowGraph is never reached.
# [] — WorkflowGraph accepts (clean fixture).
# [...] — WorkflowGraph rejects; each substring must appear in the
# emitted WorkflowError messages.
_SCENARIOS = [
("clean", [], []),
# A workflow with just a startCall and no edges is valid — startCall
# has no `min_outgoing` constraint, so a "greet then hang up" flow
# passes both audit and WorkflowGraph.
("start_only", [], []),
(
"bad_edge_into_start",
["target_max_incoming_0:startCall"],
["Start Call cannot have incoming edges"],
),
(
"bad_edge_into_webhook",
["target_max_incoming_0:webhook"],
["Webhook cannot have incoming edges"],
),
(
"bad_edge_out_of_webhook",
["source_max_outgoing_0:webhook"],
["Webhook cannot have outgoing edges"],
),
(
"bad_edge_out_of_globalnode",
["source_max_outgoing_0:globalNode"],
["Global Node cannot have outgoing edges"],
),
("bad_edge_target_missing", ["target_id_missing"], None),
("bad_edge_source_missing", ["source_id_missing"], None),
(
"no_start_node",
["no_start_node"],
["Workflow must have exactly one start node"],
),
]
@pytest.mark.parametrize(
"name,expected_reasons",
[(name, reasons) for name, reasons, _ in _SCENARIOS],
)
def test_audit_catches_violations(name, expected_reasons):
_, definition = _load(name)
violations = audit_definition(definition["nodes"], definition["edges"])
reasons = sorted(v["reason"] for v in violations)
assert reasons == sorted(expected_reasons)
@pytest.mark.parametrize(
"name,expected_graph_messages",
[
(name, messages)
for name, _, messages in _SCENARIOS
if messages is not None # skip fixtures DTO rejects upstream
],
)
def test_workflow_graph_rejects_violations(name, expected_graph_messages):
"""If WorkflowGraph accepts a definition, every save path that goes
'live' will accept it so this layer is the canonical regression
point for the rules in `NodeSpec.graph_constraints`."""
raw, _ = _load(name)
dto = ReactFlowDTO.model_validate_json(raw)
if not expected_graph_messages:
WorkflowGraph(dto)
return
with pytest.raises(ValueError) as exc_info:
WorkflowGraph(dto)
actual_messages = [w["message"] for w in exc_info.value.args[0]]
for expected in expected_graph_messages:
assert any(expected in m for m in actual_messages), (
f"Expected substring {expected!r} not found in graph errors: {actual_messages}"
)