mirror of
https://github.com/dograh-hq/dograh.git
synced 2026-07-25 12:01:04 +02:00
feat: wire provisional VAD configuration
This commit is contained in:
parent
5ea80506db
commit
01d4d07638
7 changed files with 351 additions and 133 deletions
|
|
@ -80,6 +80,7 @@ from pipecat.turns.user_mute import (
|
|||
)
|
||||
from pipecat.turns.user_start import (
|
||||
ExternalUserTurnStartStrategy,
|
||||
MinWordsUserTurnStartStrategy,
|
||||
ProvisionalVADUserTurnStartStrategy,
|
||||
)
|
||||
from pipecat.turns.user_start.vad_user_turn_start_strategy import (
|
||||
|
|
@ -99,6 +100,8 @@ ensure_tracing()
|
|||
|
||||
DEFAULT_USER_TURN_STOP_TIMEOUT = 5.0
|
||||
EXTERNAL_TURN_USER_STOP_TIMEOUT = 30.0
|
||||
DEFAULT_TURN_START_STRATEGY = "default"
|
||||
DEFAULT_TURN_START_MIN_WORDS = 3
|
||||
|
||||
|
||||
def _resolve_user_turn_stop_timeout(
|
||||
|
|
@ -111,6 +114,42 @@ def _resolve_user_turn_stop_timeout(
|
|||
return DEFAULT_USER_TURN_STOP_TIMEOUT
|
||||
|
||||
|
||||
def _resolve_turn_start_min_words(run_configs: dict) -> int:
|
||||
return max(
|
||||
1,
|
||||
int(run_configs.get("turn_start_min_words", DEFAULT_TURN_START_MIN_WORDS)),
|
||||
)
|
||||
|
||||
|
||||
def _create_non_realtime_user_turn_start_strategies(
|
||||
run_configs: dict, *, uses_external_turns: bool
|
||||
):
|
||||
"""Return user turn start strategies for non-realtime pipelines."""
|
||||
|
||||
turn_start_strategy = run_configs.get(
|
||||
"turn_start_strategy", DEFAULT_TURN_START_STRATEGY
|
||||
)
|
||||
|
||||
if turn_start_strategy == "min_words":
|
||||
return [
|
||||
MinWordsUserTurnStartStrategy(
|
||||
min_words=_resolve_turn_start_min_words(run_configs)
|
||||
)
|
||||
]
|
||||
|
||||
if turn_start_strategy == "provisional_vad":
|
||||
return [ProvisionalVADUserTurnStartStrategy()]
|
||||
|
||||
if uses_external_turns:
|
||||
# The STT emits its own turn boundaries and owns interruptions. Local
|
||||
# VAD is deliberately kept out of the default start strategies: it would
|
||||
# win the race on raw voice activity and start the turn before the STT
|
||||
# confirms a real turn.
|
||||
return [ExternalUserTurnStartStrategy(enable_interruptions=True)]
|
||||
|
||||
return [VADUserTurnStartStrategy()]
|
||||
|
||||
|
||||
def _create_realtime_user_turn_config(provider: str):
|
||||
"""Return user turn strategies and optional local VAD for realtime providers."""
|
||||
|
||||
|
|
@ -734,21 +773,38 @@ async def _run_pipeline_impl(
|
|||
# follows those external signals. Other models use configurable turn
|
||||
# detection.
|
||||
uses_external_turns = stt_uses_external_turns(user_config)
|
||||
user_turn_start_strategies = _create_non_realtime_user_turn_start_strategies(
|
||||
run_configs,
|
||||
uses_external_turns=uses_external_turns,
|
||||
)
|
||||
turn_start_strategy = run_configs.get(
|
||||
"turn_start_strategy", DEFAULT_TURN_START_STRATEGY
|
||||
)
|
||||
turn_start_strategy_names = [
|
||||
strategy.__class__.__name__ for strategy in user_turn_start_strategies
|
||||
]
|
||||
turn_start_min_words = (
|
||||
_resolve_turn_start_min_words(run_configs)
|
||||
if turn_start_strategy == "min_words"
|
||||
else None
|
||||
)
|
||||
logger.info(
|
||||
f"[run {workflow_run_id}] Non-realtime interrupt strategy "
|
||||
f"requested={turn_start_strategy} "
|
||||
f"effective={turn_start_strategy_names} "
|
||||
f"min_words={turn_start_min_words} "
|
||||
f"uses_external_turns={uses_external_turns}"
|
||||
)
|
||||
if uses_external_turns:
|
||||
user_turn_strategies = UserTurnStrategies(
|
||||
start=[
|
||||
VADUserTurnStartStrategy(),
|
||||
ExternalUserTurnStartStrategy(enable_interruptions=True),
|
||||
],
|
||||
start=user_turn_start_strategies,
|
||||
stop=[ExternalUserTurnStopStrategy()],
|
||||
)
|
||||
elif turn_stop_strategy == "turn_analyzer":
|
||||
# Smart Turn Analyzer: best for longer responses with natural pauses
|
||||
smart_turn_params = SmartTurnParams(stop_secs=smart_turn_stop_secs)
|
||||
user_turn_strategies = UserTurnStrategies(
|
||||
start=[
|
||||
ProvisionalVADUserTurnStartStrategy(),
|
||||
],
|
||||
start=user_turn_start_strategies,
|
||||
stop=[
|
||||
TurnAnalyzerUserTurnStopStrategy(
|
||||
turn_analyzer=LocalSmartTurnAnalyzerV3(params=smart_turn_params)
|
||||
|
|
@ -758,9 +814,7 @@ async def _run_pipeline_impl(
|
|||
else:
|
||||
# Transcription-based (default): best for short 1-2 word responses
|
||||
user_turn_strategies = UserTurnStrategies(
|
||||
start=[
|
||||
ProvisionalVADUserTurnStartStrategy(),
|
||||
],
|
||||
start=user_turn_start_strategies,
|
||||
stop=[SpeechTimeoutUserTurnStopStrategy()],
|
||||
)
|
||||
|
||||
|
|
@ -982,12 +1036,6 @@ async def _run_pipeline_impl(
|
|||
|
||||
register_audio_data_handler(audio_buffer, workflow_run_id, in_memory_audio_buffer)
|
||||
|
||||
# Mark this run active so a deploy can drain it before stopping this worker:
|
||||
# the orchestrator polls GET /api/v1/health/active-calls and waits for zero
|
||||
# before sending SIGTERM (uvicorn force-closes live call WebSockets on
|
||||
# SIGTERM, so the wait has to come first). See api/services/pipecat/
|
||||
# active_calls.py and scripts/rolling_update.sh.
|
||||
register_active_call(workflow_run_id)
|
||||
try:
|
||||
# Run the pipeline
|
||||
await run_pipeline_worker(task)
|
||||
|
|
@ -995,7 +1043,6 @@ async def _run_pipeline_impl(
|
|||
except asyncio.CancelledError:
|
||||
logger.warning("Received CancelledError in _run_pipeline")
|
||||
finally:
|
||||
unregister_active_call(workflow_run_id)
|
||||
# Close MCP sessions here, not in engine.cleanup(). The anyio cancel
|
||||
# scopes opened by MCPClient.start() in engine.initialize() are
|
||||
# task-affine; this finally runs in the same task as initialize(),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||
from pipecat.turns.user_start import (
|
||||
ExternalUserTurnStartStrategy,
|
||||
MinWordsUserTurnStartStrategy,
|
||||
ProvisionalVADUserTurnStartStrategy,
|
||||
)
|
||||
from pipecat.turns.user_start.vad_user_turn_start_strategy import (
|
||||
VADUserTurnStartStrategy,
|
||||
|
|
@ -12,8 +14,10 @@ from pipecat.turns.user_stop import (
|
|||
|
||||
from api.services.configuration.registry import ServiceProviders
|
||||
from api.services.pipecat.run_pipeline import (
|
||||
DEFAULT_TURN_START_MIN_WORDS,
|
||||
DEFAULT_USER_TURN_STOP_TIMEOUT,
|
||||
EXTERNAL_TURN_USER_STOP_TIMEOUT,
|
||||
_create_non_realtime_user_turn_start_strategies,
|
||||
_create_realtime_user_turn_config,
|
||||
_resolve_user_turn_stop_timeout,
|
||||
)
|
||||
|
|
@ -115,6 +119,70 @@ def test_unknown_realtime_providers_keep_local_vad():
|
|||
assert strategies.stop[0].wait_for_transcript is False
|
||||
|
||||
|
||||
def test_non_realtime_default_uses_external_start_for_external_turn_stt():
|
||||
strategies = _create_non_realtime_user_turn_start_strategies(
|
||||
{},
|
||||
uses_external_turns=True,
|
||||
)
|
||||
|
||||
assert len(strategies) == 1
|
||||
assert isinstance(strategies[0], ExternalUserTurnStartStrategy)
|
||||
assert strategies[0]._enable_interruptions is True
|
||||
|
||||
|
||||
def test_non_realtime_default_uses_vad_start_for_standard_stt():
|
||||
strategies = _create_non_realtime_user_turn_start_strategies(
|
||||
{},
|
||||
uses_external_turns=False,
|
||||
)
|
||||
|
||||
assert len(strategies) == 1
|
||||
assert isinstance(strategies[0], VADUserTurnStartStrategy)
|
||||
|
||||
|
||||
def test_non_realtime_can_use_min_words_start_strategy():
|
||||
strategies = _create_non_realtime_user_turn_start_strategies(
|
||||
{"turn_start_strategy": "min_words", "turn_start_min_words": 4},
|
||||
uses_external_turns=False,
|
||||
)
|
||||
|
||||
assert len(strategies) == 1
|
||||
assert isinstance(strategies[0], MinWordsUserTurnStartStrategy)
|
||||
assert strategies[0]._min_words == 4
|
||||
|
||||
|
||||
def test_non_realtime_explicit_min_words_overrides_external_turn_default():
|
||||
strategies = _create_non_realtime_user_turn_start_strategies(
|
||||
{"turn_start_strategy": "min_words", "turn_start_min_words": 4},
|
||||
uses_external_turns=True,
|
||||
)
|
||||
|
||||
assert len(strategies) == 1
|
||||
assert isinstance(strategies[0], MinWordsUserTurnStartStrategy)
|
||||
assert strategies[0]._min_words == 4
|
||||
|
||||
|
||||
def test_non_realtime_min_words_start_strategy_has_default_threshold():
|
||||
strategies = _create_non_realtime_user_turn_start_strategies(
|
||||
{"turn_start_strategy": "min_words"},
|
||||
uses_external_turns=False,
|
||||
)
|
||||
|
||||
assert len(strategies) == 1
|
||||
assert isinstance(strategies[0], MinWordsUserTurnStartStrategy)
|
||||
assert strategies[0]._min_words == DEFAULT_TURN_START_MIN_WORDS
|
||||
|
||||
|
||||
def test_non_realtime_can_use_provisional_vad_start_strategy():
|
||||
strategies = _create_non_realtime_user_turn_start_strategies(
|
||||
{"turn_start_strategy": "provisional_vad"},
|
||||
uses_external_turns=False,
|
||||
)
|
||||
|
||||
assert len(strategies) == 1
|
||||
assert isinstance(strategies[0], ProvisionalVADUserTurnStartStrategy)
|
||||
|
||||
|
||||
def test_external_turn_stt_uses_longer_stop_timeout():
|
||||
assert (
|
||||
_resolve_user_turn_stop_timeout({}, uses_external_turns=True)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue