apunkt/dograh
1
0
Fork 0
mirror of https://github.com/dograh-hq/dograh.git synced 2026-06-07 07:55:16 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat: add dictionary support for STT boosting in voice agents (#136)

Browse source 
* feat: add dictionary support for voice agents

Also fixes #132

* chore: add keyterms in evals
This commit is contained in:
Abhishek 2026-01-29 11:20:07 +05:30 • committed by GitHub
parent e3a1e0bf07
commit db75d90535
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 9666 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

16
api/services/looptalk/core/pipeline_builder.py
View file

@ -76,8 +76,22 @@ class LoopTalkPipelineBuilder:
pipeline_sample_rate=16000,
)
# Extract keyterms from workflow configurations
keyterms = None
if (
workflow.workflow_configurations
and "dictionary" in workflow.workflow_configurations
):
dictionary = workflow.workflow_configurations["dictionary"]
if dictionary and isinstance(dictionary, str):
keyterms = [
term.strip() for term in dictionary.split(",") if term.strip()
]
if keyterms:
logger.info(f"Using {len(keyterms)} keyterms for STT: {keyterms}")
# Create services
stt = create_stt_service(user_config)
stt = create_stt_service(user_config, keyterms=keyterms)
llm = create_llm_service(user_config)
tts = create_tts_service(user_config, audio_config)

22
api/services/pipecat/run_pipeline.py
View file

@ -443,12 +443,7 @@ async def _run_pipeline(
# Get user configuration
user_config = await db_client.get_user_configurations(user_id)
# Create services based on user configuration
stt = create_stt_service(user_config)
tts = create_tts_service(user_config, audio_config)
llm = create_llm_service(user_config)
# Get workflow first so we can create engine before pipeline components
# Get workflow first so we can extract configurations before creating services
workflow = await db_client.get_workflow(workflow_id, user_id)
if not workflow:
raise HTTPException(status_code=404, detail="Workflow not found")
@ -456,6 +451,7 @@ async def _run_pipeline(
# Extract configurations from workflow configurations
max_call_duration_seconds = 300 # Default 5 minutes
max_user_idle_timeout = 10.0 # Default 10 seconds
keyterms = None # Dictionary words for STT boosting
if workflow.workflow_configurations:
# Use workflow-specific max call duration if provided
@ -470,6 +466,20 @@ async def _run_pipeline(
"max_user_idle_timeout"
]
# Extract dictionary words and convert to keyterms list
if "dictionary" in workflow.workflow_configurations:
dictionary = workflow.workflow_configurations["dictionary"]
if dictionary and isinstance(dictionary, str):
# Split by comma and strip whitespace from each term
keyterms = [
term.strip() for term in dictionary.split(",") if term.strip()
]
# Create services based on user configuration
stt = create_stt_service(user_config, keyterms=keyterms)
tts = create_tts_service(user_config, audio_config)
llm = create_llm_service(user_config)
workflow_graph = WorkflowGraph(
ReactFlowDTO.model_validate(workflow.workflow_definition_with_fallback)
)

22
api/services/pipecat/service_factory.py
View file

@ -29,8 +29,13 @@ if TYPE_CHECKING:
from api.services.pipecat.audio_config import AudioConfig
def create_stt_service(user_config):
"""Create and return appropriate STT service based on user configuration"""
def create_stt_service(user_config, keyterms: list[str] | None = None):
"""Create and return appropriate STT service based on user configuration
Args:
user_config: User configuration containing STT settings
keyterms: Optional list of keyterms for speech recognition boosting (Deepgram only)
"""
logger.info(
f"Creating STT service: provider={user_config.stt.provider}, model={user_config.stt.model}"
)
@ -44,6 +49,7 @@ def create_stt_service(user_config):
params=DeepgramFluxSTTService.InputParams(
eot_timeout_ms=3000,
eot_threshold=0.7,
keyterm=keyterms or [],
),
should_interrupt=False, # Let UserAggregator take care of sending InterruptionFrame
)
@ -56,6 +62,7 @@ def create_stt_service(user_config):
profanity_filter=False,
endpointing=100,
model=user_config.stt.model,
keyterm=keyterms or [],
)
logger.debug(f"Using DeepGram Model - {user_config.stt.model}")
return DeepgramSTTService(
@ -77,6 +84,7 @@ def create_stt_service(user_config):
api_key=user_config.stt.api_key,
model=user_config.stt.model,
language=language,
keyterms=keyterms,
)
elif user_config.stt.provider == ServiceProviders.SARVAM.value:
# Map Sarvam language code to pipecat Language enum
@ -102,7 +110,10 @@ def create_stt_service(user_config):
params=SarvamSTTService.InputParams(language=pipecat_language),
)
elif user_config.stt.provider == ServiceProviders.SPEECHMATICS.value:
from pipecat.services.speechmatics.stt import OperatingPoint
from pipecat.services.speechmatics.stt import (
AdditionalVocabEntry,
OperatingPoint,
)
language = getattr(user_config.stt, "language", None) or "en"
# Map model field to operating point (standard or enhanced)
@ -111,11 +122,16 @@ def create_stt_service(user_config):
if user_config.stt.model == "enhanced"
else OperatingPoint.STANDARD
)
# Convert keyterms to AdditionalVocabEntry objects for Speechmatics
additional_vocab = []
if keyterms:
additional_vocab = [AdditionalVocabEntry(content=term) for term in keyterms]
return SpeechmaticsSTTService(
api_key=user_config.stt.api_key,
params=SpeechmaticsSTTService.InputParams(
language=language,
operating_point=operating_point,
additional_vocab=additional_vocab,
),
)
else:

BIN
evals/stt/audio/billing_department.m4a Normal file
View file

Binary file not shown.

45
evals/stt/event_capture.py
View file

@ -54,9 +54,10 @@ class EventCaptureResult:
created_at: str
events: list[CapturedEvent] = field(default_factory=list)
transcript: str = "" # Final transcript for reference
keyterms: list[str] = field(default_factory=list) # Keyterms used for recognition
def to_dict(self) -> dict[str, Any]:
return {
result = {
"audio_file": self.audio_file,
"audio_path": self.audio_path,
"provider": self.provider,
@ -65,6 +66,9 @@ class EventCaptureResult:
"events": [e.to_dict() for e in self.events],
"transcript": self.transcript,
}
if self.keyterms:
result["keyterms"] = self.keyterms
return result
EventCallback = Callable[[str, dict[str, Any]], None]
@ -86,6 +90,7 @@ async def capture_events(
provider: STTProvider,
audio_path: Path,
sample_rate: int = 8000,
keyterms: list[str] | None = None,
**kwargs: Any,
) -> EventCaptureResult:
"""Capture WebSocket events from a provider.
@ -94,6 +99,7 @@ async def capture_events(
provider: The STT provider to use
audio_path: Path to the audio file
sample_rate: Audio sample rate
keyterms: Optional list of keyterms to boost recognition
**kwargs: Additional provider parameters
Returns:
@ -120,6 +126,7 @@ async def capture_events(
result = await provider.transcribe(
audio_path,
sample_rate=sample_rate,
keyterms=keyterms,
on_event=on_event,
**kwargs,
)
@ -132,9 +139,26 @@ async def capture_events(
created_at=datetime.now().isoformat(),
events=events,
transcript=result.transcript,
keyterms=keyterms or [],
)
def _hash_keyterms(keyterms: list[str]) -> str:
"""Generate a short hash of keyterms for unique filenames.
Args:
keyterms: List of keyterms to hash
Returns:
8-character hash string
"""
import hashlib
# Sort keyterms for consistent hashing regardless of order
sorted_terms = sorted(keyterms)
content = ",".join(sorted_terms)
return hashlib.sha256(content.encode()).hexdigest()[:8]
def save_result(result: EventCaptureResult, output_dir: Path) -> Path:
"""Save capture result to JSON file.
@ -147,9 +171,10 @@ def save_result(result: EventCaptureResult, output_dir: Path) -> Path:
"""
output_dir.mkdir(parents=True, exist_ok=True)
# Format: {audio_name}-{provider}.json
# Format: {audio_name}-{provider}.json or {audio_name}-{provider}-kt-{hash}.json
audio_name = Path(result.audio_file).stem
output_file = output_dir / f"{audio_name}-{result.provider}.json"
suffix = f"-kt-{_hash_keyterms(result.keyterms)}" if result.keyterms else ""
output_file = output_dir / f"{audio_name}-{result.provider}{suffix}.json"
with open(output_file, "w") as f:
json.dump(result.to_dict(), f, indent=2)
@ -189,6 +214,12 @@ Examples:
action="store_true",
help="Enable speaker diarization",
)
parser.add_argument(
"--keyterms",
type=str,
default=None,
help="Comma-separated list of keyterms to boost recognition (e.g., 'technical support,escalation')",
)
parser.add_argument(
"--output-dir",
type=str,
@ -208,10 +239,17 @@ Examples:
print(f"Error: Audio file not found: {audio_path}")
return 1
# Parse keyterms from comma-separated string
keyterms = None
if args.keyterms:
keyterms = [term.strip() for term in args.keyterms.split(",") if term.strip()]
print(f"Audio file: {audio_path}")
print(f"Provider: {args.provider}")
print(f"Sample rate: {args.sample_rate} Hz")
print(f"Diarization: {args.diarize}")
if keyterms:
print(f"Keyterms: {keyterms}")
try:
provider = get_provider(args.provider)
@ -222,6 +260,7 @@ Examples:
audio_path,
sample_rate=args.sample_rate,
diarize=args.diarize,
keyterms=keyterms,
)
output_dir = script_dir / args.output_dir

411
evals/stt/results/billing_department-deepgram-kt-36e56883.json Normal file
View file

@ -0,0 +1,411 @@
{
"audio_file": "billing_department.m4a",
"audio_path": "../audio/billing_department.m4a",
"provider": "deepgram",
"duration": 3.366893,
"created_at": "2026-01-28T14:44:20.298166",
"events": [
{
"timestamp": 6.249174475669861e-07,
"event_type": "SpeechStarted",
"data": {
"type": "SpeechStarted",
"channel": [
0,
1
],
"timestamp": 0.69
}
},
{
"timestamp": 0.35415166709572077,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 1.0399375,
"start": 0.0,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "50820bed-7f00-4adc-8ce2-d119720030e4",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 1.4038341669365764,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 2.0799375,
"start": 0.0,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "Billing department",
"confidence": 1.0,
"words": [
{
"word": "billing",
"start": 0.71999997,
"end": 1.12,
"confidence": 0.5859375,
"punctuated_word": "Billing"
},
{
"word": "department",
"start": 1.12,
"end": 1.76,
"confidence": 1.0,
"punctuated_word": "department"
}
]
}
]
},
"metadata": {
"request_id": "50820bed-7f00-4adc-8ce2-d119720030e4",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 1.6528216670267284,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 2.29,
"start": 0.0,
"is_final": true,
"speech_final": true,
"channel": {
"alternatives": [
{
"transcript": "Billing department.",
"confidence": 0.92944336,
"words": [
{
"word": "billing",
"start": 0.71999997,
"end": 1.12,
"confidence": 0.49365234,
"punctuated_word": "Billing"
},
{
"word": "department",
"start": 1.12,
"end": 1.76,
"confidence": 0.92944336,
"punctuated_word": "department."
}
]
}
]
},
"metadata": {
"request_id": "50820bed-7f00-4adc-8ce2-d119720030e4",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 1.873247167095542,
"event_type": "SpeechStarted",
"data": {
"type": "SpeechStarted",
"channel": [
0,
1
],
"timestamp": 2.48
}
},
{
"timestamp": 2.0547706249635667,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 0.3800001,
"start": 2.29,
"is_final": true,
"speech_final": true,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "50820bed-7f00-4adc-8ce2-d119720030e4",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 3.1265713339671493,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 1.0168123,
"start": 2.67,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "50820bed-7f00-4adc-8ce2-d119720030e4",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 3.126730459043756,
"event_type": "UtteranceEnd",
"data": {
"type": "UtteranceEnd",
"channel": [
0,
1
],
"last_word_end": 1.76
}
},
{
"timestamp": 4.1764094170648605,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 2.0568123,
"start": 2.67,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "50820bed-7f00-4adc-8ce2-d119720030e4",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 5.243699250044301,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 3.0968122,
"start": 2.67,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "50820bed-7f00-4adc-8ce2-d119720030e4",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 5.883386292029172,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 3.6568122,
"start": 2.67,
"is_final": true,
"speech_final": true,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "50820bed-7f00-4adc-8ce2-d119720030e4",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 5.885270999977365,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 0.0,
"start": 6.3268123,
"is_final": true,
"speech_final": true,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "50820bed-7f00-4adc-8ce2-d119720030e4",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 5.885405584005639,
"event_type": "Metadata",
"data": {
"type": "Metadata",
"transaction_key": "deprecated",
"request_id": "50820bed-7f00-4adc-8ce2-d119720030e4",
"sha256": "cfb86929b56f55283684f9b1d1edd3fcbd991d7a0facb35503125692e7ae171f",
"created": "2026-01-28T09:14:13.297Z",
"duration": 6.3268123,
"channels": 1,
"models": [
"40bd3654-e622-47c4-a111-63a61b23bfe8"
],
"model_info": {
"40bd3654-e622-47c4-a111-63a61b23bfe8": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
}
}
}
}
],
"transcript": "Billing department.",
"keyterms": [
"billing department",
"bacon cheeseburger",
"escalation",
"technical support"
]
}

408
evals/stt/results/billing_department-deepgram-kt-da3adde5.json Normal file
View file

@ -0,0 +1,408 @@
{
"audio_file": "billing_department.m4a",
"audio_path": "../audio/billing_department.m4a",
"provider": "deepgram",
"duration": 3.366893,
"created_at": "2026-01-28T14:42:55.015635",
"events": [
{
"timestamp": 1.7499551177024841e-06,
"event_type": "SpeechStarted",
"data": {
"type": "SpeechStarted",
"channel": [
0,
1
],
"timestamp": 0.69
}
},
{
"timestamp": 0.341313665965572,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 1.0399375,
"start": 0.0,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "5d7ff50e-aad9-4ef4-b08a-a8446365baf7",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 1.4372159577906132,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 2.0799375,
"start": 0.0,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "Billing department",
"confidence": 1.0,
"words": [
{
"word": "billing",
"start": 0.71999997,
"end": 1.12,
"confidence": 0.41088867,
"punctuated_word": "Billing"
},
{
"word": "department",
"start": 1.12,
"end": 1.76,
"confidence": 1.0,
"punctuated_word": "department"
}
]
}
]
},
"metadata": {
"request_id": "5d7ff50e-aad9-4ef4-b08a-a8446365baf7",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 1.6498620409984142,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 2.29,
"start": 0.0,
"is_final": true,
"speech_final": true,
"channel": {
"alternatives": [
{
"transcript": "Billing department.",
"confidence": 0.92700195,
"words": [
{
"word": "billing",
"start": 0.71999997,
"end": 1.12,
"confidence": 0.39135742,
"punctuated_word": "Billing"
},
{
"word": "department",
"start": 1.12,
"end": 1.76,
"confidence": 0.92700195,
"punctuated_word": "department."
}
]
}
]
},
"metadata": {
"request_id": "5d7ff50e-aad9-4ef4-b08a-a8446365baf7",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 1.867587290937081,
"event_type": "SpeechStarted",
"data": {
"type": "SpeechStarted",
"channel": [
0,
1
],
"timestamp": 2.48
}
},
{
"timestamp": 2.0528456249739975,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 0.3800001,
"start": 2.29,
"is_final": true,
"speech_final": true,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "5d7ff50e-aad9-4ef4-b08a-a8446365baf7",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 3.1088090408593416,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 1.0168123,
"start": 2.67,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "5d7ff50e-aad9-4ef4-b08a-a8446365baf7",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 3.108877374790609,
"event_type": "UtteranceEnd",
"data": {
"type": "UtteranceEnd",
"channel": [
0,
1
],
"last_word_end": 1.76
}
},
{
"timestamp": 4.168927124934271,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 2.0568123,
"start": 2.67,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "5d7ff50e-aad9-4ef4-b08a-a8446365baf7",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 5.246617290889844,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 3.0968122,
"start": 2.67,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "5d7ff50e-aad9-4ef4-b08a-a8446365baf7",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 5.879137415904552,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 3.6568122,
"start": 2.67,
"is_final": true,
"speech_final": true,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "5d7ff50e-aad9-4ef4-b08a-a8446365baf7",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 5.923465499887243,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 0.0,
"start": 6.3268123,
"is_final": true,
"speech_final": true,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "5d7ff50e-aad9-4ef4-b08a-a8446365baf7",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 5.923510582884774,
"event_type": "Metadata",
"data": {
"type": "Metadata",
"transaction_key": "deprecated",
"request_id": "5d7ff50e-aad9-4ef4-b08a-a8446365baf7",
"sha256": "cfb86929b56f55283684f9b1d1edd3fcbd991d7a0facb35503125692e7ae171f",
"created": "2026-01-28T09:12:48.017Z",
"duration": 6.3268123,
"channels": 1,
"models": [
"40bd3654-e622-47c4-a111-63a61b23bfe8"
],
"model_info": {
"40bd3654-e622-47c4-a111-63a61b23bfe8": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
}
}
}
}
],
"transcript": "Billing department.",
"keyterms": [
"billing department"
]
}

405
evals/stt/results/billing_department-deepgram.json Normal file
View file

@ -0,0 +1,405 @@
{
"audio_file": "billing_department.m4a",
"audio_path": "../audio/billing_department.m4a",
"provider": "deepgram",
"duration": 3.366893,
"created_at": "2026-01-28T14:43:20.186354",
"events": [
{
"timestamp": 7.078051567077637e-07,
"event_type": "SpeechStarted",
"data": {
"type": "SpeechStarted",
"channel": [
0,
1
],
"timestamp": 0.69
}
},
{
"timestamp": 0.33802783279679716,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 1.0399375,
"start": 0.0,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "2a39d199-c0e3-4d55-8a3c-0778957656fa",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 1.4018059158697724,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 2.0799375,
"start": 0.0,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "Building department.",
"confidence": 0.9604492,
"words": [
{
"word": "building",
"start": 0.71999997,
"end": 1.12,
"confidence": 0.9604492,
"punctuated_word": "Building"
},
{
"word": "department",
"start": 1.12,
"end": 1.76,
"confidence": 0.90625,
"punctuated_word": "department."
}
]
}
]
},
"metadata": {
"request_id": "2a39d199-c0e3-4d55-8a3c-0778957656fa",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 1.648887624964118,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 2.29,
"start": 0.0,
"is_final": true,
"speech_final": true,
"channel": {
"alternatives": [
{
"transcript": "Building department.",
"confidence": 0.9633789,
"words": [
{
"word": "building",
"start": 0.71999997,
"end": 1.12,
"confidence": 0.9633789,
"punctuated_word": "Building"
},
{
"word": "department",
"start": 1.12,
"end": 1.76,
"confidence": 0.93066406,
"punctuated_word": "department."
}
]
}
]
},
"metadata": {
"request_id": "2a39d199-c0e3-4d55-8a3c-0778957656fa",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 1.8730058749206364,
"event_type": "SpeechStarted",
"data": {
"type": "SpeechStarted",
"channel": [
0,
1
],
"timestamp": 2.48
}
},
{
"timestamp": 2.049614582909271,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 0.3800001,
"start": 2.29,
"is_final": true,
"speech_final": true,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "2a39d199-c0e3-4d55-8a3c-0778957656fa",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 3.109506082953885,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 1.0168123,
"start": 2.67,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "2a39d199-c0e3-4d55-8a3c-0778957656fa",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 3.109549832995981,
"event_type": "UtteranceEnd",
"data": {
"type": "UtteranceEnd",
"channel": [
0,
1
],
"last_word_end": 1.76
}
},
{
"timestamp": 4.166177165927365,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 2.0568123,
"start": 2.67,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "2a39d199-c0e3-4d55-8a3c-0778957656fa",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 5.222685665823519,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 3.0968122,
"start": 2.67,
"is_final": false,
"speech_final": false,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "2a39d199-c0e3-4d55-8a3c-0778957656fa",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 5.8756575000006706,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 3.6568122,
"start": 2.67,
"is_final": true,
"speech_final": true,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "2a39d199-c0e3-4d55-8a3c-0778957656fa",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 5.878054332919419,
"event_type": "Results",
"data": {
"type": "Results",
"channel_index": [
0,
1
],
"duration": 0.0,
"start": 6.3268123,
"is_final": true,
"speech_final": true,
"channel": {
"alternatives": [
{
"transcript": "",
"confidence": 0.0,
"words": []
}
]
},
"metadata": {
"request_id": "2a39d199-c0e3-4d55-8a3c-0778957656fa",
"model_info": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
},
"model_uuid": "40bd3654-e622-47c4-a111-63a61b23bfe8"
},
"from_finalize": false
}
},
{
"timestamp": 5.878150374861434,
"event_type": "Metadata",
"data": {
"type": "Metadata",
"transaction_key": "deprecated",
"request_id": "2a39d199-c0e3-4d55-8a3c-0778957656fa",
"sha256": "cfb86929b56f55283684f9b1d1edd3fcbd991d7a0facb35503125692e7ae171f",
"created": "2026-01-28T09:13:13.255Z",
"duration": 6.3268123,
"channels": 1,
"models": [
"40bd3654-e622-47c4-a111-63a61b23bfe8"
],
"model_info": {
"40bd3654-e622-47c4-a111-63a61b23bfe8": {
"name": "general-nova-3",
"version": "2025-04-17.21547",
"arch": "nova-3"
}
}
}
}
],
"transcript": "Building department."
}

769
evals/stt/results/billing_department-speechmatics-kt-36e56883.json Normal file
View file

@ -0,0 +1,769 @@
{
"audio_file": "billing_department.m4a",
"audio_path": "../audio/billing_department.m4a",
"provider": "speechmatics",
"duration": 3.366893,
"created_at": "2026-01-28T14:44:44.848713",
"events": [
{
"timestamp": 1.166015863418579e-06,
"event_type": "Info",
"data": {
"message": "Info",
"type": "concurrent_session_usage",
"reason": "1 concurrent sessions active out of quota 2",
"usage": 1,
"quota": 2,
"last_updated": "2026-01-28T09:14:35Z"
}
},
{
"timestamp": 2.2519701248966157,
"event_type": "RecognitionStarted",
"data": {
"message": "RecognitionStarted",
"orchestrator_version": "2026.01.15+504a3b4d7c+14.13.0",
"id": "c726bc2a-8545-4fbe-a8e2-1b3bf47bec2b",
"language_pack_info": {
"adapted": false,
"itn": true,
"language_description": "English",
"word_delimiter": " ",
"writing_direction": "left-to-right"
}
}
},
{
"timestamp": 2.2521586660295725,
"event_type": "Info",
"data": {
"message": "Info",
"type": "recognition_quality",
"reason": "Running recognition using a broadcast model quality.",
"quality": "broadcast"
}
},
{
"timestamp": 2.525465582963079,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 1
}
},
{
"timestamp": 2.6058010410051793,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 2
}
},
{
"timestamp": 2.692795458016917,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 3
}
},
{
"timestamp": 2.769717250019312,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 4
}
},
{
"timestamp": 2.8562146660406142,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 5
}
},
{
"timestamp": 2.9351985829416662,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 6
}
},
{
"timestamp": 3.0134935828391463,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 7
}
},
{
"timestamp": 3.0981823748443276,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 8
}
},
{
"timestamp": 3.178123499965295,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 9
}
},
{
"timestamp": 3.2633222909644246,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 10
}
},
{
"timestamp": 3.340563707984984,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 11
}
},
{
"timestamp": 3.422975125024095,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 12
}
},
{
"timestamp": 3.5047679159324616,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 13
}
},
{
"timestamp": 3.5847412499133497,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 14
}
},
{
"timestamp": 3.6674655829556286,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 15
}
},
{
"timestamp": 3.7482279578689486,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 16
}
},
{
"timestamp": 3.8288538749329746,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 17
}
},
{
"timestamp": 3.914150415919721,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 18
}
},
{
"timestamp": 3.995344790862873,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 19
}
},
{
"timestamp": 4.074907124973834,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 20
}
},
{
"timestamp": 4.155347666004673,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 21
}
},
{
"timestamp": 4.242748707998544,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 22
}
},
{
"timestamp": 4.320666583022103,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 23
}
},
{
"timestamp": 4.399169499985874,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 24
}
},
{
"timestamp": 4.479986791033298,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 25
}
},
{
"timestamp": 4.561514915898442,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 26
}
},
{
"timestamp": 4.643435915932059,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 27
}
},
{
"timestamp": 4.723911207867786,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 28
}
},
{
"timestamp": 4.805873750010505,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 29
}
},
{
"timestamp": 4.887105040950701,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 30
}
},
{
"timestamp": 4.967926082899794,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 31
}
},
{
"timestamp": 5.05002929084003,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 32
}
},
{
"timestamp": 5.135565208038315,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 33
}
},
{
"timestamp": 5.212782375048846,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 34
}
},
{
"timestamp": 5.293570416048169,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 35
}
},
{
"timestamp": 5.374145332956687,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 36
}
},
{
"timestamp": 5.4545241659507155,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 37
}
},
{
"timestamp": 5.5366969159804285,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 38
}
},
{
"timestamp": 5.617715457919985,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 39
}
},
{
"timestamp": 5.698430374963209,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 40
}
},
{
"timestamp": 5.781335124978796,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 41
}
},
{
"timestamp": 5.86323649995029,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 42
}
},
{
"timestamp": 5.945636000018567,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 43
}
},
{
"timestamp": 6.0369323750492185,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 44
}
},
{
"timestamp": 6.106611500028521,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 45
}
},
{
"timestamp": 6.188817082904279,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 46
}
},
{
"timestamp": 6.270534249953926,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 47
}
},
{
"timestamp": 6.352104166056961,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 48
}
},
{
"timestamp": 6.435103040887043,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 49
}
},
{
"timestamp": 6.513447541045025,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 50
}
},
{
"timestamp": 6.5960384998470545,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 51
}
},
{
"timestamp": 6.679259999888018,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 52
}
},
{
"timestamp": 6.758154207840562,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 53
}
},
{
"timestamp": 6.841611708048731,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 54
}
},
{
"timestamp": 6.921356082893908,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 55
}
},
{
"timestamp": 7.0067201249767095,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 56
}
},
{
"timestamp": 7.083883499959484,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 57
}
},
{
"timestamp": 7.165489041013643,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 58
}
},
{
"timestamp": 7.248983249999583,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 59
}
},
{
"timestamp": 7.328776124864817,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 60
}
},
{
"timestamp": 7.415970708010718,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 61
}
},
{
"timestamp": 7.493241749936715,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 62
}
},
{
"timestamp": 7.572302374988794,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 63
}
},
{
"timestamp": 7.654820582829416,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 64
}
},
{
"timestamp": 7.736952665960416,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 65
}
},
{
"timestamp": 7.816302041057497,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 66
}
},
{
"timestamp": 7.8992077498696744,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 67
}
},
{
"timestamp": 7.980802374891937,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 68
}
},
{
"timestamp": 8.061972290975973,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 69
}
},
{
"timestamp": 8.141521875048056,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 70
}
},
{
"timestamp": 8.225720207905397,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 71
}
},
{
"timestamp": 8.239358749939129,
"event_type": "AddTranscript",
"data": {
"message": "AddTranscript",
"format": "2.9",
"results": [
{
"alternatives": [
{
"confidence": 0.61,
"content": "Building",
"language": "en"
}
],
"end_time": 1.24,
"start_time": 0.84,
"type": "word"
}
],
"metadata": {
"end_time": 1.24,
"start_time": 0.0,
"transcript": "Building "
}
}
},
{
"timestamp": 8.335481374990195,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 72
}
},
{
"timestamp": 8.386238124920055,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 73
}
},
{
"timestamp": 8.467259540921077,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 74
}
},
{
"timestamp": 8.55006245803088,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 75
}
},
{
"timestamp": 8.629941874882206,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 76
}
},
{
"timestamp": 8.713717208011076,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 77
}
},
{
"timestamp": 8.794668999966234,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 78
}
},
{
"timestamp": 8.877457082970068,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 79
}
},
{
"timestamp": 8.959119957871735,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 80
}
},
{
"timestamp": 9.482231416041031,
"event_type": "AddTranscript",
"data": {
"message": "AddTranscript",
"format": "2.9",
"results": [
{
"alternatives": [
{
"confidence": 0.84,
"content": "department",
"language": "en"
}
],
"end_time": 2.2,
"start_time": 1.24,
"type": "word"
},
{
"alternatives": [
{
"confidence": 1.0,
"content": ".",
"language": "en"
}
],
"attaches_to": "previous",
"end_time": 2.2,
"is_eos": true,
"start_time": 2.2,
"type": "punctuation"
}
],
"metadata": {
"end_time": 6.24,
"start_time": 1.24,
"transcript": "department. "
}
}
},
{
"timestamp": 9.51494679087773,
"event_type": "EndOfTranscript",
"data": {
"message": "EndOfTranscript"
}
}
],
"transcript": "Building department.",
"keyterms": [
"billing department",
"bacon cheeseburger",
"escalation",
"technical support"
]
}

766
evals/stt/results/billing_department-speechmatics-kt-da3adde5.json Normal file
View file

@ -0,0 +1,766 @@
{
"audio_file": "billing_department.m4a",
"audio_path": "../audio/billing_department.m4a",
"provider": "speechmatics",
"duration": 3.366893,
"created_at": "2026-01-28T14:45:20.000459",
"events": [
{
"timestamp": 6.249174475669861e-07,
"event_type": "Info",
"data": {
"message": "Info",
"type": "concurrent_session_usage",
"reason": "1 concurrent sessions active out of quota 2",
"usage": 1,
"quota": 2,
"last_updated": "2026-01-28T09:15:10Z"
}
},
{
"timestamp": 2.20607762504369,
"event_type": "RecognitionStarted",
"data": {
"message": "RecognitionStarted",
"orchestrator_version": "2026.01.15+504a3b4d7c+14.13.0",
"id": "f44acd32-54bd-4e04-8415-a0ad3660544f",
"language_pack_info": {
"adapted": false,
"itn": true,
"language_description": "English",
"word_delimiter": " ",
"writing_direction": "left-to-right"
}
}
},
{
"timestamp": 2.2061435419600457,
"event_type": "Info",
"data": {
"message": "Info",
"type": "recognition_quality",
"reason": "Running recognition using a broadcast model quality.",
"quality": "broadcast"
}
},
{
"timestamp": 2.43935945793055,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 1
}
},
{
"timestamp": 2.520240667043254,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 2
}
},
{
"timestamp": 2.602808583062142,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 3
}
},
{
"timestamp": 2.685762207955122,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 4
}
},
{
"timestamp": 2.766228208085522,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 5
}
},
{
"timestamp": 2.8458993749227375,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 6
}
},
{
"timestamp": 2.92684100009501,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 7
}
},
{
"timestamp": 3.0081180420238525,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 8
}
},
{
"timestamp": 3.0901281249243766,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 9
}
},
{
"timestamp": 3.170878208009526,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 10
}
},
{
"timestamp": 3.2519385830964893,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 11
}
},
{
"timestamp": 3.332821666961536,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 12
}
},
{
"timestamp": 3.413922332925722,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 13
}
},
{
"timestamp": 3.495305582880974,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 14
}
},
{
"timestamp": 3.5763502079062164,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 15
}
},
{
"timestamp": 3.657859291881323,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 16
}
},
{
"timestamp": 3.7392752920277417,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 17
}
},
{
"timestamp": 3.82034262502566,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 18
}
},
{
"timestamp": 3.902143832994625,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 19
}
},
{
"timestamp": 3.983252708101645,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 20
}
},
{
"timestamp": 4.063989083049819,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 21
}
},
{
"timestamp": 4.1459952080622315,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 22
}
},
{
"timestamp": 4.23128241696395,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 23
}
},
{
"timestamp": 4.311947041889653,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 24
}
},
{
"timestamp": 4.420789791969582,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 25
}
},
{
"timestamp": 4.476196999894455,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 26
}
},
{
"timestamp": 4.5579968329984695,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 27
}
},
{
"timestamp": 4.639027666999027,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 28
}
},
{
"timestamp": 4.720825124997646,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 29
}
},
{
"timestamp": 4.801794874947518,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 30
}
},
{
"timestamp": 4.8831967080477625,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 31
}
},
{
"timestamp": 4.9653322079684585,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 32
}
},
{
"timestamp": 5.047957124887034,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 33
}
},
{
"timestamp": 5.128045292105526,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 34
}
},
{
"timestamp": 5.2095284999813884,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 35
}
},
{
"timestamp": 5.291791666997597,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 36
}
},
{
"timestamp": 5.37210754188709,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 37
}
},
{
"timestamp": 5.4541247501038015,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 38
}
},
{
"timestamp": 5.535319749964401,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 39
}
},
{
"timestamp": 5.61644074996002,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 40
}
},
{
"timestamp": 5.698145082918927,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 41
}
},
{
"timestamp": 5.778899458004162,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 42
}
},
{
"timestamp": 5.8616157080978155,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 43
}
},
{
"timestamp": 5.941820249892771,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 44
}
},
{
"timestamp": 6.024121375055984,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 45
}
},
{
"timestamp": 6.105051416903734,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 46
}
},
{
"timestamp": 6.1863586669787765,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 47
}
},
{
"timestamp": 6.268003958044574,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 48
}
},
{
"timestamp": 6.351039875065908,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 49
}
},
{
"timestamp": 6.4314415419939905,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 50
}
},
{
"timestamp": 6.511939582880586,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 51
}
},
{
"timestamp": 6.59375641704537,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 52
}
},
{
"timestamp": 6.676572208059952,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 53
}
},
{
"timestamp": 6.75589399994351,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 54
}
},
{
"timestamp": 6.838304250035435,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 55
}
},
{
"timestamp": 6.923452458111569,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 56
}
},
{
"timestamp": 7.003655791981146,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 57
}
},
{
"timestamp": 7.083533917088062,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 58
}
},
{
"timestamp": 7.164683833019808,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 59
}
},
{
"timestamp": 7.246581458020955,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 60
}
},
{
"timestamp": 7.327494208002463,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 61
}
},
{
"timestamp": 7.409501957939938,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 62
}
},
{
"timestamp": 7.491202791919932,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 63
}
},
{
"timestamp": 7.570988958002999,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 64
}
},
{
"timestamp": 7.65196495805867,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 65
}
},
{
"timestamp": 7.733250207966194,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 66
}
},
{
"timestamp": 7.815604374976829,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 67
}
},
{
"timestamp": 7.8962331670336425,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 68
}
},
{
"timestamp": 7.979087250074372,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 69
}
},
{
"timestamp": 8.060215374920517,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 70
}
},
{
"timestamp": 8.141423000022769,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 71
}
},
{
"timestamp": 8.178827208001167,
"event_type": "AddTranscript",
"data": {
"message": "AddTranscript",
"format": "2.9",
"results": [
{
"alternatives": [
{
"confidence": 0.66,
"content": "Building",
"language": "en"
}
],
"end_time": 1.24,
"start_time": 0.84,
"type": "word"
}
],
"metadata": {
"end_time": 1.24,
"start_time": 0.0,
"transcript": "Building "
}
}
},
{
"timestamp": 8.223303125007078,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 72
}
},
{
"timestamp": 8.303685167105868,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 73
}
},
{
"timestamp": 8.385801125084981,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 74
}
},
{
"timestamp": 8.467919332906604,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 75
}
},
{
"timestamp": 8.547684249933809,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 76
}
},
{
"timestamp": 8.63002754189074,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 77
}
},
{
"timestamp": 8.711125958012417,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 78
}
},
{
"timestamp": 8.791022874880582,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 79
}
},
{
"timestamp": 8.872025707969442,
"event_type": "AudioAdded",
"data": {
"message": "AudioAdded",
"seq_no": 80
}
},
{
"timestamp": 9.597293166909367,
"event_type": "AddTranscript",
"data": {
"message": "AddTranscript",
"format": "2.9",
"results": [
{
"alternatives": [
{
"confidence": 0.87,
"content": "department",
"language": "en"
}
],
"end_time": 2.2,
"start_time": 1.24,
"type": "word"
},
{
"alternatives": [
{
"confidence": 1.0,
"content": ".",
"language": "en"
}
],
"attaches_to": "previous",
"end_time": 2.2,
"is_eos": true,
"start_time": 2.2,
"type": "punctuation"
}
],
"metadata": {
"end_time": 6.24,
"start_time": 1.24,
"transcript": "department. "
}
}
},
{
"timestamp": 9.597344167064875,
"event_type": "EndOfTranscript",
"data": {
"message": "EndOfTranscript"
}
}
],
"transcript": "Building department.",
"keyterms": [
"billing department"
]
}

6549
evals/visualizer/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

1
evals/visualizer/src/app/api/results/route.ts
View file

@ -27,6 +27,7 @@ export async function GET() {
duration: data.duration,
created_at: data.created_at,
event_count: data.events.length,
keyterms: data.keyterms,
});
} catch {
console.error(`Failed to parse ${file}`);

56
evals/visualizer/src/app/page.tsx
View file

@ -93,31 +93,45 @@ export default function Home() {
href={`/view/${result.id}`}
className="block bg-zinc-900 hover:bg-zinc-800 rounded-lg p-4 transition-colors"
>
<div className="flex items-center justify-between">
<div className="space-y-1">
<div className="flex items-center gap-3">
<span className="font-medium">{result.audio_file}</span>
<span
className={`text-xs px-2 py-0.5 rounded ${
PROVIDER_COLORS[result.provider] ||
"bg-zinc-700 text-zinc-300"
}`}
>
{result.provider}
</span>
<div className="space-y-2">
<div className="flex items-center justify-between">
<div className="space-y-1">
<div className="flex items-center gap-3">
<span className="font-medium">{result.audio_file}</span>
<span
className={`text-xs px-2 py-0.5 rounded ${
PROVIDER_COLORS[result.provider] ||
"bg-zinc-700 text-zinc-300"
}`}
>
{result.provider}
</span>
</div>
<div className="text-sm text-zinc-500">
{formatDate(result.created_at)}
</div>
</div>
<div className="text-sm text-zinc-500">
{formatDate(result.created_at)}
<div className="text-right space-y-1">
<div className="text-sm text-zinc-400">
{formatDuration(result.duration)}
</div>
<div className="text-xs text-zinc-500">
{result.event_count} events
</div>
</div>
</div>
<div className="text-right space-y-1">
<div className="text-sm text-zinc-400">
{formatDuration(result.duration)}
{result.keyterms && result.keyterms.length > 0 && (
<div className="flex flex-wrap gap-1.5">
{result.keyterms.map((term, index) => (
<span
key={index}
className="text-xs px-2 py-0.5 rounded bg-amber-500/10 text-amber-300 border border-amber-500/20"
>
{term}
</span>
))}
</div>
<div className="text-xs text-zinc-500">
{result.event_count} events
</div>
</div>
)}
</div>
</Link>
))}

17
evals/visualizer/src/app/view/[id]/page.tsx
View file

@ -104,7 +104,24 @@ export default function ViewPage() {
>
{result.provider}
</span>
{result.keyterms && result.keyterms.length > 0 && (
<span className="text-sm px-2 py-0.5 rounded bg-amber-500/20 text-amber-300">
{result.keyterms.length} keyterm{result.keyterms.length !== 1 ? "s" : ""}
</span>
)}
</div>
{result.keyterms && result.keyterms.length > 0 && (
<div className="mt-2 flex flex-wrap gap-2">
{result.keyterms.map((term, index) => (
<span
key={index}
className="text-xs px-2 py-1 rounded bg-amber-500/10 text-amber-300 border border-amber-500/30"
>
{term}
</span>
))}
</div>
)}
{result.transcript && (
<p className="text-zinc-400 mt-2 text-sm line-clamp-2">
{result.transcript}

2
evals/visualizer/src/types/index.ts
View file

@ -12,6 +12,7 @@ export interface EventCaptureResult {
created_at: string;
events: CapturedEvent[];
transcript: string;
keyterms?: string[];
}
export interface ResultSummary {
@ -21,4 +22,5 @@ export interface ResultSummary {
duration: number;
created_at: string;
event_count: number;
keyterms?: string[];
}

2
pipecat

@ -1 +1 @@
Subproject commit df1432e168570661ae418500fb04e8c62ba1335b
Subproject commit c1e410a8f3b2f34f368252a2588a17fbe61f1d32

31
ui/src/app/workflow/[workflowId]/RenderWorkflow.tsx
View file

@ -6,7 +6,7 @@ import {
Panel,
ReactFlow,
} from "@xyflow/react";
import { BrushCleaning, Maximize2, Minus, Plus, Rocket, Settings, Variable } from 'lucide-react';
import { BookA, BrushCleaning, Maximize2, Minus, Plus, Rocket, Settings, Variable } from 'lucide-react';
import React, { useEffect, useMemo, useState } from 'react';
import { listDocumentsApiV1KnowledgeBaseDocumentsGet, listToolsApiV1ToolsGet } from '@/client';
@ -20,6 +20,7 @@ import AddNodePanel from "../../../components/flow/AddNodePanel";
import CustomEdge from "../../../components/flow/edges/CustomEdge";
import { AgentNode, EndCall, GlobalNode, StartCall, TriggerNode, WebhookNode } from "../../../components/flow/nodes";
import { ConfigurationsDialog } from './components/ConfigurationsDialog';
import { DictionaryDialog } from './components/DictionaryDialog';
import { EmbedDialog } from './components/EmbedDialog';
import { PhoneCallDialog } from './components/PhoneCallDialog';
import { TemplateContextVariablesDialog } from './components/TemplateContextVariablesDialog';
@ -63,6 +64,7 @@ interface RenderWorkflowProps {
function RenderWorkflow({ initialWorkflowName, workflowId, initialFlow, initialTemplateContextVariables, initialWorkflowConfigurations, user, getAccessToken }: RenderWorkflowProps) {
const [isContextVarsDialogOpen, setIsContextVarsDialogOpen] = useState(false);
const [isConfigurationsDialogOpen, setIsConfigurationsDialogOpen] = useState(false);
const [isDictionaryDialogOpen, setIsDictionaryDialogOpen] = useState(false);
const [isEmbedDialogOpen, setIsEmbedDialogOpen] = useState(false);
const [isPhoneCallDialogOpen, setIsPhoneCallDialogOpen] = useState(false);
const [documents, setDocuments] = useState<DocumentResponseSchema[] | undefined>(undefined);
@ -88,7 +90,9 @@ function RenderWorkflow({ initialWorkflowName, workflowId, initialFlow, initialT
onNodesChange,
onRun,
saveTemplateContextVariables,
saveWorkflowConfigurations
saveWorkflowConfigurations,
dictionary,
saveDictionary
} = useWorkflowState({
initialWorkflowName,
workflowId,
@ -238,6 +242,22 @@ function RenderWorkflow({ initialWorkflowName, workflowId, initialFlow, initialT
</TooltipContent>
</Tooltip>
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="outline"
size="icon"
onClick={() => setIsDictionaryDialogOpen(true)}
className="bg-white shadow-sm hover:shadow-md"
>
<BookA className="h-4 w-4" />
</Button>
</TooltipTrigger>
<TooltipContent side="left">
<p>Dictionary</p>
</TooltipContent>
</Tooltip>
<Tooltip>
<TooltipTrigger asChild>
<Button
@ -356,6 +376,13 @@ function RenderWorkflow({ initialWorkflowName, workflowId, initialFlow, initialT
onSave={saveTemplateContextVariables}
/>
<DictionaryDialog
open={isDictionaryDialogOpen}
onOpenChange={setIsDictionaryDialogOpen}
dictionary={dictionary}
onSave={saveDictionary}
/>
<EmbedDialog
open={isEmbedDialogOpen}
onOpenChange={setIsEmbedDialogOpen}

80
ui/src/app/workflow/[workflowId]/components/DictionaryDialog.tsx Normal file
View file

@ -0,0 +1,80 @@
import { useEffect, useState } from "react";
import { Button } from "@/components/ui/button";
import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
import { Label } from "@/components/ui/label";
import { Textarea } from "@/components/ui/textarea";
interface DictionaryDialogProps {
open: boolean;
onOpenChange: (open: boolean) => void;
dictionary: string;
onSave: (dictionary: string) => Promise<void>;
}
export const DictionaryDialog = ({
open,
onOpenChange,
dictionary,
onSave
}: DictionaryDialogProps) => {
const [dictionaryValue, setDictionaryValue] = useState(dictionary);
// Sync local state with prop when dialog opens
useEffect(() => {
if (open) {
setDictionaryValue(dictionary);
}
}, [open, dictionary]);
const handleSave = async () => {
await onSave(dictionaryValue);
onOpenChange(false);
};
const handleDialogOpenChange = (isOpen: boolean) => {
onOpenChange(isOpen);
if (isOpen) {
setDictionaryValue(dictionary);
}
};
return (
<Dialog open={open} onOpenChange={handleDialogOpenChange}>
<DialogContent className="max-w-lg">
<DialogHeader>
<DialogTitle>Dictionary</DialogTitle>
<DialogDescription>
Add any specific words that you would want the bot to actively listen for. The Voice Agent learns your
unique words and names. Add expected words and phrases, company jargon, named entities, or industry-specific lingo. <br/>
Example: billing department, tretinoin etc. <br/>
(May incur extra cost depending on provider)
</DialogDescription>
</DialogHeader>
<div className="space-y-4">
<div className="space-y-2">
<Label htmlFor="dictionary" className="text-sm font-medium">Words</Label>
<Textarea
id="dictionary"
placeholder="Enter words separated by comma"
value={dictionaryValue}
onChange={(e) => setDictionaryValue(e.target.value)}
rows={4}
className="resize-none"
/>
</div>
</div>
<DialogFooter>
<div className="flex items-center gap-2">
<Button variant="outline" onClick={() => onOpenChange(false)}>
Cancel
</Button>
<Button onClick={handleSave}>
Save Dictionary
</Button>
</div>
</DialogFooter>
</DialogContent>
</Dialog>
);
};

11
ui/src/app/workflow/[workflowId]/components/TemplateContextVariablesDialog.tsx
View file

@ -1,5 +1,5 @@
import { Trash2Icon } from "lucide-react";
import { useState } from "react";
import { useEffect, useState } from "react";
import { Button } from "@/components/ui/button";
import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
@ -23,6 +23,15 @@ export const TemplateContextVariablesDialog = ({
const [newKey, setNewKey] = useState("");
const [newValue, setNewValue] = useState("");
// Sync local state with prop when dialog opens
useEffect(() => {
if (open) {
setContextVars(templateContextVariables);
setNewKey("");
setNewValue("");
}
}, [open, templateContextVariables]);
const handleAddContextVar = () => {
if (newKey && newValue) {
setContextVars(prev => ({ ...prev, [newKey]: newValue }));

62
ui/src/app/workflow/[workflowId]/hooks/useWorkflowState.ts
View file

@ -20,7 +20,7 @@ import { WorkflowError } from "@/client/types.gen";
import { FlowEdge, FlowNode, NodeType } from "@/components/flow/types";
import logger from '@/lib/logger';
import { getNextNodeId, getRandomId } from "@/lib/utils";
import { WorkflowConfigurations } from "@/types/workflow-configurations";
import { DEFAULT_WORKFLOW_CONFIGURATIONS, WorkflowConfigurations } from "@/types/workflow-configurations";
export function getDefaultAllowInterrupt(type: string = NodeType.START_CALL): boolean {
switch (type) {
@ -141,6 +141,8 @@ export const useWorkflowState = ({
setWorkflowValidationErrors,
setTemplateContextVariables,
setWorkflowConfigurations,
setDictionary,
dictionary,
clearValidationErrors,
markNodeAsInvalid,
markEdgeAsInvalid,
@ -174,7 +176,8 @@ export const useWorkflowState = ({
initialNodes,
initialFlow?.edges ?? [],
initialTemplateContextVariables,
initialWorkflowConfigurations
initialWorkflowConfigurations,
initialWorkflowConfigurations?.dictionary ?? ''
);
}, []);
@ -223,6 +226,11 @@ export const useWorkflowState = ({
selected: true, // Mark the new node as selected
};
// Deselect all existing nodes before adding the new one
const currentNodes = rfInstance.current.getNodes();
const deselectedNodes = currentNodes.map(node => ({ ...node, selected: false }));
rfInstance.current.setNodes(deselectedNodes);
// Use addNodes from ReactFlow instance
rfInstance.current.addNodes([newNode]);
setIsAddNodePanelOpen(false);
@ -326,6 +334,19 @@ export const useWorkflowState = ({
await validateWorkflow();
}, [workflowId, workflowName, setIsDirty, user, getAccessToken, validateWorkflow]);
// Set up keyboard shortcut for save (Cmd/Ctrl + S)
useEffect(() => {
const handleKeyDown = (e: KeyboardEvent) => {
if ((e.metaKey || e.ctrlKey) && e.key === 's') {
e.preventDefault();
saveWorkflow();
}
};
window.addEventListener('keydown', handleKeyDown);
return () => window.removeEventListener('keydown', handleKeyDown);
}, [saveWorkflow]);
const onConnect: OnConnect = useCallback((connection) => {
if (!rfInstance.current) return;
@ -411,6 +432,9 @@ export const useWorkflowState = ({
const saveWorkflowConfigurations = useCallback(async (configurations: WorkflowConfigurations, newWorkflowName: string) => {
if (!user) return;
const accessToken = await getAccessToken();
// Preserve the current dictionary when saving other configurations
const currentDictionary = useWorkflowStore.getState().dictionary;
const configurationsWithDictionary: WorkflowConfigurations = { ...configurations, dictionary: currentDictionary };
try {
await updateWorkflowApiV1WorkflowWorkflowIdPut({
path: {
@ -419,13 +443,13 @@ export const useWorkflowState = ({
body: {
name: newWorkflowName,
workflow_definition: null,
workflow_configurations: configurations as Record<string, unknown>,
workflow_configurations: configurationsWithDictionary as Record<string, unknown>,
},
headers: {
'Authorization': `Bearer ${accessToken}`,
},
});
setWorkflowConfigurations(configurations);
setWorkflowConfigurations(configurationsWithDictionary);
setWorkflowName(newWorkflowName);
logger.info('Workflow configurations saved successfully');
} catch (error) {
@ -434,6 +458,34 @@ export const useWorkflowState = ({
}
}, [workflowId, user, getAccessToken, setWorkflowConfigurations, setWorkflowName]);
// Save dictionary
const saveDictionary = useCallback(async (newDictionary: string) => {
if (!user) return;
const accessToken = await getAccessToken();
const currentConfigurations = useWorkflowStore.getState().workflowConfigurations ?? DEFAULT_WORKFLOW_CONFIGURATIONS;
const updatedConfigurations: WorkflowConfigurations = { ...currentConfigurations, dictionary: newDictionary };
try {
await updateWorkflowApiV1WorkflowWorkflowIdPut({
path: {
workflow_id: workflowId,
},
body: {
name: workflowName,
workflow_definition: null,
workflow_configurations: updatedConfigurations as Record<string, unknown>,
},
headers: {
'Authorization': `Bearer ${accessToken}`,
},
});
setDictionary(newDictionary);
setWorkflowConfigurations(updatedConfigurations);
} catch (error) {
logger.error(`Error saving dictionary: ${error}`);
throw error;
}
}, [workflowId, workflowName, user, getAccessToken, setDictionary, setWorkflowConfigurations]);
// Update rfInstance when it changes
useEffect(() => {
if (rfInstance.current) {
@ -456,6 +508,7 @@ export const useWorkflowState = ({
workflowValidationErrors,
templateContextVariables,
workflowConfigurations,
dictionary,
setNodes,
setIsDirty,
setIsAddNodePanelOpen,
@ -468,6 +521,7 @@ export const useWorkflowState = ({
onRun,
saveTemplateContextVariables,
saveWorkflowConfigurations,
saveDictionary,
// Export undo/redo state
undo,
redo,

38
ui/src/app/workflow/[workflowId]/stores/workflowStore.ts
View file

@ -35,6 +35,7 @@ interface WorkflowState {
// Configuration
templateContextVariables: Record<string, string>;
workflowConfigurations: WorkflowConfigurations | null;
dictionary: string;
// ReactFlow instance reference
rfInstance: ReactFlowInstance<FlowNode, FlowEdge> | null;
@ -48,7 +49,8 @@ interface WorkflowActions {
nodes: FlowNode[],
edges: FlowEdge[],
templateContextVariables?: Record<string, string>,
workflowConfigurations?: WorkflowConfigurations | null
workflowConfigurations?: WorkflowConfigurations | null,
dictionary?: string
) => void;
// History management
@ -74,6 +76,7 @@ interface WorkflowActions {
setWorkflowName: (name: string) => void;
setTemplateContextVariables: (variables: Record<string, string>) => void;
setWorkflowConfigurations: (configurations: WorkflowConfigurations) => void;
setDictionary: (dictionary: string) => void;
// UI state
setIsDirty: (isDirty: boolean) => void;
@ -110,10 +113,11 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
workflowValidationErrors: [],
templateContextVariables: {},
workflowConfigurations: DEFAULT_WORKFLOW_CONFIGURATIONS,
dictionary: '',
rfInstance: null,
// Actions
initializeWorkflow: (workflowId, workflowName, nodes, edges, templateContextVariables = {}, workflowConfigurations = DEFAULT_WORKFLOW_CONFIGURATIONS) => {
initializeWorkflow: (workflowId, workflowName, nodes, edges, templateContextVariables = {}, workflowConfigurations = DEFAULT_WORKFLOW_CONFIGURATIONS, dictionary = '') => {
const initialHistory: HistoryState = { nodes, edges, workflowName };
set({
workflowId,
@ -122,6 +126,7 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
edges,
templateContextVariables,
workflowConfigurations,
dictionary,
isDirty: false,
workflowValidationErrors: [],
history: [initialHistory],
@ -195,18 +200,28 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
setNodes: (nodes, changes) => {
// Determine whether to push to history and set isDirty based on change types
if (changes && changes.length > 0) {
// Check if any changes are user-initiated (not just selections or dimensions)
const hasDirtyChanges = changes.some(change =>
change.type === 'add' ||
change.type === 'remove' ||
(change.type === 'position' && change.dragging)
// Check for add/remove changes (always push to history)
const hasAddRemoveChanges = changes.some(change =>
change.type === 'add' || change.type === 'remove'
);
if (hasDirtyChanges) {
// Check for position changes - only push to history when drag ENDS (dragging: false)
// but still mark as dirty during dragging
const hasDragEndChanges = changes.some(change =>
change.type === 'position' && change.dragging === false
);
const isActiveDragging = changes.some(change =>
change.type === 'position' && change.dragging === true
);
if (hasAddRemoveChanges || hasDragEndChanges) {
get().pushToHistory();
set({ nodes, isDirty: true });
} else if (isActiveDragging) {
// During active dragging, update nodes but don't push to history
set({ nodes, isDirty: true });
} else {
// For selection changes or dimension updates, don't push to history
// For selection changes or dimension updates, don't push to history or set dirty
set({ nodes });
}
} else {
@ -312,6 +327,10 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
set({ workflowConfigurations });
},
setDictionary: (dictionary) => {
set({ dictionary });
},
setIsDirty: (isDirty) => {
set({ isDirty });
},
@ -375,6 +394,7 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
workflowValidationErrors: [],
templateContextVariables: {},
workflowConfigurations: DEFAULT_WORKFLOW_CONFIGURATIONS,
dictionary: '',
rfInstance: null,
});
},

2
ui/src/components/flow/edges/CustomEdge.tsx
View file

@ -227,7 +227,7 @@ export default function CustomEdge(props: CustomEdgeProps) {
: isHovered
? 'drop-shadow(0 0 6px rgba(96, 165, 250, 0.4))'
: 'none',
transition: 'all 0.2s ease',
transition: 'stroke 0.2s ease, stroke-width 0.2s ease, filter 0.2s ease',
}}
interactionWidth={20}
/>

4
ui/src/types/workflow-configurations.ts
View file

@ -15,6 +15,7 @@ export interface WorkflowConfigurations {
ambient_noise_configuration: AmbientNoiseConfiguration;
max_call_duration: number; // Maximum call duration in seconds
max_user_idle_timeout: number; // Maximum user idle time in seconds
dictionary?: string; // Comma-separated words for voice agent to listen for
[key: string]: unknown; // Allow additional properties for future configurations
}
@ -30,5 +31,6 @@ export const DEFAULT_WORKFLOW_CONFIGURATIONS: WorkflowConfigurations = {
volume: 0.3
},
max_call_duration: 600, // 10 minutes
max_user_idle_timeout: 10 // 10 seconds
max_user_idle_timeout: 10, // 10 seconds
dictionary: ''
};