apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-20 15:28:07 +02:00
Code Issues Projects Releases Packages Wiki Activity

adding docs. Still need to fix the messages list, but waiting on PR #621

Browse source
This commit is contained in:
Salman Paracha 2025-12-17 15:45:32 -08:00
parent dfd5886743
commit 9cebeb72a7
5 changed files with 418 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

31
crates/brightstaff/src/handlers/utils.rs
View file

@ -11,7 +11,7 @@ use tokio_stream::StreamExt;
use tracing::warn;
// Import tracing constants and signals
use crate::tracing::{llm, error};
use crate::tracing::{llm, error, signals as signal_constants};
use crate::signals::signals::{SignalAnalyzer, InteractionQuality, FLAG_MARKER};
use hermesllm::apis::openai::Message;
@ -153,12 +153,29 @@ impl StreamProcessor for ObservableStreamProcessor {
// Add overall quality
self.span.attributes.push(Attribute {
key: "signals.quality".to_string(),
key: signal_constants::QUALITY.to_string(),
value: AttributeValue {
string_value: Some(format!("{:?}", report.overall_quality)),
},
});
// Add repair/follow-up metrics if concerning
if report.follow_up.is_concerning || report.follow_up.repair_count > 0 {
self.span.attributes.push(Attribute {
key: signal_constants::REPAIR_COUNT.to_string(),
value: AttributeValue {
string_value: Some(report.follow_up.repair_count.to_string()),
},
});
self.span.attributes.push(Attribute {
key: signal_constants::REPAIR_RATIO.to_string(),
value: AttributeValue {
string_value: Some(format!("{:.3}", report.follow_up.repair_ratio)),
},
});
}
// Add flag marker to operation name if any concerning signal is detected
let should_flag = report.frustration.has_frustration
|| report.repetition.has_looping
@ -176,13 +193,13 @@ impl StreamProcessor for ObservableStreamProcessor {
// Add key signal metrics
if report.frustration.has_frustration {
self.span.attributes.push(Attribute {
key: "signals.frustration.count".to_string(),
key: signal_constants::FRUSTRATION_COUNT.to_string(),
value: AttributeValue {
string_value: Some(report.frustration.frustration_count.to_string()),
},
});
self.span.attributes.push(Attribute {
key: "signals.frustration.severity".to_string(),
key: signal_constants::FRUSTRATION_SEVERITY.to_string(),
value: AttributeValue {
string_value: Some(report.frustration.severity.to_string()),
},
@ -191,7 +208,7 @@ impl StreamProcessor for ObservableStreamProcessor {
if report.repetition.has_looping {
self.span.attributes.push(Attribute {
key: "signals.repetition.count".to_string(),
key: signal_constants::REPETITION_COUNT.to_string(),
value: AttributeValue {
string_value: Some(report.repetition.repetition_count.to_string()),
},
@ -200,7 +217,7 @@ impl StreamProcessor for ObservableStreamProcessor {
if report.escalation.escalation_requested {
self.span.attributes.push(Attribute {
key: "signals.escalation.requested".to_string(),
key: signal_constants::ESCALATION_REQUESTED.to_string(),
value: AttributeValue {
string_value: Some("true".to_string()),
},
@ -209,7 +226,7 @@ impl StreamProcessor for ObservableStreamProcessor {
if report.positive_feedback.has_positive_feedback {
self.span.attributes.push(Attribute {
key: "signals.positive_feedback.count".to_string(),
key: signal_constants::POSITIVE_FEEDBACK_COUNT.to_string(),
value: AttributeValue {
string_value: Some(report.positive_feedback.positive_count.to_string()),
},

39
crates/brightstaff/src/tracing/constants.rs
View file

@ -141,6 +141,45 @@ pub mod error {
pub const STACK_TRACE: &str = "error.stack_trace";
}
// =============================================================================
// Span Attributes - Agentic Signals
// =============================================================================
/// Behavioral quality indicators for agent interactions
/// These signals are computed automatically from conversation patterns
pub mod signals {
/// Overall quality assessment
/// Values: "Excellent", "Good", "Neutral", "Poor", "Severe"
pub const QUALITY: &str = "signals.quality";
/// Total number of turns in the conversation
pub const TURN_COUNT: &str = "signals.turn_count";
/// Efficiency score (0.0-1.0)
pub const EFFICIENCY_SCORE: &str = "signals.efficiency_score";
/// Number of repair attempts detected
pub const REPAIR_COUNT: &str = "signals.follow_up.repair.count";
/// Ratio of repairs to user turns
pub const REPAIR_RATIO: &str = "signals.follow_up.repair.ratio";
/// Number of frustration indicators detected
pub const FRUSTRATION_COUNT: &str = "signals.frustration.count";
/// Frustration severity level (0-3)
pub const FRUSTRATION_SEVERITY: &str = "signals.frustration.severity";
/// Number of repetition instances detected
pub const REPETITION_COUNT: &str = "signals.repetition.count";
/// Whether escalation was requested (user asked for human help)
pub const ESCALATION_REQUESTED: &str = "signals.escalation.requested";
/// Number of positive feedback indicators detected
pub const POSITIVE_FEEDBACK_COUNT: &str = "signals.positive_feedback.count";
}
// =============================================================================
// Operation Names
// =============================================================================

2
crates/brightstaff/src/tracing/mod.rs
View file

@ -1,3 +1,3 @@
mod constants;
pub use constants::{OperationNameBuilder, operation_component, http, llm, error, routing};
pub use constants::{OperationNameBuilder, operation_component, http, llm, error, routing, signals};