mirror of
https://github.com/katanemo/plano.git
synced 2026-06-26 15:39:40 +02:00
Remove deprecated legacy signal OTel attributes (#976)
This commit is contained in:
parent
558df0307c
commit
ff4f2b95d6
5 changed files with 8 additions and 142 deletions
|
|
@ -1,27 +1,21 @@
|
|||
//! Helpers for emitting `SignalReport` data to OpenTelemetry spans.
|
||||
//!
|
||||
//! Two sets of attributes are emitted:
|
||||
//!
|
||||
//! - **Legacy** keys under `signals.*` (e.g. `signals.frustration.count`),
|
||||
//! computed from the new layered counts. Preserved for one release for
|
||||
//! backward compatibility with existing dashboards.
|
||||
//! - **New** layered keys (e.g. `signals.interaction.misalignment.count`),
|
||||
//! one set of `count`/`severity` attributes per category, plus per-instance
|
||||
//! span events named `signal.<dotted_signal_type>`.
|
||||
//! Layered keys (e.g. `signals.interaction.misalignment.count`) are emitted,
|
||||
//! one set of `count`/`severity` attributes per category, plus per-instance
|
||||
//! span events named `signal.<dotted_signal_type>`.
|
||||
|
||||
use opentelemetry::trace::SpanRef;
|
||||
use opentelemetry::KeyValue;
|
||||
|
||||
use crate::signals::schemas::{SignalGroup, SignalReport, SignalType};
|
||||
use crate::signals::schemas::{SignalGroup, SignalReport};
|
||||
|
||||
/// Emit both legacy and layered OTel attributes/events for a `SignalReport`.
|
||||
/// Emit layered OTel attributes/events for a `SignalReport`.
|
||||
///
|
||||
/// Returns `true` if any "concerning" signal was found, mirroring the previous
|
||||
/// behavior used to flag the span operation name.
|
||||
pub fn emit_signals_to_span(span: &SpanRef<'_>, report: &SignalReport) -> bool {
|
||||
emit_overall(span, report);
|
||||
emit_layered_attributes(span, report);
|
||||
emit_legacy_attributes(span, report);
|
||||
emit_signal_events(span, report);
|
||||
|
||||
is_concerning(report)
|
||||
|
|
@ -90,69 +84,6 @@ fn emit_layered_attributes(span: &SpanRef<'_>, report: &SignalReport) {
|
|||
);
|
||||
}
|
||||
|
||||
fn count_of(report: &SignalReport, t: SignalType) -> usize {
|
||||
report.iter_signals().filter(|s| s.signal_type == t).count()
|
||||
}
|
||||
|
||||
/// Emit the legacy attribute keys consumed by existing dashboards. These are
|
||||
/// derived from the new `SignalReport` so no detector contract is broken.
|
||||
fn emit_legacy_attributes(span: &SpanRef<'_>, report: &SignalReport) {
|
||||
use crate::tracing::signals as legacy;
|
||||
|
||||
// signals.follow_up.repair.{count,ratio} - misalignment proxies repairs.
|
||||
let repair_count = report.interaction.misalignment.count;
|
||||
let user_turns = report.turn_metrics.user_turns.max(1) as f32;
|
||||
if repair_count > 0 {
|
||||
span.set_attribute(KeyValue::new(legacy::REPAIR_COUNT, repair_count as i64));
|
||||
let ratio = repair_count as f32 / user_turns;
|
||||
span.set_attribute(KeyValue::new(legacy::REPAIR_RATIO, format!("{:.3}", ratio)));
|
||||
}
|
||||
|
||||
// signals.frustration.{count,severity} - disengagement.negative_stance is
|
||||
// the closest legacy analog of "frustration".
|
||||
let frustration_count = count_of(report, SignalType::DisengagementNegativeStance);
|
||||
if frustration_count > 0 {
|
||||
span.set_attribute(KeyValue::new(
|
||||
legacy::FRUSTRATION_COUNT,
|
||||
frustration_count as i64,
|
||||
));
|
||||
let severity = match frustration_count {
|
||||
0 => 0,
|
||||
1..=2 => 1,
|
||||
3..=4 => 2,
|
||||
_ => 3,
|
||||
};
|
||||
span.set_attribute(KeyValue::new(legacy::FRUSTRATION_SEVERITY, severity as i64));
|
||||
}
|
||||
|
||||
// signals.repetition.count - stagnation (repetition + dragging).
|
||||
if report.interaction.stagnation.count > 0 {
|
||||
span.set_attribute(KeyValue::new(
|
||||
legacy::REPETITION_COUNT,
|
||||
report.interaction.stagnation.count as i64,
|
||||
));
|
||||
}
|
||||
|
||||
// signals.escalation.requested - any escalation/quit signal.
|
||||
let escalated = report.interaction.disengagement.signals.iter().any(|s| {
|
||||
matches!(
|
||||
s.signal_type,
|
||||
SignalType::DisengagementEscalation | SignalType::DisengagementQuit
|
||||
)
|
||||
});
|
||||
if escalated {
|
||||
span.set_attribute(KeyValue::new(legacy::ESCALATION_REQUESTED, true));
|
||||
}
|
||||
|
||||
// signals.positive_feedback.count - satisfaction signals.
|
||||
if report.interaction.satisfaction.count > 0 {
|
||||
span.set_attribute(KeyValue::new(
|
||||
legacy::POSITIVE_FEEDBACK_COUNT,
|
||||
report.interaction.satisfaction.count as i64,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_signal_events(span: &SpanRef<'_>, report: &SignalReport) {
|
||||
for sig in report.iter_signals() {
|
||||
let event_name = format!("signal.{}", sig.signal_type.as_str());
|
||||
|
|
@ -231,11 +162,4 @@ mod tests {
|
|||
let r = report_with_escalation();
|
||||
assert!(is_concerning(&r));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_of_returns_per_type_count() {
|
||||
let r = report_with_escalation();
|
||||
assert_eq!(count_of(&r, SignalType::DisengagementEscalation), 1);
|
||||
assert_eq!(count_of(&r, SignalType::DisengagementNegativeStance), 0);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -367,9 +367,7 @@ impl StreamProcessor for ObservableStreamProcessor {
|
|||
self.response_buffer.shrink_to_fit();
|
||||
|
||||
// Analyze signals if messages are available and record as span
|
||||
// attributes + per-signal events. We dual-emit legacy aggregate keys
|
||||
// and the new layered taxonomy so existing dashboards keep working
|
||||
// while new consumers can opt into the richer hierarchy.
|
||||
// attributes + per-signal events using the layered signal taxonomy.
|
||||
if let Some(ref messages) = self.messages {
|
||||
let analyzer = SignalAnalyzer::default();
|
||||
let report = analyzer.analyze_openai(messages);
|
||||
|
|
|
|||
|
|
@ -183,27 +183,6 @@ pub mod signals {
|
|||
|
||||
/// Efficiency score (0.0-1.0)
|
||||
pub const EFFICIENCY_SCORE: &str = "signals.efficiency_score";
|
||||
|
||||
/// Number of repair attempts detected
|
||||
pub const REPAIR_COUNT: &str = "signals.follow_up.repair.count";
|
||||
|
||||
/// Ratio of repairs to user turns
|
||||
pub const REPAIR_RATIO: &str = "signals.follow_up.repair.ratio";
|
||||
|
||||
/// Number of frustration indicators detected
|
||||
pub const FRUSTRATION_COUNT: &str = "signals.frustration.count";
|
||||
|
||||
/// Frustration severity level (0-3)
|
||||
pub const FRUSTRATION_SEVERITY: &str = "signals.frustration.severity";
|
||||
|
||||
/// Number of repetition instances detected
|
||||
pub const REPETITION_COUNT: &str = "signals.repetition.count";
|
||||
|
||||
/// Whether escalation was requested (user asked for human help)
|
||||
pub const ESCALATION_REQUESTED: &str = "signals.escalation.requested";
|
||||
|
||||
/// Number of positive feedback indicators detected
|
||||
pub const POSITIVE_FEEDBACK_COUNT: &str = "signals.positive_feedback.count";
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
|
|
|
|||
|
|
@ -334,35 +334,6 @@ Emitted per category, only when ``count > 0``. One ``.count`` and one
|
|||
* - ``signals.environment.exhaustion.severity``
|
||||
- "
|
||||
|
||||
Legacy attributes (deprecated, still emitted)
|
||||
---------------------------------------------
|
||||
|
||||
The following aggregate keys pre-date the paper taxonomy and are still
|
||||
emitted for one release so existing dashboards keep working. They are
|
||||
derived from the layered counts above and will be removed in a future
|
||||
release. Migrate to the layered keys when convenient.
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 50 50
|
||||
|
||||
* - Legacy attribute
|
||||
- Layered equivalent
|
||||
* - ``signals.follow_up.repair.count``
|
||||
- ``signals.interaction.misalignment.count``
|
||||
* - ``signals.follow_up.repair.ratio``
|
||||
- (computed: ``misalignment.count / max(user_turns, 1)``)
|
||||
* - ``signals.frustration.count``
|
||||
- Count of ``disengagement.negative_stance`` instances
|
||||
* - ``signals.frustration.severity``
|
||||
- Derived severity bucket of the above
|
||||
* - ``signals.repetition.count``
|
||||
- ``signals.interaction.stagnation.count``
|
||||
* - ``signals.escalation.requested``
|
||||
- True if any ``disengagement.escalation`` or ``disengagement.quit`` fired
|
||||
* - ``signals.positive_feedback.count``
|
||||
- ``signals.interaction.satisfaction.count``
|
||||
|
||||
Span Events
|
||||
===========
|
||||
|
||||
|
|
@ -520,11 +491,6 @@ event::
|
|||
signals.interaction.disengagement.count = 6
|
||||
signals.interaction.disengagement.severity = 3
|
||||
|
||||
# Legacy (deprecated, emitted while dual-emit is on)
|
||||
signals.frustration.count = 4
|
||||
signals.frustration.severity = 2
|
||||
signals.escalation.requested = true
|
||||
|
||||
# Per-instance span events
|
||||
event: signal.interaction.disengagement.escalation
|
||||
signal.type = "interaction.disengagement.escalation"
|
||||
|
|
@ -537,8 +503,7 @@ Building Dashboards
|
|||
===================
|
||||
|
||||
Use signal attributes to build monitoring dashboards in Grafana, Honeycomb,
|
||||
Datadog, etc. Prefer the layered keys — they align with the paper taxonomy
|
||||
and will outlive the legacy keys.
|
||||
Datadog, etc. The layered keys align with the paper taxonomy.
|
||||
|
||||
- **Quality distribution**: Count of traces by ``signals.quality``
|
||||
- **P95 turn count**: 95th percentile of ``signals.turn_count``
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ In your observability platform (Jaeger, Grafana Tempo, Datadog, etc.), filter tr
|
|||
- Find external issues: ``signals.environment.exhaustion.count > 0``
|
||||
- Find inefficient flows: ``signals.efficiency_score < 0.5``
|
||||
|
||||
For complete details on all 20 leaf signal types, severity scheme, legacy attribute deprecation, and best practices, see the :doc:`../../concepts/signals` guide.
|
||||
For complete details on all 20 leaf signal types, severity scheme, and best practices, see the :doc:`../../concepts/signals` guide.
|
||||
|
||||
|
||||
Custom Span Attributes
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue