mirror of
https://github.com/katanemo/plano.git
synced 2026-07-05 15:52:12 +02:00
Remove deprecated legacy signal OTel attributes (#976)
This commit is contained in:
parent
558df0307c
commit
ff4f2b95d6
5 changed files with 8 additions and 142 deletions
|
|
@ -1,27 +1,21 @@
|
||||||
//! Helpers for emitting `SignalReport` data to OpenTelemetry spans.
|
//! Helpers for emitting `SignalReport` data to OpenTelemetry spans.
|
||||||
//!
|
//!
|
||||||
//! Two sets of attributes are emitted:
|
//! Layered keys (e.g. `signals.interaction.misalignment.count`) are emitted,
|
||||||
//!
|
//! one set of `count`/`severity` attributes per category, plus per-instance
|
||||||
//! - **Legacy** keys under `signals.*` (e.g. `signals.frustration.count`),
|
//! span events named `signal.<dotted_signal_type>`.
|
||||||
//! computed from the new layered counts. Preserved for one release for
|
|
||||||
//! backward compatibility with existing dashboards.
|
|
||||||
//! - **New** layered keys (e.g. `signals.interaction.misalignment.count`),
|
|
||||||
//! one set of `count`/`severity` attributes per category, plus per-instance
|
|
||||||
//! span events named `signal.<dotted_signal_type>`.
|
|
||||||
|
|
||||||
use opentelemetry::trace::SpanRef;
|
use opentelemetry::trace::SpanRef;
|
||||||
use opentelemetry::KeyValue;
|
use opentelemetry::KeyValue;
|
||||||
|
|
||||||
use crate::signals::schemas::{SignalGroup, SignalReport, SignalType};
|
use crate::signals::schemas::{SignalGroup, SignalReport};
|
||||||
|
|
||||||
/// Emit both legacy and layered OTel attributes/events for a `SignalReport`.
|
/// Emit layered OTel attributes/events for a `SignalReport`.
|
||||||
///
|
///
|
||||||
/// Returns `true` if any "concerning" signal was found, mirroring the previous
|
/// Returns `true` if any "concerning" signal was found, mirroring the previous
|
||||||
/// behavior used to flag the span operation name.
|
/// behavior used to flag the span operation name.
|
||||||
pub fn emit_signals_to_span(span: &SpanRef<'_>, report: &SignalReport) -> bool {
|
pub fn emit_signals_to_span(span: &SpanRef<'_>, report: &SignalReport) -> bool {
|
||||||
emit_overall(span, report);
|
emit_overall(span, report);
|
||||||
emit_layered_attributes(span, report);
|
emit_layered_attributes(span, report);
|
||||||
emit_legacy_attributes(span, report);
|
|
||||||
emit_signal_events(span, report);
|
emit_signal_events(span, report);
|
||||||
|
|
||||||
is_concerning(report)
|
is_concerning(report)
|
||||||
|
|
@ -90,69 +84,6 @@ fn emit_layered_attributes(span: &SpanRef<'_>, report: &SignalReport) {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn count_of(report: &SignalReport, t: SignalType) -> usize {
|
|
||||||
report.iter_signals().filter(|s| s.signal_type == t).count()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Emit the legacy attribute keys consumed by existing dashboards. These are
|
|
||||||
/// derived from the new `SignalReport` so no detector contract is broken.
|
|
||||||
fn emit_legacy_attributes(span: &SpanRef<'_>, report: &SignalReport) {
|
|
||||||
use crate::tracing::signals as legacy;
|
|
||||||
|
|
||||||
// signals.follow_up.repair.{count,ratio} - misalignment proxies repairs.
|
|
||||||
let repair_count = report.interaction.misalignment.count;
|
|
||||||
let user_turns = report.turn_metrics.user_turns.max(1) as f32;
|
|
||||||
if repair_count > 0 {
|
|
||||||
span.set_attribute(KeyValue::new(legacy::REPAIR_COUNT, repair_count as i64));
|
|
||||||
let ratio = repair_count as f32 / user_turns;
|
|
||||||
span.set_attribute(KeyValue::new(legacy::REPAIR_RATIO, format!("{:.3}", ratio)));
|
|
||||||
}
|
|
||||||
|
|
||||||
// signals.frustration.{count,severity} - disengagement.negative_stance is
|
|
||||||
// the closest legacy analog of "frustration".
|
|
||||||
let frustration_count = count_of(report, SignalType::DisengagementNegativeStance);
|
|
||||||
if frustration_count > 0 {
|
|
||||||
span.set_attribute(KeyValue::new(
|
|
||||||
legacy::FRUSTRATION_COUNT,
|
|
||||||
frustration_count as i64,
|
|
||||||
));
|
|
||||||
let severity = match frustration_count {
|
|
||||||
0 => 0,
|
|
||||||
1..=2 => 1,
|
|
||||||
3..=4 => 2,
|
|
||||||
_ => 3,
|
|
||||||
};
|
|
||||||
span.set_attribute(KeyValue::new(legacy::FRUSTRATION_SEVERITY, severity as i64));
|
|
||||||
}
|
|
||||||
|
|
||||||
// signals.repetition.count - stagnation (repetition + dragging).
|
|
||||||
if report.interaction.stagnation.count > 0 {
|
|
||||||
span.set_attribute(KeyValue::new(
|
|
||||||
legacy::REPETITION_COUNT,
|
|
||||||
report.interaction.stagnation.count as i64,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
// signals.escalation.requested - any escalation/quit signal.
|
|
||||||
let escalated = report.interaction.disengagement.signals.iter().any(|s| {
|
|
||||||
matches!(
|
|
||||||
s.signal_type,
|
|
||||||
SignalType::DisengagementEscalation | SignalType::DisengagementQuit
|
|
||||||
)
|
|
||||||
});
|
|
||||||
if escalated {
|
|
||||||
span.set_attribute(KeyValue::new(legacy::ESCALATION_REQUESTED, true));
|
|
||||||
}
|
|
||||||
|
|
||||||
// signals.positive_feedback.count - satisfaction signals.
|
|
||||||
if report.interaction.satisfaction.count > 0 {
|
|
||||||
span.set_attribute(KeyValue::new(
|
|
||||||
legacy::POSITIVE_FEEDBACK_COUNT,
|
|
||||||
report.interaction.satisfaction.count as i64,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn emit_signal_events(span: &SpanRef<'_>, report: &SignalReport) {
|
fn emit_signal_events(span: &SpanRef<'_>, report: &SignalReport) {
|
||||||
for sig in report.iter_signals() {
|
for sig in report.iter_signals() {
|
||||||
let event_name = format!("signal.{}", sig.signal_type.as_str());
|
let event_name = format!("signal.{}", sig.signal_type.as_str());
|
||||||
|
|
@ -231,11 +162,4 @@ mod tests {
|
||||||
let r = report_with_escalation();
|
let r = report_with_escalation();
|
||||||
assert!(is_concerning(&r));
|
assert!(is_concerning(&r));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn count_of_returns_per_type_count() {
|
|
||||||
let r = report_with_escalation();
|
|
||||||
assert_eq!(count_of(&r, SignalType::DisengagementEscalation), 1);
|
|
||||||
assert_eq!(count_of(&r, SignalType::DisengagementNegativeStance), 0);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -367,9 +367,7 @@ impl StreamProcessor for ObservableStreamProcessor {
|
||||||
self.response_buffer.shrink_to_fit();
|
self.response_buffer.shrink_to_fit();
|
||||||
|
|
||||||
// Analyze signals if messages are available and record as span
|
// Analyze signals if messages are available and record as span
|
||||||
// attributes + per-signal events. We dual-emit legacy aggregate keys
|
// attributes + per-signal events using the layered signal taxonomy.
|
||||||
// and the new layered taxonomy so existing dashboards keep working
|
|
||||||
// while new consumers can opt into the richer hierarchy.
|
|
||||||
if let Some(ref messages) = self.messages {
|
if let Some(ref messages) = self.messages {
|
||||||
let analyzer = SignalAnalyzer::default();
|
let analyzer = SignalAnalyzer::default();
|
||||||
let report = analyzer.analyze_openai(messages);
|
let report = analyzer.analyze_openai(messages);
|
||||||
|
|
|
||||||
|
|
@ -183,27 +183,6 @@ pub mod signals {
|
||||||
|
|
||||||
/// Efficiency score (0.0-1.0)
|
/// Efficiency score (0.0-1.0)
|
||||||
pub const EFFICIENCY_SCORE: &str = "signals.efficiency_score";
|
pub const EFFICIENCY_SCORE: &str = "signals.efficiency_score";
|
||||||
|
|
||||||
/// Number of repair attempts detected
|
|
||||||
pub const REPAIR_COUNT: &str = "signals.follow_up.repair.count";
|
|
||||||
|
|
||||||
/// Ratio of repairs to user turns
|
|
||||||
pub const REPAIR_RATIO: &str = "signals.follow_up.repair.ratio";
|
|
||||||
|
|
||||||
/// Number of frustration indicators detected
|
|
||||||
pub const FRUSTRATION_COUNT: &str = "signals.frustration.count";
|
|
||||||
|
|
||||||
/// Frustration severity level (0-3)
|
|
||||||
pub const FRUSTRATION_SEVERITY: &str = "signals.frustration.severity";
|
|
||||||
|
|
||||||
/// Number of repetition instances detected
|
|
||||||
pub const REPETITION_COUNT: &str = "signals.repetition.count";
|
|
||||||
|
|
||||||
/// Whether escalation was requested (user asked for human help)
|
|
||||||
pub const ESCALATION_REQUESTED: &str = "signals.escalation.requested";
|
|
||||||
|
|
||||||
/// Number of positive feedback indicators detected
|
|
||||||
pub const POSITIVE_FEEDBACK_COUNT: &str = "signals.positive_feedback.count";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
|
|
|
||||||
|
|
@ -334,35 +334,6 @@ Emitted per category, only when ``count > 0``. One ``.count`` and one
|
||||||
* - ``signals.environment.exhaustion.severity``
|
* - ``signals.environment.exhaustion.severity``
|
||||||
- "
|
- "
|
||||||
|
|
||||||
Legacy attributes (deprecated, still emitted)
|
|
||||||
---------------------------------------------
|
|
||||||
|
|
||||||
The following aggregate keys pre-date the paper taxonomy and are still
|
|
||||||
emitted for one release so existing dashboards keep working. They are
|
|
||||||
derived from the layered counts above and will be removed in a future
|
|
||||||
release. Migrate to the layered keys when convenient.
|
|
||||||
|
|
||||||
.. list-table::
|
|
||||||
:header-rows: 1
|
|
||||||
:widths: 50 50
|
|
||||||
|
|
||||||
* - Legacy attribute
|
|
||||||
- Layered equivalent
|
|
||||||
* - ``signals.follow_up.repair.count``
|
|
||||||
- ``signals.interaction.misalignment.count``
|
|
||||||
* - ``signals.follow_up.repair.ratio``
|
|
||||||
- (computed: ``misalignment.count / max(user_turns, 1)``)
|
|
||||||
* - ``signals.frustration.count``
|
|
||||||
- Count of ``disengagement.negative_stance`` instances
|
|
||||||
* - ``signals.frustration.severity``
|
|
||||||
- Derived severity bucket of the above
|
|
||||||
* - ``signals.repetition.count``
|
|
||||||
- ``signals.interaction.stagnation.count``
|
|
||||||
* - ``signals.escalation.requested``
|
|
||||||
- True if any ``disengagement.escalation`` or ``disengagement.quit`` fired
|
|
||||||
* - ``signals.positive_feedback.count``
|
|
||||||
- ``signals.interaction.satisfaction.count``
|
|
||||||
|
|
||||||
Span Events
|
Span Events
|
||||||
===========
|
===========
|
||||||
|
|
||||||
|
|
@ -520,11 +491,6 @@ event::
|
||||||
signals.interaction.disengagement.count = 6
|
signals.interaction.disengagement.count = 6
|
||||||
signals.interaction.disengagement.severity = 3
|
signals.interaction.disengagement.severity = 3
|
||||||
|
|
||||||
# Legacy (deprecated, emitted while dual-emit is on)
|
|
||||||
signals.frustration.count = 4
|
|
||||||
signals.frustration.severity = 2
|
|
||||||
signals.escalation.requested = true
|
|
||||||
|
|
||||||
# Per-instance span events
|
# Per-instance span events
|
||||||
event: signal.interaction.disengagement.escalation
|
event: signal.interaction.disengagement.escalation
|
||||||
signal.type = "interaction.disengagement.escalation"
|
signal.type = "interaction.disengagement.escalation"
|
||||||
|
|
@ -537,8 +503,7 @@ Building Dashboards
|
||||||
===================
|
===================
|
||||||
|
|
||||||
Use signal attributes to build monitoring dashboards in Grafana, Honeycomb,
|
Use signal attributes to build monitoring dashboards in Grafana, Honeycomb,
|
||||||
Datadog, etc. Prefer the layered keys — they align with the paper taxonomy
|
Datadog, etc. The layered keys align with the paper taxonomy.
|
||||||
and will outlive the legacy keys.
|
|
||||||
|
|
||||||
- **Quality distribution**: Count of traces by ``signals.quality``
|
- **Quality distribution**: Count of traces by ``signals.quality``
|
||||||
- **P95 turn count**: 95th percentile of ``signals.turn_count``
|
- **P95 turn count**: 95th percentile of ``signals.turn_count``
|
||||||
|
|
|
||||||
|
|
@ -153,7 +153,7 @@ In your observability platform (Jaeger, Grafana Tempo, Datadog, etc.), filter tr
|
||||||
- Find external issues: ``signals.environment.exhaustion.count > 0``
|
- Find external issues: ``signals.environment.exhaustion.count > 0``
|
||||||
- Find inefficient flows: ``signals.efficiency_score < 0.5``
|
- Find inefficient flows: ``signals.efficiency_score < 0.5``
|
||||||
|
|
||||||
For complete details on all 20 leaf signal types, severity scheme, legacy attribute deprecation, and best practices, see the :doc:`../../concepts/signals` guide.
|
For complete details on all 20 leaf signal types, severity scheme, and best practices, see the :doc:`../../concepts/signals` guide.
|
||||||
|
|
||||||
|
|
||||||
Custom Span Attributes
|
Custom Span Attributes
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue