mirror of
https://github.com/katanemo/plano.git
synced 2026-05-02 20:32:42 +02:00
961 lines
No EOL
88 KiB
HTML
Executable file
961 lines
No EOL
88 KiB
HTML
Executable file
<!DOCTYPE html>
|
||
|
||
<html :class="{ 'dark' : darkMode === true }" data-content_root="../" lang="en" x-data="{ darkMode: $persist(window.matchMedia('(prefers-color-scheme: dark)').matches), activeSection: ''}">
|
||
<head>
|
||
<script>
|
||
(function () {
|
||
// Set initial color scheme
|
||
if ((localStorage.getItem("_x_darkMode") === "true") || (window.matchMedia("(prefers-color-scheme: dark)").matches)) {
|
||
document.documentElement.classList.add("dark");
|
||
}
|
||
|
||
// Watch for media preference changes
|
||
window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change", (event) => {
|
||
localStorage.setItem("_x_darkMode", event.matches);
|
||
document.documentElement.classList.toggle("dark", event.matches);
|
||
});
|
||
})();
|
||
</script>
|
||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||
<meta charset="utf-8"/>
|
||
<meta content="#ffffff" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||
<meta content="#030711" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||
<title>Signals™ | Plano Docs v0.4.20</title>
|
||
<meta content="Signals™ | Plano Docs v0.4.20" property="og:title"/>
|
||
<meta content="Signals™ | Plano Docs v0.4.20" name="twitter:title"/>
|
||
<link href="../_static/pygments.css?v=73db4dac" rel="stylesheet" type="text/css"/>
|
||
<link href="../_static/theme.css?v=979577e3" rel="stylesheet" type="text/css"/>
|
||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||
<link href="../_static/awesome-sphinx-design.css?v=b1d4564d" rel="stylesheet" type="text/css"/>
|
||
<link href="./docs/concepts/signals.html" rel="canonical"/>
|
||
<link href="../_static/favicon.ico" rel="icon"/>
|
||
<link href="../search.html" rel="search" title="Search"/>
|
||
<link href="../guides/orchestration.html" rel="next" title="Orchestration"/>
|
||
<link href="prompt_target.html" rel="prev" title="Prompt Target"/>
|
||
</head>
|
||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||
Skip to content
|
||
</a><header class="sticky top-0 z-40 w-full border-b shadow-xs border-border bg-background/90 backdrop-blur"><div class="container flex items-center h-14">
|
||
<div class="hidden mr-4 md:flex">
|
||
<a class="flex items-center mr-6" href="../index.html">
|
||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4.20</span>
|
||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||
</svg>
|
||
<span class="sr-only">Toggle navigation menu</span>
|
||
</button>
|
||
<div class="flex items-center justify-between flex-1 gap-2 sm:gap-4 md:justify-end">
|
||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||
<span class="text-xs">⌘</span>
|
||
K
|
||
</kbd>
|
||
</form>
|
||
</div>
|
||
<nav class="flex items-center gap-1">
|
||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||
</div>
|
||
</a>
|
||
<button @click="darkMode = !darkMode" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" title="Toggle color scheme" type="button">
|
||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="16" viewbox="0 96 960 960" width="16" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||
</svg>
|
||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="16" viewbox="0 96 960 960" width="16" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||
</svg>
|
||
</button>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</header>
|
||
<div class="flex-1"><div class="container md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||
<a class="justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4.20</span>
|
||
</a>
|
||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0">
|
||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||
|
||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||
<script>
|
||
window.dataLayer = window.dataLayer || [];
|
||
function gtag(){dataLayer.push(arguments);}
|
||
gtag('js', new Date());
|
||
|
||
gtag('config', 'G-EH2VW19FXE');
|
||
</script>
|
||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||
<ul class="current">
|
||
<li class="toctree-l1"><a class="reference internal" href="listeners.html">Listeners</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="agents.html">Agents</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="filter_chain.html">Filter Chains</a></li>
|
||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button" x-cloak=""><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-cloak="" x-show="expanded">
|
||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/client_libraries.html">Client Libraries</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/model_aliases.html">Model Aliases</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="prompt_target.html">Prompt Target</a></li>
|
||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Signals™</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../guides/orchestration.html">Orchestration</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button" x-cloak=""><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-cloak="" x-show="expanded">
|
||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Guardrails</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../guides/state.html">Conversational State</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||
<ul>
|
||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button" x-cloak=""><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-cloak="" x-show="expanded">
|
||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../resources/cli_reference.html">CLI Reference</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||
</svg>
|
||
</button>
|
||
</aside>
|
||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||
<div class="w-full min-w-0 mx-auto">
|
||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||
<span class="hidden md:inline">Plano Docs v0.4.20</span>
|
||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||
</svg>
|
||
</a>
|
||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Signals™</span>
|
||
</nav>
|
||
<div id="content" role="main">
|
||
<section id="signals">
|
||
<h1>Signals™<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#signals"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||
<p>Agentic Signals are lightweight, model-free behavioral indicators computed
|
||
from live interaction trajectories and attached to your existing
|
||
OpenTelemetry traces. They are the instrumentation layer of a closed-loop
|
||
improvement flywheel for agents — turning raw production traffic into
|
||
prioritized data that can drive prompt, routing, and model updates without
|
||
running an LLM-as-judge on every session.</p>
|
||
<p>The framework implemented here follows the taxonomy and detector design in
|
||
<em>Signals: Trajectory Sampling and Triage for Agentic Interactions</em>
|
||
(<a class="reference external" href="https://arxiv.org/abs/2604.00356" rel="nofollow noopener">Chen et al., 2026<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>). All detectors
|
||
are computed without model calls; the entire pipeline attaches structured
|
||
attributes and span events to existing spans so your dashboards and alerts
|
||
work unmodified.</p>
|
||
<section id="why-signals-matter-the-improvement-flywheel">
|
||
<h2>Why Signals Matter: The Improvement Flywheel<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#why-signals-matter-the-improvement-flywheel" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#why-signals-matter-the-improvement-flywheel'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Agentic applications are increasingly deployed at scale, yet improving them
|
||
after deployment remains difficult. Production trajectories are long,
|
||
numerous, and non-deterministic, making exhaustive human review infeasible
|
||
and auxiliary LLM evaluation expensive. As a result, teams face a
|
||
bottleneck: they cannot score every response, inspect every trace, or
|
||
reliably identify which failures and successes should inform the next model
|
||
update. Without a low-cost triage layer, the feedback loop from production
|
||
behavior to model improvement remains incomplete.</p>
|
||
<p>Signals close this loop by cheaply identifying which interactions among
|
||
millions are worth inspecting:</p>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>Instrument.</strong> Live trajectories are scored with model-free signals
|
||
attached as structured attributes on existing OpenTelemetry spans,
|
||
organized under a fixed taxonomy of interaction, execution, and
|
||
environment signals. This requires no additional model calls,
|
||
infrastructure, or changes to online agent behavior.</p></li>
|
||
<li><p><strong>Sample & triage.</strong> Signal attributes act as filters: they surface
|
||
severe failures, retrieve representative exemplars, and exclude the
|
||
uninformative middle. In our experiments, signal-based sampling
|
||
achieves 82% informativeness on <span class="math notranslate nohighlight">\(\tau\)</span>-bench, compared with 54%
|
||
for random sampling, yielding a 1.52× efficiency gain per informative
|
||
trajectory.</p></li>
|
||
<li><p><strong>Data Construction.</strong> The triaged subset becomes targeted input for
|
||
constructing preference datasets or supervised fine-tuning datasets
|
||
from production trajectories.</p></li>
|
||
<li><p><strong>Model Optimization.</strong> The resulting preference or supervised
|
||
fine-tuning data is used to update the model through methods such as
|
||
DPO, RLHF, or supervised fine-tuning, so optimization is driven by
|
||
targeted production behavior rather than undifferentiated trace noise.</p></li>
|
||
<li><p><strong>Deploy.</strong> The improved model is deployed and immediately
|
||
re-instrumented with the same signals, enabling teams to measure
|
||
whether the change improved production behavior and to feed the next
|
||
iteration.</p></li>
|
||
</ol>
|
||
<p>This loop depends on the first step being nearly free. The framework is
|
||
therefore designed around fixed-taxonomy, model-free detectors with
|
||
<span class="math notranslate nohighlight">\(O(\text{messages})\)</span> cost, no online behavior change, and no
|
||
dependence on expensive evaluator models. By making production traces
|
||
searchable and sampleable at scale, signals turn raw agent telemetry into a
|
||
practical model-optimization flywheel.</p>
|
||
</section>
|
||
<section id="what-are-behavioral-signals">
|
||
<h2>What Are Behavioral Signals?<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#what-are-behavioral-signals" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#what-are-behavioral-signals'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Behavioral signals are canaries in the coal mine — early, objective
|
||
indicators that something may have gone wrong (or gone exceptionally well).
|
||
They don’t explain <em>why</em> an agent failed, but they reliably signal <em>where</em>
|
||
attention is needed.</p>
|
||
<p>These signals emerge naturally from the rhythm of interaction:</p>
|
||
<ul class="simple">
|
||
<li><p>A user rephrasing or correcting the same request</p></li>
|
||
<li><p>Sharp increases in conversation length</p></li>
|
||
<li><p>Negative stance markers (“this doesn’t work”, ALL CAPS, excessive !!! or ???)</p></li>
|
||
<li><p>Agent repetition or tool-call loops</p></li>
|
||
<li><p>Expressions of gratitude, confirmation, or task success</p></li>
|
||
<li><p>Requests for a human agent or explicit quit intent</p></li>
|
||
<li><p>Tool errors, timeouts, rate limits, and context-window exhaustion</p></li>
|
||
</ul>
|
||
<p>Individually, these clues are shallow; together, they form a fingerprint of
|
||
agent performance. Embedded directly into traces, they make it easy to spot
|
||
friction as it happens: where users struggle, where agents loop, where tool
|
||
failures cluster, and where escalations occur.</p>
|
||
</section>
|
||
<section id="signal-taxonomy">
|
||
<h2>Signal Taxonomy<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#signal-taxonomy" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#signal-taxonomy'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Signals are organized into three top-level <strong>layers</strong>, each with its own
|
||
intent. Every detected signal belongs to exactly one leaf type under one of
|
||
seven categories. The per-category summaries and leaf-type descriptions
|
||
below are borrowed verbatim from the reference implementation at
|
||
<a class="reference external" href="https://github.com/katanemo/signals" rel="nofollow noopener">katanemo/signals<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> to keep the
|
||
documentation and the detector contract in sync.</p>
|
||
<section id="interaction-user-agent-conversational-quality">
|
||
<h3>Interaction — user ↔ agent conversational quality<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#interaction-user-agent-conversational-quality" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#interaction-user-agent-conversational-quality'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p><strong>Misalignment</strong> — Misalignment signals capture semantic or intent mismatch
|
||
between the user and the agent, such as rephrasing, corrections,
|
||
clarifications, and restated constraints. These signals do not assert that
|
||
either party is “wrong”; they only indicate that shared understanding has
|
||
not yet been established.</p>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 30.0%"/>
|
||
<col style="width: 70.0%"/>
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Leaf signal type</p></th>
|
||
<th class="head"><p>Description</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">misalignment.correction</span></code></p></td>
|
||
<td><p>Explicit corrections, negations, mistake acknowledgments.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">misalignment.rephrase</span></code></p></td>
|
||
<td><p>Rephrasing indicators, alternative explanations.</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">misalignment.clarification</span></code></p></td>
|
||
<td><p>Confusion expressions, requests for clarification.</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p><strong>Stagnation</strong> — Stagnation signals capture cases where the discourse
|
||
continues but fails to make visible progress. This includes near-duplicate
|
||
assistant responses, circular explanations, repeated scaffolding, and other
|
||
forms of linguistic degeneration.</p>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 30.0%"/>
|
||
<col style="width: 70.0%"/>
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Leaf signal type</p></th>
|
||
<th class="head"><p>Description</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">stagnation.dragging</span></code></p></td>
|
||
<td><p>Excessive turn count, conversation not progressing efficiently.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">stagnation.repetition</span></code></p></td>
|
||
<td><p>Near-duplicate or repetitive assistant responses.</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p><strong>Disengagement</strong> — Disengagement signals mark the withdrawal of
|
||
cooperative intent from the interaction. These include explicit requests to
|
||
exit the agent flow (e.g., “talk to a human”), strong negative stances, and
|
||
abandonment markers.</p>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 30.0%"/>
|
||
<col style="width: 70.0%"/>
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Leaf signal type</p></th>
|
||
<th class="head"><p>Description</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">disengagement.escalation</span></code></p></td>
|
||
<td><p>Requests for human agent or support.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">disengagement.quit</span></code></p></td>
|
||
<td><p>Notification to quit or leave.</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">disengagement.negative_stance</span></code></p></td>
|
||
<td><p>Complaints, frustration, negative sentiment.</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p><strong>Satisfaction</strong> — Satisfaction signals indicate explicit stabilization and
|
||
completion of the interaction. These include expressions of gratitude,
|
||
success confirmations, and closing utterances. We use these signals to
|
||
sample exemplar traces rather than to assign quality scores.</p>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 30.0%"/>
|
||
<col style="width: 70.0%"/>
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Leaf signal type</p></th>
|
||
<th class="head"><p>Description</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">satisfaction.gratitude</span></code></p></td>
|
||
<td><p>Expressions of thanks and appreciation.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">satisfaction.confirmation</span></code></p></td>
|
||
<td><p>Explicit satisfaction expressions.</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">satisfaction.success</span></code></p></td>
|
||
<td><p>Confirmation of task completion or understanding.</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</section>
|
||
<section id="execution-agent-caused-action-quality">
|
||
<h3>Execution — agent-caused action quality<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#execution-agent-caused-action-quality" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#execution-agent-caused-action-quality'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p><strong>Failure</strong> — Detects agent-caused failures in tool/function usage. These
|
||
are issues the agent is responsible for (as opposed to environment failures
|
||
which are external system issues). Requires tool-call traces
|
||
(<code class="docutils literal notranslate"><span class="pre">function_call</span></code> / <code class="docutils literal notranslate"><span class="pre">observation</span></code>) to fire.</p>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 30.0%"/>
|
||
<col style="width: 70.0%"/>
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Leaf signal type</p></th>
|
||
<th class="head"><p>Description</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">execution.failure.invalid_args</span></code></p></td>
|
||
<td><p>Wrong type, missing required field.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">execution.failure.bad_query</span></code></p></td>
|
||
<td><p>Empty results due to overly narrow/wrong query.</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">execution.failure.tool_not_found</span></code></p></td>
|
||
<td><p>Agent called non-existent tool.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">execution.failure.auth_misuse</span></code></p></td>
|
||
<td><p>Agent didn’t pass credentials correctly.</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">execution.failure.state_error</span></code></p></td>
|
||
<td><p>Tool called in wrong state/order.</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p><strong>Loops</strong> — Detects behavioral patterns where the agent gets stuck
|
||
repeating tool calls. These are distinct from
|
||
<code class="docutils literal notranslate"><span class="pre">interaction.stagnation</span></code> (conversation text repetition) and
|
||
<code class="docutils literal notranslate"><span class="pre">execution.failure</span></code> (single tool errors) — these detect tool-level
|
||
behavioral loops.</p>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 30.0%"/>
|
||
<col style="width: 70.0%"/>
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Leaf signal type</p></th>
|
||
<th class="head"><p>Description</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">execution.loops.retry</span></code></p></td>
|
||
<td><p>Same tool with identical args ≥3 times.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">execution.loops.parameter_drift</span></code></p></td>
|
||
<td><p>Same tool with varied args ≥3 times.</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">execution.loops.oscillation</span></code></p></td>
|
||
<td><p>Multi-tool A→B→A→B pattern ≥3 cycles.</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</section>
|
||
<section id="environment-external-system-boundary-conditions">
|
||
<h3>Environment — external system / boundary conditions<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#environment-external-system-boundary-conditions" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#environment-external-system-boundary-conditions'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p><strong>Exhaustion</strong> — Detects failures and constraints arising from the
|
||
surrounding system rather than the agent’s internal policy or reasoning.
|
||
These are external issues the agent cannot control.</p>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 30.0%"/>
|
||
<col style="width: 70.0%"/>
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Leaf signal type</p></th>
|
||
<th class="head"><p>Description</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">environment.exhaustion.api_error</span></code></p></td>
|
||
<td><p>5xx errors, service unavailable.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">environment.exhaustion.timeout</span></code></p></td>
|
||
<td><p>Connection/read timeouts.</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">environment.exhaustion.rate_limit</span></code></p></td>
|
||
<td><p>429, quota exceeded.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">environment.exhaustion.network</span></code></p></td>
|
||
<td><p>Connection refused, DNS errors.</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">environment.exhaustion.malformed_response</span></code></p></td>
|
||
<td><p>Invalid JSON, unexpected schema.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">environment.exhaustion.context_overflow</span></code></p></td>
|
||
<td><p>Token/context limit exceeded.</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</section>
|
||
</section>
|
||
<section id="how-it-works">
|
||
<h2>How It Works<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#how-it-works" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#how-it-works'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Signals are computed automatically by the gateway after each assistant
|
||
response and emitted as <strong>OpenTelemetry trace attributes</strong> and <strong>span events</strong>
|
||
on your existing spans. No additional libraries or instrumentation are
|
||
required — just configure your OTEL collector endpoint as usual.</p>
|
||
<p>Each conversation trace is enriched with layered signal attributes
|
||
(category-level counts and severities) plus one span event per detected
|
||
signal instance (with confidence, snippet, and per-detector metadata).</p>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>Signal analysis is enabled by default and runs on the request path. It
|
||
does <strong>not</strong> affect the response sent to the client. Set
|
||
<code class="docutils literal notranslate"><span class="pre">overrides.disable_signals:</span> <span class="pre">true</span></code> in your Plano config to skip this
|
||
CPU-heavy analysis (see the configuration reference).</p>
|
||
</div>
|
||
</section>
|
||
<section id="otel-span-attributes">
|
||
<h2>OTel Span Attributes<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#otel-span-attributes" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#otel-span-attributes'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Signal data is exported as structured OTel attributes. There are two tiers:
|
||
<strong>top-level</strong> attributes (always emitted on spans that carry signal
|
||
analysis) and <strong>layered</strong> attributes (emitted only when the corresponding
|
||
category has at least one signal instance).</p>
|
||
<section id="top-level-attributes">
|
||
<h3>Top-level attributes<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#top-level-attributes" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#top-level-attributes'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>Always emitted once signals are computed.</p>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 40.0%"/>
|
||
<col style="width: 15.0%"/>
|
||
<col style="width: 45.0%"/>
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Attribute</p></th>
|
||
<th class="head"><p>Type</p></th>
|
||
<th class="head"><p>Value</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.quality</span></code></p></td>
|
||
<td><p>string</p></td>
|
||
<td><p>One of <code class="docutils literal notranslate"><span class="pre">excellent</span></code>, <code class="docutils literal notranslate"><span class="pre">good</span></code>, <code class="docutils literal notranslate"><span class="pre">neutral</span></code>, <code class="docutils literal notranslate"><span class="pre">poor</span></code>, <code class="docutils literal notranslate"><span class="pre">severe</span></code>.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signals.quality_score</span></code></p></td>
|
||
<td><p>float</p></td>
|
||
<td><p>Numeric score 0.0 – 100.0 that feeds the quality bucket.</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.turn_count</span></code></p></td>
|
||
<td><p>int</p></td>
|
||
<td><p>Total number of user + assistant turns in the interaction.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signals.efficiency_score</span></code></p></td>
|
||
<td><p>float</p></td>
|
||
<td><p>Efficiency metric 0.0 – 1.0 (stays at 1.0 up to baseline turns,
|
||
then decays: <code class="docutils literal notranslate"><span class="pre">1</span> <span class="pre">/</span> <span class="pre">(1</span> <span class="pre">+</span> <span class="pre">0.3</span> <span class="pre">*</span> <span class="pre">(turns</span> <span class="pre">-</span> <span class="pre">baseline))</span></code>).</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</section>
|
||
<section id="layered-attributes">
|
||
<h3>Layered attributes<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#layered-attributes" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#layered-attributes'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>Emitted per category, only when <code class="docutils literal notranslate"><span class="pre">count</span> <span class="pre">></span> <span class="pre">0</span></code>. One <code class="docutils literal notranslate"><span class="pre">.count</span></code> and one
|
||
<code class="docutils literal notranslate"><span class="pre">.severity</span></code> attribute per category. Severity is a 0–3 bucket (see
|
||
<a class="reference internal" href="#severity-levels">Severity levels</a> below).</p>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 50.0%"/>
|
||
<col style="width: 50.0%"/>
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Attribute (emitted when fired)</p></th>
|
||
<th class="head"><p>Source</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.interaction.misalignment.count</span></code></p></td>
|
||
<td><p>Any <code class="docutils literal notranslate"><span class="pre">misalignment.*</span></code> leaf type</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signals.interaction.misalignment.severity</span></code></p></td>
|
||
<td><p>“</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.interaction.stagnation.count</span></code></p></td>
|
||
<td><p>Any <code class="docutils literal notranslate"><span class="pre">stagnation.*</span></code> leaf type</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signals.interaction.stagnation.severity</span></code></p></td>
|
||
<td><p>“</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.interaction.disengagement.count</span></code></p></td>
|
||
<td><p>Any <code class="docutils literal notranslate"><span class="pre">disengagement.*</span></code> leaf type</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signals.interaction.disengagement.severity</span></code></p></td>
|
||
<td><p>“</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.interaction.satisfaction.count</span></code></p></td>
|
||
<td><p>Any <code class="docutils literal notranslate"><span class="pre">satisfaction.*</span></code> leaf type</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signals.interaction.satisfaction.severity</span></code></p></td>
|
||
<td><p>“</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.execution.failure.count</span></code></p></td>
|
||
<td><p>Any <code class="docutils literal notranslate"><span class="pre">failure.*</span></code> leaf type</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signals.execution.failure.severity</span></code></p></td>
|
||
<td><p>“</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.execution.loops.count</span></code></p></td>
|
||
<td><p>Any <code class="docutils literal notranslate"><span class="pre">loops.*</span></code> leaf type</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signals.execution.loops.severity</span></code></p></td>
|
||
<td><p>“</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.environment.exhaustion.count</span></code></p></td>
|
||
<td><p>Any <code class="docutils literal notranslate"><span class="pre">exhaustion.*</span></code> leaf type</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signals.environment.exhaustion.severity</span></code></p></td>
|
||
<td><p>“</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</section>
|
||
<section id="legacy-attributes-deprecated-still-emitted">
|
||
<h3>Legacy attributes (deprecated, still emitted)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#legacy-attributes-deprecated-still-emitted" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#legacy-attributes-deprecated-still-emitted'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>The following aggregate keys pre-date the paper taxonomy and are still
|
||
emitted for one release so existing dashboards keep working. They are
|
||
derived from the layered counts above and will be removed in a future
|
||
release. Migrate to the layered keys when convenient.</p>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 50.0%"/>
|
||
<col style="width: 50.0%"/>
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Legacy attribute</p></th>
|
||
<th class="head"><p>Layered equivalent</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.follow_up.repair.count</span></code></p></td>
|
||
<td><p><code class="docutils literal notranslate"><span class="pre">signals.interaction.misalignment.count</span></code></p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signals.follow_up.repair.ratio</span></code></p></td>
|
||
<td><p>(computed: <code class="docutils literal notranslate"><span class="pre">misalignment.count</span> <span class="pre">/</span> <span class="pre">max(user_turns,</span> <span class="pre">1)</span></code>)</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.frustration.count</span></code></p></td>
|
||
<td><p>Count of <code class="docutils literal notranslate"><span class="pre">disengagement.negative_stance</span></code> instances</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signals.frustration.severity</span></code></p></td>
|
||
<td><p>Derived severity bucket of the above</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.repetition.count</span></code></p></td>
|
||
<td><p><code class="docutils literal notranslate"><span class="pre">signals.interaction.stagnation.count</span></code></p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signals.escalation.requested</span></code></p></td>
|
||
<td><p>True if any <code class="docutils literal notranslate"><span class="pre">disengagement.escalation</span></code> or <code class="docutils literal notranslate"><span class="pre">disengagement.quit</span></code> fired</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signals.positive_feedback.count</span></code></p></td>
|
||
<td><p><code class="docutils literal notranslate"><span class="pre">signals.interaction.satisfaction.count</span></code></p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</section>
|
||
</section>
|
||
<section id="span-events">
|
||
<h2>Span Events<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#span-events" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#span-events'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>In addition to span attributes, every detected signal instance is emitted as
|
||
a span event named <code class="docutils literal notranslate"><span class="pre">signal.<dotted-type></span></code> (e.g.
|
||
<code class="docutils literal notranslate"><span class="pre">signal.interaction.satisfaction.gratitude</span></code>). Each event carries:</p>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 30.0%"/>
|
||
<col style="width: 15.0%"/>
|
||
<col style="width: 55.0%"/>
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Event attribute</p></th>
|
||
<th class="head"><p>Type</p></th>
|
||
<th class="head"><p>Description</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signal.type</span></code></p></td>
|
||
<td><p>string</p></td>
|
||
<td><p>Full dotted signal type (same as the event name suffix).</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signal.message_index</span></code></p></td>
|
||
<td><p>int</p></td>
|
||
<td><p>Zero-based index of the message that triggered the signal.</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signal.confidence</span></code></p></td>
|
||
<td><p>float</p></td>
|
||
<td><p>Detector confidence in [0.0, 1.0].</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">signal.snippet</span></code></p></td>
|
||
<td><p>string</p></td>
|
||
<td><p>Matched substring from the source message (when available).</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">signal.metadata</span></code></p></td>
|
||
<td><p>string (JSON)</p></td>
|
||
<td><p>Per-detector metadata (pattern name, ratio values, etc.).</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p>Span events are the right surface for drill-down: attribute filters narrow
|
||
traces, then events tell you <em>which messages</em> fired <em>which signals</em> with
|
||
<em>what evidence</em>.</p>
|
||
<section id="visual-flag-marker">
|
||
<h3>Visual Flag Marker<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#visual-flag-marker" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#visual-flag-marker'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>When concerning signals are detected (disengagement present, stagnation
|
||
count > 2, any execution failure / loop, or overall quality <code class="docutils literal notranslate"><span class="pre">poor</span></code>/
|
||
<code class="docutils literal notranslate"><span class="pre">severe</span></code>), the marker 🚩 (U+1F6A9) is appended to the span’s operation
|
||
name.
|
||
This makes flagged sessions immediately visible in trace UIs without
|
||
requiring attribute filtering.</p>
|
||
</section>
|
||
<section id="querying-in-your-observability-platform">
|
||
<h3>Querying in Your Observability Platform<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#querying-in-your-observability-platform" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#querying-in-your-observability-platform'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>Example queries against the layered keys:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="n">signals</span><span class="o">.</span><span class="n">quality</span> <span class="o">=</span> <span class="s2">"severe"</span>
|
||
</span><span id="line-2"><span class="n">signals</span><span class="o">.</span><span class="n">turn_count</span> <span class="o">></span> <span class="mi">10</span>
|
||
</span><span id="line-3"><span class="n">signals</span><span class="o">.</span><span class="n">efficiency_score</span> <span class="o"><</span> <span class="mf">0.5</span>
|
||
</span><span id="line-4"><span class="n">signals</span><span class="o">.</span><span class="n">interaction</span><span class="o">.</span><span class="n">disengagement</span><span class="o">.</span><span class="n">severity</span> <span class="o">>=</span> <span class="mi">2</span>
|
||
</span><span id="line-5"><span class="n">signals</span><span class="o">.</span><span class="n">interaction</span><span class="o">.</span><span class="n">misalignment</span><span class="o">.</span><span class="n">count</span> <span class="o">></span> <span class="mi">3</span>
|
||
</span><span id="line-6"><span class="n">signals</span><span class="o">.</span><span class="n">interaction</span><span class="o">.</span><span class="n">satisfaction</span><span class="o">.</span><span class="n">count</span> <span class="o">></span> <span class="mi">0</span> <span class="n">AND</span> <span class="n">signals</span><span class="o">.</span><span class="n">quality</span> <span class="o">=</span> <span class="s2">"good"</span>
|
||
</span><span id="line-7"><span class="n">signals</span><span class="o">.</span><span class="n">execution</span><span class="o">.</span><span class="n">failure</span><span class="o">.</span><span class="n">count</span> <span class="o">></span> <span class="mi">0</span>
|
||
</span><span id="line-8"><span class="n">signals</span><span class="o">.</span><span class="n">environment</span><span class="o">.</span><span class="n">exhaustion</span><span class="o">.</span><span class="n">count</span> <span class="o">></span> <span class="mi">0</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>For flagged sessions, search for 🚩 in span names.</p>
|
||
<a class="reference internal image-reference" href="../_images/signals_trace.png"><img alt="../_images/signals_trace.png" class="align-center" src="../_images/signals_trace.png" style="width: 100%;"/>
|
||
</a>
|
||
</section>
|
||
</section>
|
||
<section id="severity-levels">
|
||
<h2>Severity Levels<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#severity-levels" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#severity-levels'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Every category aggregates its leaf signal counts into a severity bucket used
|
||
by both the layered <code class="docutils literal notranslate"><span class="pre">.severity</span></code> attribute and the overall quality score.</p>
|
||
<ul class="simple">
|
||
<li><p><strong>None (0)</strong>: 0 instances</p></li>
|
||
<li><p><strong>Mild (1)</strong>: 1–2 instances</p></li>
|
||
<li><p><strong>Moderate (2)</strong>: 3–4 instances</p></li>
|
||
<li><p><strong>Severe (3)</strong>: 5+ instances</p></li>
|
||
</ul>
|
||
<p>Severity is always computed per-category. For example, three instances of
|
||
<code class="docutils literal notranslate"><span class="pre">misalignment.rephrase</span></code> plus two of <code class="docutils literal notranslate"><span class="pre">misalignment.correction</span></code> yield
|
||
<code class="docutils literal notranslate"><span class="pre">signals.interaction.misalignment.severity</span> <span class="pre">=</span> <span class="pre">3</span></code> (5 instances total).</p>
|
||
</section>
|
||
<section id="overall-quality-assessment">
|
||
<h2>Overall Quality Assessment<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#overall-quality-assessment" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#overall-quality-assessment'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Signals are aggregated into an overall interaction quality on a 5-point
|
||
scale. The scoring model starts at 50.0 (neutral), adds positive weight for
|
||
satisfaction, and subtracts weight for disengagement, misalignment (when
|
||
ratio > 30% of user turns), stagnation (when count > 2), execution failures,
|
||
execution loops, and environment exhaustion.</p>
|
||
<p>The resulting numeric score maps to the bucket emitted in <code class="docutils literal notranslate"><span class="pre">signals.quality</span></code>:</p>
|
||
<dl class="simple">
|
||
<dt><strong>Excellent (75 – 100)</strong></dt><dd><p>Strong positive signals, efficient resolution, low friction.</p>
|
||
</dd>
|
||
<dt><strong>Good (60 – 74)</strong></dt><dd><p>Mostly positive with minor clarifications; some back-and-forth but
|
||
successful.</p>
|
||
</dd>
|
||
<dt><strong>Neutral (40 – 59)</strong></dt><dd><p>Mixed signals; neither clearly good nor bad.</p>
|
||
</dd>
|
||
<dt><strong>Poor (25 – 39)</strong></dt><dd><p>Concerning negative patterns (high friction, multiple misalignments,
|
||
moderate disengagement, tool failures). High abandonment risk.</p>
|
||
</dd>
|
||
<dt><strong>Severe (0 – 24)</strong></dt><dd><p>Critical issues — escalation requested, severe disengagement, severe
|
||
stagnation, or compounding failures. Requires immediate attention.</p>
|
||
</dd>
|
||
</dl>
|
||
<p>The raw numeric score is available under <code class="docutils literal notranslate"><span class="pre">signals.quality_score</span></code>.</p>
|
||
</section>
|
||
<section id="sampling-and-prioritization">
|
||
<h2>Sampling and Prioritization<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#sampling-and-prioritization" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#sampling-and-prioritization'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>In production, trace data is overwhelming. Signals provide a lightweight
|
||
first layer of triage to select the small fraction of trajectories that are
|
||
most likely to be informative. Per the paper, signal-based sampling reaches
|
||
82% informativeness on τ-bench versus 54% for random sampling — a 1.52×
|
||
efficiency gain per informative trajectory.</p>
|
||
<p>Workflow:</p>
|
||
<ol class="arabic simple">
|
||
<li><p>Gateway captures conversation messages and computes signals</p></li>
|
||
<li><p>Signal attributes and per-instance events are emitted to OTEL spans</p></li>
|
||
<li><p>Your observability platform ingests and indexes the attributes</p></li>
|
||
<li><p>Query / filter by signal attributes to surface outliers and exemplars</p></li>
|
||
<li><p>Review high-information traces to identify improvement opportunities</p></li>
|
||
<li><p>Update prompts, routing, or policies based on findings</p></li>
|
||
<li><p>Redeploy and monitor signal metrics to validate improvements</p></li>
|
||
</ol>
|
||
<p>This creates a reinforcement loop where traces become both diagnostic data
|
||
and training signal for prompt engineering, routing policies, and
|
||
preference-data construction.</p>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>An in-gateway triage sampler that selects informative trajectories
|
||
inline — with configurable per-category weights and budgets — is planned
|
||
as a follow-up to this release. Today, sampling is consumer-side: your
|
||
observability platform filters on the signal attributes described above.</p>
|
||
</div>
|
||
</section>
|
||
<section id="example-span">
|
||
<h2>Example Span<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-span" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#example-span'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>A concerning session, showing both layered attributes and a per-instance
|
||
event:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Span name: "POST /v1/chat/completions gpt-5.2 🚩"</span>
|
||
</span><span id="line-2">
|
||
</span><span id="line-3"><span class="c1"># Top-level</span>
|
||
</span><span id="line-4"><span class="n">signals</span><span class="o">.</span><span class="n">quality</span> <span class="o">=</span> <span class="s2">"severe"</span>
|
||
</span><span id="line-5"><span class="n">signals</span><span class="o">.</span><span class="n">quality_score</span> <span class="o">=</span> <span class="mf">0.0</span>
|
||
</span><span id="line-6"><span class="n">signals</span><span class="o">.</span><span class="n">turn_count</span> <span class="o">=</span> <span class="mi">4</span>
|
||
</span><span id="line-7"><span class="n">signals</span><span class="o">.</span><span class="n">efficiency_score</span> <span class="o">=</span> <span class="mf">1.0</span>
|
||
</span><span id="line-8">
|
||
</span><span id="line-9"><span class="c1"># Layered (only non-zero categories are emitted)</span>
|
||
</span><span id="line-10"><span class="n">signals</span><span class="o">.</span><span class="n">interaction</span><span class="o">.</span><span class="n">disengagement</span><span class="o">.</span><span class="n">count</span> <span class="o">=</span> <span class="mi">6</span>
|
||
</span><span id="line-11"><span class="n">signals</span><span class="o">.</span><span class="n">interaction</span><span class="o">.</span><span class="n">disengagement</span><span class="o">.</span><span class="n">severity</span> <span class="o">=</span> <span class="mi">3</span>
|
||
</span><span id="line-12">
|
||
</span><span id="line-13"><span class="c1"># Legacy (deprecated, emitted while dual-emit is on)</span>
|
||
</span><span id="line-14"><span class="n">signals</span><span class="o">.</span><span class="n">frustration</span><span class="o">.</span><span class="n">count</span> <span class="o">=</span> <span class="mi">4</span>
|
||
</span><span id="line-15"><span class="n">signals</span><span class="o">.</span><span class="n">frustration</span><span class="o">.</span><span class="n">severity</span> <span class="o">=</span> <span class="mi">2</span>
|
||
</span><span id="line-16"><span class="n">signals</span><span class="o">.</span><span class="n">escalation</span><span class="o">.</span><span class="n">requested</span> <span class="o">=</span> <span class="n">true</span>
|
||
</span><span id="line-17">
|
||
</span><span id="line-18"><span class="c1"># Per-instance span events</span>
|
||
</span><span id="line-19"><span class="n">event</span><span class="p">:</span> <span class="n">signal</span><span class="o">.</span><span class="n">interaction</span><span class="o">.</span><span class="n">disengagement</span><span class="o">.</span><span class="n">escalation</span>
|
||
</span><span id="line-20"> <span class="n">signal</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="s2">"interaction.disengagement.escalation"</span>
|
||
</span><span id="line-21"> <span class="n">signal</span><span class="o">.</span><span class="n">message_index</span> <span class="o">=</span> <span class="mi">6</span>
|
||
</span><span id="line-22"> <span class="n">signal</span><span class="o">.</span><span class="n">confidence</span> <span class="o">=</span> <span class="mf">1.0</span>
|
||
</span><span id="line-23"> <span class="n">signal</span><span class="o">.</span><span class="n">snippet</span> <span class="o">=</span> <span class="s2">"get me a human"</span>
|
||
</span><span id="line-24"> <span class="n">signal</span><span class="o">.</span><span class="n">metadata</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"pattern_type"</span><span class="p">:</span><span class="s2">"escalation"</span><span class="p">}</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="building-dashboards">
|
||
<h2>Building Dashboards<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#building-dashboards" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#building-dashboards'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Use signal attributes to build monitoring dashboards in Grafana, Honeycomb,
|
||
Datadog, etc. Prefer the layered keys — they align with the paper taxonomy
|
||
and will outlive the legacy keys.</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Quality distribution</strong>: Count of traces by <code class="docutils literal notranslate"><span class="pre">signals.quality</span></code></p></li>
|
||
<li><p><strong>P95 turn count</strong>: 95th percentile of <code class="docutils literal notranslate"><span class="pre">signals.turn_count</span></code></p></li>
|
||
<li><p><strong>Average efficiency</strong>: Mean of <code class="docutils literal notranslate"><span class="pre">signals.efficiency_score</span></code></p></li>
|
||
<li><p><strong>High misalignment rate</strong>: Percentage where
|
||
<code class="docutils literal notranslate"><span class="pre">signals.interaction.misalignment.count</span> <span class="pre">></span> <span class="pre">3</span></code></p></li>
|
||
<li><p><strong>Disengagement rate</strong>: Percentage where
|
||
<code class="docutils literal notranslate"><span class="pre">signals.interaction.disengagement.severity</span> <span class="pre">>=</span> <span class="pre">2</span></code></p></li>
|
||
<li><p><strong>Satisfaction rate</strong>: Percentage where
|
||
<code class="docutils literal notranslate"><span class="pre">signals.interaction.satisfaction.count</span> <span class="pre">>=</span> <span class="pre">1</span></code></p></li>
|
||
<li><p><strong>Escalation rate</strong>: Percentage where a <code class="docutils literal notranslate"><span class="pre">disengagement.escalation</span></code> or
|
||
<code class="docutils literal notranslate"><span class="pre">disengagement.quit</span></code> event fired (via span-event filter)</p></li>
|
||
<li><p><strong>Tool-failure rate</strong>: Percentage where
|
||
<code class="docutils literal notranslate"><span class="pre">signals.execution.failure.count</span> <span class="pre">></span> <span class="pre">0</span></code></p></li>
|
||
<li><p><strong>Environment issue rate</strong>: Percentage where
|
||
<code class="docutils literal notranslate"><span class="pre">signals.environment.exhaustion.count</span> <span class="pre">></span> <span class="pre">0</span></code></p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="creating-alerts">
|
||
<h2>Creating Alerts<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#creating-alerts" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#creating-alerts'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Set up alerts based on signal thresholds:</p>
|
||
<ul class="simple">
|
||
<li><p>Alert when <code class="docutils literal notranslate"><span class="pre">signals.quality</span> <span class="pre">=</span> <span class="pre">"severe"</span></code> count exceeds threshold in a
|
||
1-hour window</p></li>
|
||
<li><p>Alert on sudden spike in
|
||
<code class="docutils literal notranslate"><span class="pre">signals.interaction.disengagement.severity</span> <span class="pre">>=</span> <span class="pre">2</span></code> (>2× baseline)</p></li>
|
||
<li><p>Alert on sustained <code class="docutils literal notranslate"><span class="pre">signals.execution.failure.count</span> <span class="pre">></span> <span class="pre">0</span></code> — agent-caused
|
||
tool issues</p></li>
|
||
<li><p>Alert on spikes in <code class="docutils literal notranslate"><span class="pre">signals.environment.exhaustion.count</span></code> — external
|
||
system degradation</p></li>
|
||
<li><p>Alert on degraded efficiency (P95 <code class="docutils literal notranslate"><span class="pre">signals.turn_count</span></code> up > 50%)</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="best-practices">
|
||
<h2>Best Practices<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#best-practices" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#best-practices'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Start simple:</p>
|
||
<ul class="simple">
|
||
<li><p>Alert or page on <code class="docutils literal notranslate"><span class="pre">severe</span></code> sessions (or on spikes in <code class="docutils literal notranslate"><span class="pre">severe</span></code> rate)</p></li>
|
||
<li><p>Review <code class="docutils literal notranslate"><span class="pre">poor</span></code> sessions within 24 hours</p></li>
|
||
<li><p>Sample <code class="docutils literal notranslate"><span class="pre">excellent</span></code> sessions as exemplars</p></li>
|
||
</ul>
|
||
<p>Combine multiple signals to infer failure modes:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Silent loop</strong>: <code class="docutils literal notranslate"><span class="pre">signals.interaction.stagnation.severity</span> <span class="pre">>=</span> <span class="pre">2</span></code> +
|
||
<code class="docutils literal notranslate"><span class="pre">signals.turn_count</span></code> above baseline</p></li>
|
||
<li><p><strong>User giving up</strong>: <code class="docutils literal notranslate"><span class="pre">signals.interaction.disengagement.severity</span> <span class="pre">>=</span> <span class="pre">2</span></code> +
|
||
any escalation event</p></li>
|
||
<li><p><strong>Misunderstood intent</strong>:
|
||
<code class="docutils literal notranslate"><span class="pre">signals.interaction.misalignment.count</span> <span class="pre">/</span> <span class="pre">user_turns</span> <span class="pre">></span> <span class="pre">0.3</span></code></p></li>
|
||
<li><p><strong>Agent-caused friction</strong>: <code class="docutils literal notranslate"><span class="pre">signals.execution.failure.count</span> <span class="pre">></span> <span class="pre">0</span></code> +
|
||
<code class="docutils literal notranslate"><span class="pre">signals.interaction.misalignment.count</span> <span class="pre">></span> <span class="pre">0</span></code></p></li>
|
||
<li><p><strong>External degradation, not agent fault</strong>:
|
||
<code class="docutils literal notranslate"><span class="pre">signals.environment.exhaustion.count</span> <span class="pre">></span> <span class="pre">0</span></code> while
|
||
<code class="docutils literal notranslate"><span class="pre">signals.execution.failure.count</span> <span class="pre">=</span> <span class="pre">0</span></code></p></li>
|
||
<li><p><strong>Working well</strong>: <code class="docutils literal notranslate"><span class="pre">signals.interaction.satisfaction.count</span> <span class="pre">>=</span> <span class="pre">1</span></code> +
|
||
<code class="docutils literal notranslate"><span class="pre">signals.efficiency_score</span> <span class="pre">></span> <span class="pre">0.8</span></code> + no disengagement</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="limitations-and-considerations">
|
||
<h2>Limitations and Considerations<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#limitations-and-considerations" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#limitations-and-considerations'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Signals don’t capture:</p>
|
||
<ul class="simple">
|
||
<li><p>Task completion / real outcomes</p></li>
|
||
<li><p>Factual or domain correctness</p></li>
|
||
<li><p>Silent abandonment (user leaves without expressing frustration)</p></li>
|
||
<li><p>Non-English nuance (pattern libraries are English-oriented)</p></li>
|
||
</ul>
|
||
<p>Mitigation strategies:</p>
|
||
<ul class="simple">
|
||
<li><p>Periodically sample flagged sessions and measure false positives / negatives</p></li>
|
||
<li><p>Tune baselines per use case and user population</p></li>
|
||
<li><p>Add domain-specific phrase libraries where needed</p></li>
|
||
<li><p>Combine signals with non-text metrics (tool failures, disconnects, latency)</p></li>
|
||
</ul>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>Behavioral signals complement — but do not replace — domain-specific
|
||
response quality evaluation. Use signals to prioritize which traces to
|
||
inspect, then apply domain expertise and outcome checks to diagnose root
|
||
causes.</p>
|
||
</div>
|
||
<div class="admonition tip">
|
||
<p class="admonition-title">Tip</p>
|
||
<p>The 🚩 marker in the span name provides instant visual feedback in
|
||
trace UIs, while the structured attributes (<code class="docutils literal notranslate"><span class="pre">signals.quality</span></code>,
|
||
<code class="docutils literal notranslate"><span class="pre">signals.interaction.disengagement.severity</span></code>, etc.) and per-instance
|
||
span events enable powerful querying and drill-down in your observability
|
||
platform.</p>
|
||
</div>
|
||
</section>
|
||
<section id="see-also">
|
||
<h2>See Also<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#see-also" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#see-also'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<ul class="simple">
|
||
<li><p><a class="reference external" href="https://arxiv.org/abs/2604.00356" rel="nofollow noopener">Signals: Trajectory Sampling and Triage for Agentic Interactions<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> — the paper this framework implements</p></li>
|
||
<li><p><a class="reference internal" href="../guides/observability/tracing.html"><span class="doc">Tracing</span></a> — Distributed tracing for agent
|
||
systems</p></li>
|
||
<li><p><a class="reference internal" href="../guides/observability/monitoring.html"><span class="doc">Monitoring</span></a> — Metrics and dashboards</p></li>
|
||
<li><p><a class="reference internal" href="../guides/observability/access_logging.html"><span class="doc">Access Logging</span></a> — Request / response logging</p></li>
|
||
<li><p><a class="reference internal" href="../guides/observability/observability.html"><span class="doc">Observability</span></a> — Complete observability guide</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||
<div class="mr-auto">
|
||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="prompt_target.html">
|
||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<polyline points="15 18 9 12 15 6"></polyline>
|
||
</svg>
|
||
Prompt Target
|
||
</a>
|
||
</div>
|
||
<div class="ml-auto">
|
||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../guides/orchestration.html">
|
||
Orchestration
|
||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<polyline points="9 18 15 12 9 6"></polyline>
|
||
</svg>
|
||
</a>
|
||
</div>
|
||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] h-full overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||
<ul>
|
||
<li><a :data-current="activeSection === '#why-signals-matter-the-improvement-flywheel'" class="reference internal" href="#why-signals-matter-the-improvement-flywheel">Why Signals Matter: The Improvement Flywheel</a></li>
|
||
<li><a :data-current="activeSection === '#what-are-behavioral-signals'" class="reference internal" href="#what-are-behavioral-signals">What Are Behavioral Signals?</a></li>
|
||
<li><a :data-current="activeSection === '#signal-taxonomy'" class="reference internal" href="#signal-taxonomy">Signal Taxonomy</a><ul>
|
||
<li><a :data-current="activeSection === '#interaction-user-agent-conversational-quality'" class="reference internal" href="#interaction-user-agent-conversational-quality">Interaction — user ↔ agent conversational quality</a></li>
|
||
<li><a :data-current="activeSection === '#execution-agent-caused-action-quality'" class="reference internal" href="#execution-agent-caused-action-quality">Execution — agent-caused action quality</a></li>
|
||
<li><a :data-current="activeSection === '#environment-external-system-boundary-conditions'" class="reference internal" href="#environment-external-system-boundary-conditions">Environment — external system / boundary conditions</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a :data-current="activeSection === '#how-it-works'" class="reference internal" href="#how-it-works">How It Works</a></li>
|
||
<li><a :data-current="activeSection === '#otel-span-attributes'" class="reference internal" href="#otel-span-attributes">OTel Span Attributes</a><ul>
|
||
<li><a :data-current="activeSection === '#top-level-attributes'" class="reference internal" href="#top-level-attributes">Top-level attributes</a></li>
|
||
<li><a :data-current="activeSection === '#layered-attributes'" class="reference internal" href="#layered-attributes">Layered attributes</a></li>
|
||
<li><a :data-current="activeSection === '#legacy-attributes-deprecated-still-emitted'" class="reference internal" href="#legacy-attributes-deprecated-still-emitted">Legacy attributes (deprecated, still emitted)</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a :data-current="activeSection === '#span-events'" class="reference internal" href="#span-events">Span Events</a><ul>
|
||
<li><a :data-current="activeSection === '#visual-flag-marker'" class="reference internal" href="#visual-flag-marker">Visual Flag Marker</a></li>
|
||
<li><a :data-current="activeSection === '#querying-in-your-observability-platform'" class="reference internal" href="#querying-in-your-observability-platform">Querying in Your Observability Platform</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a :data-current="activeSection === '#severity-levels'" class="reference internal" href="#severity-levels">Severity Levels</a></li>
|
||
<li><a :data-current="activeSection === '#overall-quality-assessment'" class="reference internal" href="#overall-quality-assessment">Overall Quality Assessment</a></li>
|
||
<li><a :data-current="activeSection === '#sampling-and-prioritization'" class="reference internal" href="#sampling-and-prioritization">Sampling and Prioritization</a></li>
|
||
<li><a :data-current="activeSection === '#example-span'" class="reference internal" href="#example-span">Example Span</a></li>
|
||
<li><a :data-current="activeSection === '#building-dashboards'" class="reference internal" href="#building-dashboards">Building Dashboards</a></li>
|
||
<li><a :data-current="activeSection === '#creating-alerts'" class="reference internal" href="#creating-alerts">Creating Alerts</a></li>
|
||
<li><a :data-current="activeSection === '#best-practices'" class="reference internal" href="#best-practices">Best Practices</a></li>
|
||
<li><a :data-current="activeSection === '#limitations-and-considerations'" class="reference internal" href="#limitations-and-considerations">Limitations and Considerations</a></li>
|
||
<li><a :data-current="activeSection === '#see-also'" class="reference internal" href="#see-also">See Also</a></li>
|
||
</ul>
|
||
</div>
|
||
</aside>
|
||
</main>
|
||
</div>
|
||
</div><footer class="py-6 border-t border-border md:py-0">
|
||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2026, Katanemo Labs, a DigitalOcean Company Last updated: Apr 24, 2026. </p>
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
</div>
|
||
<script src="../_static/documentation_options.js?v=6df19ec7"></script>
|
||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script defer="defer" src="../_static/theme.js?v=d6a9845b"></script>
|
||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||
<script async="async" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<script src="../_static/js/fix-copy.js?v=2f5cab98"></script>
|
||
</body>
|
||
</html> |