mirror of
https://github.com/katanemo/plano.git
synced 2026-04-25 00:36:34 +02:00
807 lines
No EOL
102 KiB
HTML
Executable file
807 lines
No EOL
102 KiB
HTML
Executable file
<!DOCTYPE html>
|
||
|
||
<html :class="{ 'dark' : darkMode === true }" data-content_root="../../" lang="en" x-data="{ darkMode: $persist(window.matchMedia('(prefers-color-scheme: dark)').matches), activeSection: ''}">
|
||
<head>
|
||
<script>
|
||
(function () {
|
||
// Set initial color scheme
|
||
if ((localStorage.getItem("_x_darkMode") === "true") || (window.matchMedia("(prefers-color-scheme: dark)").matches)) {
|
||
document.documentElement.classList.add("dark");
|
||
}
|
||
|
||
// Watch for media preference changes
|
||
window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change", (event) => {
|
||
localStorage.setItem("_x_darkMode", event.matches);
|
||
document.documentElement.classList.toggle("dark", event.matches);
|
||
});
|
||
})();
|
||
</script>
|
||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||
<meta charset="utf-8"/>
|
||
<meta content="#ffffff" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||
<meta content="#030711" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||
<title>Tracing | Plano Docs v0.4.14</title>
|
||
<meta content="Tracing | Plano Docs v0.4.14" property="og:title"/>
|
||
<meta content="Tracing | Plano Docs v0.4.14" name="twitter:title"/>
|
||
<link href="../../_static/pygments.css?v=73db4dac" rel="stylesheet" type="text/css"/>
|
||
<link href="../../_static/theme.css?v=73505e79" rel="stylesheet" type="text/css"/>
|
||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||
<link href="../../_static/awesome-sphinx-design.css?v=c54898a4" rel="stylesheet" type="text/css"/>
|
||
<link href="./docs/guides/observability/tracing.html" rel="canonical"/>
|
||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||
<link href="../../search.html" rel="search" title="Search"/>
|
||
<link href="monitoring.html" rel="next" title="Monitoring"/>
|
||
<link href="observability.html" rel="prev" title="Observability"/>
|
||
</head>
|
||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||
Skip to content
|
||
</a><header class="sticky top-0 z-40 w-full border-b shadow-xs border-border bg-background/90 backdrop-blur"><div class="container flex items-center h-14">
|
||
<div class="hidden mr-4 md:flex">
|
||
<a class="flex items-center mr-6" href="../../index.html">
|
||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4.14</span>
|
||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||
</svg>
|
||
<span class="sr-only">Toggle navigation menu</span>
|
||
</button>
|
||
<div class="flex items-center justify-between flex-1 gap-2 sm:gap-4 md:justify-end">
|
||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||
<span class="text-xs">⌘</span>
|
||
K
|
||
</kbd>
|
||
</form>
|
||
</div>
|
||
<nav class="flex items-center gap-1">
|
||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||
</div>
|
||
</a>
|
||
<button @click="darkMode = !darkMode" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" title="Toggle color scheme" type="button">
|
||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="16" viewbox="0 96 960 960" width="16" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||
</svg>
|
||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="16" viewbox="0 96 960 960" width="16" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||
</svg>
|
||
</button>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</header>
|
||
<div class="flex-1"><div class="container md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||
<a class="justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4.14</span>
|
||
</a>
|
||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0">
|
||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||
|
||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||
<script>
|
||
window.dataLayer = window.dataLayer || [];
|
||
function gtag(){dataLayer.push(arguments);}
|
||
gtag('js', new Date());
|
||
|
||
gtag('config', 'G-EH2VW19FXE');
|
||
</script>
|
||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/listeners.html">Listeners</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/agents.html">Agents</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/filter_chain.html">Filter Chains</a></li>
|
||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button" x-cloak=""><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-cloak="" x-show="expanded">
|
||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/prompt_target.html">Prompt Target</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/signals.html">Signals™</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||
<ul class="current">
|
||
<li class="toctree-l1"><a class="reference internal" href="../orchestration.html">Orchestration</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../llm_router.html">LLM Routing</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../function_calling.html">Function Calling</a></li>
|
||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button" x-cloak=""><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Tracing</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="monitoring.html">Monitoring</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="access_logging.html">Access Logging</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../prompt_guard.html">Guardrails</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../state.html">Conversational State</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||
<ul>
|
||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button" x-cloak=""><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-cloak="" x-show="expanded">
|
||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../resources/cli_reference.html">CLI Reference</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../resources/llms_txt.html">llms.txt</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||
</svg>
|
||
</button>
|
||
</aside>
|
||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||
<div class="w-full min-w-0 mx-auto">
|
||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||
<span class="hidden md:inline">Plano Docs v0.4.14</span>
|
||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||
</svg>
|
||
</a>
|
||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="observability.html">Observability</a>
|
||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Tracing</span>
|
||
</nav>
|
||
<div id="content" role="main">
|
||
<section id="tracing">
|
||
<span id="plano-overview-tracing"></span><h1>Tracing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#tracing"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||
<section id="overview">
|
||
<h2>Overview<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#overview" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#overview'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p><a class="reference external" href="https://opentelemetry.io/" rel="nofollow noopener">OpenTelemetry<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> is an open-source observability framework providing APIs
|
||
and instrumentation for generating, collecting, processing, and exporting telemetry data, such as traces,
|
||
metrics, and logs. Its flexible design supports a wide range of backends and seamlessly integrates with
|
||
modern application tools. A key feature of OpenTelemetry is its commitment to standards like the
|
||
<a class="reference external" href="https://www.w3.org/TR/trace-context/" rel="nofollow noopener">W3C Trace Context<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p>
|
||
<p><strong>Tracing</strong> is a critical tool that allows developers to visualize and understand the flow of
|
||
requests in an AI application. With tracing, you can capture a detailed view of how requests propagate
|
||
through various services and components, which is crucial for <strong>debugging</strong>, <strong>performance optimization</strong>,
|
||
and understanding complex AI agent architectures like Co-pilots.</p>
|
||
<p><strong>Plano</strong> propagates trace context using the W3C Trace Context standard, specifically through the
|
||
<code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header. This allows each component in the system to record its part of the request
|
||
flow, enabling <strong>end-to-end tracing</strong> across the entire application. By using OpenTelemetry, Plano ensures
|
||
that developers can capture this trace data consistently and in a format compatible with various observability
|
||
tools.</p>
|
||
<a class="reference internal image-reference" href="../../_images/tracing.png"><img alt="../../_images/tracing.png" class="align-center" src="../../_images/tracing.png" style="width: 100%;"/>
|
||
</a>
|
||
</section>
|
||
<section id="understanding-plano-traces">
|
||
<h2>Understanding Plano Traces<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#understanding-plano-traces" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#understanding-plano-traces'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Plano creates structured traces that capture the complete flow of requests through your AI system. Each trace consists of multiple spans representing different stages of processing.</p>
|
||
<section id="inbound-request-handling">
|
||
<h3>Inbound Request Handling<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#inbound-request-handling" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#inbound-request-handling'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>When a request enters Plano, it creates an <strong>inbound span</strong> (<code class="docutils literal notranslate"><span class="pre">plano(inbound)</span></code>) that represents the initial request reception and processing. This span captures:</p>
|
||
<ul class="simple">
|
||
<li><p>HTTP request details (method, path, headers)</p></li>
|
||
<li><p>Request payload size</p></li>
|
||
<li><p>Initial validation and authentication</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="orchestration-routing">
|
||
<h3>Orchestration & Routing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#orchestration-routing" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#orchestration-routing'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>For agent systems, Plano performs intelligent routing through orchestration spans:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Agent Orchestration</strong> (<code class="docutils literal notranslate"><span class="pre">plano(orchestrator)</span></code>): When multiple agents are available, Plano uses an LLM to analyze the user’s intent and select the most appropriate agent. This span captures the orchestration decision-making process.</p></li>
|
||
<li><p><strong>LLM Routing</strong> (<code class="docutils literal notranslate"><span class="pre">plano(routing)</span></code>): For direct LLM requests, Plano determines the optimal endpoint based on your routing strategy (round-robin, least-latency, cost-optimized). This span includes:</p>
|
||
<ul>
|
||
<li><p>Routing strategy used</p></li>
|
||
<li><p>Selected upstream endpoint</p></li>
|
||
<li><p>Route determination time</p></li>
|
||
<li><p>Fallback indicators (if applicable)</p></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</section>
|
||
<section id="agent-processing">
|
||
<h3>Agent Processing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#agent-processing" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#agent-processing'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>When requests are routed to agents, Plano creates spans for agent execution:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Agent Filter Chains</strong> (<code class="docutils literal notranslate"><span class="pre">plano(filter)</span></code>): If filters are configured (guardrails, context enrichment, query rewriting), each filter execution is captured in its own span, showing the transformation pipeline.</p></li>
|
||
<li><p><strong>Agent Execution</strong> (<code class="docutils literal notranslate"><span class="pre">plano(agent)</span></code>): The main agent processing span that captures the agent’s work, including any tools invoked and intermediate reasoning steps.</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="outbound-llm-calls">
|
||
<h3>Outbound LLM Calls<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#outbound-llm-calls" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#outbound-llm-calls'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>All LLM calls—whether from Plano’s routing layer or from agents—are traced with <strong>LLM spans</strong> (<code class="docutils literal notranslate"><span class="pre">plano(llm)</span></code>) that capture:</p>
|
||
<ul class="simple">
|
||
<li><p>Model name and provider (e.g., <code class="docutils literal notranslate"><span class="pre">gpt-4</span></code>, <code class="docutils literal notranslate"><span class="pre">claude-3-sonnet</span></code>)</p></li>
|
||
<li><p>Request parameters (temperature, max_tokens, top_p)</p></li>
|
||
<li><p>Token usage (prompt_tokens, completion_tokens)</p></li>
|
||
<li><p>Streaming indicators and time-to-first-token</p></li>
|
||
<li><p>Response metadata</p></li>
|
||
</ul>
|
||
<p><strong>Example Span Attributes</strong>:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># LLM call span</span>
|
||
</span><span id="line-2"><span class="n">llm</span><span class="o">.</span><span class="n">model</span> <span class="o">=</span> <span class="s2">"gpt-4"</span>
|
||
</span><span id="line-3"><span class="n">llm</span><span class="o">.</span><span class="n">provider</span> <span class="o">=</span> <span class="s2">"openai"</span>
|
||
</span><span id="line-4"><span class="n">llm</span><span class="o">.</span><span class="n">usage</span><span class="o">.</span><span class="n">prompt_tokens</span> <span class="o">=</span> <span class="mi">150</span>
|
||
</span><span id="line-5"><span class="n">llm</span><span class="o">.</span><span class="n">usage</span><span class="o">.</span><span class="n">completion_tokens</span> <span class="o">=</span> <span class="mi">75</span>
|
||
</span><span id="line-6"><span class="n">llm</span><span class="o">.</span><span class="n">duration_ms</span> <span class="o">=</span> <span class="mi">1250</span>
|
||
</span><span id="line-7"><span class="n">llm</span><span class="o">.</span><span class="n">time_to_first_token</span> <span class="o">=</span> <span class="mi">320</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="handoff-to-upstream-services">
|
||
<h3>Handoff to Upstream Services<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#handoff-to-upstream-services" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#handoff-to-upstream-services'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>When Plano forwards requests to upstream services (agents, APIs, or LLM providers), it creates <strong>handoff spans</strong> (<code class="docutils literal notranslate"><span class="pre">plano(handoff)</span></code>) that capture:</p>
|
||
<ul class="simple">
|
||
<li><p>Upstream endpoint URL</p></li>
|
||
<li><p>Request/response sizes</p></li>
|
||
<li><p>HTTP status codes</p></li>
|
||
<li><p>Upstream response times</p></li>
|
||
</ul>
|
||
<p>This creates a complete end-to-end trace showing the full request lifecycle through all system components.</p>
|
||
</section>
|
||
</section>
|
||
<section id="behavioral-signals-in-traces">
|
||
<h2>Behavioral Signals in Traces<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#behavioral-signals-in-traces" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#behavioral-signals-in-traces'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Plano automatically enriches OpenTelemetry traces with <a class="reference internal" href="../../concepts/signals.html"><span class="doc">Signals™</span></a> — behavioral quality indicators computed from conversation patterns. These signals are attached as span attributes, providing immediate visibility into interaction quality.</p>
|
||
<p><strong>What Signals Provide</strong></p>
|
||
<p>Signals act as early warning indicators embedded in your traces:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Quality Assessment</strong>: Overall interaction quality (Excellent/Good/Neutral/Poor/Severe)</p></li>
|
||
<li><p><strong>Efficiency Metrics</strong>: Turn count, efficiency scores, repair frequency</p></li>
|
||
<li><p><strong>User Sentiment</strong>: Frustration indicators, positive feedback, escalation requests</p></li>
|
||
<li><p><strong>Agent Behavior</strong>: Repetition detection, looping patterns</p></li>
|
||
</ul>
|
||
<p><strong>Visual Flag Markers</strong></p>
|
||
<p>When concerning signals are detected (frustration, looping, escalation, or poor/severe quality), Plano automatically appends a flag marker <strong>🚩</strong> to the span’s operation name. This makes problematic traces immediately visible in your tracing UI without requiring additional queries.</p>
|
||
<p><strong>Example Span with Signals</strong>:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Span name: "POST /v1/chat/completions gpt-4 🚩"</span>
|
||
</span><span id="line-2"><span class="c1"># Standard LLM attributes:</span>
|
||
</span><span id="line-3"><span class="n">llm</span><span class="o">.</span><span class="n">model</span> <span class="o">=</span> <span class="s2">"gpt-4"</span>
|
||
</span><span id="line-4"><span class="n">llm</span><span class="o">.</span><span class="n">usage</span><span class="o">.</span><span class="n">total_tokens</span> <span class="o">=</span> <span class="mi">225</span>
|
||
</span><span id="line-5">
|
||
</span><span id="line-6"><span class="c1"># Behavioral signal attributes:</span>
|
||
</span><span id="line-7"><span class="n">signals</span><span class="o">.</span><span class="n">quality</span> <span class="o">=</span> <span class="s2">"Severe"</span>
|
||
</span><span id="line-8"><span class="n">signals</span><span class="o">.</span><span class="n">turn_count</span> <span class="o">=</span> <span class="mi">15</span>
|
||
</span><span id="line-9"><span class="n">signals</span><span class="o">.</span><span class="n">efficiency_score</span> <span class="o">=</span> <span class="mf">0.234</span>
|
||
</span><span id="line-10"><span class="n">signals</span><span class="o">.</span><span class="n">frustration</span><span class="o">.</span><span class="n">severity</span> <span class="o">=</span> <span class="mi">3</span>
|
||
</span><span id="line-11"><span class="n">signals</span><span class="o">.</span><span class="n">escalation</span><span class="o">.</span><span class="n">requested</span> <span class="o">=</span> <span class="s2">"true"</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p><strong>Querying Signal Data</strong></p>
|
||
<p>In your observability platform (Jaeger, Grafana Tempo, Datadog, etc.), filter traces by signal attributes:</p>
|
||
<ul class="simple">
|
||
<li><p>Find severe interactions: <code class="docutils literal notranslate"><span class="pre">signals.quality</span> <span class="pre">=</span> <span class="pre">"Severe"</span></code></p></li>
|
||
<li><p>Find frustrated users: <code class="docutils literal notranslate"><span class="pre">signals.frustration.severity</span> <span class="pre">>=</span> <span class="pre">2</span></code></p></li>
|
||
<li><p>Find inefficient flows: <code class="docutils literal notranslate"><span class="pre">signals.efficiency_score</span> <span class="pre"><</span> <span class="pre">0.5</span></code></p></li>
|
||
<li><p>Find escalations: <code class="docutils literal notranslate"><span class="pre">signals.escalation.requested</span> <span class="pre">=</span> <span class="pre">"true"</span></code></p></li>
|
||
</ul>
|
||
<p>For complete details on all available signals, detection methods, and best practices, see the <a class="reference internal" href="../../concepts/signals.html"><span class="doc">Signals™</span></a> guide.</p>
|
||
</section>
|
||
<section id="custom-span-attributes">
|
||
<h2>Custom Span Attributes<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#custom-span-attributes" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#custom-span-attributes'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Plano can automatically attach <strong>custom span attributes</strong> derived from request headers and <strong>static</strong> attributes
|
||
defined in configuration. This lets you stamp
|
||
traces with identifiers like workspace, tenant, or user IDs without changing application code or adding
|
||
custom instrumentation.</p>
|
||
<p><strong>Why This Is Useful</strong></p>
|
||
<ul class="simple">
|
||
<li><p><strong>Tenant-aware debugging</strong>: Filter traces by <code class="docutils literal notranslate"><span class="pre">workspace.id</span></code> or <code class="docutils literal notranslate"><span class="pre">tenant.id</span></code>.</p></li>
|
||
<li><p><strong>Customer-specific visibility</strong>: Attribute performance or errors to a specific customer.</p></li>
|
||
<li><p><strong>Low overhead</strong>: No code changes in agents or clients—just headers.</p></li>
|
||
</ul>
|
||
<section id="how-it-works">
|
||
<h3>How It Works<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#how-it-works" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#how-it-works'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>You configure one or more header prefixes. Any incoming HTTP header whose name starts with one of these
|
||
prefixes is captured as a span attribute. You can also provide static attributes that are always injected.</p>
|
||
<ul class="simple">
|
||
<li><p>The <strong>prefix is only for matching</strong>, not the resulting attribute key.</p></li>
|
||
<li><p>The attribute key is the header name <strong>with the prefix removed</strong>, then hyphens converted to dots.</p></li>
|
||
</ul>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>Custom span attributes are attached to LLM spans when handling <code class="docutils literal notranslate"><span class="pre">/v1/...</span></code> requests via <code class="docutils literal notranslate"><span class="pre">llm_chat</span></code>. For orchestrator requests to <code class="docutils literal notranslate"><span class="pre">/agents/...</span></code>,
|
||
these attributes are added to both the orchestrator selection span and to each agent span created by <code class="docutils literal notranslate"><span class="pre">agent_chat</span></code>.</p>
|
||
</div>
|
||
<p><strong>Example</strong></p>
|
||
<p>Configured prefix:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="n">tracing</span><span class="p">:</span>
|
||
</span><span id="line-2"> <span class="n">span_attributes</span><span class="p">:</span>
|
||
</span><span id="line-3"> <span class="n">header_prefixes</span><span class="p">:</span>
|
||
</span><span id="line-4"> <span class="o">-</span> <span class="n">x</span><span class="o">-</span><span class="n">katanemo</span><span class="o">-</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>Incoming headers:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="n">X</span><span class="o">-</span><span class="n">Katanemo</span><span class="o">-</span><span class="n">Workspace</span><span class="o">-</span><span class="n">Id</span><span class="p">:</span> <span class="n">ws_123</span>
|
||
</span><span id="line-2"><span class="n">X</span><span class="o">-</span><span class="n">Katanemo</span><span class="o">-</span><span class="n">Tenant</span><span class="o">-</span><span class="n">Id</span><span class="p">:</span> <span class="n">ten_456</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>Resulting span attributes:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="n">workspace</span><span class="o">.</span><span class="n">id</span> <span class="o">=</span> <span class="s2">"ws_123"</span>
|
||
</span><span id="line-2"><span class="n">tenant</span><span class="o">.</span><span class="n">id</span> <span class="o">=</span> <span class="s2">"ten_456"</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="configuration">
|
||
<h3>Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configuration" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configuration'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>Add the prefix list under <code class="docutils literal notranslate"><span class="pre">tracing</span></code> in your config:</p>
|
||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">tracing</span><span class="p">:</span>
|
||
</span><span id="line-2"><span class="w"> </span><span class="nt">random_sampling</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">100</span>
|
||
</span><span id="line-3"><span class="w"> </span><span class="nt">span_attributes</span><span class="p">:</span>
|
||
</span><span id="line-4"><span class="w"> </span><span class="nt">header_prefixes</span><span class="p">:</span>
|
||
</span><span id="line-5"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">x-katanemo-</span>
|
||
</span><span id="line-6"><span class="w"> </span><span class="nt">static</span><span class="p">:</span>
|
||
</span><span id="line-7"><span class="w"> </span><span class="nt">environment</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">production</span>
|
||
</span><span id="line-8"><span class="w"> </span><span class="nt">service.version</span><span class="p">:</span><span class="w"> </span><span class="s">"1.0.0"</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>Static attributes are always injected alongside any header-derived attributes. If a header-derived
|
||
attribute key matches a static key, the header value overrides the static value.</p>
|
||
<p>You can provide multiple prefixes:</p>
|
||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">tracing</span><span class="p">:</span>
|
||
</span><span id="line-2"><span class="w"> </span><span class="nt">span_attributes</span><span class="p">:</span>
|
||
</span><span id="line-3"><span class="w"> </span><span class="nt">header_prefixes</span><span class="p">:</span>
|
||
</span><span id="line-4"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">x-katanemo-</span>
|
||
</span><span id="line-5"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">x-tenant-</span>
|
||
</span><span id="line-6"><span class="w"> </span><span class="nt">static</span><span class="p">:</span>
|
||
</span><span id="line-7"><span class="w"> </span><span class="nt">environment</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">production</span>
|
||
</span><span id="line-8"><span class="w"> </span><span class="nt">service.version</span><span class="p">:</span><span class="w"> </span><span class="s">"1.0.0"</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="notes-and-examples">
|
||
<h3>Notes and Examples<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#notes-and-examples" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#notes-and-examples'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<ul class="simple">
|
||
<li><p><strong>Prefix must match exactly</strong>: <code class="docutils literal notranslate"><span class="pre">katanemo-</span></code> does not match <code class="docutils literal notranslate"><span class="pre">x-katanemo-</span></code> headers.</p></li>
|
||
<li><p><strong>Trailing dash is recommended</strong>: Without it, <code class="docutils literal notranslate"><span class="pre">x-katanemo</span></code> would also match <code class="docutils literal notranslate"><span class="pre">x-katanemo-foo</span></code> and
|
||
<code class="docutils literal notranslate"><span class="pre">x-katanemofoo</span></code>.</p></li>
|
||
<li><p><strong>Keys are always strings</strong>: Values are captured as string attributes.</p></li>
|
||
</ul>
|
||
<p><strong>Prefix mismatch example</strong></p>
|
||
<p>Config:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="n">tracing</span><span class="p">:</span>
|
||
</span><span id="line-2"> <span class="n">span_attributes</span><span class="p">:</span>
|
||
</span><span id="line-3"> <span class="n">header_prefixes</span><span class="p">:</span>
|
||
</span><span id="line-4"> <span class="o">-</span> <span class="n">x</span><span class="o">-</span><span class="n">katanemo</span><span class="o">-</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>Request headers:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="n">X</span><span class="o">-</span><span class="n">Other</span><span class="o">-</span><span class="n">User</span><span class="o">-</span><span class="n">Id</span><span class="p">:</span> <span class="n">usr_999</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>Result: no attributes are captured from <code class="docutils literal notranslate"><span class="pre">X-Other-User-Id</span></code>.</p>
|
||
</section>
|
||
</section>
|
||
<section id="benefits-of-using-traceparent-headers">
|
||
<h2>Benefits of Using <code class="docutils literal notranslate"><span class="pre">Traceparent</span></code> Headers<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#benefits-of-using-traceparent-headers" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#benefits-of-using-traceparent-headers'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<ul class="simple">
|
||
<li><p><strong>Standardization</strong>: The W3C Trace Context standard ensures compatibility across ecosystem tools, allowing
|
||
traces to be propagated uniformly through different layers of the system.</p></li>
|
||
<li><p><strong>Ease of Integration</strong>: OpenTelemetry’s design allows developers to easily integrate tracing with minimal
|
||
changes to their codebase, enabling quick adoption of end-to-end observability.</p></li>
|
||
<li><p><strong>Interoperability</strong>: Works seamlessly with popular tracing tools like AWS X-Ray, Datadog, Jaeger, and many others,
|
||
making it easy to visualize traces in the tools you’re already usi</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="how-to-initiate-a-trace">
|
||
<h2>How to Initiate A Trace<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#how-to-initiate-a-trace" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#how-to-initiate-a-trace'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>Enable Tracing Configuration</strong>: Simply add the <code class="docutils literal notranslate"><span class="pre">random_sampling</span></code> in <code class="docutils literal notranslate"><span class="pre">tracing</span></code> section to 100`` flag to in the <a class="reference internal" href="../../concepts/listeners.html#plano-overview-listeners"><span class="std std-ref">listener</span></a> config</p></li>
|
||
<li><p><strong>Trace Context Propagation</strong>: Plano automatically propagates the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header. When a request is received, Plano will:</p>
|
||
<ul class="simple">
|
||
<li><p>Generate a new <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header if one is not present.</p></li>
|
||
<li><p>Extract the trace context from the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header if it exists.</p></li>
|
||
<li><p>Start a new span representing its processing of the request.</p></li>
|
||
<li><p>Forward the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header to downstream services.</p></li>
|
||
</ul>
|
||
</li>
|
||
<li><p><strong>Sampling Policy</strong>: The 100 in <code class="docutils literal notranslate"><span class="pre">random_sampling:</span> <span class="pre">100</span></code> means that all the requests as sampled for tracing.
|
||
You can adjust this value from 0-100.</p></li>
|
||
</ol>
|
||
</section>
|
||
<section id="tracing-with-the-cli">
|
||
<h2>Tracing with the CLI<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#tracing-with-the-cli" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#tracing-with-the-cli'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>The Plano CLI ships with a local OTLP/gRPC listener and a trace viewer so you can inspect spans without wiring a full observability backend. This is ideal for development, debugging, and quick QA.</p>
|
||
<section id="quick-start">
|
||
<h3>Quick Start<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#quick-start" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#quick-start'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>You can enable tracing in either of these ways:</p>
|
||
<ol class="arabic simple">
|
||
<li><p>Start the local listener explicitly:</p></li>
|
||
</ol>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>planoai<span class="w"> </span>trace<span class="w"> </span>listen
|
||
</span></code></pre></div>
|
||
</div>
|
||
<ol class="arabic simple" start="2">
|
||
<li><p>Or start Plano with tracing enabled (auto-starts the local OTLP listener):</p></li>
|
||
</ol>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>planoai<span class="w"> </span>up<span class="w"> </span>--with-tracing
|
||
</span><span id="line-2">
|
||
</span><span id="line-3"><span class="gp"># </span>Optional:<span class="w"> </span>choose<span class="w"> </span>a<span class="w"> </span>different<span class="w"> </span>listener<span class="w"> </span>port
|
||
</span><span id="line-4"><span class="gp">$ </span>planoai<span class="w"> </span>up<span class="w"> </span>--with-tracing<span class="w"> </span>--tracing-port<span class="w"> </span><span class="m">4318</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
<ol class="arabic simple" start="3">
|
||
<li><p>Send requests through Plano as usual. The listener accepts OTLP/gRPC on:</p>
|
||
<ul class="simple">
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">0.0.0.0:4317</span></code> (default)</p></li>
|
||
</ul>
|
||
</li>
|
||
<li><p>View the most recent trace:</p></li>
|
||
</ol>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>planoai<span class="w"> </span>trace
|
||
</span></code></pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="inspect-and-filter-traces">
|
||
<h3>Inspect and Filter Traces<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#inspect-and-filter-traces" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#inspect-and-filter-traces'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>List available trace IDs:</p>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>planoai<span class="w"> </span>trace<span class="w"> </span>--list
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>Open a specific trace (full or short trace ID):</p>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>planoai<span class="w"> </span>trace<span class="w"> </span>7f4e9a1c
|
||
</span><span id="line-2"><span class="gp">$ </span>planoai<span class="w"> </span>trace<span class="w"> </span>7f4e9a1c0d9d4a0bb9bf5a8a7d13f62a
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>Filter by attributes and time window:</p>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>planoai<span class="w"> </span>trace<span class="w"> </span>--where<span class="w"> </span>llm.model<span class="o">=</span>gpt-4o-mini<span class="w"> </span>--since<span class="w"> </span>30m
|
||
</span><span id="line-2"><span class="gp">$ </span>planoai<span class="w"> </span>trace<span class="w"> </span>--filter<span class="w"> </span><span class="s2">"http.*"</span><span class="w"> </span>--limit<span class="w"> </span><span class="m">5</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>Return JSON for automation:</p>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>planoai<span class="w"> </span>trace<span class="w"> </span>--json
|
||
</span><span id="line-2"><span class="gp">$ </span>planoai<span class="w"> </span>trace<span class="w"> </span>--list<span class="w"> </span>--json
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>Show full span attributes (disable default compact view):</p>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>planoai<span class="w"> </span>trace<span class="w"> </span>--verbose
|
||
</span><span id="line-2"><span class="gp">$ </span>planoai<span class="w"> </span>trace<span class="w"> </span>-v
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>Point the CLI at a different local listener port:</p>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span><span class="nb">export</span><span class="w"> </span><span class="nv">PLANO_TRACE_PORT</span><span class="o">=</span><span class="m">50051</span>
|
||
</span><span id="line-2"><span class="gp">$ </span>planoai<span class="w"> </span>trace<span class="w"> </span>--list
|
||
</span></code></pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="notes">
|
||
<h3>Notes<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#notes" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#notes'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<ul class="simple">
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">--where</span></code> accepts repeatable <code class="docutils literal notranslate"><span class="pre">key=value</span></code> filters and uses AND semantics.</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">--filter</span></code> supports wildcards (<code class="docutils literal notranslate"><span class="pre">*</span></code>) to limit displayed attributes.</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">--no-interactive</span></code> disables prompts when listing traces.</p></li>
|
||
<li><p>By default, inbound/outbound spans use a compact attribute view.</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="trace-propagation">
|
||
<h2>Trace Propagation<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#trace-propagation" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#trace-propagation'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Plano uses the W3C Trace Context standard for trace propagation, which relies on the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header.
|
||
This header carries tracing information in a standardized format, enabling interoperability between different
|
||
tracing systems.</p>
|
||
<section id="header-format">
|
||
<h3>Header Format<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#header-format" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#header-format'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header has the following format:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="n">traceparent</span><span class="p">:</span> <span class="p">{</span><span class="n">version</span><span class="p">}</span><span class="o">-</span><span class="p">{</span><span class="n">trace</span><span class="o">-</span><span class="nb">id</span><span class="p">}</span><span class="o">-</span><span class="p">{</span><span class="n">parent</span><span class="o">-</span><span class="nb">id</span><span class="p">}</span><span class="o">-</span><span class="p">{</span><span class="n">trace</span><span class="o">-</span><span class="n">flags</span><span class="p">}</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
<ul class="simple">
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">{version}</span></code>: The version of the Trace Context specification (e.g., <code class="docutils literal notranslate"><span class="pre">00</span></code>).</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">{trace-id}</span></code>: A 16-byte (32-character hexadecimal) unique identifier for the trace.</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">{parent-id}</span></code>: An 8-byte (16-character hexadecimal) identifier for the parent span.</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">{trace-flags}</span></code>: Flags indicating trace options (e.g., sampling).</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="instrumentation">
|
||
<h3>Instrumentation<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#instrumentation" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#instrumentation'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>To integrate AI tracing, your application needs to follow a few simple steps. The steps
|
||
below are very common practice, and not unique to Plano, when you reading tracing headers and export
|
||
<a class="reference external" href="https://docs.lightstep.com/docs/understand-distributed-tracing" rel="nofollow noopener">spans<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> for distributed tracing.</p>
|
||
<ul class="simple">
|
||
<li><p>Read the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header from incoming requests.</p></li>
|
||
<li><p>Start new spans as children of the extracted context.</p></li>
|
||
<li><p>Include the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header in outbound requests to propagate trace context.</p></li>
|
||
<li><p>Send tracing data to a collector or tracing backend to export spans</p></li>
|
||
</ul>
|
||
<section id="example-with-opentelemetry-in-python">
|
||
<h4>Example with OpenTelemetry in Python<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-with-opentelemetry-in-python"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||
<p>Install OpenTelemetry packages:</p>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>pip<span class="w"> </span>install<span class="w"> </span>opentelemetry-api<span class="w"> </span>opentelemetry-sdk<span class="w"> </span>opentelemetry-exporter-otlp
|
||
</span><span id="line-2"><span class="gp">$ </span>pip<span class="w"> </span>install<span class="w"> </span>opentelemetry-instrumentation-requests
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>Set up the tracer and exporter:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">from</span><span class="w"> </span><span class="nn">opentelemetry</span><span class="w"> </span><span class="kn">import</span> <span class="n">trace</span>
|
||
</span><span id="line-2"><span class="kn">from</span><span class="w"> </span><span class="nn">opentelemetry.exporter.otlp.proto.grpc.trace_exporter</span><span class="w"> </span><span class="kn">import</span> <span class="n">OTLPSpanExporter</span>
|
||
</span><span id="line-3"><span class="kn">from</span><span class="w"> </span><span class="nn">opentelemetry.instrumentation.requests</span><span class="w"> </span><span class="kn">import</span> <span class="n">RequestsInstrumentor</span>
|
||
</span><span id="line-4"><span class="kn">from</span><span class="w"> </span><span class="nn">opentelemetry.sdk.resources</span><span class="w"> </span><span class="kn">import</span> <span class="n">Resource</span>
|
||
</span><span id="line-5"><span class="kn">from</span><span class="w"> </span><span class="nn">opentelemetry.sdk.trace</span><span class="w"> </span><span class="kn">import</span> <span class="n">TracerProvider</span>
|
||
</span><span id="line-6"><span class="kn">from</span><span class="w"> </span><span class="nn">opentelemetry.sdk.trace.export</span><span class="w"> </span><span class="kn">import</span> <span class="n">BatchSpanProcessor</span>
|
||
</span><span id="line-7">
|
||
</span><span id="line-8"><span class="c1"># Define the service name</span>
|
||
</span><span id="line-9"><span class="n">resource</span> <span class="o">=</span> <span class="n">Resource</span><span class="p">(</span><span class="n">attributes</span><span class="o">=</span><span class="p">{</span>
|
||
</span><span id="line-10"> <span class="s2">"service.name"</span><span class="p">:</span> <span class="s2">"customer-support-agent"</span>
|
||
</span><span id="line-11"><span class="p">})</span>
|
||
</span><span id="line-12">
|
||
</span><span id="line-13"><span class="c1"># Set up the tracer provider and exporter</span>
|
||
</span><span id="line-14"><span class="n">tracer_provider</span> <span class="o">=</span> <span class="n">TracerProvider</span><span class="p">(</span><span class="n">resource</span><span class="o">=</span><span class="n">resource</span><span class="p">)</span>
|
||
</span><span id="line-15"><span class="n">otlp_exporter</span> <span class="o">=</span> <span class="n">OTLPSpanExporter</span><span class="p">(</span><span class="n">endpoint</span><span class="o">=</span><span class="s2">"otel-collector:4317"</span><span class="p">,</span> <span class="n">insecure</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
</span><span id="line-16"><span class="n">span_processor</span> <span class="o">=</span> <span class="n">BatchSpanProcessor</span><span class="p">(</span><span class="n">otlp_exporter</span><span class="p">)</span>
|
||
</span><span id="line-17"><span class="n">tracer_provider</span><span class="o">.</span><span class="n">add_span_processor</span><span class="p">(</span><span class="n">span_processor</span><span class="p">)</span>
|
||
</span><span id="line-18"><span class="n">trace</span><span class="o">.</span><span class="n">set_tracer_provider</span><span class="p">(</span><span class="n">tracer_provider</span><span class="p">)</span>
|
||
</span><span id="line-19">
|
||
</span><span id="line-20"><span class="c1"># Instrument HTTP requests</span>
|
||
</span><span id="line-21"><span class="n">RequestsInstrumentor</span><span class="p">()</span><span class="o">.</span><span class="n">instrument</span><span class="p">()</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
<p>Handle incoming requests:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">from</span><span class="w"> </span><span class="nn">opentelemetry</span><span class="w"> </span><span class="kn">import</span> <span class="n">trace</span>
|
||
</span><span id="line-2"><span class="kn">from</span><span class="w"> </span><span class="nn">opentelemetry.propagate</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract</span><span class="p">,</span> <span class="n">inject</span>
|
||
</span><span id="line-3"><span class="kn">import</span><span class="w"> </span><span class="nn">requests</span>
|
||
</span><span id="line-4">
|
||
</span><span id="line-5"><span class="k">def</span><span class="w"> </span><span class="nf">handle_request</span><span class="p">(</span><span class="n">request</span><span class="p">):</span>
|
||
</span><span id="line-6"> <span class="c1"># Extract the trace context</span>
|
||
</span><span id="line-7"> <span class="n">context</span> <span class="o">=</span> <span class="n">extract</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span>
|
||
</span><span id="line-8"> <span class="n">tracer</span> <span class="o">=</span> <span class="n">trace</span><span class="o">.</span><span class="n">get_tracer</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
|
||
</span><span id="line-9">
|
||
</span><span id="line-10"> <span class="k">with</span> <span class="n">tracer</span><span class="o">.</span><span class="n">start_as_current_span</span><span class="p">(</span><span class="s2">"process_customer_request"</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">):</span>
|
||
</span><span id="line-11"> <span class="c1"># Example of processing a customer request</span>
|
||
</span><span id="line-12"> <span class="nb">print</span><span class="p">(</span><span class="s2">"Processing customer request..."</span><span class="p">)</span>
|
||
</span><span id="line-13">
|
||
</span><span id="line-14"> <span class="c1"># Prepare headers for outgoing request to payment service</span>
|
||
</span><span id="line-15"> <span class="n">headers</span> <span class="o">=</span> <span class="p">{}</span>
|
||
</span><span id="line-16"> <span class="n">inject</span><span class="p">(</span><span class="n">headers</span><span class="p">)</span>
|
||
</span><span id="line-17">
|
||
</span><span id="line-18"> <span class="c1"># Make outgoing request to external service (e.g., payment gateway)</span>
|
||
</span><span id="line-19"> <span class="n">response</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"http://payment-service/api"</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
|
||
</span><span id="line-20">
|
||
</span><span id="line-21"> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Payment service response: </span><span class="si">{</span><span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
</section>
|
||
<section id="integrating-with-tracing-tools">
|
||
<h2>Integrating with Tracing Tools<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#integrating-with-tracing-tools" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#integrating-with-tracing-tools'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<section id="aws-x-ray">
|
||
<h3>AWS X-Ray<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#aws-x-ray" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#aws-x-ray'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>To send tracing data to <a class="reference external" href="https://aws.amazon.com/xray/" rel="nofollow noopener">AWS X-Ray<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> :</p>
|
||
<ol class="arabic">
|
||
<li><p><strong>Configure OpenTelemetry Collector</strong>: Set up the collector to export traces to AWS X-Ray.</p>
|
||
<p>Collector configuration (<code class="docutils literal notranslate"><span class="pre">otel-collector-config.yaml</span></code>):</p>
|
||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">receivers</span><span class="p">:</span>
|
||
</span><span id="line-2"><span class="w"> </span><span class="nt">otlp</span><span class="p">:</span>
|
||
</span><span id="line-3"><span class="w"> </span><span class="nt">protocols</span><span class="p">:</span>
|
||
</span><span id="line-4"><span class="w"> </span><span class="nt">grpc</span><span class="p">:</span>
|
||
</span><span id="line-5">
|
||
</span><span id="line-6"><span class="nt">processors</span><span class="p">:</span>
|
||
</span><span id="line-7"><span class="w"> </span><span class="nt">batch</span><span class="p">:</span>
|
||
</span><span id="line-8">
|
||
</span><span id="line-9"><span class="nt">exporters</span><span class="p">:</span>
|
||
</span><span id="line-10"><span class="w"> </span><span class="nt">awsxray</span><span class="p">:</span>
|
||
</span><span id="line-11"><span class="w"> </span><span class="nt">region</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain"><Your-Aws-Region></span>
|
||
</span><span id="line-12">
|
||
</span><span id="line-13"><span class="nt">service</span><span class="p">:</span>
|
||
</span><span id="line-14"><span class="w"> </span><span class="nt">pipelines</span><span class="p">:</span>
|
||
</span><span id="line-15"><span class="w"> </span><span class="nt">traces</span><span class="p">:</span>
|
||
</span><span id="line-16"><span class="w"> </span><span class="nt">receivers</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">otlp</span><span class="p p-Indicator">]</span>
|
||
</span><span id="line-17"><span class="w"> </span><span class="nt">processors</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">batch</span><span class="p p-Indicator">]</span>
|
||
</span><span id="line-18"><span class="w"> </span><span class="nt">exporters</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">awsxray</span><span class="p p-Indicator">]</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p><strong>Deploy the Collector</strong>: Run the collector as a Docker container, Kubernetes pod, or standalone service.</p></li>
|
||
<li><p><strong>Ensure AWS Credentials</strong>: Provide AWS credentials to the collector, preferably via IAM roles.</p></li>
|
||
<li><p><strong>Verify Traces</strong>: Access the AWS X-Ray console to view your traces.</p></li>
|
||
</ol>
|
||
</section>
|
||
<section id="datadog">
|
||
<h3>Datadog<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#datadog" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#datadog'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>Datadog</p>
|
||
<p>To send tracing data to <a class="reference external" href="https://docs.datadoghq.com/getting_started/tracing/" rel="nofollow noopener">Datadog<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>:</p>
|
||
<ol class="arabic">
|
||
<li><p><strong>Configure OpenTelemetry Collector</strong>: Set up the collector to export traces to Datadog.</p>
|
||
<p>Collector configuration (<code class="docutils literal notranslate"><span class="pre">otel-collector-config.yaml</span></code>):</p>
|
||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">receivers</span><span class="p">:</span>
|
||
</span><span id="line-2"><span class="w"> </span><span class="nt">otlp</span><span class="p">:</span>
|
||
</span><span id="line-3"><span class="w"> </span><span class="nt">protocols</span><span class="p">:</span>
|
||
</span><span id="line-4"><span class="w"> </span><span class="nt">grpc</span><span class="p">:</span>
|
||
</span><span id="line-5">
|
||
</span><span id="line-6"><span class="nt">processors</span><span class="p">:</span>
|
||
</span><span id="line-7"><span class="w"> </span><span class="nt">batch</span><span class="p">:</span>
|
||
</span><span id="line-8">
|
||
</span><span id="line-9"><span class="nt">exporters</span><span class="p">:</span>
|
||
</span><span id="line-10"><span class="w"> </span><span class="nt">datadog</span><span class="p">:</span>
|
||
</span><span id="line-11"><span class="w"> </span><span class="nt">api</span><span class="p">:</span>
|
||
</span><span id="line-12"><span class="w"> </span><span class="nt">key</span><span class="p">:</span><span class="w"> </span><span class="s">"${<Your-Datadog-Api-Key>}"</span>
|
||
</span><span id="line-13"><span class="w"> </span><span class="nt">site</span><span class="p">:</span><span class="w"> </span><span class="s">"${DD_SITE}"</span>
|
||
</span><span id="line-14">
|
||
</span><span id="line-15"><span class="nt">service</span><span class="p">:</span>
|
||
</span><span id="line-16"><span class="w"> </span><span class="nt">pipelines</span><span class="p">:</span>
|
||
</span><span id="line-17"><span class="w"> </span><span class="nt">traces</span><span class="p">:</span>
|
||
</span><span id="line-18"><span class="w"> </span><span class="nt">receivers</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">otlp</span><span class="p p-Indicator">]</span>
|
||
</span><span id="line-19"><span class="w"> </span><span class="nt">processors</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">batch</span><span class="p p-Indicator">]</span>
|
||
</span><span id="line-20"><span class="w"> </span><span class="nt">exporters</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">datadog</span><span class="p p-Indicator">]</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p><strong>Set Environment Variables</strong>: Provide your Datadog API key and site.</p>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span><span class="nb">export</span><span class="w"> </span><Your-Datadog-Api-Key><span class="o">=</span><Your-Datadog-Api-Key>
|
||
</span><span id="line-2"><span class="gp">$ </span><span class="nb">export</span><span class="w"> </span><span class="nv">DD_SITE</span><span class="o">=</span>datadoghq.com<span class="w"> </span><span class="c1"># Or datadoghq.eu</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p><strong>Deploy the Collector</strong>: Run the collector in your environment.</p></li>
|
||
<li><p><strong>Verify Traces</strong>: Access the Datadog APM dashboard to view your traces.</p></li>
|
||
</ol>
|
||
</section>
|
||
<section id="langtrace">
|
||
<h3>Langtrace<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#langtrace" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#langtrace'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>Langtrace is an observability tool designed specifically for large language models (LLMs). It helps you capture, analyze, and understand how LLMs are used in your applications including those built using Plano.</p>
|
||
<p>To send tracing data to <a class="reference external" href="https://docs.langtrace.ai/supported-integrations/llm-tools/arch" rel="nofollow noopener">Langtrace<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>:</p>
|
||
<ol class="arabic">
|
||
<li><p><strong>Configure Plano</strong>: Make sure Plano is installed and setup correctly. For more information, refer to the <a class="reference external" href="https://github.com/katanemo/archgw?tab=readme-ov-file#prerequisites" rel="nofollow noopener">installation guide<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>.</p></li>
|
||
<li><p><strong>Install Langtrace</strong>: Install the Langtrace SDK.:</p>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>pip<span class="w"> </span>install<span class="w"> </span>langtrace-python-sdk
|
||
</span></code></pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p><strong>Set Environment Variables</strong>: Provide your Langtrace API key.</p>
|
||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span><span class="nb">export</span><span class="w"> </span><span class="nv">LANGTRACE_API_KEY</span><span class="o">=</span><Your-Langtrace-Api-Key>
|
||
</span></code></pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p><strong>Trace Requests</strong>: Once you have Langtrace set up, you can start tracing requests.</p>
|
||
<p>Here’s an example of how to trace a request using the Langtrace Python SDK:</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||
</span><span id="line-2"><span class="kn">from</span><span class="w"> </span><span class="nn">langtrace_python_sdk</span><span class="w"> </span><span class="kn">import</span> <span class="n">langtrace</span> <span class="c1"># Must precede any llm module imports</span>
|
||
</span><span id="line-3"><span class="kn">from</span><span class="w"> </span><span class="nn">openai</span><span class="w"> </span><span class="kn">import</span> <span class="n">OpenAI</span>
|
||
</span><span id="line-4">
|
||
</span><span id="line-5"><span class="n">langtrace</span><span class="o">.</span><span class="n">init</span><span class="p">(</span><span class="n">api_key</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'LANGTRACE_API_KEY'</span><span class="p">])</span>
|
||
</span><span id="line-6">
|
||
</span><span id="line-7"><span class="n">client</span> <span class="o">=</span> <span class="n">OpenAI</span><span class="p">(</span><span class="n">api_key</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'OPENAI_API_KEY'</span><span class="p">],</span> <span class="n">base_url</span><span class="o">=</span><span class="s2">"http://localhost:12000/v1"</span><span class="p">)</span>
|
||
</span><span id="line-8">
|
||
</span><span id="line-9"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||
</span><span id="line-10"> <span class="n">model</span><span class="o">=</span><span class="s2">"gpt-4o-mini"</span><span class="p">,</span>
|
||
</span><span id="line-11"> <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
|
||
</span><span id="line-12"> <span class="p">{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"system"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"You are a helpful assistant"</span><span class="p">},</span>
|
||
</span><span id="line-13"> <span class="p">{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Hello"</span><span class="p">},</span>
|
||
</span><span id="line-14"> <span class="p">]</span>
|
||
</span><span id="line-15"><span class="p">)</span>
|
||
</span><span id="line-16">
|
||
</span><span id="line-17"><span class="nb">print</span><span class="p">(</span><span class="n">chat_completion</span><span class="o">.</span><span class="n">choices</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p><strong>Verify Traces</strong>: Access the Langtrace dashboard to view your traces.</p></li>
|
||
</ol>
|
||
</section>
|
||
</section>
|
||
<section id="best-practices">
|
||
<h2>Best Practices<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#best-practices" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#best-practices'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<ul class="simple">
|
||
<li><p><strong>Consistent Instrumentation</strong>: Ensure all services propagate the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header.</p></li>
|
||
<li><p><strong>Secure Configuration</strong>: Protect sensitive data and secure communication between services.</p></li>
|
||
<li><p><strong>Performance Monitoring</strong>: Be mindful of the performance impact and adjust sampling rates accordingly.</p></li>
|
||
<li><p><strong>Error Handling</strong>: Implement proper error handling to prevent tracing issues from affecting your application.</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="summary">
|
||
<h2>Summary<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#summary" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#summary'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>By leveraging the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header for trace context propagation, Plano enables developers to implement
|
||
tracing efficiently. This approach simplifies the process of collecting and analyzing tracing data in common
|
||
tools like AWS X-Ray and Datadog, enhancing observability and facilitating faster debugging and optimization.</p>
|
||
</section>
|
||
<section id="additional-resources">
|
||
<h2>Additional Resources<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#additional-resources" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#additional-resources'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>For full command documentation (including <code class="docutils literal notranslate"><span class="pre">planoai</span> <span class="pre">trace</span></code> and all other CLI commands), see <a class="reference internal" href="../../resources/cli_reference.html#cli-reference"><span class="std std-ref">CLI Reference</span></a>.</p>
|
||
<section id="external-references">
|
||
<h3>External References<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#external-references" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#external-references'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<ul class="simple">
|
||
<li><p><a class="reference external" href="https://opentelemetry.io/docs/" rel="nofollow noopener">OpenTelemetry Documentation<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p></li>
|
||
<li><p><a class="reference external" href="https://www.w3.org/TR/trace-context/" rel="nofollow noopener">W3C Trace Context Specification<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p></li>
|
||
<li><p><a class="reference external" href="https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/awsxrayexporter" rel="nofollow noopener">AWS X-Ray Exporter<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p></li>
|
||
<li><p><a class="reference external" href="https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/datadogexporter" rel="nofollow noopener">Datadog Exporter<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p></li>
|
||
<li><p><a class="reference external" href="https://docs.langtrace.ai/introduction" rel="nofollow noopener">Langtrace Documentation<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p></li>
|
||
</ul>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>Replace placeholders such as <code class="docutils literal notranslate"><span class="pre"><Your-Aws-Region></span></code> and <code class="docutils literal notranslate"><span class="pre"><Your-Datadog-Api-Key></span></code> with your actual configurations.</p>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
</section>
|
||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||
<div class="mr-auto">
|
||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="observability.html">
|
||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<polyline points="15 18 9 12 15 6"></polyline>
|
||
</svg>
|
||
Observability
|
||
</a>
|
||
</div>
|
||
<div class="ml-auto">
|
||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="monitoring.html">
|
||
Monitoring
|
||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<polyline points="9 18 15 12 9 6"></polyline>
|
||
</svg>
|
||
</a>
|
||
</div>
|
||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] h-full overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||
<ul>
|
||
<li><a :data-current="activeSection === '#overview'" class="reference internal" href="#overview">Overview</a></li>
|
||
<li><a :data-current="activeSection === '#understanding-plano-traces'" class="reference internal" href="#understanding-plano-traces">Understanding Plano Traces</a><ul>
|
||
<li><a :data-current="activeSection === '#inbound-request-handling'" class="reference internal" href="#inbound-request-handling">Inbound Request Handling</a></li>
|
||
<li><a :data-current="activeSection === '#orchestration-routing'" class="reference internal" href="#orchestration-routing">Orchestration & Routing</a></li>
|
||
<li><a :data-current="activeSection === '#agent-processing'" class="reference internal" href="#agent-processing">Agent Processing</a></li>
|
||
<li><a :data-current="activeSection === '#outbound-llm-calls'" class="reference internal" href="#outbound-llm-calls">Outbound LLM Calls</a></li>
|
||
<li><a :data-current="activeSection === '#handoff-to-upstream-services'" class="reference internal" href="#handoff-to-upstream-services">Handoff to Upstream Services</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a :data-current="activeSection === '#behavioral-signals-in-traces'" class="reference internal" href="#behavioral-signals-in-traces">Behavioral Signals in Traces</a></li>
|
||
<li><a :data-current="activeSection === '#custom-span-attributes'" class="reference internal" href="#custom-span-attributes">Custom Span Attributes</a><ul>
|
||
<li><a :data-current="activeSection === '#how-it-works'" class="reference internal" href="#how-it-works">How It Works</a></li>
|
||
<li><a :data-current="activeSection === '#configuration'" class="reference internal" href="#configuration">Configuration</a></li>
|
||
<li><a :data-current="activeSection === '#notes-and-examples'" class="reference internal" href="#notes-and-examples">Notes and Examples</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a :data-current="activeSection === '#benefits-of-using-traceparent-headers'" class="reference internal" href="#benefits-of-using-traceparent-headers">Benefits of Using <code class="docutils literal notranslate"><span class="pre">Traceparent</span></code> Headers</a></li>
|
||
<li><a :data-current="activeSection === '#how-to-initiate-a-trace'" class="reference internal" href="#how-to-initiate-a-trace">How to Initiate A Trace</a></li>
|
||
<li><a :data-current="activeSection === '#tracing-with-the-cli'" class="reference internal" href="#tracing-with-the-cli">Tracing with the CLI</a><ul>
|
||
<li><a :data-current="activeSection === '#quick-start'" class="reference internal" href="#quick-start">Quick Start</a></li>
|
||
<li><a :data-current="activeSection === '#inspect-and-filter-traces'" class="reference internal" href="#inspect-and-filter-traces">Inspect and Filter Traces</a></li>
|
||
<li><a :data-current="activeSection === '#notes'" class="reference internal" href="#notes">Notes</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a :data-current="activeSection === '#trace-propagation'" class="reference internal" href="#trace-propagation">Trace Propagation</a><ul>
|
||
<li><a :data-current="activeSection === '#header-format'" class="reference internal" href="#header-format">Header Format</a></li>
|
||
<li><a :data-current="activeSection === '#instrumentation'" class="reference internal" href="#instrumentation">Instrumentation</a><ul>
|
||
<li><a :data-current="activeSection === '#example-with-opentelemetry-in-python'" class="reference internal" href="#example-with-opentelemetry-in-python">Example with OpenTelemetry in Python</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li><a :data-current="activeSection === '#integrating-with-tracing-tools'" class="reference internal" href="#integrating-with-tracing-tools">Integrating with Tracing Tools</a><ul>
|
||
<li><a :data-current="activeSection === '#aws-x-ray'" class="reference internal" href="#aws-x-ray">AWS X-Ray</a></li>
|
||
<li><a :data-current="activeSection === '#datadog'" class="reference internal" href="#datadog">Datadog</a></li>
|
||
<li><a :data-current="activeSection === '#langtrace'" class="reference internal" href="#langtrace">Langtrace</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a :data-current="activeSection === '#best-practices'" class="reference internal" href="#best-practices">Best Practices</a></li>
|
||
<li><a :data-current="activeSection === '#summary'" class="reference internal" href="#summary">Summary</a></li>
|
||
<li><a :data-current="activeSection === '#additional-resources'" class="reference internal" href="#additional-resources">Additional Resources</a><ul>
|
||
<li><a :data-current="activeSection === '#external-references'" class="reference internal" href="#external-references">External References</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</aside>
|
||
</main>
|
||
</div>
|
||
</div><footer class="py-6 border-t border-border md:py-0">
|
||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Mar 30, 2026. </p>
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
</div>
|
||
<script src="../../_static/documentation_options.js?v=b7c0077d"></script>
|
||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script defer="defer" src="../../_static/theme.js?v=582b20c5"></script>
|
||
<script src="../../_static/design-tabs.js?v=f930bc37"></script>
|
||
<script src="../../_static/js/fix-copy.js?v=2f5cab98"></script>
|
||
</body>
|
||
</html> |