plano/guides/observability/tracing.html
2024-12-21 19:33:41 +00:00

546 lines
No EOL
64 KiB
HTML
Executable file
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!DOCTYPE html>
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' &amp;&amp; window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val =&gt; localStorage.setItem('darkMode', val))">
<head>
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
<meta charset="utf-8"/>
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
<meta content="width=device-width, initial-scale=1" name="viewport"/>
<title>Tracing | Arch Docs v0.1.7</title>
<meta content="Tracing | Arch Docs v0.1.7" property="og:title"/>
<meta content="Tracing | Arch Docs v0.1.7" name="twitter:title"/>
<link href="../../_static/pygments.css?v=75ebff74" rel="stylesheet" type="text/css"/>
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
<link href="./docs/guides/observability/tracing.html" rel="canonical"/>
<link href="../../_static/favicon.ico" rel="icon"/>
<link href="../../search.html" rel="search" title="Search"/>
<link href="monitoring.html" rel="next" title="Monitoring"/>
<link href="observability.html" rel="prev" title="Observability"/>
<script>
<!-- Prevent Flash of wrong theme -->
const userPreference = localStorage.getItem('darkMode');
let mode;
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
mode = 'dark';
document.documentElement.classList.add('dark');
} else {
mode = 'light';
}
if (!userPreference) {localStorage.setItem('darkMode', mode)}
</script>
</head>
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
Skip to content
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
<div class="hidden mr-4 md:flex">
<a class="flex items-center mr-6" href="../../index.html">
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.1.7</span>
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
</svg>
<span class="sr-only">Toggle navigation menu</span>
</button>
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../../search.html" class="relative flex items-center group" id="searchbox" method="get">
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
<span class="text-xs"></span>
K
</kbd>
</form>
</div>
<nav class="flex items-center space-x-1">
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
</div>
</a>
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
</svg>
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
</svg>
</button>
</nav>
</div>
</div>
</header>
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.1.7</span>
</a>
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
<div class="overflow-y-auto h-full w-full relative pr-6">
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-K2LXXSX6HB');
</script>
<nav class="table w-full min-w-full my-6 lg:my-8">
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
<ul>
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/terminology.html">Terminology</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/threading_model.html">Threading Model</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/listener.html">Listener</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/prompt.html">Prompts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/model_serving.html">Model Serving</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/error_target.html">Error Target</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../concepts/llm_provider.html">LLM Provider</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../concepts/prompt_target.html">Prompt Target</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../prompt_guard.html">Prompt Guard</a></li>
<li class="toctree-l1"><a class="reference internal" href="../function_calling.html">Function Calling</a></li>
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
<li class="toctree-l2 current"><a class="current reference internal" href="#">Tracing</a></li>
<li class="toctree-l2"><a class="reference internal" href="monitoring.html">Monitoring</a></li>
<li class="toctree-l2"><a class="reference internal" href="access_logging.html">Access Logging</a></li>
</ul>
</li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
</ul>
</nav>
</div>
</div>
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
</svg>
</button>
</aside>
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
<div class="w-full min-w-0 mx-auto">
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
<span class="hidden md:inline">Arch Docs v0.1.7</span>
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
</svg>
</a>
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="observability.html">Observability</a>
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Tracing</span>
</nav>
<div id="content" role="main">
<section id="tracing">
<span id="arch-overview-tracing"></span><h1>Tracing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#tracing"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
<section id="overview">
<h2>Overview<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#overview" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#overview'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
<p><a class="reference external" href="https://opentelemetry.io/" rel="nofollow noopener">OpenTelemetry<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> is an open-source observability framework providing APIs
and instrumentation for generating, collecting, processing, and exporting telemetry data, such as traces,
metrics, and logs. Its flexible design supports a wide range of backends and seamlessly integrates with
modern application tools. A key feature of OpenTelemetry is its commitment to standards like the
<a class="reference external" href="https://www.w3.org/TR/trace-context/" rel="nofollow noopener">W3C Trace Context<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p>
<p><strong>Tracing</strong> is a critical tool that allows developers to visualize and understand the flow of
requests in an AI application. With tracing, you can capture a detailed view of how requests propagate
through various services and components, which is crucial for <strong>debugging</strong>, <strong>performance optimization</strong>,
and understanding complex AI agent architectures like Co-pilots.</p>
<p><strong>Arch</strong> propagates trace context using the W3C Trace Context standard, specifically through the
<code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header. This allows each component in the system to record its part of the request
flow, enabling <strong>end-to-end tracing</strong> across the entire application. By using OpenTelemetry, Arch ensures
that developers can capture this trace data consistently and in a format compatible with various observability
tools.</p>
<a class="reference internal image-reference" href="../../_images/tracing.png"><img alt="../../_images/tracing.png" class="align-center" src="../../_images/tracing.png" style="width: 100%;"/>
</a>
</section>
<section id="benefits-of-using-traceparent-headers">
<h2>Benefits of Using <code class="docutils literal notranslate"><span class="pre">Traceparent</span></code> Headers<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#benefits-of-using-traceparent-headers" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#benefits-of-using-traceparent-headers'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
<ul class="simple">
<li><p><strong>Standardization</strong>: The W3C Trace Context standard ensures compatibility across ecosystem tools, allowing
traces to be propagated uniformly through different layers of the system.</p></li>
<li><p><strong>Ease of Integration</strong>: OpenTelemetrys design allows developers to easily integrate tracing with minimal
changes to their codebase, enabling quick adoption of end-to-end observability.</p></li>
<li><p><strong>Interoperability</strong>: Works seamlessly with popular tracing tools like AWS X-Ray, Datadog, Jaeger, and many others,
making it easy to visualize traces in the tools youre already usi</p></li>
</ul>
</section>
<section id="how-to-initiate-a-trace">
<h2>How to Initiate A Trace<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#how-to-initiate-a-trace" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#how-to-initiate-a-trace'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
<ol class="arabic simple">
<li><p><strong>Enable Tracing Configuration</strong>: Simply add the <code class="docutils literal notranslate"><span class="pre">random_sampling</span></code> in <code class="docutils literal notranslate"><span class="pre">tracing</span></code> section to 100`` flag to in the <a class="reference internal" href="../../concepts/tech_overview/listener.html#arch-overview-listeners"><span class="std std-ref">listener</span></a> config</p></li>
<li><p><strong>Trace Context Propagation</strong>: Arch automatically propagates the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header. When a request is received, Arch will:</p>
<ul class="simple">
<li><p>Generate a new <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header if one is not present.</p></li>
<li><p>Extract the trace context from the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header if it exists.</p></li>
<li><p>Start a new span representing its processing of the request.</p></li>
<li><p>Forward the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header to downstream services.</p></li>
</ul>
</li>
<li><p><strong>Sampling Policy</strong>: The 100 in <code class="docutils literal notranslate"><span class="pre">random_sampling:</span> <span class="pre">100</span></code> means that all the requests as sampled for tracing.
You can adjust this value from 0-100.</p></li>
</ol>
</section>
<section id="trace-propagation">
<h2>Trace Propagation<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#trace-propagation" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#trace-propagation'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
<p>Arch uses the W3C Trace Context standard for trace propagation, which relies on the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header.
This header carries tracing information in a standardized format, enabling interoperability between different
tracing systems.</p>
<section id="header-format">
<h3>Header Format<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#header-format" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#header-format'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
<p>The <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header has the following format:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="n">traceparent</span><span class="p">:</span> <span class="p">{</span><span class="n">version</span><span class="p">}</span><span class="o">-</span><span class="p">{</span><span class="n">trace</span><span class="o">-</span><span class="nb">id</span><span class="p">}</span><span class="o">-</span><span class="p">{</span><span class="n">parent</span><span class="o">-</span><span class="nb">id</span><span class="p">}</span><span class="o">-</span><span class="p">{</span><span class="n">trace</span><span class="o">-</span><span class="n">flags</span><span class="p">}</span>
</span></code></pre></div>
</div>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">{version}</span></code>: The version of the Trace Context specification (e.g., <code class="docutils literal notranslate"><span class="pre">00</span></code>).</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">{trace-id}</span></code>: A 16-byte (32-character hexadecimal) unique identifier for the trace.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">{parent-id}</span></code>: An 8-byte (16-character hexadecimal) identifier for the parent span.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">{trace-flags}</span></code>: Flags indicating trace options (e.g., sampling).</p></li>
</ul>
</section>
<section id="instrumentation">
<h3>Instrumentation<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#instrumentation" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#instrumentation'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
<p>To integrate AI tracing, your application needs to follow a few simple steps. The steps
below are very common practice, and not unique to Arch, when you reading tracing headers and export
<a class="reference external" href="https://docs.lightstep.com/docs/understand-distributed-tracing" rel="nofollow noopener">spans<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> for distributed tracing.</p>
<ul class="simple">
<li><p>Read the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header from incoming requests.</p></li>
<li><p>Start new spans as children of the extracted context.</p></li>
<li><p>Include the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header in outbound requests to propagate trace context.</p></li>
<li><p>Send tracing data to a collector or tracing backend to export spans</p></li>
</ul>
<section id="example-with-opentelemetry-in-python">
<h4>Example with OpenTelemetry in Python<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-with-opentelemetry-in-python"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
<p>Install OpenTelemetry packages:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>pip<span class="w"> </span>install<span class="w"> </span>opentelemetry-api<span class="w"> </span>opentelemetry-sdk<span class="w"> </span>opentelemetry-exporter-otlp
</span><span id="line-2"><span class="gp">$ </span>pip<span class="w"> </span>install<span class="w"> </span>opentelemetry-instrumentation-requests
</span></code></pre></div>
</div>
<p>Set up the tracer and exporter:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">from</span> <span class="nn">opentelemetry</span> <span class="kn">import</span> <span class="n">trace</span>
</span><span id="line-2"><span class="kn">from</span> <span class="nn">opentelemetry.exporter.otlp.proto.grpc.trace_exporter</span> <span class="kn">import</span> <span class="n">OTLPSpanExporter</span>
</span><span id="line-3"><span class="kn">from</span> <span class="nn">opentelemetry.instrumentation.requests</span> <span class="kn">import</span> <span class="n">RequestsInstrumentor</span>
</span><span id="line-4"><span class="kn">from</span> <span class="nn">opentelemetry.sdk.resources</span> <span class="kn">import</span> <span class="n">Resource</span>
</span><span id="line-5"><span class="kn">from</span> <span class="nn">opentelemetry.sdk.trace</span> <span class="kn">import</span> <span class="n">TracerProvider</span>
</span><span id="line-6"><span class="kn">from</span> <span class="nn">opentelemetry.sdk.trace.export</span> <span class="kn">import</span> <span class="n">BatchSpanProcessor</span>
</span><span id="line-7">
</span><span id="line-8"><span class="c1"># Define the service name</span>
</span><span id="line-9"><span class="n">resource</span> <span class="o">=</span> <span class="n">Resource</span><span class="p">(</span><span class="n">attributes</span><span class="o">=</span><span class="p">{</span>
</span><span id="line-10"> <span class="s2">"service.name"</span><span class="p">:</span> <span class="s2">"customer-support-agent"</span>
</span><span id="line-11"><span class="p">})</span>
</span><span id="line-12">
</span><span id="line-13"><span class="c1"># Set up the tracer provider and exporter</span>
</span><span id="line-14"><span class="n">tracer_provider</span> <span class="o">=</span> <span class="n">TracerProvider</span><span class="p">(</span><span class="n">resource</span><span class="o">=</span><span class="n">resource</span><span class="p">)</span>
</span><span id="line-15"><span class="n">otlp_exporter</span> <span class="o">=</span> <span class="n">OTLPSpanExporter</span><span class="p">(</span><span class="n">endpoint</span><span class="o">=</span><span class="s2">"otel-collector:4317"</span><span class="p">,</span> <span class="n">insecure</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
</span><span id="line-16"><span class="n">span_processor</span> <span class="o">=</span> <span class="n">BatchSpanProcessor</span><span class="p">(</span><span class="n">otlp_exporter</span><span class="p">)</span>
</span><span id="line-17"><span class="n">tracer_provider</span><span class="o">.</span><span class="n">add_span_processor</span><span class="p">(</span><span class="n">span_processor</span><span class="p">)</span>
</span><span id="line-18"><span class="n">trace</span><span class="o">.</span><span class="n">set_tracer_provider</span><span class="p">(</span><span class="n">tracer_provider</span><span class="p">)</span>
</span><span id="line-19">
</span><span id="line-20"><span class="c1"># Instrument HTTP requests</span>
</span><span id="line-21"><span class="n">RequestsInstrumentor</span><span class="p">()</span><span class="o">.</span><span class="n">instrument</span><span class="p">()</span>
</span></code></pre></div>
</div>
<p>Handle incoming requests:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">from</span> <span class="nn">opentelemetry</span> <span class="kn">import</span> <span class="n">trace</span>
</span><span id="line-2"><span class="kn">from</span> <span class="nn">opentelemetry.propagate</span> <span class="kn">import</span> <span class="n">extract</span><span class="p">,</span> <span class="n">inject</span>
</span><span id="line-3"><span class="kn">import</span> <span class="nn">requests</span>
</span><span id="line-4">
</span><span id="line-5"><span class="k">def</span> <span class="nf">handle_request</span><span class="p">(</span><span class="n">request</span><span class="p">):</span>
</span><span id="line-6"> <span class="c1"># Extract the trace context</span>
</span><span id="line-7"> <span class="n">context</span> <span class="o">=</span> <span class="n">extract</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span>
</span><span id="line-8"> <span class="n">tracer</span> <span class="o">=</span> <span class="n">trace</span><span class="o">.</span><span class="n">get_tracer</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
</span><span id="line-9">
</span><span id="line-10"> <span class="k">with</span> <span class="n">tracer</span><span class="o">.</span><span class="n">start_as_current_span</span><span class="p">(</span><span class="s2">"process_customer_request"</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">):</span>
</span><span id="line-11"> <span class="c1"># Example of processing a customer request</span>
</span><span id="line-12"> <span class="nb">print</span><span class="p">(</span><span class="s2">"Processing customer request..."</span><span class="p">)</span>
</span><span id="line-13">
</span><span id="line-14"> <span class="c1"># Prepare headers for outgoing request to payment service</span>
</span><span id="line-15"> <span class="n">headers</span> <span class="o">=</span> <span class="p">{}</span>
</span><span id="line-16"> <span class="n">inject</span><span class="p">(</span><span class="n">headers</span><span class="p">)</span>
</span><span id="line-17">
</span><span id="line-18"> <span class="c1"># Make outgoing request to external service (e.g., payment gateway)</span>
</span><span id="line-19"> <span class="n">response</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"http://payment-service/api"</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
</span><span id="line-20">
</span><span id="line-21"> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Payment service response: </span><span class="si">{</span><span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
</span></code></pre></div>
</div>
</section>
</section>
<section id="ai-agent-tracing-visualization-example">
<h3>AI Agent Tracing Visualization Example<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#ai-agent-tracing-visualization-example" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#ai-agent-tracing-visualization-example'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
<p>The following is an example of tracing for an AI-powered customer support system.
A customer interacts with AI agents, which forward their requests through different
specialized services and external systems.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="o">+--------------------------+</span>
</span><span id="line-2"><span class="o">|</span> <span class="n">Customer</span> <span class="n">Interaction</span> <span class="o">|</span>
</span><span id="line-3"><span class="o">+--------------------------+</span>
</span><span id="line-4"> <span class="o">|</span>
</span><span id="line-5"> <span class="n">v</span>
</span><span id="line-6"><span class="o">+--------------------------+</span> <span class="o">+--------------------------+</span>
</span><span id="line-7"><span class="o">|</span> <span class="n">Agent</span> <span class="mi">1</span> <span class="p">(</span><span class="n">Main</span> <span class="o">-</span> <span class="n">Arch</span><span class="p">)</span> <span class="o">|</span> <span class="o">----&gt;</span> <span class="o">|</span> <span class="n">External</span> <span class="n">Payment</span> <span class="n">Service</span> <span class="o">|</span>
</span><span id="line-8"><span class="o">+--------------------------+</span> <span class="o">+--------------------------+</span>
</span><span id="line-9"> <span class="o">|</span> <span class="o">|</span>
</span><span id="line-10"> <span class="n">v</span> <span class="n">v</span>
</span><span id="line-11"><span class="o">+--------------------------+</span> <span class="o">+--------------------------+</span>
</span><span id="line-12"><span class="o">|</span> <span class="n">Agent</span> <span class="mi">2</span> <span class="p">(</span><span class="n">Support</span> <span class="o">-</span> <span class="n">Arch</span><span class="p">)</span><span class="o">|</span> <span class="o">----&gt;</span> <span class="o">|</span> <span class="n">Internal</span> <span class="n">Tech</span> <span class="n">Support</span> <span class="o">|</span>
</span><span id="line-13"><span class="o">+--------------------------+</span> <span class="o">+--------------------------+</span>
</span><span id="line-14"> <span class="o">|</span> <span class="o">|</span>
</span><span id="line-15"> <span class="n">v</span> <span class="n">v</span>
</span><span id="line-16"><span class="o">+--------------------------+</span> <span class="o">+--------------------------+</span>
</span><span id="line-17"><span class="o">|</span> <span class="n">Agent</span> <span class="mi">3</span> <span class="p">(</span><span class="n">Orders</span><span class="o">-</span> <span class="n">Arch</span><span class="p">)</span> <span class="o">|</span> <span class="o">----&gt;</span> <span class="o">|</span> <span class="n">Inventory</span> <span class="n">Management</span> <span class="o">|</span>
</span><span id="line-18"><span class="o">+--------------------------+</span> <span class="o">+--------------------------+</span>
</span></code></pre></div>
</div>
<section id="trace-breakdown">
<h4>Trace Breakdown:<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#trace-breakdown"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
<ul class="simple">
<li><dl class="simple">
<dt>Customer Interaction:</dt><dd><ul>
<li><p>Span 1: Customer initiates a request via the AI-powered chatbot for billing support (e.g., asking for payment details).</p></li>
</ul>
</dd>
</dl>
</li>
<li><dl class="simple">
<dt>AI Agent 1 (Main - Arch):</dt><dd><ul>
<li><p>Span 2: AI Agent 1 (Main) processes the request and identifies it as related to billing, forwarding the request
to an external payment service.</p></li>
<li><p>Span 3: AI Agent 1 determines that additional technical support is needed for processing and forwards the request
to AI Agent 2.</p></li>
</ul>
</dd>
</dl>
</li>
<li><dl class="simple">
<dt>External Payment Service:</dt><dd><ul>
<li><p>Span 4: The external payment service processes the payment-related request (e.g., verifying payment status) and sends
the response back to AI Agent 1.</p></li>
</ul>
</dd>
</dl>
</li>
<li><dl class="simple">
<dt>AI Agent 2 (Tech - Arch):</dt><dd><ul>
<li><p>Span 5: AI Agent 2, responsible for technical queries, processes a request forwarded from AI Agent 1 (e.g., checking for
any account issues).</p></li>
<li><p>Span 6: AI Agent 2 forwards the query to Internal Tech Support for further investigation.</p></li>
</ul>
</dd>
</dl>
</li>
<li><dl class="simple">
<dt>Internal Tech Support:</dt><dd><ul>
<li><p>Span 7: Internal Tech Support processes the request (e.g., resolving account access issues) and responds to AI Agent 2.</p></li>
</ul>
</dd>
</dl>
</li>
<li><dl class="simple">
<dt>AI Agent 3 (Orders - Arch):</dt><dd><ul>
<li><p>Span 8: AI Agent 3 handles order-related queries. AI Agent 1 forwards the request to AI Agent 3 after payment verification
is completed.</p></li>
<li><p>Span 9: AI Agent 3 forwards a request to the Inventory Management system to confirm product availability for a pending order.</p></li>
</ul>
</dd>
</dl>
</li>
<li><dl class="simple">
<dt>Inventory Management:</dt><dd><ul>
<li><p>Span 10: The Inventory Management system checks stock and availability and returns the information to AI Agent 3.</p></li>
</ul>
</dd>
</dl>
</li>
</ul>
</section>
</section>
</section>
<section id="integrating-with-tracing-tools">
<h2>Integrating with Tracing Tools<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#integrating-with-tracing-tools" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#integrating-with-tracing-tools'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
<section id="aws-x-ray">
<h3>AWS X-Ray<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#aws-x-ray" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#aws-x-ray'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
<p>To send tracing data to <a class="reference external" href="https://aws.amazon.com/xray/" rel="nofollow noopener">AWS X-Ray<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> :</p>
<ol class="arabic">
<li><p><strong>Configure OpenTelemetry Collector</strong>: Set up the collector to export traces to AWS X-Ray.</p>
<p>Collector configuration (<code class="docutils literal notranslate"><span class="pre">otel-collector-config.yaml</span></code>):</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">receivers</span><span class="p">:</span>
</span><span id="line-2"><span class="w"> </span><span class="nt">otlp</span><span class="p">:</span>
</span><span id="line-3"><span class="w"> </span><span class="nt">protocols</span><span class="p">:</span>
</span><span id="line-4"><span class="w"> </span><span class="nt">grpc</span><span class="p">:</span>
</span><span id="line-5">
</span><span id="line-6"><span class="nt">processors</span><span class="p">:</span>
</span><span id="line-7"><span class="w"> </span><span class="nt">batch</span><span class="p">:</span>
</span><span id="line-8">
</span><span id="line-9"><span class="nt">exporters</span><span class="p">:</span>
</span><span id="line-10"><span class="w"> </span><span class="nt">awsxray</span><span class="p">:</span>
</span><span id="line-11"><span class="w"> </span><span class="nt">region</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">&lt;Your-Aws-Region&gt;</span>
</span><span id="line-12">
</span><span id="line-13"><span class="nt">service</span><span class="p">:</span>
</span><span id="line-14"><span class="w"> </span><span class="nt">pipelines</span><span class="p">:</span>
</span><span id="line-15"><span class="w"> </span><span class="nt">traces</span><span class="p">:</span>
</span><span id="line-16"><span class="w"> </span><span class="nt">receivers</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">otlp</span><span class="p p-Indicator">]</span>
</span><span id="line-17"><span class="w"> </span><span class="nt">processors</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">batch</span><span class="p p-Indicator">]</span>
</span><span id="line-18"><span class="w"> </span><span class="nt">exporters</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">awsxray</span><span class="p p-Indicator">]</span>
</span></code></pre></div>
</div>
</li>
<li><p><strong>Deploy the Collector</strong>: Run the collector as a Docker container, Kubernetes pod, or standalone service.</p></li>
<li><p><strong>Ensure AWS Credentials</strong>: Provide AWS credentials to the collector, preferably via IAM roles.</p></li>
<li><p><strong>Verify Traces</strong>: Access the AWS X-Ray console to view your traces.</p></li>
</ol>
</section>
<section id="datadog">
<h3>Datadog<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#datadog" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#datadog'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
<p>Datadog</p>
<p>To send tracing data to <a class="reference external" href="https://docs.datadoghq.com/getting_started/tracing/" rel="nofollow noopener">Datadog<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>:</p>
<ol class="arabic">
<li><p><strong>Configure OpenTelemetry Collector</strong>: Set up the collector to export traces to Datadog.</p>
<p>Collector configuration (<code class="docutils literal notranslate"><span class="pre">otel-collector-config.yaml</span></code>):</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">receivers</span><span class="p">:</span>
</span><span id="line-2"><span class="w"> </span><span class="nt">otlp</span><span class="p">:</span>
</span><span id="line-3"><span class="w"> </span><span class="nt">protocols</span><span class="p">:</span>
</span><span id="line-4"><span class="w"> </span><span class="nt">grpc</span><span class="p">:</span>
</span><span id="line-5">
</span><span id="line-6"><span class="nt">processors</span><span class="p">:</span>
</span><span id="line-7"><span class="w"> </span><span class="nt">batch</span><span class="p">:</span>
</span><span id="line-8">
</span><span id="line-9"><span class="nt">exporters</span><span class="p">:</span>
</span><span id="line-10"><span class="w"> </span><span class="nt">datadog</span><span class="p">:</span>
</span><span id="line-11"><span class="w"> </span><span class="nt">api</span><span class="p">:</span>
</span><span id="line-12"><span class="w"> </span><span class="nt">key</span><span class="p">:</span><span class="w"> </span><span class="s">"${&lt;Your-Datadog-Api-Key&gt;}"</span>
</span><span id="line-13"><span class="w"> </span><span class="nt">site</span><span class="p">:</span><span class="w"> </span><span class="s">"${DD_SITE}"</span>
</span><span id="line-14">
</span><span id="line-15"><span class="nt">service</span><span class="p">:</span>
</span><span id="line-16"><span class="w"> </span><span class="nt">pipelines</span><span class="p">:</span>
</span><span id="line-17"><span class="w"> </span><span class="nt">traces</span><span class="p">:</span>
</span><span id="line-18"><span class="w"> </span><span class="nt">receivers</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">otlp</span><span class="p p-Indicator">]</span>
</span><span id="line-19"><span class="w"> </span><span class="nt">processors</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">batch</span><span class="p p-Indicator">]</span>
</span><span id="line-20"><span class="w"> </span><span class="nt">exporters</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">datadog</span><span class="p p-Indicator">]</span>
</span></code></pre></div>
</div>
</li>
<li><p><strong>Set Environment Variables</strong>: Provide your Datadog API key and site.</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span><span class="nb">export</span><span class="w"> </span>&lt;Your-Datadog-Api-Key&gt;<span class="o">=</span>&lt;Your-Datadog-Api-Key&gt;
</span><span id="line-2"><span class="gp">$ </span><span class="nb">export</span><span class="w"> </span><span class="nv">DD_SITE</span><span class="o">=</span>datadoghq.com<span class="w"> </span><span class="c1"># Or datadoghq.eu</span>
</span></code></pre></div>
</div>
</li>
<li><p><strong>Deploy the Collector</strong>: Run the collector in your environment.</p></li>
<li><p><strong>Verify Traces</strong>: Access the Datadog APM dashboard to view your traces.</p></li>
</ol>
</section>
</section>
<section id="best-practices">
<h2>Best Practices<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#best-practices" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#best-practices'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
<ul class="simple">
<li><p><strong>Consistent Instrumentation</strong>: Ensure all services propagate the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header.</p></li>
<li><p><strong>Secure Configuration</strong>: Protect sensitive data and secure communication between services.</p></li>
<li><p><strong>Performance Monitoring</strong>: Be mindful of the performance impact and adjust sampling rates accordingly.</p></li>
<li><p><strong>Error Handling</strong>: Implement proper error handling to prevent tracing issues from affecting your application.</p></li>
</ul>
</section>
<section id="summary">
<h2>Summary<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#summary" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#summary'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
<p>By leveraging the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header for trace context propagation, Arch enables developers to implement
tracing efficiently. This approach simplifies the process of collecting and analyzing tracing data in common
tools like AWS X-Ray and Datadog, enhancing observability and facilitating faster debugging and optimization.</p>
</section>
<section id="additional-resources">
<h2>Additional Resources<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() =&gt; $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#additional-resources" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#additional-resources'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
<ul class="simple">
<li><p><a class="reference external" href="https://opentelemetry.io/docs/" rel="nofollow noopener">OpenTelemetry Documentation<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p></li>
<li><p><a class="reference external" href="https://www.w3.org/TR/trace-context/" rel="nofollow noopener">W3C Trace Context Specification<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p></li>
<li><p><a class="reference external" href="https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/awsxrayexporter" rel="nofollow noopener">AWS X-Ray Exporter<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p></li>
<li><p><a class="reference external" href="https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/datadogexporter" rel="nofollow noopener">Datadog Exporter<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p></li>
</ul>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Replace placeholders such as <code class="docutils literal notranslate"><span class="pre">&lt;Your-Aws-Region&gt;</span></code> and <code class="docutils literal notranslate"><span class="pre">&lt;Your-Datadog-Api-Key&gt;</span></code> with your actual configurations.</p>
</div>
</section>
</section>
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
<div class="mr-auto">
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="observability.html">
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
<polyline points="15 18 9 12 15 6"></polyline>
</svg>
Observability
</a>
</div>
<div class="ml-auto">
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="monitoring.html">
Monitoring
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
<polyline points="9 18 15 12 9 6"></polyline>
</svg>
</a>
</div>
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
<ul>
<li><a :data-current="activeSection === '#overview'" class="reference internal" href="#overview">Overview</a></li>
<li><a :data-current="activeSection === '#benefits-of-using-traceparent-headers'" class="reference internal" href="#benefits-of-using-traceparent-headers">Benefits of Using <code class="docutils literal notranslate"><span class="pre">Traceparent</span></code> Headers</a></li>
<li><a :data-current="activeSection === '#how-to-initiate-a-trace'" class="reference internal" href="#how-to-initiate-a-trace">How to Initiate A Trace</a></li>
<li><a :data-current="activeSection === '#trace-propagation'" class="reference internal" href="#trace-propagation">Trace Propagation</a><ul>
<li><a :data-current="activeSection === '#header-format'" class="reference internal" href="#header-format">Header Format</a></li>
<li><a :data-current="activeSection === '#instrumentation'" class="reference internal" href="#instrumentation">Instrumentation</a><ul>
<li><a :data-current="activeSection === '#example-with-opentelemetry-in-python'" class="reference internal" href="#example-with-opentelemetry-in-python">Example with OpenTelemetry in Python</a></li>
</ul>
</li>
<li><a :data-current="activeSection === '#ai-agent-tracing-visualization-example'" class="reference internal" href="#ai-agent-tracing-visualization-example">AI Agent Tracing Visualization Example</a><ul>
<li><a :data-current="activeSection === '#trace-breakdown'" class="reference internal" href="#trace-breakdown">Trace Breakdown:</a></li>
</ul>
</li>
</ul>
</li>
<li><a :data-current="activeSection === '#integrating-with-tracing-tools'" class="reference internal" href="#integrating-with-tracing-tools">Integrating with Tracing Tools</a><ul>
<li><a :data-current="activeSection === '#aws-x-ray'" class="reference internal" href="#aws-x-ray">AWS X-Ray</a></li>
<li><a :data-current="activeSection === '#datadog'" class="reference internal" href="#datadog">Datadog</a></li>
</ul>
</li>
<li><a :data-current="activeSection === '#best-practices'" class="reference internal" href="#best-practices">Best Practices</a></li>
<li><a :data-current="activeSection === '#summary'" class="reference internal" href="#summary">Summary</a></li>
<li><a :data-current="activeSection === '#additional-resources'" class="reference internal" href="#additional-resources">Additional Resources</a></li>
</ul>
</div>
</aside>
</main>
</div>
</div><footer class="py-6 border-t border-border md:py-0">
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2024, Katanemo Labs, Inc Last updated: Dec 21, 2024. </p>
</div>
</div>
</footer>
</div>
<script src="../../_static/documentation_options.js?v=d94a9a28"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
<script src="../../_static/design-tabs.js?v=f930bc37"></script>
</body>
</html>