mirror of
https://github.com/katanemo/plano.git
synced 2026-04-25 08:46:24 +02:00
500 lines
No EOL
63 KiB
HTML
Executable file
500 lines
No EOL
63 KiB
HTML
Executable file
<!DOCTYPE html>
|
||
|
||
<html :class="{ 'dark' : darkMode === true }" data-content_root="../../" lang="en" x-data="{ darkMode: $persist(window.matchMedia('(prefers-color-scheme: dark)').matches), activeSection: ''}">
|
||
<head>
|
||
<script>
|
||
(function () {
|
||
// Set initial color scheme
|
||
if ((localStorage.getItem("_x_darkMode") === "true") || (window.matchMedia("(prefers-color-scheme: dark)").matches)) {
|
||
document.documentElement.classList.add("dark");
|
||
}
|
||
|
||
// Watch for media preference changes
|
||
window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change", (event) => {
|
||
localStorage.setItem("_x_darkMode", event.matches);
|
||
document.documentElement.classList.toggle("dark", event.matches);
|
||
});
|
||
})();
|
||
</script>
|
||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||
<meta charset="utf-8"/>
|
||
<meta content="#ffffff" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||
<meta content="#030711" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||
<title>Request Lifecycle | Plano Docs v0.4.12</title>
|
||
<meta content="Request Lifecycle | Plano Docs v0.4.12" property="og:title"/>
|
||
<meta content="Request Lifecycle | Plano Docs v0.4.12" name="twitter:title"/>
|
||
<link href="../../_static/pygments.css?v=73db4dac" rel="stylesheet" type="text/css"/>
|
||
<link href="../../_static/theme.css?v=73505e79" rel="stylesheet" type="text/css"/>
|
||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||
<link href="../../_static/awesome-sphinx-design.css?v=c54898a4" rel="stylesheet" type="text/css"/>
|
||
<link href="./docs/resources/tech_overview/request_lifecycle.html" rel="canonical"/>
|
||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||
<link href="../../search.html" rel="search" title="Search"/>
|
||
<link href="model_serving.html" rel="next" title="Bright Staff"/>
|
||
<link href="tech_overview.html" rel="prev" title="Tech Overview"/>
|
||
</head>
|
||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||
Skip to content
|
||
</a><header class="sticky top-0 z-40 w-full border-b shadow-xs border-border bg-background/90 backdrop-blur"><div class="container flex items-center h-14">
|
||
<div class="hidden mr-4 md:flex">
|
||
<a class="flex items-center mr-6" href="../../index.html">
|
||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4.12</span>
|
||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||
</svg>
|
||
<span class="sr-only">Toggle navigation menu</span>
|
||
</button>
|
||
<div class="flex items-center justify-between flex-1 gap-2 sm:gap-4 md:justify-end">
|
||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||
<span class="text-xs">⌘</span>
|
||
K
|
||
</kbd>
|
||
</form>
|
||
</div>
|
||
<nav class="flex items-center gap-1">
|
||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||
</div>
|
||
</a>
|
||
<button @click="darkMode = !darkMode" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" title="Toggle color scheme" type="button">
|
||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="16" viewbox="0 96 960 960" width="16" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||
</svg>
|
||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="16" viewbox="0 96 960 960" width="16" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||
</svg>
|
||
</button>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</header>
|
||
<div class="flex-1"><div class="container md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||
<a class="justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4.12</span>
|
||
</a>
|
||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0">
|
||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||
|
||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||
<script>
|
||
window.dataLayer = window.dataLayer || [];
|
||
function gtag(){dataLayer.push(arguments);}
|
||
gtag('js', new Date());
|
||
|
||
gtag('config', 'G-EH2VW19FXE');
|
||
</script>
|
||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/listeners.html">Listeners</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/agents.html">Agents</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/filter_chain.html">Filter Chains</a></li>
|
||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button" x-cloak=""><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-cloak="" x-show="expanded">
|
||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/prompt_target.html">Prompt Target</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/signals.html">Signals™</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../guides/orchestration.html">Orchestration</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button" x-cloak=""><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-cloak="" x-show="expanded">
|
||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Guardrails</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../guides/state.html">Conversational State</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||
<ul class="current">
|
||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button" x-cloak=""><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Request Lifecycle</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html">Bright Staff</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../deployment.html">Deployment</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../configuration_reference.html">Configuration Reference</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../cli_reference.html">CLI Reference</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../llms_txt.html">llms.txt</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||
</svg>
|
||
</button>
|
||
</aside>
|
||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||
<div class="w-full min-w-0 mx-auto">
|
||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||
<span class="hidden md:inline">Plano Docs v0.4.12</span>
|
||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||
</svg>
|
||
</a>
|
||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="tech_overview.html">Tech Overview</a>
|
||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Request Lifecycle</span>
|
||
</nav>
|
||
<div id="content" role="main">
|
||
<section id="request-lifecycle">
|
||
<span id="lifecycle-of-a-request"></span><h1>Request Lifecycle<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#request-lifecycle"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||
<p>Below we describe the events in the lifecycle of a request passing through a Plano instance. We first
|
||
describe how Plano fits into the request path and then the internal events that take place following
|
||
the arrival of a request at Plano from downstream clients. We follow the request until the corresponding
|
||
dispatch upstream and the response path.</p>
|
||
<a class="reference internal image-reference" href="../../_images/network-topology-ingress-egress.png"><img alt="../../_images/network-topology-ingress-egress.png" class="align-center" src="../../_images/network-topology-ingress-egress.png" style="width: 100%;"/>
|
||
</a>
|
||
<section id="network-topology">
|
||
<h2>Network topology<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#network-topology" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#network-topology'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>How a request flows through the components in a network (including Plano) depends on the network’s topology.
|
||
Plano can be used in a wide variety of networking topologies. We focus on the inner operations of Plano below,
|
||
but briefly we address how Plano relates to the rest of the network in this section.</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Downstream(Ingress)</strong> listeners take requests from upstream clients like a web UI or clients that forward
|
||
prompts to you local application responses from the application flow back through Plano to the downstream.</p></li>
|
||
<li><p><strong>Upstream(Egress)</strong> listeners take requests from the application and forward them to LLMs.</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="high-level-architecture">
|
||
<h2>High level architecture<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#high-level-architecture" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#high-level-architecture'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Plano is a set of <strong>two</strong> self-contained processes that are designed to run alongside your application servers
|
||
(or on a separate server connected to your application servers via a network).</p>
|
||
<p>The first process is designated to manage HTTP-level networking and connection management concerns (protocol management, request id generation, header sanitization, etc.), and the other process is a <strong>controller</strong>, which helps Plano make intelligent decisions about the incoming prompts. The controller hosts the purpose-built LLMs to manage several critical, but undifferentiated, prompt related tasks on behalf of developers.</p>
|
||
<p>The request processing path in Plano has three main parts:</p>
|
||
<ul class="simple">
|
||
<li><p><a class="reference internal" href="../../concepts/listeners.html#plano-overview-listeners"><span class="std std-ref">Listener subsystem</span></a> which handles <strong>downstream</strong> and <strong>upstream</strong> request
|
||
processing. It is responsible for managing the inbound(edge) and outbound(egress) request lifecycle. The downstream and upstream HTTP/2 codec lives here. This also includes the lifecycle of any <strong>upstream</strong> connection to an LLM provider or tool backend. The listenser subsystmem manages connection pools, load balancing, retries, and failover.</p></li>
|
||
<li><p><a class="reference internal" href="model_serving.html#bright-staff"><span class="std std-ref">Bright Staff controller subsystem</span></a> is Plano’s memory-efficient, lightweight controller for agentic traffic. It sits inside the Plano data plane and makes real-time decisions about how prompts are handled, forwarded, and processed.</p></li>
|
||
</ul>
|
||
<p>These two subsystems are bridged with either the HTTP router filter, and the cluster manager subsystems of Envoy.</p>
|
||
<p>Also, Plano utilizes <a class="reference external" href="https://blog.envoyproxy.io/envoy-threading-model-a8d44b922310" rel="nofollow noopener">Envoy event-based thread model<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>. A main thread is responsible for the server lifecycle, configuration processing, stats, etc. and some number of <a class="reference internal" href="threading_model.html#plano-overview-threading"><span class="std std-ref">worker threads</span></a> process requests. All threads operate around an event loop (<a class="reference external" href="https://libevent.org/" rel="nofollow noopener">libevent<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>) and any given downstream TCP connection will be handled by exactly one worker thread for its lifetime. Each worker thread maintains its own pool of TCP connections to upstream endpoints.</p>
|
||
<p>Worker threads rarely share state and operate in a trivially parallel fashion. This threading model
|
||
enables scaling to very high core count CPUs.</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||
</span><span id="line-2">│ P L A N O │
|
||
</span><span id="line-3">│ AI-native proxy and data plane for agentic applications │
|
||
</span><span id="line-4">│ │
|
||
</span><span id="line-5">│ ┌─────────────────────┐ │
|
||
</span><span id="line-6">│ │ YOUR CLIENTS │ │
|
||
</span><span id="line-7">│ │ (apps· agents · UI) │ │
|
||
</span><span id="line-8">│ └──────────┬──────────┘ │
|
||
</span><span id="line-9">│ │ │
|
||
</span><span id="line-10">│ ┌──────────────────────────────┼──────────────────────────┐ │
|
||
</span><span id="line-11">│ │ │ │ │
|
||
</span><span id="line-12">│ ┌──────▼──────────┐ ┌─────────▼────────┐ ┌────────▼─────────┐ │
|
||
</span><span id="line-13">│ │ Agent Port(s) │ │ Model Port │ │ Function-Call │ │
|
||
</span><span id="line-14">│ │ :8001+ │ │ :12000 │ │ Port :10000 │ │
|
||
</span><span id="line-15">│ │ │ │ │ │ │ │
|
||
</span><span id="line-16">│ │ route your │ │ direct LLM │ │ prompt-target / │ │
|
||
</span><span id="line-17">│ │ prompts to │ │ calls with │ │ tool dispatch │ │
|
||
</span><span id="line-18">│ │ the right │ │ model-alias │ │ with parameter │ │
|
||
</span><span id="line-19">│ │ agent │ │ translation │ │ extraction │ │
|
||
</span><span id="line-20">│ └──────┬──────────┘ └─────────┬────────┘ └────────┬─────────┘ │
|
||
</span><span id="line-21">│ └──────────────────────────────┼─────────────────────────┘ │
|
||
</span><span id="line-22">│ │ │
|
||
</span><span id="line-23">│ ╔══════════════════════════════════════▼══════════════════════════════════════╗ │
|
||
</span><span id="line-24">│ ║ BRIGHTSTAFF (SUBSYSTEM) — Agentic Control Plane ║ │
|
||
</span><span id="line-25">│ ║ Async · non-blocking · parallel per-request Tokio tasks ║ │
|
||
</span><span id="line-26">│ ║ ║ │
|
||
</span><span id="line-27">│ ║ ┌─────────────────────────────────────────────────────────────────────┐ ║ │
|
||
</span><span id="line-28">│ ║ │ Agentic ROUTER │ ║ │
|
||
</span><span id="line-29">│ ║ │ Reads listener config · maps incoming request to execution path │ ║ │
|
||
</span><span id="line-30">│ ║ │ │ ║ │
|
||
</span><span id="line-31">│ ║ │ /agents/* ──────────────────────► AGENT PATH │ ║ │
|
||
</span><span id="line-32">│ ║ │ /v1/chat|messages|responses ──────► LLM PATH │ ║ │
|
||
</span><span id="line-33">│ ║ └─────────────────────────────────────────────────────────────────────┘ ║ │
|
||
</span><span id="line-34">│ ║ ║ │
|
||
</span><span id="line-35">│ ║ ─────────────────────── AGENT PATH ──────────────────────────────────── ║ │
|
||
</span><span id="line-36">│ ║ ║ │
|
||
</span><span id="line-37">│ ║ ┌──────────────────────────────────────────────────────────────────────┐ ║ │
|
||
</span><span id="line-38">│ ║ │ FILTER CHAIN (pipeline_processor.rs) │ ║ │
|
||
</span><span id="line-39">│ ║ │ │ ║ │
|
||
</span><span id="line-40">│ ║ │ prompt ──► [input_guards] ──► [query_rewrite] ──► [context_builder] │ ║ │
|
||
</span><span id="line-41">│ ║ │ guardrails prompt mutation RAG / enrichment │ ║ │
|
||
</span><span id="line-42">│ ║ │ │ ║ │
|
||
</span><span id="line-43">│ ║ │ Each filter: HTTP or MCP · can mutate, enrich, or short-circuit │ ║ │
|
||
</span><span id="line-44">│ ║ └──────────────────────────────────┬───────────────────────────────────┘ ║ │
|
||
</span><span id="line-45">│ ║ │ ║ │
|
||
</span><span id="line-46">│ ║ ┌──────────────────────────────────▼───────────────────────────────────┐ ║ │
|
||
</span><span id="line-47">│ ║ │ AGENT ORCHESTRATOR (agent_chat_completions.rs) │ ║ │
|
||
</span><span id="line-48">│ ║ │ Select agent · forward enriched request · manage conversation state │ ║ │
|
||
</span><span id="line-49">│ ║ │ Stream response back · multi-turn aware │ ║ │
|
||
</span><span id="line-50">│ ║ └──────────────────────────────────────────────────────────────────────┘ ║ │
|
||
</span><span id="line-51">│ ║ ║ │
|
||
</span><span id="line-52">│ ║ ─────────────────────── LLM PATH ────────────────────────────────────── ║ │
|
||
</span><span id="line-53">│ ║ ║ │
|
||
</span><span id="line-54">│ ║ ┌──────────────────────────────────────────────────────────────────────┐ ║ │
|
||
</span><span id="line-55">│ ║ │ MODEL ROUTER (llm_router.rs + router_chat.rs) │ ║ │
|
||
</span><span id="line-56">│ ║ │ Model alias resolution · preference-based provider selection │ ║ │
|
||
</span><span id="line-57">│ ║ │ "fast-llm" → gpt-4o-mini · "smart-llm" → gpt-4o │ ║ │
|
||
</span><span id="line-58">│ ║ └──────────────────────────────────────────────────────────────────────┘ ║ │
|
||
</span><span id="line-59">│ ║ ║ │
|
||
</span><span id="line-60">│ ║ ─────────────────── ALWAYS ON (every request) ───────────────────────── ║ │
|
||
</span><span id="line-61">│ ║ ║ │
|
||
</span><span id="line-62">│ ║ ┌────────────────────┐ ┌─────────────────────┐ ┌──────────────────┐ ║ │
|
||
</span><span id="line-63">│ ║ │ SIGNALS ANALYZER │ │ STATE STORAGE │ │ OTEL TRACING │ ║ │
|
||
</span><span id="line-64">│ ║ │ loop detection │ │ memory / postgres │ │ traceparent │ ║ │
|
||
</span><span id="line-65">│ ║ │ repetition score │ │ /v1/responses │ │ span injection │ ║ │
|
||
</span><span id="line-66">│ ║ │ quality indicators│ │ stateful API │ │ trace export │ ║ │
|
||
</span><span id="line-67">│ ║ └────────────────────┘ └─────────────────────┘ └──────────────────┘ ║ │
|
||
</span><span id="line-68">│ ╚═════════════════════════════════════╤═══════════════════════════════════════╝ │
|
||
</span><span id="line-69">│ │ │
|
||
</span><span id="line-70">│ ┌─────────────────────────────────────▼──────────────────────────────────────┐ │
|
||
</span><span id="line-71">│ │ LLM GATEWAY (llm_gateway.wasm — embedded in Envoy egress filter chain) │ │
|
||
</span><span id="line-72">│ │ │ │
|
||
</span><span id="line-73">│ │ Rate limiting · Provider format translation · TTFT metrics │ │
|
||
</span><span id="line-74">│ │ OpenAI → Anthropic · Gemini · Mistral · Groq · DeepSeek · xAI · Bedrock │ │
|
||
</span><span id="line-75">│ │ │ │
|
||
</span><span id="line-76">│ │ Envoy handles beneath this: TLS origination · SNI · retry + backoff │ │
|
||
</span><span id="line-77">│ │ connection pooling · LOGICAL_DNS · structured access logs │ │
|
||
</span><span id="line-78">│ └─────────────────────────────────────┬──────────────────────────────────────┘ │
|
||
</span><span id="line-79">│ │ │
|
||
</span><span id="line-80">└─────────────────────────────────────────┼───────────────────────────────────────────┘
|
||
</span><span id="line-81"> │
|
||
</span><span id="line-82"> ┌───────────────────────────┼────────────────────────────┐
|
||
</span><span id="line-83"> │ │ │
|
||
</span><span id="line-84"> ┌─────────▼──────────┐ ┌────────────▼──────────┐ ┌────────────▼──────────┐
|
||
</span><span id="line-85"> │ LLM PROVIDERS │ │ EXTERNAL AGENTS │ │ TOOL / API BACKENDS │
|
||
</span><span id="line-86"> │ OpenAI · Anthropic│ │ (filter chain svc) │ │ (endpoint clusters) │
|
||
</span><span id="line-87"> │ Gemini · Mistral │ │ HTTP / MCP :10500+ │ │ user-defined hosts │
|
||
</span><span id="line-88"> │ Groq · DeepSeek │ │ input_guards │ │ │
|
||
</span><span id="line-89"> │ xAI · Together.ai │ │ query_rewriter │ │ │
|
||
</span><span id="line-90"> └────────────────────┘ │ context_builder │ └───────────────────────┘
|
||
</span><span id="line-91"> └───────────────────────┘
|
||
</span><span id="line-92">
|
||
</span><span id="line-93">
|
||
</span><span id="line-94"> HOW PLANO IS DIFFERENT
|
||
</span><span id="line-95"> ─────────────────────────────────────────────────────────────────────────────────
|
||
</span><span id="line-96"> Brightstaff is the entire agentic brain — one async Rust binary that handles
|
||
</span><span id="line-97"> agent selection, filter chain orchestration, model routing, state, and signals
|
||
</span><span id="line-98"> without blocking a thread per request.
|
||
</span><span id="line-99">
|
||
</span><span id="line-100"> Filter chains are programmable dataplane steps — reusable HTTP/MCP services
|
||
</span><span id="line-101"> you wire into any agent, executing in-path before the agent ever sees the prompt.
|
||
</span><span id="line-102">
|
||
</span><span id="line-103"> The LLM gateway is a zero-overhead WASM plugin inside Envoy — format translation
|
||
</span><span id="line-104"> and rate limiting happen in-process with the proxy, not as a separate service hop.
|
||
</span><span id="line-105">
|
||
</span><span id="line-106"> Envoy provides the transport substrate (TLS, HTTP codecs, retries, connection
|
||
</span><span id="line-107"> pools, access logs) so Plano never reimplements solved infrastructure problems.
|
||
</span></code></pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="request-flow-ingress">
|
||
<h2>Request Flow (Ingress)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#request-flow-ingress" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#request-flow-ingress'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>A brief outline of the lifecycle of a request and response using the example configuration above:</p>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>TCP Connection Establishment</strong>:
|
||
A TCP connection from downstream is accepted by an Plano listener running on a worker thread.
|
||
The listener filter chain provides SNI and other pre-TLS information. The transport socket, typically TLS,
|
||
decrypts incoming data for processing.</p></li>
|
||
</ol>
|
||
<ol class="arabic simple" start="3">
|
||
<li><p><strong>Routing Decision (Agent vs Prompt Target)</strong>:
|
||
The decrypted data stream is de-framed by the HTTP/2 codec in Plano’s HTTP connection manager. Plano performs
|
||
intent matching (via the Bright Staff controller and prompt-handling logic) using the configured agents and
|
||
<a class="reference internal" href="../../concepts/prompt_target.html#prompt-target"><span class="std std-ref">prompt targets</span></a>, determining whether this request should be handled by an agent workflow
|
||
(with optional <a class="reference internal" href="../../concepts/filter_chain.html#filter-chain"><span class="std std-ref">Filter Chains</span></a>) or by a deterministic prompt target.</p></li>
|
||
</ol>
|
||
<p>4a. <strong>Agent Path: Orchestration and Filter Chains</strong></p>
|
||
<blockquote>
|
||
<div><p>If the request is routed to an <strong>agent</strong>, Plano executes any attached <a class="reference internal" href="../../concepts/filter_chain.html#filter-chain"><span class="std std-ref">Filter Chains</span></a> first. These filters can apply guardrails, rewrite prompts, or enrich context (for example, RAG retrieval) before the agent runs. Once filters complete, the Bright Staff controller orchestrates which downstream tools, APIs, or LLMs the agent should call and in what sequence.</p>
|
||
<ul class="simple">
|
||
<li><p>Plano may call one or more backend APIs or tools on behalf of the agent.</p></li>
|
||
<li><p>If an endpoint cluster is identified, load balancing is performed, circuit breakers are checked, and the request is proxied to the appropriate upstream endpoint.</p></li>
|
||
<li><p>If no specific endpoint is required, the prompt is sent to an upstream LLM using Plano’s model proxy for
|
||
completion or summarization.</p></li>
|
||
</ul>
|
||
<p>For more on agent workflows and orchestration, see <a class="reference internal" href="../../concepts/prompt_target.html#prompt-target"><span class="std std-ref">Prompt Targets and Agents</span></a> and
|
||
<a class="reference internal" href="../../concepts/filter_chain.html#filter-chain"><span class="std std-ref">Agent Filter Chains</span></a>.</p>
|
||
</div></blockquote>
|
||
<p>4b. <strong>Prompt Target Path: Deterministic Tool/API Calls</strong></p>
|
||
<blockquote>
|
||
<div><p>If the request is routed to a <strong>prompt target</strong>, Plano treats it as a deterministic, task-specific call.
|
||
Plano engages its function-calling and parameter-gathering capabilities to extract the necessary details
|
||
from the incoming prompt(s) and produce the structured inputs your backend expects.</p>
|
||
<ul class="simple">
|
||
<li><p><strong>Parameter Gathering</strong>: Plano extracts and validates parameters defined on the prompt target (for example,
|
||
currency symbols, dates, or entity identifiers) so your backend does not need to parse natural language.</p></li>
|
||
<li><p><strong>API Call Execution</strong>: Plano then routes the call to the configured backend endpoint. If an endpoint cluster is identified, load balancing and circuit-breaker checks are applied before proxying the request upstream.</p></li>
|
||
</ul>
|
||
<p>For more on how to design and configure prompt targets, see <a class="reference internal" href="../../concepts/prompt_target.html#prompt-target"><span class="std std-ref">Prompt Target</span></a>.</p>
|
||
</div></blockquote>
|
||
<ol class="arabic simple" start="5">
|
||
<li><p><strong>Error Handling and Forwarding</strong>:
|
||
Errors encountered during processing, such as failed function calls or guardrail detections, are forwarded to
|
||
designated error targets. Error details are communicated through specific headers to the application:</p>
|
||
<ul class="simple">
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">X-Function-Error-Code</span></code>: Code indicating the type of function call error.</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">X-Prompt-Guard-Error-Code</span></code>: Code specifying violations detected by prompt guardrails.</p></li>
|
||
<li><p>Additional headers carry messages and timestamps to aid in debugging and logging.</p></li>
|
||
</ul>
|
||
</li>
|
||
<li><p><strong>Response Handling</strong>:
|
||
The upstream endpoint’s TLS transport socket encrypts the response, which is then proxied back downstream.
|
||
Responses pass through HTTP filters in reverse order, ensuring any necessary processing or modification before final delivery.</p></li>
|
||
</ol>
|
||
</section>
|
||
<section id="request-flow-egress">
|
||
<h2>Request Flow (Egress)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#request-flow-egress" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#request-flow-egress'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>A brief outline of the lifecycle of a request and response in the context of egress traffic from an application to Large Language Models (LLMs) via Plano:</p>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>HTTP Connection Establishment to LLM</strong>:
|
||
Plano initiates an HTTP connection to the upstream LLM service. This connection is handled by Plano’s egress listener running on a worker thread. The connection typically uses a secure transport protocol such as HTTPS, ensuring the prompt data is encrypted before being sent to the LLM service.</p></li>
|
||
<li><p><strong>Rate Limiting</strong>:
|
||
Before sending the request to the LLM, Plano applies rate-limiting policies to ensure that the upstream LLM service is not overwhelmed by excessive traffic. Rate limits are enforced per client or service, ensuring fair usage and preventing accidental or malicious overload. If the rate limit is exceeded, Plano may return an appropriate HTTP error (e.g., 429 Too Many Requests) without sending the prompt to the LLM.</p></li>
|
||
<li><p><strong>Seamless Request Transformation and Smart Routing</strong>:
|
||
After rate limiting, Plano normalizes the outgoing request into a provider-agnostic shape and applies smart routing decisions using the configured <a class="reference internal" href="../../concepts/llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">LLM Providers</span></a>. This includes translating client-specific conventions into a unified OpenAI-style contract, enriching or overriding parameters (for example, temperature or max tokens) based on policy, and choosing the best target model or provider using <a class="reference internal" href="../../concepts/llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">model-based, alias-based, or preference-aligned routing</span></a>.</p></li>
|
||
<li><p><strong>Load Balancing to (hosted) LLM Endpoints</strong>:
|
||
After smart routing selects the target provider/model, Plano routes the prompt to the appropriate LLM endpoint.
|
||
If multiple LLM provider instances are available, load balancing is performed to distribute traffic evenly
|
||
across the instances. Plano checks the health of the LLM endpoints using circuit breakers and health checks,
|
||
ensuring that the prompt is only routed to a healthy, responsive instance.</p></li>
|
||
<li><p><strong>Response Reception and Forwarding</strong>:
|
||
Once the LLM processes the prompt, Plano receives the response from the LLM service. The response is typically a generated text, completion, or summarization. Upon reception, Plano decrypts (if necessary) and handles the response, passing it through any egress processing pipeline defined by the application, such as logging or additional response filtering.</p></li>
|
||
</ol>
|
||
<section id="post-request-processing">
|
||
<h3>Post-request processing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#post-request-processing" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#post-request-processing'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||
<p>Once a request completes, the stream is destroyed. The following also takes places:</p>
|
||
<ul class="simple">
|
||
<li><p>The post-request <a class="reference internal" href="../../guides/observability/monitoring.html#monitoring"><span class="std std-ref">monitoring</span></a> are updated (e.g. timing, active requests, upgrades, health checks).
|
||
Some statistics are updated earlier however, during request processing. Stats are batched and written by the main
|
||
thread periodically.</p></li>
|
||
<li><p><a class="reference internal" href="../../guides/observability/access_logging.html#plano-access-logging"><span class="std std-ref">Access logs</span></a> are written to the access log</p></li>
|
||
<li><p><a class="reference internal" href="../../guides/observability/tracing.html#plano-overview-tracing"><span class="std std-ref">Trace</span></a> spans are finalized. If our example request was traced, a
|
||
trace span, describing the duration and details of the request would be created by the HCM when
|
||
processing request headers and then finalized by the HCM during post-request processing.</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="configuration">
|
||
<h2>Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configuration" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configuration'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||
<p>Today, only support a static bootstrap configuration file for simplicity today:</p>
|
||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.2.0</span>
|
||
</span><span id="line-2">
|
||
</span><span id="line-3"><span class="nt">listeners</span><span class="p">:</span>
|
||
</span><span id="line-4"><span class="w"> </span><span class="nt">ingress_traffic</span><span class="p">:</span>
|
||
</span><span id="line-5"><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||
</span><span id="line-6"><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10000</span>
|
||
</span><span id="line-7">
|
||
</span><span id="line-8"><span class="c1"># Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way</span>
|
||
</span><span id="line-9"><span class="nt">model_providers</span><span class="p">:</span>
|
||
</span><span id="line-10"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||
</span><span id="line-11"><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||
</span><span id="line-12"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||
</span><span id="line-13">
|
||
</span><span id="line-14"><span class="nt">prompt_targets</span><span class="p">:</span>
|
||
</span><span id="line-15"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">information_extraction</span>
|
||
</span><span id="line-16"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||
</span><span id="line-17"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">handel all scenarios that are question and answer in nature. Like summarization, information extraction, etc.</span>
|
||
</span><span id="line-18"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||
</span><span id="line-19"><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||
</span><span id="line-20"><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/summary</span>
|
||
</span><span id="line-21"><span class="w"> </span><span class="c1"># Plano uses the default LLM and treats the response from the endpoint as the prompt to send to the LLM</span>
|
||
</span><span id="line-22"><span class="w"> </span><span class="nt">auto_llm_dispatch_on_response</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||
</span><span id="line-23"><span class="w"> </span><span class="c1"># override system prompt for this prompt target</span>
|
||
</span><span id="line-24"><span class="w"> </span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">You are a helpful information extraction assistant. Use the information that is provided to you.</span>
|
||
</span><span id="line-25">
|
||
</span><span id="line-26"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">reboot_network_device</span>
|
||
</span><span id="line-27"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Reboot a specific network device</span>
|
||
</span><span id="line-28"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||
</span><span id="line-29"><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||
</span><span id="line-30"><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/action</span>
|
||
</span><span id="line-31"><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||
</span><span id="line-32"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">device_id</span>
|
||
</span><span id="line-33"><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">str</span>
|
||
</span><span id="line-34"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Identifier of the network device to reboot.</span>
|
||
</span><span id="line-35"><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||
</span><span id="line-36"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">confirmation</span>
|
||
</span><span id="line-37"><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">bool</span>
|
||
</span><span id="line-38"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Confirmation flag to proceed with reboot.</span>
|
||
</span><span id="line-39"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">false</span>
|
||
</span><span id="line-40"><span class="w"> </span><span class="nt">enum</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">true</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">false</span><span class="p p-Indicator">]</span>
|
||
</span><span id="line-41">
|
||
</span><span id="line-42"><span class="c1"># Plano creates a round-robin load balancing between different endpoints, managed via the cluster subsystem.</span>
|
||
</span><span id="line-43"><span class="nt">endpoints</span><span class="p">:</span>
|
||
</span><span id="line-44"><span class="w"> </span><span class="nt">app_server</span><span class="p">:</span>
|
||
</span><span id="line-45"><span class="w"> </span><span class="c1"># value could be ip address or a hostname with port</span>
|
||
</span><span id="line-46"><span class="w"> </span><span class="c1"># this could also be a list of endpoints for load balancing</span>
|
||
</span><span id="line-47"><span class="w"> </span><span class="c1"># for example endpoint: [ ip1:port, ip2:port ]</span>
|
||
</span><span id="line-48"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">127.0.0.1:80</span>
|
||
</span><span id="line-49"><span class="w"> </span><span class="c1"># max time to wait for a connection to be established</span>
|
||
</span><span id="line-50"><span class="w"> </span><span class="nt">connect_timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.005s</span>
|
||
</span></code></pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||
<div class="mr-auto">
|
||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="tech_overview.html">
|
||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<polyline points="15 18 9 12 15 6"></polyline>
|
||
</svg>
|
||
Tech Overview
|
||
</a>
|
||
</div>
|
||
<div class="ml-auto">
|
||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="model_serving.html">
|
||
Bright Staff
|
||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<polyline points="9 18 15 12 9 6"></polyline>
|
||
</svg>
|
||
</a>
|
||
</div>
|
||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] h-full overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||
<ul>
|
||
<li><a :data-current="activeSection === '#network-topology'" class="reference internal" href="#network-topology">Network topology</a></li>
|
||
<li><a :data-current="activeSection === '#high-level-architecture'" class="reference internal" href="#high-level-architecture">High level architecture</a></li>
|
||
<li><a :data-current="activeSection === '#request-flow-ingress'" class="reference internal" href="#request-flow-ingress">Request Flow (Ingress)</a></li>
|
||
<li><a :data-current="activeSection === '#request-flow-egress'" class="reference internal" href="#request-flow-egress">Request Flow (Egress)</a><ul>
|
||
<li><a :data-current="activeSection === '#post-request-processing'" class="reference internal" href="#post-request-processing">Post-request processing</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a :data-current="activeSection === '#configuration'" class="reference internal" href="#configuration">Configuration</a></li>
|
||
</ul>
|
||
</div>
|
||
</aside>
|
||
</main>
|
||
</div>
|
||
</div><footer class="py-6 border-t border-border md:py-0">
|
||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Mar 19, 2026. </p>
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
</div>
|
||
<script src="../../_static/documentation_options.js?v=3611e0a4"></script>
|
||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script defer="defer" src="../../_static/theme.js?v=582b20c5"></script>
|
||
<script src="../../_static/design-tabs.js?v=f930bc37"></script>
|
||
<script src="../../_static/js/fix-copy.js?v=2f5cab98"></script>
|
||
</body>
|
||
</html> |