mirror of
https://github.com/katanemo/plano.git
synced 2026-07-23 16:51:04 +02:00
deploy: e224cba3e3
This commit is contained in:
adilhafeez
parent
0898a9aa15
commit
c25ce33efc
68 changed files with 9799 additions and 4588 deletions
|
|
@ -1,4 +1,4 @@
|
|||
# Sphinx build info version 1
|
||||
# This file records the configuration used when building these files. When it is not found, a full rebuild will be done.
|
||||
config: 9db6364d8186d5eaae6b4148a1288a59
|
||||
config: d54d7379a33d5f6b3c1acac04a881e38
|
||||
tags: 645f666f9bcd5a90fca523b33c5a78b7
|
||||
|
|
|
|||
2
CNAME
2
CNAME
|
|
@ -1 +1 @@
|
|||
docs.planoai.dev
|
||||
docs.planoai.dev
|
||||
|
|
|
|||
|
|
@ -1,100 +1,110 @@
|
|||
version: v0.1
|
||||
|
||||
# Arch Gateway configuration version
|
||||
version: v0.3.0
|
||||
|
||||
|
||||
# External HTTP agents - API type is controlled by request path (/v1/responses, /v1/messages, /v1/chat/completions)
|
||||
agents:
|
||||
- id: weather_agent # Example agent for weather
|
||||
url: http://host.docker.internal:10510
|
||||
|
||||
- id: flight_agent # Example agent for flights
|
||||
url: http://host.docker.internal:10520
|
||||
|
||||
|
||||
# MCP filters applied to requests/responses (e.g., input validation, query rewriting)
|
||||
filters:
|
||||
- id: input_guards # Example filter for input validation
|
||||
url: http://host.docker.internal:10500
|
||||
# type: mcp (default)
|
||||
# transport: streamable-http (default)
|
||||
# tool: input_guards (default - same as filter id)
|
||||
|
||||
|
||||
# LLM provider configurations with API keys and model routing
|
||||
model_providers:
|
||||
- model: openai/gpt-4o
|
||||
access_key: $OPENAI_API_KEY
|
||||
default: true
|
||||
|
||||
- model: openai/gpt-4o-mini
|
||||
access_key: $OPENAI_API_KEY
|
||||
|
||||
- model: anthropic/claude-sonnet-4-0
|
||||
access_key: $ANTHROPIC_API_KEY
|
||||
|
||||
- model: mistral/ministral-3b-latest
|
||||
access_key: $MISTRAL_API_KEY
|
||||
|
||||
|
||||
# Model aliases - use friendly names instead of full provider model names
|
||||
model_aliases:
|
||||
fast-llm:
|
||||
target: gpt-4o-mini
|
||||
|
||||
smart-llm:
|
||||
target: gpt-4o
|
||||
|
||||
|
||||
# HTTP listeners - entry points for agent routing, prompt targets, and direct LLM access
|
||||
listeners:
|
||||
ingress_traffic:
|
||||
# Agent listener for routing requests to multiple agents
|
||||
- type: agent
|
||||
name: travel_booking_service
|
||||
port: 8001
|
||||
router: plano_orchestrator_v1
|
||||
address: 0.0.0.0
|
||||
port: 10000
|
||||
message_format: openai
|
||||
timeout: 5s
|
||||
egress_traffic:
|
||||
agents:
|
||||
- id: rag_agent
|
||||
description: virtual assistant for retrieval augmented generation tasks
|
||||
filter_chain:
|
||||
- input_guards
|
||||
|
||||
# Model listener for direct LLM access
|
||||
- type: model
|
||||
name: model_1
|
||||
address: 0.0.0.0
|
||||
port: 12000
|
||||
message_format: openai
|
||||
timeout: 5s
|
||||
|
||||
# Arch creates a round-robin load balancing between different endpoints, managed via the cluster subsystem.
|
||||
# Prompt listener for function calling (for prompt_targets)
|
||||
- type: prompt
|
||||
name: prompt_function_listener
|
||||
address: 0.0.0.0
|
||||
port: 10000
|
||||
# This listener is used for prompt_targets and function calling
|
||||
|
||||
|
||||
# Reusable service endpoints
|
||||
endpoints:
|
||||
app_server:
|
||||
# value could be ip address or a hostname with port
|
||||
# this could also be a list of endpoints for load balancing
|
||||
# for example endpoint: [ ip1:port, ip2:port ]
|
||||
endpoint: 127.0.0.1:80
|
||||
# max time to wait for a connection to be established
|
||||
connect_timeout: 0.005s
|
||||
|
||||
mistral_local:
|
||||
endpoint: 127.0.0.1:8001
|
||||
|
||||
error_target:
|
||||
endpoint: error_target_1
|
||||
|
||||
# Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way
|
||||
llm_providers:
|
||||
- name: openai/gpt-4o
|
||||
access_key: $OPENAI_API_KEY
|
||||
model: openai/gpt-4o
|
||||
default: true
|
||||
|
||||
- access_key: $MISTRAL_API_KEY
|
||||
model: mistral/mistral-8x7b
|
||||
|
||||
- model: mistral/mistral-7b-instruct
|
||||
base_url: http://mistral_local
|
||||
|
||||
# Model aliases - friendly names that map to actual provider names
|
||||
model_aliases:
|
||||
# Alias for summarization tasks -> fast/cheap model
|
||||
arch.summarize.v1:
|
||||
target: gpt-4o
|
||||
|
||||
# Alias for general purpose tasks -> latest model
|
||||
arch.v1:
|
||||
target: mistral-8x7b
|
||||
|
||||
# provides a way to override default settings for the arch system
|
||||
overrides:
|
||||
# By default Arch uses an NLI + embedding approach to match an incoming prompt to a prompt target.
|
||||
# The intent matching threshold is kept at 0.80, you can override this behavior if you would like
|
||||
prompt_target_intent_matching_threshold: 0.60
|
||||
|
||||
# default system prompt used by all prompt targets
|
||||
system_prompt: You are a network assistant that just offers facts; not advice on manufacturers or purchasing decisions.
|
||||
|
||||
prompt_guards:
|
||||
input_guards:
|
||||
jailbreak:
|
||||
on_exception:
|
||||
message: Looks like you're curious about my abilities, but I can only provide assistance within my programmed parameters.
|
||||
|
||||
# Prompt targets for function calling and API orchestration
|
||||
prompt_targets:
|
||||
- name: information_extraction
|
||||
default: true
|
||||
description: handel all scenarios that are question and answer in nature. Like summarization, information extraction, etc.
|
||||
endpoint:
|
||||
name: app_server
|
||||
path: /agent/summary
|
||||
http_method: POST
|
||||
# Arch uses the default LLM and treats the response from the endpoint as the prompt to send to the LLM
|
||||
auto_llm_dispatch_on_response: true
|
||||
# override system prompt for this prompt target
|
||||
system_prompt: You are a helpful information extraction assistant. Use the information that is provided to you.
|
||||
|
||||
- name: reboot_network_device
|
||||
description: Reboot a specific network device
|
||||
endpoint:
|
||||
name: app_server
|
||||
path: /agent/action
|
||||
- name: get_current_weather
|
||||
description: Get current weather at a location.
|
||||
parameters:
|
||||
- name: device_id
|
||||
type: str
|
||||
description: Identifier of the network device to reboot.
|
||||
- name: location
|
||||
description: The location to get the weather for
|
||||
required: true
|
||||
- name: confirmation
|
||||
type: bool
|
||||
description: Confirmation flag to proceed with reboot.
|
||||
default: false
|
||||
enum: [true, false]
|
||||
type: string
|
||||
format: City, State
|
||||
- name: days
|
||||
description: the number of days for the request
|
||||
required: true
|
||||
type: int
|
||||
endpoint:
|
||||
name: app_server
|
||||
path: /weather
|
||||
http_method: POST
|
||||
|
||||
|
||||
# OpenTelemetry tracing configuration
|
||||
tracing:
|
||||
# sampling rate. Note by default Arch works on OpenTelemetry compatible tracing.
|
||||
sampling_rate: 0.1
|
||||
# Random sampling percentage (1-100)
|
||||
random_sampling: 100
|
||||
|
|
|
|||
56
_images/PlanoTagline.svg
Executable file
56
_images/PlanoTagline.svg
Executable file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 49 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 181 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 389 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 107 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 297 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 264 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 281 KiB |
BIN
_images/network-topology-ingress-egress.png
Executable file
BIN
_images/network-topology-ingress-egress.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 365 KiB |
BIN
_images/plano-system-architecture.png
Executable file
BIN
_images/plano-system-architecture.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 193 KiB |
BIN
_images/plano_network_diagram_high_level.png
Executable file
BIN
_images/plano_network_diagram_high_level.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 382 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 692 KiB After Width: | Height: | Size: 8.6 MiB |
6
_static/css/custom.css
Executable file
6
_static/css/custom.css
Executable file
|
|
@ -0,0 +1,6 @@
|
|||
/* Prevent sphinxawesome-theme's Tailwind utility `dark:invert` from inverting the header logo. */
|
||||
.dark header img[alt="Logo"],
|
||||
.dark #left-sidebar img[alt="Logo"] {
|
||||
--tw-invert: invert(0%) !important;
|
||||
filter: none !important;
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
const DOCUMENTATION_OPTIONS = {
|
||||
VERSION: ' v0.3.22',
|
||||
VERSION: ' v0.4',
|
||||
LANGUAGE: 'en',
|
||||
COLLAPSE_INDEX: false,
|
||||
BUILDER: 'html',
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 2.9 KiB |
56
_static/img/PlanoTagline.svg
Executable file
56
_static/img/PlanoTagline.svg
Executable file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 49 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 181 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 289 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 389 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 107 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 264 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 281 KiB |
BIN
_static/img/network-topology-ingress-egress.png
Executable file
BIN
_static/img/network-topology-ingress-egress.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 365 KiB |
BIN
_static/img/plano-system-architecture.png
Executable file
BIN
_static/img/plano-system-architecture.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 193 KiB |
BIN
_static/img/plano_network_diagram_high_level.png
Executable file
BIN
_static/img/plano_network_diagram_high_level.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 382 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 692 KiB After Width: | Height: | Size: 8.6 MiB |
|
|
@ -1,432 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Agentic Apps | Arch Docs v0.3.22</title>
|
||||
<meta content="Agentic Apps | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Agentic Apps | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/build_with_arch/agent.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="rag.html" rel="next" title="RAG Apps"/>
|
||||
<link href="../guides/observability/access_logging.html" rel="prev" title="Access Logging"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
let mode;
|
||||
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
||||
mode = 'dark';
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
mode = 'light';
|
||||
}
|
||||
if (!userPreference) {localStorage.setItem('darkMode', mode)}
|
||||
</script>
|
||||
</head>
|
||||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
K
|
||||
</kbd>
|
||||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
</a>
|
||||
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
|
||||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||||
</svg>
|
||||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="multi_turn.html">Multi-Turn</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</aside>
|
||||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Agentic Apps</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="agentic-apps">
|
||||
<span id="arch-agent-guide"></span><h1>Agentic Apps<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#agentic-apps"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Arch helps you build personalized agentic applications by calling application-specific (API) functions via user prompts.
|
||||
This involves any predefined functions or APIs you want to expose to users to perform tasks, gather information,
|
||||
or manipulate data. This capability is generally referred to as <a class="reference internal" href="../guides/function_calling.html#function-calling"><span class="std std-ref">function calling</span></a>, where
|
||||
you can support “agentic” apps tailored to specific use cases - from updating insurance claims to creating ad campaigns - via prompts.</p>
|
||||
<p>Arch analyzes prompts, extracts critical information from prompts, engages in lightweight conversation with the user to
|
||||
gather any missing parameters and makes API calls so that you can focus on writing business logic. Arch does this via its
|
||||
purpose-built <a class="reference external" href="https://huggingface.co/collections/katanemo/arch-function-66f209a693ea8df14317ad68" rel="nofollow noopener">Arch-Function<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> -
|
||||
the fastest (200ms p50 - 12x faser than GPT-4o) and cheapest (44x than GPT-4o) function calling LLM that matches or outperforms
|
||||
frontier LLMs.</p>
|
||||
<a class="reference internal image-reference" href="../_images/function-calling-flow.jpg"><img alt="../_images/function-calling-flow.jpg" class="align-center" src="../_images/function-calling-flow.jpg" style="width: 100%;"/>
|
||||
</a>
|
||||
<section id="single-function-call">
|
||||
<h2>Single Function Call<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#single-function-call" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#single-function-call'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>In the most common scenario, users will request a single action via prompts, and Arch efficiently processes the
|
||||
request by extracting relevant parameters, validating the input, and calling the designated function or API. Here
|
||||
is how you would go about enabling this scenario with Arch:</p>
|
||||
<section id="step-1-define-prompt-targets">
|
||||
<h3>Step 1: Define Prompt Targets<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-1-define-prompt-targets" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-1-define-prompt-targets'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<div class="literal-block-wrapper docutils container" id="id2">
|
||||
<div class="code-block-caption"><span class="caption-text">Prompt Target Example Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span><span class="nt">listener</span><span class="p">:</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">127.0.0.1</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">8080</span><span class="w"> </span><span class="c1">#If you configure port 443, you'll need to update the listener with tls_certificates</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">huggingface</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="c1"># Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">OpenAI</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="w"> </span><span class="nt">provider</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-3.5-turbo</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span><span class="c1"># default system prompt used by all prompt targets</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">|</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span><span class="w"> </span><span class="no">You are a network assistant that just offers facts; not advice on manufacturers or purchasing decisions.</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span>
|
||||
</span><span id="line-19"><mark><span class="linenos">19</span><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</mark></span><span id="line-20"><mark><span class="linenos">20</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">network_qa</span>
|
||||
</mark></span><span id="line-21"><mark><span class="linenos">21</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</mark></span><span id="line-22"><mark><span class="linenos">22</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</mark></span><span id="line-23"><mark><span class="linenos">23</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/network_summary</span>
|
||||
</mark></span><span id="line-24"><mark><span class="linenos">24</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Handle general Q/A related to networking.</span>
|
||||
</mark></span><span id="line-25"><mark><span class="linenos">25</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</mark></span><span id="line-26"><mark><span class="linenos">26</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">reboot_devices</span>
|
||||
</mark></span><span id="line-27"><mark><span class="linenos">27</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Reboot specific devices or device groups</span>
|
||||
</mark></span><span id="line-28"><mark><span class="linenos">28</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</mark></span><span id="line-29"><mark><span class="linenos">29</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</mark></span><span id="line-30"><mark><span class="linenos">30</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/device_reboot</span>
|
||||
</mark></span><span id="line-31"><mark><span class="linenos">31</span><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</mark></span><span id="line-32"><mark><span class="linenos">32</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">device_ids</span>
|
||||
</mark></span><span id="line-33"><mark><span class="linenos">33</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">list</span>
|
||||
</mark></span><span id="line-34"><mark><span class="linenos">34</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">A list of device identifiers (IDs) to reboot.</span>
|
||||
</mark></span><span id="line-35"><mark><span class="linenos">35</span><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</mark></span><span id="line-36"><mark><span class="linenos">36</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">device_summary</span>
|
||||
</mark></span><span id="line-37"><mark><span class="linenos">37</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Retrieve statistics for specific devices within a time range</span>
|
||||
</mark></span><span id="line-38"><mark><span class="linenos">38</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</mark></span><span id="line-39"><mark><span class="linenos">39</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</mark></span><span id="line-40"><mark><span class="linenos">40</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/device_summary</span>
|
||||
</mark></span><span id="line-41"><mark><span class="linenos">41</span><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</mark></span><span id="line-42"><mark><span class="linenos">42</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">device_ids</span>
|
||||
</mark></span><span id="line-43"><mark><span class="linenos">43</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">list</span>
|
||||
</mark></span><span id="line-44"><mark><span class="linenos">44</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">A list of device identifiers (IDs) to retrieve statistics for.</span>
|
||||
</mark></span><span id="line-45"><mark><span class="linenos">45</span><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span><span class="w"> </span><span class="c1"># device_ids are required to get device statistics</span>
|
||||
</mark></span><span id="line-46"><mark><span class="linenos">46</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">time_range</span>
|
||||
</mark></span><span id="line-47"><mark><span class="linenos">47</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">int</span>
|
||||
</mark></span><span id="line-48"><mark><span class="linenos">48</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Time range in days for which to gather device statistics. Defaults to 7.</span>
|
||||
</mark></span><span id="line-49"><mark><span class="linenos">49</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">7</span>
|
||||
</mark></span><span id="line-50"><span class="linenos">50</span>
|
||||
</span><span id="line-51"><span class="linenos">51</span><span class="c1"># Arch creates a round-robin load balancing between different endpoints, managed via the cluster subsystem.</span>
|
||||
</span><span id="line-52"><span class="linenos">52</span><span class="nt">endpoints</span><span class="p">:</span>
|
||||
</span><span id="line-53"><span class="linenos">53</span><span class="w"> </span><span class="nt">app_server</span><span class="p">:</span>
|
||||
</span><span id="line-54"><span class="linenos">54</span><span class="w"> </span><span class="c1"># value could be ip address or a hostname with port</span>
|
||||
</span><span id="line-55"><span class="linenos">55</span><span class="w"> </span><span class="c1"># this could also be a list of endpoints for load balancing</span>
|
||||
</span><span id="line-56"><span class="linenos">56</span><span class="w"> </span><span class="c1"># for example endpoint: [ ip1:port, ip2:port ]</span>
|
||||
</span><span id="line-57"><span class="linenos">57</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">host.docker.internal:18083</span>
|
||||
</span><span id="line-58"><span class="linenos">58</span><span class="w"> </span><span class="c1"># max time to wait for a connection to be established</span>
|
||||
</span><span id="line-59"><span class="linenos">59</span><span class="w"> </span><span class="nt">connect_timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.005s</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="step-2-process-request-parameters">
|
||||
<h3>Step 2: Process Request Parameters<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-2-process-request-parameters" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-2-process-request-parameters'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Once the prompt targets are configured as above, handling those parameters is</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id3">
|
||||
<div class="code-block-caption"><span class="caption-text">Parameter handling with Flask</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id3"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="kn">from</span><span class="w"> </span><span class="nn">flask</span><span class="w"> </span><span class="kn">import</span> <span class="n">Flask</span><span class="p">,</span> <span class="n">request</span><span class="p">,</span> <span class="n">jsonify</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="n">app</span> <span class="o">=</span> <span class="n">Flask</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="nd">@app</span><span class="o">.</span><span class="n">route</span><span class="p">(</span><span class="s2">"/agent/device_summary"</span><span class="p">,</span> <span class="n">methods</span><span class="o">=</span><span class="p">[</span><span class="s2">"POST"</span><span class="p">])</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="k">def</span><span class="w"> </span><span class="nf">get_device_summary</span><span class="p">():</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="w"> </span><span class="sd">"""</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="sd"> Endpoint to retrieve device statistics based on device IDs and an optional time range.</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="sd"> """</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span> <span class="n">data</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">get_json</span><span class="p">()</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span> <span class="c1"># Validate 'device_ids' parameter</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span> <span class="n">device_ids</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"device_ids"</span><span class="p">)</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">device_ids</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">device_ids</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span> <span class="k">return</span> <span class="p">(</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span> <span class="n">jsonify</span><span class="p">({</span><span class="s2">"error"</span><span class="p">:</span> <span class="s2">"'device_ids' parameter is required and must be a list"</span><span class="p">}),</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span> <span class="mi">400</span><span class="p">,</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span> <span class="p">)</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span> <span class="c1"># Validate 'time_range' parameter (optional, defaults to 7)</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span> <span class="n">time_range</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"time_range"</span><span class="p">,</span> <span class="mi">7</span><span class="p">)</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">time_range</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span> <span class="k">return</span> <span class="n">jsonify</span><span class="p">({</span><span class="s2">"error"</span><span class="p">:</span> <span class="s2">"'time_range' must be an integer"</span><span class="p">}),</span> <span class="mi">400</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span> <span class="c1"># Simulate retrieving statistics for the given device IDs and time range</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span> <span class="c1"># In a real application, you would query your database or external service here</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span> <span class="n">statistics</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span> <span class="k">for</span> <span class="n">device_id</span> <span class="ow">in</span> <span class="n">device_ids</span><span class="p">:</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span> <span class="c1"># Placeholder for actual data retrieval</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span> <span class="n">stats</span> <span class="o">=</span> <span class="p">{</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span> <span class="s2">"device_id"</span><span class="p">:</span> <span class="n">device_id</span><span class="p">,</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span> <span class="s2">"time_range"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"Last </span><span class="si">{</span><span class="n">time_range</span><span class="si">}</span><span class="s2"> days"</span><span class="p">,</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span> <span class="s2">"data"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"Statistics data for device </span><span class="si">{</span><span class="n">device_id</span><span class="si">}</span><span class="s2"> over the last </span><span class="si">{</span><span class="n">time_range</span><span class="si">}</span><span class="s2"> days."</span><span class="p">,</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span> <span class="p">}</span>
|
||||
</span><span id="line-36"><span class="linenos">36</span> <span class="n">statistics</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">stats</span><span class="p">)</span>
|
||||
</span><span id="line-37"><span class="linenos">37</span>
|
||||
</span><span id="line-38"><span class="linenos">38</span> <span class="n">response</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"statistics"</span><span class="p">:</span> <span class="n">statistics</span><span class="p">}</span>
|
||||
</span><span id="line-39"><span class="linenos">39</span>
|
||||
</span><span id="line-40"><span class="linenos">40</span> <span class="k">return</span> <span class="n">jsonify</span><span class="p">(</span><span class="n">response</span><span class="p">),</span> <span class="mi">200</span>
|
||||
</span><span id="line-41"><span class="linenos">41</span>
|
||||
</span><span id="line-42"><span class="linenos">42</span>
|
||||
</span><span id="line-43"><span class="linenos">43</span><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
|
||||
</span><span id="line-44"><span class="linenos">44</span> <span class="n">app</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">debug</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section id="parallel-multiple-function-calling">
|
||||
<h2>Parallel & Multiple Function Calling<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#parallel-multiple-function-calling" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#parallel-multiple-function-calling'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>In more complex use cases, users may request multiple actions or need multiple APIs/functions to be called
|
||||
simultaneously or sequentially. With Arch, you can handle these scenarios efficiently using parallel or multiple
|
||||
function calling. This allows your application to engage in a broader range of interactions, such as updating
|
||||
different datasets, triggering events across systems, or collecting results from multiple services in one prompt.</p>
|
||||
<p>Arch-FC1B is built to manage these parallel tasks efficiently, ensuring low latency and high throughput, even
|
||||
when multiple functions are invoked. It provides two mechanisms to handle these cases:</p>
|
||||
<section id="id1">
|
||||
<h3>Step 1: Define Prompt Targets<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id1" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#id1'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>When enabling multiple function calling, define the prompt targets in a way that supports multiple functions or
|
||||
API calls based on the user’s prompt. These targets can be triggered in parallel or sequentially, depending on
|
||||
the user’s intent.</p>
|
||||
<p>Example of Multiple Prompt Targets in YAML:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id4">
|
||||
<div class="code-block-caption"><span class="caption-text">Prompt Target Example Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id4"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span><span class="nt">listener</span><span class="p">:</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">127.0.0.1</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">8080</span><span class="w"> </span><span class="c1">#If you configure port 443, you'll need to update the listener with tls_certificates</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">huggingface</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="c1"># Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">OpenAI</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="w"> </span><span class="nt">provider</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-3.5-turbo</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span><span class="c1"># default system prompt used by all prompt targets</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">|</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span><span class="w"> </span><span class="no">You are a network assistant that just offers facts; not advice on manufacturers or purchasing decisions.</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span>
|
||||
</span><span id="line-19"><mark><span class="linenos">19</span><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</mark></span><span id="line-20"><mark><span class="linenos">20</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">network_qa</span>
|
||||
</mark></span><span id="line-21"><mark><span class="linenos">21</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</mark></span><span id="line-22"><mark><span class="linenos">22</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</mark></span><span id="line-23"><mark><span class="linenos">23</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/network_summary</span>
|
||||
</mark></span><span id="line-24"><mark><span class="linenos">24</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Handle general Q/A related to networking.</span>
|
||||
</mark></span><span id="line-25"><mark><span class="linenos">25</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</mark></span><span id="line-26"><mark><span class="linenos">26</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">reboot_devices</span>
|
||||
</mark></span><span id="line-27"><mark><span class="linenos">27</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Reboot specific devices or device groups</span>
|
||||
</mark></span><span id="line-28"><mark><span class="linenos">28</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</mark></span><span id="line-29"><mark><span class="linenos">29</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</mark></span><span id="line-30"><mark><span class="linenos">30</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/device_reboot</span>
|
||||
</mark></span><span id="line-31"><mark><span class="linenos">31</span><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</mark></span><span id="line-32"><mark><span class="linenos">32</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">device_ids</span>
|
||||
</mark></span><span id="line-33"><mark><span class="linenos">33</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">list</span>
|
||||
</mark></span><span id="line-34"><mark><span class="linenos">34</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">A list of device identifiers (IDs) to reboot.</span>
|
||||
</mark></span><span id="line-35"><mark><span class="linenos">35</span><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</mark></span><span id="line-36"><mark><span class="linenos">36</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">device_summary</span>
|
||||
</mark></span><span id="line-37"><mark><span class="linenos">37</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Retrieve statistics for specific devices within a time range</span>
|
||||
</mark></span><span id="line-38"><mark><span class="linenos">38</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</mark></span><span id="line-39"><mark><span class="linenos">39</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</mark></span><span id="line-40"><mark><span class="linenos">40</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/device_summary</span>
|
||||
</mark></span><span id="line-41"><mark><span class="linenos">41</span><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</mark></span><span id="line-42"><mark><span class="linenos">42</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">device_ids</span>
|
||||
</mark></span><span id="line-43"><mark><span class="linenos">43</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">list</span>
|
||||
</mark></span><span id="line-44"><mark><span class="linenos">44</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">A list of device identifiers (IDs) to retrieve statistics for.</span>
|
||||
</mark></span><span id="line-45"><mark><span class="linenos">45</span><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span><span class="w"> </span><span class="c1"># device_ids are required to get device statistics</span>
|
||||
</mark></span><span id="line-46"><mark><span class="linenos">46</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">time_range</span>
|
||||
</mark></span><span id="line-47"><mark><span class="linenos">47</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">int</span>
|
||||
</mark></span><span id="line-48"><mark><span class="linenos">48</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Time range in days for which to gather device statistics. Defaults to 7.</span>
|
||||
</mark></span><span id="line-49"><mark><span class="linenos">49</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">7</span>
|
||||
</mark></span><span id="line-50"><span class="linenos">50</span>
|
||||
</span><span id="line-51"><span class="linenos">51</span><span class="c1"># Arch creates a round-robin load balancing between different endpoints, managed via the cluster subsystem.</span>
|
||||
</span><span id="line-52"><span class="linenos">52</span><span class="nt">endpoints</span><span class="p">:</span>
|
||||
</span><span id="line-53"><span class="linenos">53</span><span class="w"> </span><span class="nt">app_server</span><span class="p">:</span>
|
||||
</span><span id="line-54"><span class="linenos">54</span><span class="w"> </span><span class="c1"># value could be ip address or a hostname with port</span>
|
||||
</span><span id="line-55"><span class="linenos">55</span><span class="w"> </span><span class="c1"># this could also be a list of endpoints for load balancing</span>
|
||||
</span><span id="line-56"><span class="linenos">56</span><span class="w"> </span><span class="c1"># for example endpoint: [ ip1:port, ip2:port ]</span>
|
||||
</span><span id="line-57"><span class="linenos">57</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">host.docker.internal:18083</span>
|
||||
</span><span id="line-58"><span class="linenos">58</span><span class="w"> </span><span class="c1"># max time to wait for a connection to be established</span>
|
||||
</span><span id="line-59"><span class="linenos">59</span><span class="w"> </span><span class="nt">connect_timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.005s</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../guides/observability/access_logging.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Access Logging
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="rag.html">
|
||||
RAG Apps
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#single-function-call'" class="reference internal" href="#single-function-call">Single Function Call</a><ul>
|
||||
<li><a :data-current="activeSection === '#step-1-define-prompt-targets'" class="reference internal" href="#step-1-define-prompt-targets">Step 1: Define Prompt Targets</a></li>
|
||||
<li><a :data-current="activeSection === '#step-2-process-request-parameters'" class="reference internal" href="#step-2-process-request-parameters">Step 2: Process Request Parameters</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#parallel-multiple-function-calling'" class="reference internal" href="#parallel-multiple-function-calling">Parallel & Multiple Function Calling</a><ul>
|
||||
<li><a :data-current="activeSection === '#id1'" class="reference internal" href="#id1">Step 1: Define Prompt Targets</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,368 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Multi-Turn | Arch Docs v0.3.22</title>
|
||||
<meta content="Multi-Turn | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Multi-Turn | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/build_with_arch/multi_turn.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="../resources/deployment.html" rel="next" title="Deployment"/>
|
||||
<link href="rag.html" rel="prev" title="RAG Apps"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
let mode;
|
||||
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
||||
mode = 'dark';
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
mode = 'light';
|
||||
}
|
||||
if (!userPreference) {localStorage.setItem('darkMode', mode)}
|
||||
</script>
|
||||
</head>
|
||||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
K
|
||||
</kbd>
|
||||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
</a>
|
||||
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
|
||||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||||
</svg>
|
||||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Multi-Turn</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</aside>
|
||||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Multi-Turn</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="multi-turn">
|
||||
<span id="arch-multi-turn-guide"></span><h1>Multi-Turn<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#multi-turn"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Developers often <a class="reference external" href="https://www.reddit.com/r/LocalLLaMA/comments/18mqwg6/best_practice_for_rag_with_followup_chat/" rel="nofollow noopener">struggle<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> to efficiently handle
|
||||
<code class="docutils literal notranslate"><span class="pre">follow-up</span></code> or <code class="docutils literal notranslate"><span class="pre">clarification</span></code> questions. Specifically, when users ask for changes or additions to previous responses, it requires developers to
|
||||
re-write prompts using LLMs with precise prompt engineering techniques. This process is slow, manual, error prone and adds latency and token cost for
|
||||
common scenarios that can be managed more efficiently.</p>
|
||||
<p>Arch is highly capable of accurately detecting and processing prompts in multi-turn scenarios so that you can buil fast and accurate agents in minutes.
|
||||
Below are some cnversational examples that you can build via Arch. Each example is enriched with annotations (via ** [Arch] ** ) that illustrates how Arch
|
||||
processess conversational messages on your behalf.</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>The following section assumes that you have some knowledge about the core concepts of Arch, such as <a class="reference internal" href="../concepts/tech_overview/prompt.html#arch-overview-prompt-handling"><span class="std std-ref">prompt_targets</span></a>.
|
||||
If you haven’t familizaried yourself with Arch’s concepts, we recommend you first read the <a class="reference internal" href="../concepts/tech_overview/tech_overview.html#tech-overview"><span class="std std-ref">tech overview</span></a> section firtst.
|
||||
Additionally, the conversation examples below assume the usage of the following <a class="reference internal" href="#multi-turn-subsection-prompt-target"><span class="std std-ref">arch_config.yaml</span></a> file.</p>
|
||||
</div>
|
||||
<section id="example-1-adjusting-retrieval">
|
||||
<h2>Example 1: Adjusting Retrieval<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-1-adjusting-retrieval" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#example-1-adjusting-retrieval'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">User: What are the benefits of renewable energy?
|
||||
</span><span id="line-2">**[Arch]**: Check if there is an available <prompt_target> that can handle this user query.
|
||||
</span><span id="line-3">**[Arch]**: Found "get_info_for_energy_source" prompt_target in arch_config.yaml. Forward prompt to the endpoint configured in "get_info_for_energy_source"
|
||||
</span><span id="line-4">...
|
||||
</span><span id="line-5">Assistant: Renewable energy reduces greenhouse gas emissions, lowers air pollution, and provides sustainable power sources like solar and wind.
|
||||
</span><span id="line-6">
|
||||
</span><span id="line-7">User: Include cost considerations in the response.
|
||||
</span><span id="line-8">**[Arch]**: Follow-up detected. Forward prompt history to the "get_info_for_energy_source" prompt_target and post the following parameters consideration="cost"
|
||||
</span><span id="line-9">...
|
||||
</span><span id="line-10">Assistant: Renewable energy reduces greenhouse gas emissions, lowers air pollution, and provides sustainable power sources like solar and wind. While the initial setup costs can be high, long-term savings from reduced fuel expenses and government incentives make it cost-effective.
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="example-2-switching-intent">
|
||||
<h2>Example 2: Switching Intent<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-2-switching-intent" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#example-2-switching-intent'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">User: What are the symptoms of diabetes?
|
||||
</span><span id="line-2">**[Arch]**: Check if there is an available <prompt_target> that can handle this user query.
|
||||
</span><span id="line-3">**[Arch]**: Found "diseases_symptoms" prompt_target in arch_config.yaml. Forward disease=diabeteres to "diseases_symptoms" prompt target
|
||||
</span><span id="line-4">...
|
||||
</span><span id="line-5">Assistant: Common symptoms include frequent urination, excessive thirst, fatigue, and blurry vision.
|
||||
</span><span id="line-6">
|
||||
</span><span id="line-7">User: How is it diagnosed?
|
||||
</span><span id="line-8">**[Arch]**: New intent detected.
|
||||
</span><span id="line-9">**[Arch]**: Found "disease_diagnoses" prompt_target in arch_config.yaml. Forward disease=diabeteres to "disease_diagnoses" prompt target
|
||||
</span><span id="line-10">...
|
||||
</span><span id="line-11">Assistant: Diabetes is diagnosed through blood tests like fasting blood sugar, A1C, or an oral glucose tolerance test.
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="build-multi-turn-rag-apps">
|
||||
<h2>Build Multi-Turn RAG Apps<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#build-multi-turn-rag-apps" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#build-multi-turn-rag-apps'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>The following section describes how you can easilly add support for multi-turn scenarios via Arch. You process and manage multi-turn prompts
|
||||
just like you manage single-turn ones. Arch handles the conpleixity of detecting the correct intent based on the last user prompt and
|
||||
the covnersational history, extracts relevant parameters needed by downstream APIs, and dipatches calls to any upstream LLMs to summarize the
|
||||
response from your APIs.</p>
|
||||
<section id="step-1-define-arch-config">
|
||||
<span id="multi-turn-subsection-prompt-target"></span><h3>Step 1: Define Arch Config<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-1-define-arch-config" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-1-define-arch-config'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<div class="literal-block-wrapper docutils container" id="id2">
|
||||
<div class="code-block-caption"><span class="caption-text">Arch Config</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span><span class="nt">listener</span><span class="p">:</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">127.0.0.1</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">8080</span><span class="w"> </span><span class="c1">#If you configure port 443, you'll need to update the listener with tls_certificates</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">huggingface</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="c1"># Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">OpenAI</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="w"> </span><span class="nt">provider</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-3.5-turbo</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span><span class="c1"># default system prompt used by all prompt targets</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">|</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span><span class="w"> </span><span class="no">You are a helpful assistant and can offer information about energy sources. You will get a JSON object with energy_source and consideration fields. Focus on answering using those fields</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">get_info_for_energy_source</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">get information about an energy source</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">energy_source</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">str</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">a source of energy</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span><span class="w"> </span><span class="nt">enum</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">renewable</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">fossil</span><span class="p p-Indicator">]</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">consideration</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">str</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">a specific type of consideration for an energy source</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span><span class="w"> </span><span class="nt">enum</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">cost</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">economic</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">technology</span><span class="p p-Indicator">]</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">rag_energy_source_agent</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/energy_source_info</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span><span class="w"> </span><span class="nt">http_method</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">POST</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="step-2-process-request-in-flask">
|
||||
<h3>Step 2: Process Request in Flask<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-2-process-request-in-flask" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-2-process-request-in-flask'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Once the prompt targets are configured as above, handle parameters across multi-turn as if its a single-turn request</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id3">
|
||||
<div class="code-block-caption"><span class="caption-text">Parameter handling with Flask</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id3"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span><span class="kn">import</span><span class="w"> </span><span class="nn">gradio</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">gr</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="kn">from</span><span class="w"> </span><span class="nn">fastapi</span><span class="w"> </span><span class="kn">import</span> <span class="n">FastAPI</span><span class="p">,</span> <span class="n">HTTPException</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="kn">from</span><span class="w"> </span><span class="nn">pydantic</span><span class="w"> </span><span class="kn">import</span> <span class="n">BaseModel</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Optional</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="kn">from</span><span class="w"> </span><span class="nn">openai</span><span class="w"> </span><span class="kn">import</span> <span class="n">OpenAI</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="kn">from</span><span class="w"> </span><span class="nn">common</span><span class="w"> </span><span class="kn">import</span> <span class="n">create_gradio_app</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="n">app</span> <span class="o">=</span> <span class="n">FastAPI</span><span class="p">()</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="c1"># Define the request model</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span><span class="k">class</span><span class="w"> </span><span class="nc">EnergySourceRequest</span><span class="p">(</span><span class="n">BaseModel</span><span class="p">):</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span> <span class="n">energy_source</span><span class="p">:</span> <span class="nb">str</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span> <span class="n">consideration</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span><span class="k">class</span><span class="w"> </span><span class="nc">EnergySourceResponse</span><span class="p">(</span><span class="n">BaseModel</span><span class="p">):</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span> <span class="n">energy_source</span><span class="p">:</span> <span class="nb">str</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span> <span class="n">consideration</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span><span class="c1"># Post method for device summary</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span><span class="nd">@app</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="s2">"/agent/energy_source_info"</span><span class="p">)</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span><span class="k">def</span><span class="w"> </span><span class="nf">get_workforce</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">EnergySourceRequest</span><span class="p">):</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span><span class="w"> </span><span class="sd">"""</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span><span class="sd"> Endpoint to get details about energy source</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span><span class="sd"> """</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span> <span class="n">considertion</span> <span class="o">=</span> <span class="s2">"You don't have any specific consideration. Feel free to talk in a more open ended fashion"</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span> <span class="k">if</span> <span class="n">request</span><span class="o">.</span><span class="n">consideration</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span> <span class="n">considertion</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Add specific focus on the following consideration when you summarize the content for the energy source: </span><span class="si">{</span><span class="n">request</span><span class="o">.</span><span class="n">consideration</span><span class="si">}</span><span class="s2">"</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span> <span class="n">response</span> <span class="o">=</span> <span class="p">{</span>
|
||||
</span><span id="line-36"><span class="linenos">36</span> <span class="s2">"energy_source"</span><span class="p">:</span> <span class="n">request</span><span class="o">.</span><span class="n">energy_source</span><span class="p">,</span>
|
||||
</span><span id="line-37"><span class="linenos">37</span> <span class="s2">"consideration"</span><span class="p">:</span> <span class="n">considertion</span><span class="p">,</span>
|
||||
</span><span id="line-38"><span class="linenos">38</span> <span class="p">}</span>
|
||||
</span><span id="line-39"><span class="linenos">39</span> <span class="k">return</span> <span class="n">response</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="demo-app">
|
||||
<h3>Demo App<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#demo-app" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#demo-app'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>For your convenience, we’ve built a <a class="reference external" href="https://github.com/katanemo/archgw/tree/main/demos/samples_python/multi_turn_rag_agent" rel="nofollow noopener">demo app<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>
|
||||
that you can test and modify locally for multi-turn RAG scenarios.</p>
|
||||
<figure class="align-center" id="id4">
|
||||
<a class="reference internal image-reference" href="../_images/mutli-turn-example.png"><img alt="../_images/mutli-turn-example.png" src="../_images/mutli-turn-example.png" style="width: 100%;"/>
|
||||
</a>
|
||||
<figcaption>
|
||||
<p><span class="caption-text">Example multi-turn user conversation showing adjusting retrieval</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id4"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></p>
|
||||
</figcaption>
|
||||
</figure>
|
||||
</section>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="rag.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
RAG Apps
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../resources/deployment.html">
|
||||
Deployment
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#example-1-adjusting-retrieval'" class="reference internal" href="#example-1-adjusting-retrieval">Example 1: Adjusting Retrieval</a></li>
|
||||
<li><a :data-current="activeSection === '#example-2-switching-intent'" class="reference internal" href="#example-2-switching-intent">Example 2: Switching Intent</a></li>
|
||||
<li><a :data-current="activeSection === '#build-multi-turn-rag-apps'" class="reference internal" href="#build-multi-turn-rag-apps">Build Multi-Turn RAG Apps</a><ul>
|
||||
<li><a :data-current="activeSection === '#step-1-define-arch-config'" class="reference internal" href="#step-1-define-arch-config">Step 1: Define Arch Config</a></li>
|
||||
<li><a :data-current="activeSection === '#step-2-process-request-in-flask'" class="reference internal" href="#step-2-process-request-in-flask">Step 2: Process Request in Flask</a></li>
|
||||
<li><a :data-current="activeSection === '#demo-app'" class="reference internal" href="#demo-app">Demo App</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,316 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>RAG Apps | Arch Docs v0.3.22</title>
|
||||
<meta content="RAG Apps | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="RAG Apps | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/build_with_arch/rag.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="multi_turn.html" rel="next" title="Multi-Turn"/>
|
||||
<link href="agent.html" rel="prev" title="Agentic Apps"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
let mode;
|
||||
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
||||
mode = 'dark';
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
mode = 'light';
|
||||
}
|
||||
if (!userPreference) {localStorage.setItem('darkMode', mode)}
|
||||
</script>
|
||||
</head>
|
||||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
K
|
||||
</kbd>
|
||||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
</a>
|
||||
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
|
||||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||||
</svg>
|
||||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="multi_turn.html">Multi-Turn</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</aside>
|
||||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">RAG Apps</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="rag-apps">
|
||||
<span id="arch-rag-guide"></span><h1>RAG Apps<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#rag-apps"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>The following section describes how Arch can help you build faster, smarter and more accurate
|
||||
Retrieval-Augmented Generation (RAG) applications, including fast and accurate RAG in multi-turn
|
||||
converational scenarios.</p>
|
||||
<section id="what-is-retrieval-augmented-generation-rag">
|
||||
<h2>What is Retrieval-Augmented Generation (RAG)?<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#what-is-retrieval-augmented-generation-rag" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#what-is-retrieval-augmented-generation-rag'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>RAG applications combine retrieval-based methods with generative AI models to provide more accurate,
|
||||
contextually relevant, and reliable outputs. These applications leverage external data sources to augment
|
||||
the capabilities of Large Language Models (LLMs), enabling them to retrieve and integrate specific information
|
||||
rather than relying solely on the LLM’s internal knowledge.</p>
|
||||
</section>
|
||||
<section id="parameter-extraction-for-rag">
|
||||
<h2>Parameter Extraction for RAG<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#parameter-extraction-for-rag" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#parameter-extraction-for-rag'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>To build RAG (Retrieval Augmented Generation) applications, you can configure prompt targets with parameters,
|
||||
enabling Arch to retrieve critical information in a structured way for processing. This approach improves the
|
||||
retrieval quality and speed of your application. By extracting parameters from the conversation, you can pull
|
||||
the appropriate chunks from a vector database or SQL-like data store to enhance accuracy. With Arch, you can
|
||||
streamline data retrieval and processing to build more efficient and precise RAG applications.</p>
|
||||
<section id="step-1-define-prompt-targets">
|
||||
<h3>Step 1: Define Prompt Targets<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-1-define-prompt-targets" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-1-define-prompt-targets'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<div class="literal-block-wrapper docutils container" id="id1">
|
||||
<div class="code-block-caption"><span class="caption-text">Prompt Targets</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id1"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">get_device_statistics</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Retrieve and present the relevant data based on the specified devices and time range</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/device_summary</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">device_ids</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">list</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">A list of device identifiers (IDs) to reboot.</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">time_range</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">int</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">The number of days in the past over which to retrieve device statistics</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">false</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">7</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="step-2-process-request-parameters-in-flask">
|
||||
<h3>Step 2: Process Request Parameters in Flask<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-2-process-request-parameters-in-flask" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-2-process-request-parameters-in-flask'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Once the prompt targets are configured as above, handling those parameters is</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id2">
|
||||
<div class="code-block-caption"><span class="caption-text">Parameter handling with Flask</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="kn">from</span><span class="w"> </span><span class="nn">flask</span><span class="w"> </span><span class="kn">import</span> <span class="n">Flask</span><span class="p">,</span> <span class="n">request</span><span class="p">,</span> <span class="n">jsonify</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="n">app</span> <span class="o">=</span> <span class="n">Flask</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="nd">@app</span><span class="o">.</span><span class="n">route</span><span class="p">(</span><span class="s2">"/agent/device_summary"</span><span class="p">,</span> <span class="n">methods</span><span class="o">=</span><span class="p">[</span><span class="s2">"POST"</span><span class="p">])</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="k">def</span><span class="w"> </span><span class="nf">get_device_summary</span><span class="p">():</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="w"> </span><span class="sd">"""</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="sd"> Endpoint to retrieve device statistics based on device IDs and an optional time range.</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="sd"> """</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span> <span class="n">data</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">get_json</span><span class="p">()</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span> <span class="c1"># Validate 'device_ids' parameter</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span> <span class="n">device_ids</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"device_ids"</span><span class="p">)</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">device_ids</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">device_ids</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span> <span class="k">return</span> <span class="p">(</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span> <span class="n">jsonify</span><span class="p">({</span><span class="s2">"error"</span><span class="p">:</span> <span class="s2">"'device_ids' parameter is required and must be a list"</span><span class="p">}),</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span> <span class="mi">400</span><span class="p">,</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span> <span class="p">)</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span> <span class="c1"># Validate 'time_range' parameter (optional, defaults to 7)</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span> <span class="n">time_range</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"time_range"</span><span class="p">,</span> <span class="mi">7</span><span class="p">)</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">time_range</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span> <span class="k">return</span> <span class="n">jsonify</span><span class="p">({</span><span class="s2">"error"</span><span class="p">:</span> <span class="s2">"'time_range' must be an integer"</span><span class="p">}),</span> <span class="mi">400</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span> <span class="c1"># Simulate retrieving statistics for the given device IDs and time range</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span> <span class="c1"># In a real application, you would query your database or external service here</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span> <span class="n">statistics</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span> <span class="k">for</span> <span class="n">device_id</span> <span class="ow">in</span> <span class="n">device_ids</span><span class="p">:</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span> <span class="c1"># Placeholder for actual data retrieval</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span> <span class="n">stats</span> <span class="o">=</span> <span class="p">{</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span> <span class="s2">"device_id"</span><span class="p">:</span> <span class="n">device_id</span><span class="p">,</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span> <span class="s2">"time_range"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"Last </span><span class="si">{</span><span class="n">time_range</span><span class="si">}</span><span class="s2"> days"</span><span class="p">,</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span> <span class="s2">"data"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"Statistics data for device </span><span class="si">{</span><span class="n">device_id</span><span class="si">}</span><span class="s2"> over the last </span><span class="si">{</span><span class="n">time_range</span><span class="si">}</span><span class="s2"> days."</span><span class="p">,</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span> <span class="p">}</span>
|
||||
</span><span id="line-36"><span class="linenos">36</span> <span class="n">statistics</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">stats</span><span class="p">)</span>
|
||||
</span><span id="line-37"><span class="linenos">37</span>
|
||||
</span><span id="line-38"><span class="linenos">38</span> <span class="n">response</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"statistics"</span><span class="p">:</span> <span class="n">statistics</span><span class="p">}</span>
|
||||
</span><span id="line-39"><span class="linenos">39</span>
|
||||
</span><span id="line-40"><span class="linenos">40</span> <span class="k">return</span> <span class="n">jsonify</span><span class="p">(</span><span class="n">response</span><span class="p">),</span> <span class="mi">200</span>
|
||||
</span><span id="line-41"><span class="linenos">41</span>
|
||||
</span><span id="line-42"><span class="linenos">42</span>
|
||||
</span><span id="line-43"><span class="linenos">43</span><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
|
||||
</span><span id="line-44"><span class="linenos">44</span> <span class="n">app</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">debug</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section id="multi-turn-rag-follow-up-questions">
|
||||
<h2>Multi-Turn RAG (Follow-up Questions)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#multi-turn-rag-follow-up-questions" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#multi-turn-rag-follow-up-questions'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Developers often <a class="reference external" href="https://www.reddit.com/r/LocalLLaMA/comments/18mqwg6/best_practice_for_rag_with_followup_chat/" rel="nofollow noopener">struggle<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> to efficiently handle
|
||||
<code class="docutils literal notranslate"><span class="pre">follow-up</span></code> or <code class="docutils literal notranslate"><span class="pre">clarification</span></code> questions. Specifically, when users ask for changes or additions to previous responses, it requires developers to
|
||||
re-write prompts using LLMs with precise prompt engineering techniques. This process is slow, manual, error prone and adds signifcant latency to the
|
||||
user experience.</p>
|
||||
<p>Arch is highly capable of accurately detecting and processing prompts in a multi-turn scenarios so that you can buil fast and accurate RAG apps in
|
||||
minutes. For additional details on how to build multi-turn RAG applications please refer to our <a class="reference internal" href="multi_turn.html#arch-multi-turn-guide"><span class="std std-ref">multi-turn</span></a> docs.</p>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="agent.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Agentic Apps
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="multi_turn.html">
|
||||
Multi-Turn
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#what-is-retrieval-augmented-generation-rag'" class="reference internal" href="#what-is-retrieval-augmented-generation-rag">What is Retrieval-Augmented Generation (RAG)?</a></li>
|
||||
<li><a :data-current="activeSection === '#parameter-extraction-for-rag'" class="reference internal" href="#parameter-extraction-for-rag">Parameter Extraction for RAG</a><ul>
|
||||
<li><a :data-current="activeSection === '#step-1-define-prompt-targets'" class="reference internal" href="#step-1-define-prompt-targets">Step 1: Define Prompt Targets</a></li>
|
||||
<li><a :data-current="activeSection === '#step-2-process-request-parameters-in-flask'" class="reference internal" href="#step-2-process-request-parameters-in-flask">Step 2: Process Request Parameters in Flask</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#multi-turn-rag-follow-up-questions'" class="reference internal" href="#multi-turn-rag-follow-up-questions">Multi-Turn RAG (Follow-up Questions)</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
277
concepts/agents.html
Executable file
277
concepts/agents.html
Executable file
|
|
@ -0,0 +1,277 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Agents | Plano Docs v0.4</title>
|
||||
<meta content="Agents | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Agents | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/agents.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="filter_chain.html" rel="next" title="Filter Chains"/>
|
||||
<link href="listeners.html" rel="prev" title="Listeners"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
let mode;
|
||||
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
||||
mode = 'dark';
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
mode = 'light';
|
||||
}
|
||||
if (!userPreference) {localStorage.setItem('darkMode', mode)}
|
||||
</script>
|
||||
</head>
|
||||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
K
|
||||
</kbd>
|
||||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
</a>
|
||||
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
|
||||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||||
</svg>
|
||||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</aside>
|
||||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Agents</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="agents">
|
||||
<span id="id1"></span><h1>Agents<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#agents"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Agents are autonomous systems that handle wide-ranging, open-ended tasks by calling models in a loop until the work is complete. Unlike deterministic <a class="reference internal" href="prompt_target.html#prompt-target"><span class="std std-ref">prompt targets</span></a>, agents have access to tools, reason about which actions to take, and adapt their behavior based on intermediate results—making them ideal for complex workflows that require multi-step reasoning, external API calls, and dynamic decision-making.</p>
|
||||
<p>Plano helps developers build and scale multi-agent systems by managing the orchestration layer—deciding which agent(s) or LLM(s) should handle each request, and in what sequence—while developers focus on implementing agent logic in any language or framework they choose.</p>
|
||||
<section id="agent-orchestration">
|
||||
<h2>Agent Orchestration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#agent-orchestration" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#agent-orchestration'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p><strong>Plano-Orchestrator</strong> is a family of state-of-the-art routing and orchestration models that decide which agent(s) should handle each request, and in what sequence. Built for real-world multi-agent deployments, it analyzes user intent and conversation context to make precise routing and orchestration decisions while remaining efficient enough for low-latency production use across general chat, coding, and long-context multi-turn conversations.</p>
|
||||
<p>This allows development teams to:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Scale multi-agent systems</strong>: Route requests across multiple specialized agents without hardcoding routing logic in application code.</p></li>
|
||||
<li><p><strong>Improve performance</strong>: Direct requests to the most appropriate agent based on intent, reducing unnecessary handoffs and improving response quality.</p></li>
|
||||
<li><p><strong>Enhance debuggability</strong>: Centralized routing decisions are observable through Plano’s tracing and logging, making it easier to understand why a particular agent was selected.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="inner-loop-vs-outer-loop">
|
||||
<h2>Inner Loop vs. Outer Loop<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#inner-loop-vs-outer-loop" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#inner-loop-vs-outer-loop'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Plano distinguishes between the <strong>inner loop</strong> (agent implementation logic) and the <strong>outer loop</strong> (orchestration and routing):</p>
|
||||
<section id="inner-loop-agent-logic">
|
||||
<h3>Inner Loop (Agent Logic)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#inner-loop-agent-logic" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#inner-loop-agent-logic'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>The inner loop is where your agent lives—the business logic that decides which tools to call, how to interpret results, and when the task is complete. You implement this in any language or framework:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Python agents</strong>: Using frameworks like LangChain, LlamaIndex, CrewAI, or custom Python code.</p></li>
|
||||
<li><p><strong>JavaScript/TypeScript agents</strong>: Using frameworks like LangChain.js or custom Node.js implementations.</p></li>
|
||||
<li><p><strong>Any other AI famreowkr</strong>: Agents are just HTTP services that Plano can route to.</p></li>
|
||||
</ul>
|
||||
<p>Your agent controls:</p>
|
||||
<ul class="simple">
|
||||
<li><p>Which tools or APIs to call in response to a prompt.</p></li>
|
||||
<li><p>How to interpret tool results and decide next steps.</p></li>
|
||||
<li><p>When to call the LLM for reasoning or summarization.</p></li>
|
||||
<li><p>When the task is complete and what response to return.</p></li>
|
||||
</ul>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p><strong>Making LLM Calls from Agents</strong></p>
|
||||
<p>When your agent needs to call an LLM for reasoning, summarization, or completion, you should route those calls through Plano’s Model Proxy rather than calling LLM providers directly. This gives you:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Consistent responses</strong>: Normalized response formats across all <a class="reference internal" href="llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">LLM providers</span></a>, whether you’re using OpenAI, Anthropic, Azure OpenAI, or any OpenAI-compatible provider.</p></li>
|
||||
<li><p><strong>Rich agentic signals</strong>: Automatic capture of function calls, tool usage, reasoning steps, and model behavior—surfaced through traces and metrics without instrumenting your agent code.</p></li>
|
||||
<li><p><strong>Smart model routing</strong>: Leverage <a class="reference internal" href="llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">model-based, alias-based, or preference-aligned routing</span></a> to dynamically select the best model for each task based on cost, performance, or custom policies.</p></li>
|
||||
</ul>
|
||||
<p>By routing LLM calls through the Model Proxy, your agents remain decoupled from specific providers and can benefit from centralized policy enforcement, observability, and intelligent routing—all managed in the outer loop. For a step-by-step guide, see <a class="reference internal" href="../guides/llm_router.html#llm-router"><span class="std std-ref">LLM Routing</span></a> in the LLM Router guide.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="outer-loop-orchestration">
|
||||
<h3>Outer Loop (Orchestration)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#outer-loop-orchestration" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#outer-loop-orchestration'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>The outer loop is Plano’s orchestration layer—it manages the lifecycle of requests across agents and LLMs:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Intent analysis</strong>: Plano-Orchestrator analyzes incoming prompts to determine user intent and conversation context.</p></li>
|
||||
<li><p><strong>Routing decisions</strong>: Routes requests to the appropriate agent(s) or LLM(s) based on capabilities, context, and availability.</p></li>
|
||||
<li><p><strong>Sequencing</strong>: Determines whether multiple agents need to collaborate and in what order.</p></li>
|
||||
<li><p><strong>Lifecycle management</strong>: Handles retries, failover, circuit breaking, and load balancing across agent instances.</p></li>
|
||||
</ul>
|
||||
<p>By managing the outer loop, Plano allows you to:</p>
|
||||
<ul class="simple">
|
||||
<li><p>Add new agents without changing routing logic in existing agents.</p></li>
|
||||
<li><p>Run multiple versions or variants of agents for A/B testing or canary deployments.</p></li>
|
||||
<li><p>Apply consistent <a class="reference internal" href="filter_chain.html#filter-chain"><span class="std std-ref">filter chains</span></a> (guardrails, context enrichment) before requests reach agents.</p></li>
|
||||
<li><p>Monitor and debug multi-agent workflows through centralized observability.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
</section>
|
||||
<section id="key-benefits">
|
||||
<h2>Key Benefits<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#key-benefits" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#key-benefits'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Language and framework agnostic</strong>: Write agents in any language; Plano orchestrates them via HTTP.</p></li>
|
||||
<li><p><strong>Reduced complexity</strong>: Agents focus on task logic; Plano handles routing, retries, and cross-cutting concerns.</p></li>
|
||||
<li><p><strong>Better observability</strong>: Centralized tracing shows which agents were called, in what sequence, and why.</p></li>
|
||||
<li><p><strong>Easier scaling</strong>: Add more agent instances or new agent types without refactoring existing code.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="listeners.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Listeners
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="filter_chain.html">
|
||||
Filter Chains
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#agent-orchestration'" class="reference internal" href="#agent-orchestration">Agent Orchestration</a></li>
|
||||
<li><a :data-current="activeSection === '#inner-loop-vs-outer-loop'" class="reference internal" href="#inner-loop-vs-outer-loop">Inner Loop vs. Outer Loop</a><ul>
|
||||
<li><a :data-current="activeSection === '#inner-loop-agent-logic'" class="reference internal" href="#inner-loop-agent-logic">Inner Loop (Agent Logic)</a></li>
|
||||
<li><a :data-current="activeSection === '#outer-loop-orchestration'" class="reference internal" href="#outer-loop-orchestration">Outer Loop (Orchestration)</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#key-benefits'" class="reference internal" href="#key-benefits">Key Benefits</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
304
concepts/filter_chain.html
Executable file
304
concepts/filter_chain.html
Executable file
|
|
@ -0,0 +1,304 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Filter Chains | Plano Docs v0.4</title>
|
||||
<meta content="Filter Chains | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Filter Chains | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/filter_chain.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="llm_providers/llm_providers.html" rel="next" title="Model (LLM) Providers"/>
|
||||
<link href="agents.html" rel="prev" title="Agents"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
let mode;
|
||||
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
||||
mode = 'dark';
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
mode = 'light';
|
||||
}
|
||||
if (!userPreference) {localStorage.setItem('darkMode', mode)}
|
||||
</script>
|
||||
</head>
|
||||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
K
|
||||
</kbd>
|
||||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
</a>
|
||||
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
|
||||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||||
</svg>
|
||||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="agents.html">Agents</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</aside>
|
||||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Filter Chains</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="filter-chains">
|
||||
<span id="filter-chain"></span><h1>Filter Chains<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#filter-chains"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Filter chains are Plano’s way of capturing <strong>reusable workflow steps</strong> in the dataplane, without duplication and coupling logic into application code. A filter chain is an ordered list of <strong>mutations</strong> that a request flows through before reaching its final destination —such as an agent, an LLM, or a tool backend. Each filter is a network-addressable service/path that can:</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Inspect the incoming prompt, metadata, and conversation state.</p></li>
|
||||
<li><p>Mutate or enrich the request (for example, rewrite queries or build context).</p></li>
|
||||
<li><p>Short-circuit the flow and return a response early (for example, block a request on a compliance failure).</p></li>
|
||||
<li><p>Emit structured logs and traces so you can debug and continuously improve your agents.</p></li>
|
||||
</ol>
|
||||
<p>In other words, filter chains provide a lightweight programming model over HTTP for building reusable steps
|
||||
in your agent architectures.</p>
|
||||
<section id="typical-use-cases">
|
||||
<h2>Typical Use Cases<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#typical-use-cases" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#typical-use-cases'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Without a dataplane programming model, teams tend to spread logic like query rewriting, compliance checks,
|
||||
context building, and routing decisions across many agents and frameworks. This quickly becomes hard to reason
|
||||
about and even harder to evolve.</p>
|
||||
<p>Filter chains show up most often in patterns like:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Guardrails and Compliance</strong>: Enforcing content policies, stripping or masking sensitive data, and blocking obviously unsafe or off-topic requests before they reach an agent.</p></li>
|
||||
<li><p><strong>Query rewriting, RAG, and Memory</strong>: Rewriting user queries for retrieval, normalizing entities, and assembling RAG context envelopes while pulling in relevant memory (for example, conversation history, user profiles, or prior tool results) before calling a model or tool.</p></li>
|
||||
<li><p><strong>Cross-cutting Observability</strong>: Injecting correlation IDs, sampling traces, or logging enriched request metadata at consistent points in the request path.</p></li>
|
||||
</ul>
|
||||
<p>Because these behaviors live in the dataplane rather than inside individual agents, you define them once, attach them to many agents and prompt targets, and can add, remove, or reorder them without changing application code.</p>
|
||||
</section>
|
||||
<section id="configuration-example">
|
||||
<h2>Configuration example<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configuration-example" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configuration-example'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>The example below shows a configuration where an agent uses a filter chain with two filters: a query rewriter,
|
||||
and a context builder that prepares retrieval context before the agent runs.</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id1">
|
||||
<div class="code-block-caption"><span class="caption-text">Example Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id1"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.3.0</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">rag_agent</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://host.docker.internal:10505</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span>
|
||||
</span><span id="line-7"><mark><span class="linenos"> 7</span><span class="nt">filters</span><span class="p">:</span>
|
||||
</mark></span><span id="line-8"><mark><span class="linenos"> 8</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">query_rewriter</span>
|
||||
</mark></span><span id="line-9"><mark><span class="linenos"> 9</span><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://host.docker.internal:10501</span>
|
||||
</mark></span><span id="line-10"><mark><span class="linenos">10</span><span class="w"> </span><span class="c1"># type: mcp # default is mcp</span>
|
||||
</mark></span><span id="line-11"><mark><span class="linenos">11</span><span class="w"> </span><span class="c1"># transport: streamable-http # default is streamable-http</span>
|
||||
</mark></span><span id="line-12"><mark><span class="linenos">12</span><span class="w"> </span><span class="c1"># tool: query_rewriter # default name is the filter id</span>
|
||||
</mark></span><span id="line-13"><mark><span class="linenos">13</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">context_builder</span>
|
||||
</mark></span><span id="line-14"><mark><span class="linenos">14</span><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://host.docker.internal:10502</span>
|
||||
</mark></span><span id="line-15"><span class="linenos">15</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span><span class="nt">model_providers</span><span class="p">:</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o-mini</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span><span class="nt">model_aliases</span><span class="p">:</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span><span class="w"> </span><span class="nt">fast-llm</span><span class="p">:</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-4o-mini</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span><span class="w"> </span><span class="nt">smart-llm</span><span class="p">:</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-4o</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">agent</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">agent_1</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">8001</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span><span class="w"> </span><span class="nt">router</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">arch_agent_router</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span><span class="w"> </span><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">rag_agent</span>
|
||||
</span><span id="line-36"><span class="linenos">36</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">virtual assistant for retrieval augmented generation tasks</span>
|
||||
</span><span id="line-37"><mark><span class="linenos">37</span><span class="w"> </span><span class="nt">filter_chain</span><span class="p">:</span>
|
||||
</mark></span><span id="line-38"><mark><span class="linenos">38</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">query_rewriter</span>
|
||||
</mark></span><span id="line-39"><mark><span class="linenos">39</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">context_builder</span>
|
||||
</mark></span><span id="line-40"><span class="linenos">40</span><span class="nt">tracing</span><span class="p">:</span>
|
||||
</span><span id="line-41"><span class="linenos">41</span><span class="w"> </span><span class="nt">random_sampling</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">100</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p>In this setup:</p>
|
||||
<ul class="simple">
|
||||
<li><p>The <code class="docutils literal notranslate"><span class="pre">filters</span></code> section defines the reusable filters, each running as its own HTTP/MCP service.</p></li>
|
||||
<li><p>The <code class="docutils literal notranslate"><span class="pre">listeners</span></code> section wires the <code class="docutils literal notranslate"><span class="pre">rag_agent</span></code> behind an <code class="docutils literal notranslate"><span class="pre">agent</span></code> listener and attaches a <code class="docutils literal notranslate"><span class="pre">filter_chain</span></code> with <code class="docutils literal notranslate"><span class="pre">query_rewriter</span></code> followed by <code class="docutils literal notranslate"><span class="pre">context_builder</span></code>.</p></li>
|
||||
<li><p>When a request arrives at <code class="docutils literal notranslate"><span class="pre">agent_1</span></code>, Plano executes the filters in order before handing control to <code class="docutils literal notranslate"><span class="pre">rag_agent</span></code>.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="filter-chain-programming-model-http-and-mcp">
|
||||
<h2>Filter Chain Programming Model (HTTP and MCP)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#filter-chain-programming-model-http-and-mcp" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#filter-chain-programming-model-http-and-mcp'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Filters are implemented as simple RESTful endpoints reachable via HTTP. If you want to use the <a class="reference external" href="https://modelcontextprotocol.io/" rel="nofollow noopener">Model Context Protocol (MCP)<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, you can configure that as well, which makes it easy to write filters in any language. However, you can also write a filter as a plain HTTP service.</p>
|
||||
<p>When defining a filter in Plano configuration, the following fields are optional:</p>
|
||||
<ul class="simple">
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">type</span></code>: Controls the filter runtime. Use <code class="docutils literal notranslate"><span class="pre">mcp</span></code> for Model Context Protocol filters, or <code class="docutils literal notranslate"><span class="pre">http</span></code> for plain HTTP filters. Defaults to <code class="docutils literal notranslate"><span class="pre">mcp</span></code>.</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">transport</span></code>: Controls how Plano talks to the filter (defaults to <code class="docutils literal notranslate"><span class="pre">streamable-http</span></code> for efficient streaming interactions over HTTP). You can omit this for standard HTTP transport.</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">tool</span></code>: Names the MCP tool Plano will invoke (by default, the filter <code class="docutils literal notranslate"><span class="pre">id</span></code>). You can omit this if the tool name matches your filter id.</p></li>
|
||||
</ul>
|
||||
<p>In practice, you typically only need to specify <code class="docutils literal notranslate"><span class="pre">id</span></code> and <code class="docutils literal notranslate"><span class="pre">url</span></code> to get started. Plano’s sensible defaults mean a filter can be as simple as an HTTP endpoint. If you want to customize the runtime or protocol, those fields are there, but they’re optional.</p>
|
||||
<p>Filters communicate the outcome of their work via HTTP status codes:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>HTTP 200 (Success)</strong>: The filter successfully processed the request. If the filter mutated the request (e.g., rewrote a query or enriched context), those mutations are passed downstream.</p></li>
|
||||
<li><p><strong>HTTP 4xx (User Error)</strong>: The request violates a filter’s rules or constraints—for example, content moderation policies or compliance checks. The request is terminated, and the error is returned to the caller. This is <em>not</em> a fatal error; it represents expected user-facing policy enforcement.</p></li>
|
||||
<li><p><strong>HTTP 5xx (Fatal Error)</strong>: An unexpected failure in the filter itself (for example, a crash or misconfiguration). Plano will surface the error back to the caller and record it in logs and traces.</p></li>
|
||||
</ul>
|
||||
<p>This semantics allows filters to enforce guardrails and policies (4xx) without blocking the entire system, while still surfacing critical failures (5xx) for investigation.</p>
|
||||
<p>If any filter fails or decides to terminate the request early (for example, after a policy violation), Plano will
|
||||
surface that outcome back to the caller and record it in logs and traces. This makes filter chains a safe and
|
||||
powerful abstraction for evolving your agent workflows over time.</p>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="agents.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Agents
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="llm_providers/llm_providers.html">
|
||||
Model (LLM) Providers
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#typical-use-cases'" class="reference internal" href="#typical-use-cases">Typical Use Cases</a></li>
|
||||
<li><a :data-current="activeSection === '#configuration-example'" class="reference internal" href="#configuration-example">Configuration example</a></li>
|
||||
<li><a :data-current="activeSection === '#filter-chain-programming-model-http-and-mcp'" class="reference internal" href="#filter-chain-programming-model-http-and-mcp">Filter Chain Programming Model (HTTP and MCP)</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,25 +1,25 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Listener | Arch Docs v0.3.22</title>
|
||||
<meta content="Listener | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Listener | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/tech_overview/listener.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../../search.html" rel="search" title="Search"/>
|
||||
<link href="prompt.html" rel="next" title="Prompts"/>
|
||||
<link href="threading_model.html" rel="prev" title="Threading Model"/>
|
||||
<title>Listeners | Plano Docs v0.4</title>
|
||||
<meta content="Listeners | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Listeners | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/listeners.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="agents.html" rel="next" title="Agents"/>
|
||||
<link href="../get_started/quickstart.html" rel="prev" title="Quickstart"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -38,8 +38,8 @@
|
|||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -47,7 +47,7 @@
|
|||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -74,71 +74,66 @@
|
|||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="error_target.html">Error Target</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_target.html">Prompt Target</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -152,77 +147,102 @@
|
|||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="tech_overview.html">Tech Overview</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Listener</span>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Listeners</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="listener">
|
||||
<span id="arch-overview-listeners"></span><h1>Listener<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#listener"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p><strong>Listener</strong> is a top level primitive in Arch, which simplifies the configuration required to bind incoming
|
||||
connections from downstream clients, and for egress connections to LLMs (hosted or API)</p>
|
||||
<p>Arch builds on Envoy’s Listener subsystem to streamline connection management for developers. Arch minimizes
|
||||
the complexity of Envoy’s listener setup by using best-practices and exposing only essential settings,
|
||||
making it easier for developers to bind connections without deep knowledge of Envoy’s configuration model. This
|
||||
simplification ensures that connections are secure, reliable, and optimized for performance.</p>
|
||||
<section id="downstream-ingress">
|
||||
<h2>Downstream (Ingress)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#downstream-ingress" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#downstream-ingress'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Developers can configure Arch to accept connections from downstream clients. A downstream listener acts as the
|
||||
primary entry point for incoming traffic, handling initial connection setup, including network filtering, guardrails,
|
||||
and additional network security checks. For more details on prompt security and safety,
|
||||
see <a class="reference internal" href="prompt.html#arch-overview-prompt-handling"><span class="std std-ref">here</span></a>.</p>
|
||||
<section id="listeners">
|
||||
<span id="plano-overview-listeners"></span><h1>Listeners<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#listeners"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p><strong>Listeners</strong> are a top-level primitive in Plano that bind network traffic to the dataplane. They simplify the
|
||||
configuration required to accept incoming connections from downstream clients (edge) and to expose a unified egress
|
||||
endpoint for calls from your applications to upstream LLMs.</p>
|
||||
<p>Plano builds on Envoy’s Listener subsystem to streamline connection management for developers. It hides most of
|
||||
Envoy’s complexity behind sensible defaults and a focused configuration surface, so you can bind listeners without
|
||||
deep knowledge of Envoy’s configuration model while still getting secure, reliable, and performant connections.</p>
|
||||
<p>Listeners are modular building blocks: you can configure only inbound listeners (for edge proxying and guardrails),
|
||||
only outbound/model-proxy listeners (for LLM routing from your services), or both together. This lets you fit Plano
|
||||
cleanly into existing architectures, whether you need it at the edge, behind the firewall, or across the full
|
||||
request path.</p>
|
||||
<section id="network-topology">
|
||||
<h2>Network Topology<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#network-topology" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#network-topology'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>The diagram below shows how inbound and outbound traffic flow through Plano and how listeners relate to agents,
|
||||
prompt targets, and upstream LLMs:</p>
|
||||
<a class="reference internal image-reference" href="../_images/network-topology-ingress-egress.png"><img alt="../_images/network-topology-ingress-egress.png" class="align-center" src="../_images/network-topology-ingress-egress.png" style="width: 100%;"/>
|
||||
</a>
|
||||
</section>
|
||||
<section id="upstream-egress">
|
||||
<h2>Upstream (Egress)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#upstream-egress" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#upstream-egress'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Arch automatically configures a listener to route requests from your application to upstream LLM API providers (or hosts).
|
||||
When you start Arch, it creates a listener for egress traffic based on the presence of the <code class="docutils literal notranslate"><span class="pre">listener</span></code> configuration
|
||||
section in the configuration file. Arch binds itself to a local address such as <code class="docutils literal notranslate"><span class="pre">127.0.0.1:12000/v1</span></code> or a DNS-based
|
||||
address like <code class="docutils literal notranslate"><span class="pre">arch.local:12000/v1</span></code> for outgoing traffic. For more details on LLM providers, read <a class="reference internal" href="../llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">here</span></a>.</p>
|
||||
<section id="inbound-agent-prompt-target">
|
||||
<h2>Inbound (Agent & Prompt Target)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#inbound-agent-prompt-target" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#inbound-agent-prompt-target'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Developers configure <strong>inbound listeners</strong> to accept connections from clients such as web frontends, backend
|
||||
services, or other gateways. An inbound listener acts as the primary entry point for prompt traffic, handling
|
||||
initial connection setup, TLS termination, guardrails, and forwarding incoming traffic to the appropriate prompt
|
||||
targets or agents.</p>
|
||||
<p>There are two primary types of inbound connections exposed via listeners:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Agent Inbound (Edge)</strong>: Clients (web/mobile apps or other services) connect to Plano, send prompts, and receive
|
||||
responses. This is typically your public/edge listener where Plano applies guardrails, routing, and orchestration
|
||||
before returning results to the caller.</p></li>
|
||||
<li><p><strong>Prompt Target Inbound (Edge)</strong>: Your application server calls Plano’s internal listener targeting
|
||||
<a class="reference internal" href="prompt_target.html#prompt-target"><span class="std std-ref">prompt targets</span></a> that can invoke tools and LLMs directly on its behalf.</p></li>
|
||||
</ul>
|
||||
<p>Inbound listeners are where you attach <a class="reference internal" href="filter_chain.html#filter-chain"><span class="std std-ref">Filter Chains</span></a> so that safety and context-building happen
|
||||
consistently at the edge.</p>
|
||||
</section>
|
||||
<section id="configure-listener">
|
||||
<h2>Configure Listener<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configure-listener" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configure-listener'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>To configure a Downstream (Ingress) Listener, simply add the <code class="docutils literal notranslate"><span class="pre">listener</span></code> directive to your configuration file:</p>
|
||||
<section id="outbound-model-proxy-egress">
|
||||
<h2>Outbound (Model Proxy & Egress)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#outbound-model-proxy-egress" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#outbound-model-proxy-egress'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Plano also exposes an <strong>egress listener</strong> that your applications call when sending requests to upstream LLM providers
|
||||
or self-hosted models. From your application’s perspective this looks like a single OpenAI-compatible HTTP endpoint
|
||||
(for example, <code class="docutils literal notranslate"><span class="pre">http://127.0.0.1:12000/v1</span></code>), while Plano handles provider selection, retries, and failover behind
|
||||
the scenes.</p>
|
||||
<p>Under the hood, Plano opens outbound HTTP(S) connections to upstream LLM providers using its unified API surface and
|
||||
smart model routing. For more details on how Plano talks to models and how providers are configured, see
|
||||
<a class="reference internal" href="llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">LLM providers</span></a>.</p>
|
||||
</section>
|
||||
<section id="configure-listeners">
|
||||
<h2>Configure Listeners<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configure-listeners" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configure-listeners'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Listeners are configured via the <code class="docutils literal notranslate"><span class="pre">listeners</span></code> block in your Plano configuration. You can define one or more inbound
|
||||
listeners (for example, <code class="docutils literal notranslate"><span class="pre">type:edge</span></code>) or one or more outbound/model listeners (for example, <code class="docutils literal notranslate"><span class="pre">type:model</span></code>), or both
|
||||
in the same deployment.</p>
|
||||
<p>To configure an inbound (edge) listener, add a <code class="docutils literal notranslate"><span class="pre">listeners</span></code> block to your configuration file and define at least one
|
||||
listener with address, port, and protocol details:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id1">
|
||||
<div class="code-block-caption"><span class="caption-text">Example Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id1"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1.0</span>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.2.0</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span>
|
||||
</span><span id="line-3"><mark><span class="linenos"> 3</span><span class="nt">listeners</span><span class="p">:</span>
|
||||
</mark></span><span id="line-4"><mark><span class="linenos"> 4</span><span class="w"> </span><span class="nt">ingress_traffic</span><span class="p">:</span>
|
||||
</mark></span><span id="line-5"><mark><span class="linenos"> 5</span><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</mark></span><span id="line-6"><mark><span class="linenos"> 6</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10000</span>
|
||||
</mark></span><span id="line-7"><mark><span class="linenos"> 7</span><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</mark></span><span id="line-8"><span class="linenos"> 8</span><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="c1"># Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span><span class="c1"># default system prompt used by all prompt targets</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">You are a network assistant that just offers facts; not advice on manufacturers or purchasing decisions.</span>
|
||||
</mark></span><span id="line-7"><mark><span class="linenos"> 7</span>
|
||||
</mark></span><span id="line-8"><span class="linenos"> 8</span><span class="c1"># Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="nt">model_providers</span><span class="p">:</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p>When you start Plano, you specify a listener address/port that you want to bind downstream. Plano also exposes a
|
||||
predefined internal listener (<code class="docutils literal notranslate"><span class="pre">127.0.0.1:12000</span></code>) that you can use to proxy egress calls originating from your
|
||||
application to LLMs (API-based or hosted) via prompt targets.</p>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="threading_model.html">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../get_started/quickstart.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Threading Model
|
||||
Quickstart
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="prompt.html">
|
||||
Prompts
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="agents.html">
|
||||
Agents
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
|
|
@ -231,9 +251,10 @@ address like <code class="docutils literal notranslate"><span class="pre">arch.l
|
|||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#downstream-ingress'" class="reference internal" href="#downstream-ingress">Downstream (Ingress)</a></li>
|
||||
<li><a :data-current="activeSection === '#upstream-egress'" class="reference internal" href="#upstream-egress">Upstream (Egress)</a></li>
|
||||
<li><a :data-current="activeSection === '#configure-listener'" class="reference internal" href="#configure-listener">Configure Listener</a></li>
|
||||
<li><a :data-current="activeSection === '#network-topology'" class="reference internal" href="#network-topology">Network Topology</a></li>
|
||||
<li><a :data-current="activeSection === '#inbound-agent-prompt-target'" class="reference internal" href="#inbound-agent-prompt-target">Inbound (Agent & Prompt Target)</a></li>
|
||||
<li><a :data-current="activeSection === '#outbound-model-proxy-egress'" class="reference internal" href="#outbound-model-proxy-egress">Outbound (Model Proxy & Egress)</a></li>
|
||||
<li><a :data-current="activeSection === '#configure-listeners'" class="reference internal" href="#configure-listeners">Configure Listeners</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
|
|
@ -242,15 +263,15 @@ address like <code class="docutils literal notranslate"><span class="pre">arch.l
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../../_static/design-tabs.js?v=f930bc37"></script>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -7,13 +7,13 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Client Libraries | Arch Docs v0.3.22</title>
|
||||
<meta content="Client Libraries | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Client Libraries | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Client Libraries | Plano Docs v0.4</title>
|
||||
<meta content="Client Libraries | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Client Libraries | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/llm_providers/client_libraries.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,18 +148,18 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="llm_providers.html">LLM Providers</a>
|
||||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="llm_providers.html">Model (LLM) Providers</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Client Libraries</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="client-libraries">
|
||||
<span id="id1"></span><h1>Client Libraries<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#client-libraries"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Arch provides a unified interface that works seamlessly with multiple client libraries and tools. You can use your preferred client library without changing your existing code - just point it to Arch’s gateway endpoints.</p>
|
||||
<p>Plano provides a unified interface that works seamlessly with multiple client libraries and tools. You can use your preferred client library without changing your existing code - just point it to Plano’s gateway endpoints.</p>
|
||||
<section id="supported-clients">
|
||||
<h2>Supported Clients<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#supported-clients" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#supported-clients'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ul class="simple">
|
||||
|
|
@ -176,7 +171,7 @@
|
|||
</section>
|
||||
<section id="gateway-endpoints">
|
||||
<h2>Gateway Endpoints<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#gateway-endpoints" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#gateway-endpoints'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Arch exposes two main endpoints:</p>
|
||||
<p>Plano exposes three main endpoints:</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 40.0%"/>
|
||||
|
|
@ -191,7 +186,10 @@
|
|||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">http://127.0.0.1:12000/v1/chat/completions</span></code></p></td>
|
||||
<td><p>OpenAI-compatible chat completions (LLM Gateway)</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">http://127.0.0.1:12000/v1/messages</span></code></p></td>
|
||||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">http://127.0.0.1:12000/v1/responses</span></code></p></td>
|
||||
<td><p>OpenAI Responses API with <a class="reference internal" href="../../guides/state.html#managing-conversational-state"><span class="std std-ref">conversational state management</span></a> (LLM Gateway)</p></td>
|
||||
</tr>
|
||||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">http://127.0.0.1:12000/v1/messages</span></code></p></td>
|
||||
<td><p>Anthropic-compatible messages (LLM Gateway)</p></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
|
|
@ -199,7 +197,7 @@
|
|||
</section>
|
||||
<section id="openai-python-sdk">
|
||||
<h2>OpenAI (Python) SDK<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#openai-python-sdk" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#openai-python-sdk'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>The OpenAI SDK works with any provider through Arch’s OpenAI-compatible endpoint.</p>
|
||||
<p>The OpenAI SDK works with any provider through Plano’s OpenAI-compatible endpoint.</p>
|
||||
<p><strong>Installation:</strong></p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">pip<span class="w"> </span>install<span class="w"> </span>openai
|
||||
</span></code></pre></div>
|
||||
|
|
@ -207,7 +205,7 @@
|
|||
<p><strong>Basic Usage:</strong></p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">from</span><span class="w"> </span><span class="nn">openai</span><span class="w"> </span><span class="kn">import</span> <span class="n">OpenAI</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="c1"># Point to Arch's LLM Gateway</span>
|
||||
</span><span id="line-3"><span class="c1"># Point to Plano's LLM Gateway</span>
|
||||
</span><span id="line-4"><span class="n">client</span> <span class="o">=</span> <span class="n">OpenAI</span><span class="p">(</span>
|
||||
</span><span id="line-5"> <span class="n">api_key</span><span class="o">=</span><span class="s2">"test-key"</span><span class="p">,</span> <span class="c1"># Can be any value for local testing</span>
|
||||
</span><span id="line-6"> <span class="n">base_url</span><span class="o">=</span><span class="s2">"http://127.0.0.1:12000/v1"</span>
|
||||
|
|
@ -255,7 +253,7 @@
|
|||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Using with Non-OpenAI Models:</strong></p>
|
||||
<p>The OpenAI SDK can be used with any provider configured in Arch:</p>
|
||||
<p>The OpenAI SDK can be used with any provider configured in Plano:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Using Claude model through OpenAI SDK</span>
|
||||
</span><span id="line-2"><span class="n">completion</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-3"> <span class="n">model</span><span class="o">=</span><span class="s2">"claude-3-5-sonnet-20241022"</span><span class="p">,</span>
|
||||
|
|
@ -282,9 +280,82 @@
|
|||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="openai-responses-api-conversational-state">
|
||||
<h2>OpenAI Responses API (Conversational State)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#openai-responses-api-conversational-state" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#openai-responses-api-conversational-state'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>The OpenAI Responses API (<code class="docutils literal notranslate"><span class="pre">v1/responses</span></code>) enables multi-turn conversations with automatic state management. Plano handles conversation history for you, so you don’t need to manually include previous messages in each request.</p>
|
||||
<p>See <a class="reference internal" href="../../guides/state.html#managing-conversational-state"><span class="std std-ref">Conversational State</span></a> for detailed configuration and storage backend options.</p>
|
||||
<p><strong>Installation:</strong></p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">pip<span class="w"> </span>install<span class="w"> </span>openai
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Basic Multi-Turn Conversation:</strong></p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">from</span><span class="w"> </span><span class="nn">openai</span><span class="w"> </span><span class="kn">import</span> <span class="n">OpenAI</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="c1"># Point to Plano's LLM Gateway</span>
|
||||
</span><span id="line-4"><span class="n">client</span> <span class="o">=</span> <span class="n">OpenAI</span><span class="p">(</span>
|
||||
</span><span id="line-5"> <span class="n">api_key</span><span class="o">=</span><span class="s2">"test-key"</span><span class="p">,</span>
|
||||
</span><span id="line-6"> <span class="n">base_url</span><span class="o">=</span><span class="s2">"http://127.0.0.1:12000/v1"</span>
|
||||
</span><span id="line-7"><span class="p">)</span>
|
||||
</span><span id="line-8">
|
||||
</span><span id="line-9"><span class="c1"># First turn - creates a new conversation</span>
|
||||
</span><span id="line-10"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-11"> <span class="n">model</span><span class="o">=</span><span class="s2">"gpt-4o-mini"</span><span class="p">,</span>
|
||||
</span><span id="line-12"> <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
|
||||
</span><span id="line-13"> <span class="p">{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"My name is Alice"</span><span class="p">}</span>
|
||||
</span><span id="line-14"> <span class="p">]</span>
|
||||
</span><span id="line-15"><span class="p">)</span>
|
||||
</span><span id="line-16">
|
||||
</span><span id="line-17"><span class="c1"># Extract response_id for conversation continuity</span>
|
||||
</span><span id="line-18"><span class="n">response_id</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">id</span>
|
||||
</span><span id="line-19"><span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Assistant: </span><span class="si">{</span><span class="n">response</span><span class="o">.</span><span class="n">choices</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">content</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
</span><span id="line-20">
|
||||
</span><span id="line-21"><span class="c1"># Second turn - continues the conversation</span>
|
||||
</span><span id="line-22"><span class="c1"># Plano automatically retrieves and merges previous context</span>
|
||||
</span><span id="line-23"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-24"> <span class="n">model</span><span class="o">=</span><span class="s2">"gpt-4o-mini"</span><span class="p">,</span>
|
||||
</span><span id="line-25"> <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
|
||||
</span><span id="line-26"> <span class="p">{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"What's my name?"</span><span class="p">}</span>
|
||||
</span><span id="line-27"> <span class="p">],</span>
|
||||
</span><span id="line-28"> <span class="n">metadata</span><span class="o">=</span><span class="p">{</span><span class="s2">"response_id"</span><span class="p">:</span> <span class="n">response_id</span><span class="p">}</span> <span class="c1"># Reference previous conversation</span>
|
||||
</span><span id="line-29"><span class="p">)</span>
|
||||
</span><span id="line-30">
|
||||
</span><span id="line-31"><span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Assistant: </span><span class="si">{</span><span class="n">response</span><span class="o">.</span><span class="n">choices</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">content</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
</span><span id="line-32"><span class="c1"># Output: "Your name is Alice"</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Using with Any Provider:</strong></p>
|
||||
<p>The Responses API works with any LLM provider configured in Plano:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Multi-turn conversation with Claude</span>
|
||||
</span><span id="line-2"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-3"> <span class="n">model</span><span class="o">=</span><span class="s2">"claude-3-5-sonnet-20241022"</span><span class="p">,</span>
|
||||
</span><span id="line-4"> <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
|
||||
</span><span id="line-5"> <span class="p">{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Let's discuss quantum physics"</span><span class="p">}</span>
|
||||
</span><span id="line-6"> <span class="p">]</span>
|
||||
</span><span id="line-7"><span class="p">)</span>
|
||||
</span><span id="line-8">
|
||||
</span><span id="line-9"><span class="n">response_id</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">id</span>
|
||||
</span><span id="line-10">
|
||||
</span><span id="line-11"><span class="c1"># Continue conversation - Plano manages state regardless of provider</span>
|
||||
</span><span id="line-12"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-13"> <span class="n">model</span><span class="o">=</span><span class="s2">"claude-3-5-sonnet-20241022"</span><span class="p">,</span>
|
||||
</span><span id="line-14"> <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
|
||||
</span><span id="line-15"> <span class="p">{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Tell me more about entanglement"</span><span class="p">}</span>
|
||||
</span><span id="line-16"> <span class="p">],</span>
|
||||
</span><span id="line-17"> <span class="n">metadata</span><span class="o">=</span><span class="p">{</span><span class="s2">"response_id"</span><span class="p">:</span> <span class="n">response_id</span><span class="p">}</span>
|
||||
</span><span id="line-18"><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Key Benefits:</strong></p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Reduced payload size</strong>: No need to send full conversation history in each request</p></li>
|
||||
<li><p><strong>Provider flexibility</strong>: Use any configured LLM provider with state management</p></li>
|
||||
<li><p><strong>Automatic context merging</strong>: Plano handles conversation continuity behind the scenes</p></li>
|
||||
<li><p><strong>Production-ready storage</strong>: Configure <a class="reference internal" href="../../guides/state.html#managing-conversational-state"><span class="std std-ref">PostgreSQL or memory storage</span></a> based on your needs</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="anthropic-python-sdk">
|
||||
<h2>Anthropic (Python) SDK<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#anthropic-python-sdk" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#anthropic-python-sdk'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>The Anthropic SDK works with any provider through Arch’s Anthropic-compatible endpoint.</p>
|
||||
<p>The Anthropic SDK works with any provider through Plano’s Anthropic-compatible endpoint.</p>
|
||||
<p><strong>Installation:</strong></p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">pip<span class="w"> </span>install<span class="w"> </span>anthropic
|
||||
</span></code></pre></div>
|
||||
|
|
@ -292,7 +363,7 @@
|
|||
<p><strong>Basic Usage:</strong></p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">import</span><span class="w"> </span><span class="nn">anthropic</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="c1"># Point to Arch's LLM Gateway</span>
|
||||
</span><span id="line-3"><span class="c1"># Point to Plano's LLM Gateway</span>
|
||||
</span><span id="line-4"><span class="n">client</span> <span class="o">=</span> <span class="n">anthropic</span><span class="o">.</span><span class="n">Anthropic</span><span class="p">(</span>
|
||||
</span><span id="line-5"> <span class="n">api_key</span><span class="o">=</span><span class="s2">"test-key"</span><span class="p">,</span> <span class="c1"># Can be any value for local testing</span>
|
||||
</span><span id="line-6"> <span class="n">base_url</span><span class="o">=</span><span class="s2">"http://127.0.0.1:12000"</span>
|
||||
|
|
@ -341,7 +412,7 @@
|
|||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Using with Non-Anthropic Models:</strong></p>
|
||||
<p>The Anthropic SDK can be used with any provider configured in Arch:</p>
|
||||
<p>The Anthropic SDK can be used with any provider configured in Plano:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Using OpenAI model through Anthropic SDK</span>
|
||||
</span><span id="line-2"><span class="n">message</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">messages</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-3"> <span class="n">model</span><span class="o">=</span><span class="s2">"gpt-4o-mini"</span><span class="p">,</span>
|
||||
|
|
@ -426,7 +497,7 @@
|
|||
</section>
|
||||
<section id="cross-client-compatibility">
|
||||
<h2>Cross-Client Compatibility<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#cross-client-compatibility" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#cross-client-compatibility'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>One of Arch’s key features is cross-client compatibility. You can:</p>
|
||||
<p>One of Plano’s key features is cross-client compatibility. You can:</p>
|
||||
<p><strong>Use OpenAI SDK with Claude Models:</strong></p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># OpenAI client calling Claude model</span>
|
||||
</span><span id="line-2"><span class="kn">from</span><span class="w"> </span><span class="nn">openai</span><span class="w"> </span><span class="kn">import</span> <span class="n">OpenAI</span>
|
||||
|
|
@ -570,6 +641,7 @@ Implement fallback logic for better reliability:</p>
|
|||
<li><a :data-current="activeSection === '#supported-clients'" class="reference internal" href="#supported-clients">Supported Clients</a></li>
|
||||
<li><a :data-current="activeSection === '#gateway-endpoints'" class="reference internal" href="#gateway-endpoints">Gateway Endpoints</a></li>
|
||||
<li><a :data-current="activeSection === '#openai-python-sdk'" class="reference internal" href="#openai-python-sdk">OpenAI (Python) SDK</a></li>
|
||||
<li><a :data-current="activeSection === '#openai-responses-api-conversational-state'" class="reference internal" href="#openai-responses-api-conversational-state">OpenAI Responses API (Conversational State)</a></li>
|
||||
<li><a :data-current="activeSection === '#anthropic-python-sdk'" class="reference internal" href="#anthropic-python-sdk">Anthropic (Python) SDK</a></li>
|
||||
<li><a :data-current="activeSection === '#curl-examples'" class="reference internal" href="#curl-examples">cURL Examples</a></li>
|
||||
<li><a :data-current="activeSection === '#cross-client-compatibility'" class="reference internal" href="#cross-client-compatibility">Cross-Client Compatibility</a></li>
|
||||
|
|
@ -584,12 +656,12 @@ Implement fallback logic for better reliability:</p>
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>LLM Providers | Arch Docs v0.3.22</title>
|
||||
<meta content="LLM Providers | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="LLM Providers | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Model (LLM) Providers | Plano Docs v0.4</title>
|
||||
<meta content="Model (LLM) Providers | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Model (LLM) Providers | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/llm_providers/llm_providers.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../../search.html" rel="search" title="Search"/>
|
||||
<link href="supported_providers.html" rel="next" title="Supported Providers & Configuration"/>
|
||||
<link href="../tech_overview/error_target.html" rel="prev" title="Error Target"/>
|
||||
<link href="../filter_chain.html" rel="prev" title="Filter Chains"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="current reference internal expandable" href="#">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="current reference internal expandable" href="#">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,53 +148,54 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">LLM Providers</span>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Model (LLM) Providers</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="llm-providers">
|
||||
<span id="id1"></span><h1>LLM Providers<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#llm-providers"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p><strong>LLM Providers</strong> are a top-level primitive in Arch, helping developers centrally define, secure, observe,
|
||||
and manage the usage of their LLMs. Arch builds on Envoy’s reliable <a class="reference external" href="https://www.envoyproxy.io/docs/envoy/v1.31.2/intro/arch_overview/upstream/cluster_manager" rel="nofollow noopener">cluster subsystem<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>
|
||||
to manage egress traffic to LLMs, which includes intelligent routing, retry and fail-over mechanisms,
|
||||
ensuring high availability and fault tolerance. This abstraction also enables developers to seamlessly
|
||||
switch between LLM providers or upgrade LLM versions, simplifying the integration and scaling of LLMs
|
||||
across applications.</p>
|
||||
<p>Today, we are enabling you to connect to 11+ different AI providers through a unified interface with advanced routing and management capabilities.
|
||||
Whether you’re using OpenAI, Anthropic, Azure OpenAI, local Ollama models, or any OpenAI-compatible provider, Arch provides seamless integration with enterprise-grade features.</p>
|
||||
<section id="model-llm-providers">
|
||||
<span id="llm-providers"></span><h1>Model (LLM) Providers<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#model-llm-providers"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p><strong>Model Providers</strong> are a top-level primitive in Plano, helping developers centrally define, secure, observe,
|
||||
and manage the usage of their models. Plano builds on Envoy’s reliable <a class="reference external" href="https://www.envoyproxy.io/docs/envoy/v1.31.2/intro/arch_overview/upstream/cluster_manager" rel="nofollow noopener">cluster subsystem<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> to manage egress traffic to models, which includes intelligent routing, retry and fail-over mechanisms,
|
||||
ensuring high availability and fault tolerance. This abstraction also enables developers to seamlessly switch between model providers or upgrade model versions, simplifying the integration and scaling of models across applications.</p>
|
||||
<p>Today, we are enable you to connect to 15+ different AI providers through a unified interface with advanced routing and management capabilities.
|
||||
Whether you’re using OpenAI, Anthropic, Azure OpenAI, local Ollama models, or any OpenAI-compatible provider, Plano provides seamless integration with enterprise-grade features.</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>Please refer to the quickstart guide <a class="reference internal" href="../../get_started/quickstart.html#llm-routing-quickstart"><span class="std std-ref">here</span></a> to configure and use LLM providers via common client libraries like OpenAI and Anthropic Python SDKs, or via direct HTTP/cURL requests.</p>
|
||||
</div>
|
||||
<section id="core-capabilities">
|
||||
<h2>Core Capabilities<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#core-capabilities" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#core-capabilities'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p><strong>Multi-Provider Support</strong>
|
||||
Connect to any combination of providers simultaneously (see <a class="reference internal" href="supported_providers.html#supported-providers"><span class="std std-ref">Supported Providers & Configuration</span></a> for full details):</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>First-Class Providers</strong>: Native integrations with OpenAI, Anthropic, DeepSeek, Mistral, Groq, Google Gemini, Together AI, xAI, Azure OpenAI, and Ollama</p></li>
|
||||
<li><p><strong>OpenAI-Compatible Providers</strong>: Any provider implementing the OpenAI Chat Completions API standard</p></li>
|
||||
<li><p>First-Class Providers: Native integrations with OpenAI, Anthropic, DeepSeek, Mistral, Groq, Google Gemini, Together AI, xAI, Azure OpenAI, and Ollama</p></li>
|
||||
<li><p>OpenAI-Compatible Providers: Any provider implementing the OpenAI Chat Completions API standard</p></li>
|
||||
</ul>
|
||||
<p><strong>Intelligent Routing</strong>
|
||||
Three powerful routing approaches to optimize model selection:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Model-based Routing</strong>: Direct routing to specific models using provider/model names (see <a class="reference internal" href="supported_providers.html#supported-providers"><span class="std std-ref">Supported Providers & Configuration</span></a>)</p></li>
|
||||
<li><p><strong>Alias-based Routing</strong>: Semantic routing using custom aliases (see <a class="reference internal" href="model_aliases.html#model-aliases"><span class="std std-ref">Model Aliases</span></a>)</p></li>
|
||||
<li><p><strong>Preference-aligned Routing</strong>: Intelligent routing using the Arch-Router model (see <a class="reference internal" href="../../guides/llm_router.html#preference-aligned-routing"><span class="std std-ref">Preference-aligned Routing (Arch-Router)</span></a>)</p></li>
|
||||
<li><p>Model-based Routing: Direct routing to specific models using provider/model names (see <a class="reference internal" href="supported_providers.html#supported-providers"><span class="std std-ref">Supported Providers & Configuration</span></a>)</p></li>
|
||||
<li><p>Alias-based Routing: Semantic routing using custom aliases (see <a class="reference internal" href="model_aliases.html#model-aliases"><span class="std std-ref">Model Aliases</span></a>)</p></li>
|
||||
<li><p>Preference-aligned Routing: Intelligent routing using the Plano-Router model (see <a class="reference internal" href="../../guides/llm_router.html#preference-aligned-routing"><span class="std std-ref">Preference-aligned routing (Arch-Router)</span></a>)</p></li>
|
||||
</ul>
|
||||
<p><strong>Unified Client Interface</strong>
|
||||
Use your preferred client library without changing existing code (see <a class="reference internal" href="client_libraries.html#client-libraries"><span class="std std-ref">Client Libraries</span></a> for details):</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>OpenAI Python SDK</strong>: Full compatibility with all providers</p></li>
|
||||
<li><p><strong>Anthropic Python SDK</strong>: Native support with cross-provider capabilities</p></li>
|
||||
<li><p><strong>cURL & HTTP Clients</strong>: Direct REST API access for any programming language</p></li>
|
||||
<li><p><strong>Custom Integrations</strong>: Standard HTTP interfaces for seamless integration</p></li>
|
||||
<li><p>OpenAI Python SDK: Full compatibility with all providers</p></li>
|
||||
<li><p>Anthropic Python SDK: Native support with cross-provider capabilities</p></li>
|
||||
<li><p>cURL & HTTP Clients: Direct REST API access for any programming language</p></li>
|
||||
<li><p>Custom Integrations: Standard HTTP interfaces for seamless integration</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="key-benefits">
|
||||
<h2>Key Benefits<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#key-benefits" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#key-benefits'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Provider Flexibility</strong>: Switch between providers without changing client code</p></li>
|
||||
<li><p><strong>Three Routing Methods</strong>: Choose from model-based, alias-based, or preference-aligned routing (using <a class="reference external" href="https://huggingface.co/katanemo/Arch-Router-1.5B" rel="nofollow noopener">Arch-Router-1.5B<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>) strategies</p></li>
|
||||
<li><p><strong>Three Routing Methods</strong>: Choose from model-based, alias-based, or preference-aligned routing (using <a class="reference external" href="https://huggingface.co/katanemo/Plano-Router-1.5B" rel="nofollow noopener">Plano-Router-1.5B<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>) strategies</p></li>
|
||||
<li><p><strong>Cost Optimization</strong>: Route requests to cost-effective models based on complexity</p></li>
|
||||
<li><p><strong>Performance Optimization</strong>: Use fast models for simple tasks, powerful models for complex reasoning</p></li>
|
||||
<li><p><strong>Environment Management</strong>: Configure different models for different environments</p></li>
|
||||
|
|
@ -224,7 +220,7 @@ Use your preferred client library without changing existing code (see <a class="
|
|||
<section id="advanced-features">
|
||||
<h2>Advanced Features<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#advanced-features" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#advanced-features'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p><a class="reference internal" href="../../guides/llm_router.html#preference-aligned-routing"><span class="std std-ref">Preference-aligned Routing (Arch-Router)</span></a> - Learn about preference-aligned dynamic routing and intelligent model selection</p></li>
|
||||
<li><p><a class="reference internal" href="../../guides/llm_router.html#preference-aligned-routing"><span class="std std-ref">Preference-aligned routing (Arch-Router)</span></a> - Learn about preference-aligned dynamic routing and intelligent model selection</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="getting-started">
|
||||
|
|
@ -248,6 +244,7 @@ Use your preferred client library without changing existing code (see <a class="
|
|||
<li class="toctree-l2"><a class="reference internal" href="client_libraries.html#supported-clients">Supported Clients</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="client_libraries.html#gateway-endpoints">Gateway Endpoints</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="client_libraries.html#openai-python-sdk">OpenAI (Python) SDK</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="client_libraries.html#openai-responses-api-conversational-state">OpenAI Responses API (Conversational State)</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="client_libraries.html#anthropic-python-sdk">Anthropic (Python) SDK</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="client_libraries.html#curl-examples">cURL Examples</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="client_libraries.html#cross-client-compatibility">Cross-Client Compatibility</a></li>
|
||||
|
|
@ -271,11 +268,11 @@ Use your preferred client library without changing existing code (see <a class="
|
|||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../tech_overview/error_target.html">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../filter_chain.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Error Target
|
||||
Filter Chains
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
|
|
@ -302,12 +299,12 @@ Use your preferred client library without changing existing code (see <a class="
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -7,13 +7,13 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Model Aliases | Arch Docs v0.3.22</title>
|
||||
<meta content="Model Aliases | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Model Aliases | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Model Aliases | Plano Docs v0.4</title>
|
||||
<meta content="Model Aliases | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Model Aliases | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/llm_providers/model_aliases.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,12 +148,12 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="llm_providers.html">LLM Providers</a>
|
||||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="llm_providers.html">Model (LLM) Providers</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Model Aliases</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
|
|
@ -396,7 +391,7 @@
|
|||
<section id="see-also">
|
||||
<h2>See Also<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#see-also" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#see-also'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p><a class="reference internal" href="llm_providers.html#llm-providers"><span class="std std-ref">LLM Providers</span></a> - Learn about configuring LLM providers</p></li>
|
||||
<li><p><a class="reference internal" href="llm_providers.html#llm-providers"><span class="std std-ref">Model (LLM) Providers</span></a> - Learn about configuring LLM providers</p></li>
|
||||
<li><p><a class="reference internal" href="../../guides/llm_router.html#llm-router"><span class="std std-ref">LLM Routing</span></a> - Understand how aliases work with intelligent routing</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
|
|
@ -435,12 +430,12 @@
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Supported Providers & Configuration | Arch Docs v0.3.22</title>
|
||||
<meta content="Supported Providers & Configuration | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Supported Providers & Configuration | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Supported Providers & Configuration | Plano Docs v0.4</title>
|
||||
<meta content="Supported Providers & Configuration | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Supported Providers & Configuration | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/llm_providers/supported_providers.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../../search.html" rel="search" title="Search"/>
|
||||
<link href="client_libraries.html" rel="next" title="Client Libraries"/>
|
||||
<link href="llm_providers.html" rel="prev" title="LLM Providers"/>
|
||||
<link href="llm_providers.html" rel="prev" title="Model (LLM) Providers"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,39 +148,31 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="llm_providers.html">LLM Providers</a>
|
||||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="llm_providers.html">Model (LLM) Providers</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Supported Providers & Configuration</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="supported-providers-configuration">
|
||||
<span id="supported-providers"></span><h1>Supported Providers & Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#supported-providers-configuration"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Arch provides first-class support for multiple LLM providers through native integrations and OpenAI-compatible interfaces. This comprehensive guide covers all supported providers, their available chat models, and detailed configuration instructions.</p>
|
||||
<p>Plano provides first-class support for multiple LLM providers through native integrations and OpenAI-compatible interfaces. This comprehensive guide covers all supported providers, their available chat models, and detailed configuration instructions.</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p><strong>Model Support:</strong> Arch supports all chat models from each provider, not just the examples shown in this guide. The configurations below demonstrate common models for reference, but you can use any chat model available from your chosen provider.</p>
|
||||
<p><strong>Model Support:</strong> Plano supports all chat models from each provider, not just the examples shown in this guide. The configurations below demonstrate common models for reference, but you can use any chat model available from your chosen provider.</p>
|
||||
<p>Please refer to the quuickstart guide <a class="reference internal" href="../../get_started/quickstart.html#llm-routing-quickstart"><span class="std std-ref">here</span></a> to configure and use LLM providers via common client libraries like OpenAI and Anthropic Python SDKs, or via direct HTTP/cURL requests.</p>
|
||||
</div>
|
||||
<section id="configuration-structure">
|
||||
<h2>Configuration Structure<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configuration-structure" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configuration-structure'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>All providers are configured in the <code class="docutils literal notranslate"><span class="pre">llm_providers</span></code> section of your <code class="docutils literal notranslate"><span class="pre">arch_config.yaml</span></code> file:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">egress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">12000</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-8"><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-9">
|
||||
</span><span id="line-10"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="c1"># Provider configurations go here</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">provider/model-name</span>
|
||||
</span><span id="line-13"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$API_KEY</span>
|
||||
</span><span id="line-14"><span class="w"> </span><span class="c1"># Additional provider-specific options</span>
|
||||
<p>All providers are configured in the <code class="docutils literal notranslate"><span class="pre">llm_providers</span></code> section of your <code class="docutils literal notranslate"><span class="pre">plano_config.yaml</span></code> file:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="c1"># Provider configurations go here</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">provider/model-name</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$API_KEY</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="c1"># Additional provider-specific options</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Common Configuration Fields:</strong></p>
|
||||
|
|
@ -206,7 +193,7 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
</section>
|
||||
<section id="supported-api-endpoints">
|
||||
<h2>Supported API Endpoints<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#supported-api-endpoints" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#supported-api-endpoints'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Arch supports the following standardized endpoints across providers:</p>
|
||||
<p>Plano supports the following standardized endpoints across providers:</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 30.0%"/>
|
||||
|
|
@ -228,6 +215,10 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
<td><p>Anthropic-style messages</p></td>
|
||||
<td><p>Anthropic SDK, cURL, custom clients</p></td>
|
||||
</tr>
|
||||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">/v1/responses</span></code></p></td>
|
||||
<td><p>Unified response endpoint for agentic apps</p></td>
|
||||
<td><p>All SDKs, cURL, custom clients</p></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
|
@ -238,7 +229,7 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
<p><strong>Provider Prefix:</strong> <code class="docutils literal notranslate"><span class="pre">openai/</span></code></p>
|
||||
<p><strong>API Endpoint:</strong> <code class="docutils literal notranslate"><span class="pre">/v1/chat/completions</span></code></p>
|
||||
<p><strong>Authentication:</strong> API Key - Get your OpenAI API key from <a class="reference external" href="https://platform.openai.com/api-keys" rel="nofollow noopener">OpenAI Platform<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>.</p>
|
||||
<p><strong>Supported Chat Models:</strong> All OpenAI chat models including GPT-5, GPT-4o, GPT-4, GPT-3.5-turbo, and all future releases.</p>
|
||||
<p><strong>Supported Chat Models:</strong> All OpenAI chat models including GPT-5.2, GPT-5, GPT-4o, and all future releases.</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 30.0%"/>
|
||||
|
|
@ -252,31 +243,27 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr class="row-even"><td><p>GPT-5</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">openai/gpt-5</span></code></p></td>
|
||||
<tr class="row-even"><td><p>GPT-5.2</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">openai/gpt-5.2</span></code></p></td>
|
||||
<td><p>Next-generation model (use any model name from OpenAI’s API)</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>GPT-4o</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">openai/gpt-4o</span></code></p></td>
|
||||
<tr class="row-odd"><td><p>GPT-5</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">openai/gpt-5</span></code></p></td>
|
||||
<td><p>Latest multimodal model</p></td>
|
||||
</tr>
|
||||
<tr class="row-even"><td><p>GPT-4o mini</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">openai/gpt-4o-mini</span></code></p></td>
|
||||
<td><p>Fast, cost-effective model</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>GPT-4</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">openai/gpt-4</span></code></p></td>
|
||||
<tr class="row-odd"><td><p>GPT-4o</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">openai/gpt-4o</span></code></p></td>
|
||||
<td><p>High-capability reasoning model</p></td>
|
||||
</tr>
|
||||
<tr class="row-even"><td><p>GPT-3.5 Turbo</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">openai/gpt-3.5-turbo</span></code></p></td>
|
||||
<td><p>Balanced performance and cost</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>o3-mini</p></td>
|
||||
<tr class="row-even"><td><p>o3-mini</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">openai/o3-mini</span></code></p></td>
|
||||
<td><p>Reasoning-focused model (preview)</p></td>
|
||||
</tr>
|
||||
<tr class="row-even"><td><p>o3</p></td>
|
||||
<tr class="row-odd"><td><p>o3</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">openai/o3</span></code></p></td>
|
||||
<td><p>Advanced reasoning model (preview)</p></td>
|
||||
</tr>
|
||||
|
|
@ -285,15 +272,15 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
<p><strong>Configuration Examples:</strong></p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="c1"># Latest models (examples - use any OpenAI chat model)</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o-mini</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-5.2</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-6">
|
||||
</span><span id="line-7"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-5</span>
|
||||
</span><span id="line-8"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-9">
|
||||
</span><span id="line-10"><span class="w"> </span><span class="c1"># Use any model name from OpenAI's API</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-5</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
|
|
@ -303,7 +290,7 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
<p><strong>Provider Prefix:</strong> <code class="docutils literal notranslate"><span class="pre">anthropic/</span></code></p>
|
||||
<p><strong>API Endpoint:</strong> <code class="docutils literal notranslate"><span class="pre">/v1/messages</span></code></p>
|
||||
<p><strong>Authentication:</strong> API Key - Get your Anthropic API key from <a class="reference external" href="https://console.anthropic.com/settings/keys" rel="nofollow noopener">Anthropic Console<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>.</p>
|
||||
<p><strong>Supported Chat Models:</strong> All Anthropic Claude models including Claude Sonnet 4, Claude 3.5 Sonnet, Claude 3.5 Haiku, Claude 3 Opus, and all future releases.</p>
|
||||
<p><strong>Supported Chat Models:</strong> All Anthropic Claude models including Claude Sonnet 4.5, Claude Opus 4.5, Claude Haiku 4.5, and all future releases.</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 30.0%"/>
|
||||
|
|
@ -317,43 +304,35 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr class="row-even"><td><p>Claude Sonnet 4</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">anthropic/claude-sonnet-4</span></code></p></td>
|
||||
<td><p>Next-generation model (use any model name from Anthropic’s API)</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>Claude 3.5 Sonnet</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">anthropic/claude-3-5-sonnet-20241022</span></code></p></td>
|
||||
<td><p>Latest high-performance model</p></td>
|
||||
</tr>
|
||||
<tr class="row-even"><td><p>Claude 3.5 Haiku</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">anthropic/claude-3-5-haiku-20241022</span></code></p></td>
|
||||
<td><p>Fast and efficient model</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>Claude 3 Opus</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">anthropic/claude-3-opus-20240229</span></code></p></td>
|
||||
<tr class="row-even"><td><p>Claude Opus 4.5</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">anthropic/claude-opus-4-5</span></code></p></td>
|
||||
<td><p>Most capable model for complex tasks</p></td>
|
||||
</tr>
|
||||
<tr class="row-even"><td><p>Claude 3 Sonnet</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">anthropic/claude-3-sonnet-20240229</span></code></p></td>
|
||||
<tr class="row-odd"><td><p>Claude Sonnet 4.5</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">anthropic/claude-sonnet-4-5</span></code></p></td>
|
||||
<td><p>Balanced performance model</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>Claude 3 Haiku</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">anthropic/claude-3-haiku-20240307</span></code></p></td>
|
||||
<td><p>Fastest model</p></td>
|
||||
<tr class="row-even"><td><p>Claude Haiku 4.5</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">anthropic/claude-haiku-4-5</span></code></p></td>
|
||||
<td><p>Fast and efficient model</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>Claude Sonnet 3.5</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">anthropic/claude-sonnet-3-5</span></code></p></td>
|
||||
<td><p>Complex agents and coding</p></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<p><strong>Configuration Examples:</strong></p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="c1"># Latest models (examples - use any Anthropic chat model)</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-3-5-sonnet-20241022</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-opus-4-5</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$ANTHROPIC_API_KEY</span>
|
||||
</span><span id="line-5">
|
||||
</span><span id="line-6"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-3-5-haiku-20241022</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-sonnet-4-5</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$ANTHROPIC_API_KEY</span>
|
||||
</span><span id="line-8">
|
||||
</span><span id="line-9"><span class="w"> </span><span class="c1"># Use any model name from Anthropic's API</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-sonnet-4</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-haiku-4-5</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$ANTHROPIC_API_KEY</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
|
|
@ -450,7 +429,7 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
<p><strong>Provider Prefix:</strong> <code class="docutils literal notranslate"><span class="pre">groq/</span></code></p>
|
||||
<p><strong>API Endpoint:</strong> <code class="docutils literal notranslate"><span class="pre">/openai/v1/chat/completions</span></code> (transformed internally)</p>
|
||||
<p><strong>Authentication:</strong> API Key - Get your Groq API key from <a class="reference external" href="https://console.groq.com/keys" rel="nofollow noopener">Groq Console<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>.</p>
|
||||
<p><strong>Supported Chat Models:</strong> All Groq chat models including Llama 3, Mixtral, Gemma, and all future releases.</p>
|
||||
<p><strong>Supported Chat Models:</strong> All Groq chat models including Llama 4, GPT OSS, Mixtral, Gemma, and all future releases.</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 30.0%"/>
|
||||
|
|
@ -464,27 +443,30 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr class="row-even"><td><p>Llama 3.1 8B</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">groq/llama3-8b-8192</span></code></p></td>
|
||||
<tr class="row-even"><td><p>Llama 4 Maverick 17B</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">groq/llama-4-maverick-17b-128e-instruct</span></code></p></td>
|
||||
<td><p>Fast inference Llama model</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>Llama 3.1 70B</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">groq/llama3-70b-8192</span></code></p></td>
|
||||
<td><p>Larger Llama model</p></td>
|
||||
<tr class="row-odd"><td><p>Llama 4 Scout 8B</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">groq/llama-4-scout-8b-128e-instruct</span></code></p></td>
|
||||
<td><p>Smaller Llama model</p></td>
|
||||
</tr>
|
||||
<tr class="row-even"><td><p>Mixtral 8x7B</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">groq/mixtral-8x7b-32768</span></code></p></td>
|
||||
<td><p>Mixture of experts model</p></td>
|
||||
<tr class="row-even"><td><p>GPT OSS 20B</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">groq/gpt-oss-20b</span></code></p></td>
|
||||
<td><p>Open source GPT model</p></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<p><strong>Configuration Examples:</strong></p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">groq/llama3-8b-8192</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">groq/llama-4-maverick-17b-128e-instruct</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$GROQ_API_KEY</span>
|
||||
</span><span id="line-4">
|
||||
</span><span id="line-5"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">groq/mixtral-8x7b-32768</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">groq/llama-4-scout-8b-128e-instruct</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$GROQ_API_KEY</span>
|
||||
</span><span id="line-7">
|
||||
</span><span id="line-8"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">groq/gpt-oss-20b</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$GROQ_API_KEY</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
|
|
@ -493,7 +475,7 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
<p><strong>Provider Prefix:</strong> <code class="docutils literal notranslate"><span class="pre">gemini/</span></code></p>
|
||||
<p><strong>API Endpoint:</strong> <code class="docutils literal notranslate"><span class="pre">/v1beta/openai/chat/completions</span></code> (transformed internally)</p>
|
||||
<p><strong>Authentication:</strong> API Key - Get your Google AI API key from <a class="reference external" href="https://aistudio.google.com/app/apikey" rel="nofollow noopener">Google AI Studio<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>.</p>
|
||||
<p><strong>Supported Chat Models:</strong> All Google Gemini chat models including Gemini 1.5 Pro, Gemini 1.5 Flash, and all future releases.</p>
|
||||
<p><strong>Supported Chat Models:</strong> All Google Gemini chat models including Gemini 3 Pro, Gemini 3 Flash, and all future releases.</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 30.0%"/>
|
||||
|
|
@ -507,22 +489,22 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr class="row-even"><td><p>Gemini 1.5 Pro</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">gemini/gemini-1.5-pro</span></code></p></td>
|
||||
<tr class="row-even"><td><p>Gemini 3 Pro</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">gemini/gemini-3-pro</span></code></p></td>
|
||||
<td><p>Advanced reasoning and creativity</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>Gemini 1.5 Flash</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">gemini/gemini-1.5-flash</span></code></p></td>
|
||||
<tr class="row-odd"><td><p>Gemini 3 Flash</p></td>
|
||||
<td><p><code class="docutils literal notranslate"><span class="pre">gemini/gemini-3-flash</span></code></p></td>
|
||||
<td><p>Fast and efficient model</p></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<p><strong>Configuration Examples:</strong></p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gemini/gemini-1.5-pro</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gemini/gemini-3-pro</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$GOOGLE_API_KEY</span>
|
||||
</span><span id="line-4">
|
||||
</span><span id="line-5"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gemini/gemini-1.5-flash</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gemini/gemini-3-flash</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$GOOGLE_API_KEY</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
|
|
@ -724,7 +706,7 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
<h3>Amazon Bedrock<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#amazon-bedrock" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#amazon-bedrock'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p><strong>Provider Prefix:</strong> <code class="docutils literal notranslate"><span class="pre">amazon_bedrock/</span></code></p>
|
||||
<dl class="simple">
|
||||
<dt><strong>API Endpoint:</strong> Arch automatically constructs the endpoint as:</dt><dd><ul class="simple">
|
||||
<dt><strong>API Endpoint:</strong> Plano automatically constructs the endpoint as:</dt><dd><ul class="simple">
|
||||
<li><p>Non-streaming: <code class="docutils literal notranslate"><span class="pre">/model/{model-id}/converse</span></code></p></li>
|
||||
<li><p>Streaming: <code class="docutils literal notranslate"><span class="pre">/model/{model-id}/converse-stream</span></code></p></li>
|
||||
</ul>
|
||||
|
|
@ -894,7 +876,7 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
<h3>Routing Preferences<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#routing-preferences" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#routing-preferences'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Configure routing preferences for dynamic model selection:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-5.2</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">routing_preferences</span><span class="p">:</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">complex_reasoning</span>
|
||||
|
|
@ -902,7 +884,7 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
</span><span id="line-7"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">code_review</span>
|
||||
</span><span id="line-8"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">reviewing and analyzing existing code for bugs and improvements</span>
|
||||
</span><span id="line-9">
|
||||
</span><span id="line-10"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-3-5-sonnet-20241022</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-sonnet-4-5</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$ANTHROPIC_API_KEY</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="nt">routing_preferences</span><span class="p">:</span>
|
||||
</span><span id="line-13"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">creative_writing</span>
|
||||
|
|
@ -914,14 +896,14 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
<section id="model-selection-guidelines">
|
||||
<h2>Model Selection Guidelines<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#model-selection-guidelines" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#model-selection-guidelines'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p><strong>For Production Applications:</strong>
|
||||
- <strong>High Performance</strong>: OpenAI GPT-4o, Anthropic Claude 3.5 Sonnet
|
||||
- <strong>Cost-Effective</strong>: OpenAI GPT-4o mini, Anthropic Claude 3.5 Haiku
|
||||
- <strong>High Performance</strong>: OpenAI GPT-5.2, Anthropic Claude Sonnet 4.5
|
||||
- <strong>Cost-Effective</strong>: OpenAI GPT-5, Anthropic Claude Haiku 4.5
|
||||
- <strong>Code Tasks</strong>: DeepSeek Coder, Together AI Code Llama
|
||||
- <strong>Local Deployment</strong>: Ollama with Llama 3.1 or Code Llama</p>
|
||||
<p><strong>For Development/Testing:</strong>
|
||||
- <strong>Fast Iteration</strong>: Groq models (optimized inference)
|
||||
- <strong>Local Testing</strong>: Ollama models
|
||||
- <strong>Cost Control</strong>: Smaller models like GPT-4o mini or Mistral Small</p>
|
||||
- <strong>Cost Control</strong>: Smaller models like GPT-4o or Mistral Small</p>
|
||||
</section>
|
||||
<section id="see-also">
|
||||
<h2>See Also<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#see-also" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#see-also'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
|
|
@ -940,7 +922,7 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
LLM Providers
|
||||
Model (LLM) Providers
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
|
|
@ -995,12 +977,12 @@ Any provider that implements the OpenAI API interface can be configured using cu
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -7,18 +7,18 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Prompt Target | Arch Docs v0.3.22</title>
|
||||
<meta content="Prompt Target | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Prompt Target | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Prompt Target | Plano Docs v0.4</title>
|
||||
<meta content="Prompt Target | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Prompt Target | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/prompt_target.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="../guides/prompt_guard.html" rel="next" title="Prompt Guard"/>
|
||||
<link href="../guides/orchestration.html" rel="next" title="Orchestration"/>
|
||||
<link href="llm_providers/model_aliases.html" rel="prev" title="Model Aliases"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -163,11 +158,15 @@
|
|||
<div id="content" role="main">
|
||||
<section id="prompt-target">
|
||||
<span id="id1"></span><h1>Prompt Target<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#prompt-target"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p><strong>Prompt Targets</strong> are a core concept in Arch, empowering developers to clearly define how user prompts are interpreted, processed, and routed within their generative AI applications. Prompts can seamlessly be routed either to specialized AI agents capable of handling sophisticated, context-driven tasks or to targeted tools provided by your application, offering users a fast, precise, and personalized experience.</p>
|
||||
<p>This section covers the essentials of prompt targets—what they are, how to configure them, their practical uses, and recommended best practices—to help you fully utilize this feature in your applications.</p>
|
||||
<section id="what-are-prompt-targets">
|
||||
<h2>What Are Prompt Targets?<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#what-are-prompt-targets" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#what-are-prompt-targets'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Prompt targets are endpoints within Arch that handle specific types of user prompts. They act as the bridge between user inputs and your backend agents or tools (APIs), enabling Arch to route, process, and manage prompts efficiently. Defining prompt targets helps you decouple your application’s core logic from processing and handling complexities, leading to clearer code organization, better scalability, and easier maintenance.</p>
|
||||
<p>A Prompt Target is a deterministic, task-specific backend function or API endpoint that your application calls via Plano.
|
||||
Unlike agents (which handle wide-ranging, open-ended tasks), prompt targets are designed for focused, specific workloads where Plano can add value through input clarification and validation.</p>
|
||||
<p>Plano helps by:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Clarifying and validating input</strong>: Plano enriches incoming prompts with metadata (e.g., detecting follow-ups or clarifying requests) and can extract structured parameters from natural language before passing them to your backend.</p></li>
|
||||
<li><p><strong>Enabling high determinism</strong>: Since the task is specific and well-defined, Plano can reliably extract the information your backend needs without ambiguity.</p></li>
|
||||
<li><p><strong>Reducing backend work</strong>: Your backend receives clean, validated, structured inputs—so you can focus on business logic instead of parsing and validation.</p></li>
|
||||
</ul>
|
||||
<p>For example, a prompt target might be “schedule a meeting” (specific task, deterministic inputs like date, time, attendees) or “retrieve documents” (well-defined RAG query with clear intent). Prompt targets are typically called from your application code via Plano’s internal listener.</p>
|
||||
<table class="docutils align-default" style="width: 100%">
|
||||
<thead>
|
||||
<tr class="row-odd"><th class="head"><p><strong>Capability</strong></p></th>
|
||||
|
|
@ -190,23 +189,19 @@
|
|||
</tbody>
|
||||
</table>
|
||||
<section id="key-features">
|
||||
<h3>Key Features<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#key-features" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#key-features'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<h2>Key Features<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#key-features" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#key-features'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Below are the key features of prompt targets that empower developers to build efficient, scalable, and personalized GenAI solutions:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Design Scenarios</strong>: Define prompt targets to effectively handle specific agentic scenarios.</p></li>
|
||||
<li><p><strong>Input Management</strong>: Specify required and optional parameters for each target.</p></li>
|
||||
<li><p><strong>Tools Integration</strong>: Seamlessly connect prompts to backend APIs or functions.</p></li>
|
||||
<li><p><strong>Error Handling</strong>: Direct errors to designated handlers for streamlined troubleshooting.</p></li>
|
||||
<li><p><strong>Metadata Enrichment</strong>: Attach additional context to prompts for enhanced processing.</p></li>
|
||||
<li><p><strong>Multi-Turn Support</strong>: Manage follow-up prompts and clarifications in conversational flows.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
</section>
|
||||
<section id="configuring-prompt-targets">
|
||||
<h2>Configuring Prompt Targets<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configuring-prompt-targets" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configuring-prompt-targets'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Configuring prompt targets involves defining them in Arch’s configuration file. Each Prompt target specifies how a particular type of prompt should be handled, including the endpoint to invoke and any parameters required.</p>
|
||||
<section id="basic-configuration">
|
||||
<h3>Basic Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#basic-configuration" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#basic-configuration'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>A prompt target configuration includes the following elements:</p>
|
||||
<h2>Basic Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#basic-configuration" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#basic-configuration'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Configuring prompt targets involves defining them in Plano’s configuration file. Each Prompt target specifies how a particular type of prompt should be handled, including the endpoint to invoke and any parameters required. A prompt target configuration includes the following elements:</p>
|
||||
<ul class="simple">
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">name</span></code>: A unique identifier for the prompt target.</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">description</span></code>: A brief explanation of what the prompt target does.</p></li>
|
||||
|
|
@ -215,9 +210,9 @@
|
|||
</ul>
|
||||
</section>
|
||||
<section id="defining-parameters">
|
||||
<span id="defining-prompt-target-parameters"></span><h3>Defining Parameters<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#defining-parameters" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#defining-parameters'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Parameters are the pieces of information that Arch needs to extract from the user’s prompt to perform the desired action.
|
||||
Each parameter can be marked as required or optional. Here is a full list of parameter attributes that Arch can support:</p>
|
||||
<span id="defining-prompt-target-parameters"></span><h2>Defining Parameters<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#defining-parameters" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#defining-parameters'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Parameters are the pieces of information that Plano needs to extract from the user’s prompt to perform the desired action.
|
||||
Each parameter can be marked as required or optional. Here is a full list of parameter attributes that Plano can support:</p>
|
||||
<table class="docutils align-default" style="width: 100%">
|
||||
<thead>
|
||||
<tr class="row-odd"><th class="head"><p><strong>Attribute</strong></p></th>
|
||||
|
|
@ -256,9 +251,9 @@ Each parameter can be marked as required or optional. Here is a full list of par
|
|||
</table>
|
||||
</section>
|
||||
<section id="example-configuration-for-tools">
|
||||
<h3>Example Configuration For Tools<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-configuration-for-tools" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#example-configuration-for-tools'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<div class="literal-block-wrapper docutils container" id="id2">
|
||||
<div class="code-block-caption"><span class="caption-text">Tools and Function Calling Configuration Example</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<h2>Example Configuration For Tools<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-configuration-for-tools" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#example-configuration-for-tools'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<div class="literal-block-wrapper docutils container" id="id3">
|
||||
<div class="code-block-caption"><span class="caption-text">Tools and Function Calling Configuration Example</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id3"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">get_weather</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Get the current weather for a location</span>
|
||||
|
|
@ -279,52 +274,158 @@ Each parameter can be marked as required or optional. Here is a full list of par
|
|||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="example-configuration-for-agents">
|
||||
<h3>Example Configuration For Agents<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-configuration-for-agents" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#example-configuration-for-agents'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<div class="literal-block-wrapper docutils container" id="id3">
|
||||
<div class="code-block-caption"><span class="caption-text">Agent Orchestration Configuration Example</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id3"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">overrides</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">use_agent_orchestrator</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-3">
|
||||
</span><span id="line-4"><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">sales_agent</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">handles queries related to sales and purchases</span>
|
||||
</span><span id="line-7">
|
||||
</span><span id="line-8"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">issues_and_repairs</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">handles issues, repairs, or refunds</span>
|
||||
</span><span id="line-10">
|
||||
</span><span id="line-11"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">escalate_to_human</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">escalates to human agent</span>
|
||||
<section id="multi-turn">
|
||||
<span id="plano-multi-turn-guide"></span><h2>Multi-Turn<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#multi-turn" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#multi-turn'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Developers often <a class="reference external" href="https://www.reddit.com/r/LocalLLaMA/comments/18mqwg6/best_practice_for_rag_with_followup_chat/" rel="nofollow noopener">struggle<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> to efficiently handle
|
||||
<code class="docutils literal notranslate"><span class="pre">follow-up</span></code> or <code class="docutils literal notranslate"><span class="pre">clarification</span></code> questions. Specifically, when users ask for changes or additions to previous responses, it requires developers to
|
||||
re-write prompts using LLMs with precise prompt engineering techniques. This process is slow, manual, error prone and adds latency and token cost for
|
||||
common scenarios that can be managed more efficiently.</p>
|
||||
<p>Plano is highly capable of accurately detecting and processing prompts in multi-turn scenarios so that you can buil fast and accurate agents in minutes.
|
||||
Below are some cnversational examples that you can build via Plano. Each example is enriched with annotations (via ** [Plano] ** ) that illustrates how Plano
|
||||
processess conversational messages on your behalf.</p>
|
||||
<p>Example 1: Adjusting Retrieval</p>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">User: What are the benefits of renewable energy?
|
||||
</span><span id="line-2">**[Plano]**: Check if there is an available <prompt_target> that can handle this user query.
|
||||
</span><span id="line-3">**[Plano]**: Found "get_info_for_energy_source" prompt_target in arch_config.yaml. Forward prompt to the endpoint configured in "get_info_for_energy_source"
|
||||
</span><span id="line-4">...
|
||||
</span><span id="line-5">Assistant: Renewable energy reduces greenhouse gas emissions, lowers air pollution, and provides sustainable power sources like solar and wind.
|
||||
</span><span id="line-6">
|
||||
</span><span id="line-7">User: Include cost considerations in the response.
|
||||
</span><span id="line-8">**[Plano]**: Follow-up detected. Forward prompt history to the "get_info_for_energy_source" prompt_target and post the following parameters consideration="cost"
|
||||
</span><span id="line-9">...
|
||||
</span><span id="line-10">Assistant: Renewable energy reduces greenhouse gas emissions, lowers air pollution, and provides sustainable power sources like solar and wind. While the initial setup costs can be high, long-term savings from reduced fuel expenses and government incentives make it cost-effective.
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<section id="example-2-switching-intent">
|
||||
<h3>Example 2: Switching Intent<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-2-switching-intent" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#example-2-switching-intent'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">User: What are the symptoms of diabetes?
|
||||
</span><span id="line-2">**[Plano]**: Check if there is an available <prompt_target> that can handle this user query.
|
||||
</span><span id="line-3">**[Plano]**: Found "diseases_symptoms" prompt_target in arch_config.yaml. Forward disease=diabeteres to "diseases_symptoms" prompt target
|
||||
</span><span id="line-4">...
|
||||
</span><span id="line-5">Assistant: Common symptoms include frequent urination, excessive thirst, fatigue, and blurry vision.
|
||||
</span><span id="line-6">
|
||||
</span><span id="line-7">User: How is it diagnosed?
|
||||
</span><span id="line-8">**[Plano]**: New intent detected.
|
||||
</span><span id="line-9">**[Plano]**: Found "disease_diagnoses" prompt_target in arch_config.yaml. Forward disease=diabeteres to "disease_diagnoses" prompt target
|
||||
</span><span id="line-10">...
|
||||
</span><span id="line-11">Assistant: Diabetes is diagnosed through blood tests like fasting blood sugar, A1C, or an oral glucose tolerance test.
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="build-multi-turn-rag-apps">
|
||||
<h3>Build Multi-Turn RAG Apps<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#build-multi-turn-rag-apps" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#build-multi-turn-rag-apps'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>The following section describes how you can easilly add support for multi-turn scenarios via Plano. You process and manage multi-turn prompts
|
||||
just like you manage single-turn ones. Plano handles the conpleixity of detecting the correct intent based on the last user prompt and
|
||||
the covnersational history, extracts relevant parameters needed by downstream APIs, and dipatches calls to any upstream LLMs to summarize the
|
||||
response from your APIs.</p>
|
||||
</section>
|
||||
<section id="step-1-define-plano-config">
|
||||
<span id="multi-turn-subsection-prompt-target"></span><h3>Step 1: Define Plano Config<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-1-define-plano-config" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-1-define-plano-config'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<div class="literal-block-wrapper docutils container" id="id4">
|
||||
<div class="code-block-caption"><span class="caption-text">Plano Config</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id4"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span><span class="nt">listener</span><span class="p">:</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">127.0.0.1</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">8080</span><span class="w"> </span><span class="c1">#If you configure port 443, you'll need to update the listener with tls_certificates</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">huggingface</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="c1"># Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">OpenAI</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="w"> </span><span class="nt">provider</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-3.5-turbo</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span><span class="c1"># default system prompt used by all prompt targets</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">|</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span><span class="w"> </span><span class="no">You are a helpful assistant and can offer information about energy sources. You will get a JSON object with energy_source and consideration fields. Focus on answering using those fields</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">get_info_for_energy_source</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">get information about an energy source</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">energy_source</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">str</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">a source of energy</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span><span class="w"> </span><span class="nt">enum</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">renewable</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">fossil</span><span class="p p-Indicator">]</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">consideration</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">str</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">a specific type of consideration for an energy source</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span><span class="w"> </span><span class="nt">enum</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">cost</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">economic</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">technology</span><span class="p p-Indicator">]</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">rag_energy_source_agent</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/energy_source_info</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span><span class="w"> </span><span class="nt">http_method</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">POST</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>Today, you can use Arch to coordinate more specific agentic scenarios via tools and function calling, or use it for high-level agent routing and hand off scenarios. In the future, we plan to offer you the ability to combine these two approaches for more complex scenarios. Please see <a class="reference external" href="https://github.com/katanemo/archgw/issues/442" rel="nofollow noopener">github issues<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> for more details.</p>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section id="routing-logic">
|
||||
<h2>Routing Logic<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#routing-logic" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#routing-logic'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Prompt targets determine where and how user prompts are processed. Arch uses intelligent routing logic to ensure that prompts are directed to the appropriate targets based on their intent and context.</p>
|
||||
<section id="default-targets">
|
||||
<h3>Default Targets<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#default-targets" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#default-targets'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>For general-purpose prompts that do not match any specific prompt target, Arch routes them to a designated default target. This is useful for handling open-ended queries like document summarization or information extraction.</p>
|
||||
</section>
|
||||
<section id="intent-matching">
|
||||
<h3>Intent Matching<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#intent-matching" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#intent-matching'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Arch analyzes the user’s prompt to determine its intent and matches it with the most suitable prompt target based on the name and description defined in the configuration.</p>
|
||||
<p>For example:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">Prompt:<span class="w"> </span><span class="s2">"Can you reboot the router?"</span>
|
||||
</span><span id="line-2">Matching<span class="w"> </span>Target:<span class="w"> </span>reboot_device<span class="w"> </span><span class="o">(</span>based<span class="w"> </span>on<span class="w"> </span>description<span class="w"> </span>matching<span class="w"> </span><span class="s2">"reboot devices"</span><span class="o">)</span>
|
||||
<section id="step-2-process-request-in-flask">
|
||||
<h3>Step 2: Process Request in Flask<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-2-process-request-in-flask" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-2-process-request-in-flask'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Once the prompt targets are configured as above, handle parameters across multi-turn as if its a single-turn request</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id5">
|
||||
<div class="code-block-caption"><span class="caption-text">Parameter handling with Flask</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id5"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span><span class="kn">import</span><span class="w"> </span><span class="nn">gradio</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">gr</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="kn">from</span><span class="w"> </span><span class="nn">fastapi</span><span class="w"> </span><span class="kn">import</span> <span class="n">FastAPI</span><span class="p">,</span> <span class="n">HTTPException</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="kn">from</span><span class="w"> </span><span class="nn">pydantic</span><span class="w"> </span><span class="kn">import</span> <span class="n">BaseModel</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Optional</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="kn">from</span><span class="w"> </span><span class="nn">openai</span><span class="w"> </span><span class="kn">import</span> <span class="n">OpenAI</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="kn">from</span><span class="w"> </span><span class="nn">common</span><span class="w"> </span><span class="kn">import</span> <span class="n">create_gradio_app</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="n">app</span> <span class="o">=</span> <span class="n">FastAPI</span><span class="p">()</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="c1"># Define the request model</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span><span class="k">class</span><span class="w"> </span><span class="nc">EnergySourceRequest</span><span class="p">(</span><span class="n">BaseModel</span><span class="p">):</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span> <span class="n">energy_source</span><span class="p">:</span> <span class="nb">str</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span> <span class="n">consideration</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span><span class="k">class</span><span class="w"> </span><span class="nc">EnergySourceResponse</span><span class="p">(</span><span class="n">BaseModel</span><span class="p">):</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span> <span class="n">energy_source</span><span class="p">:</span> <span class="nb">str</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span> <span class="n">consideration</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span><span class="c1"># Post method for device summary</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span><span class="nd">@app</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="s2">"/agent/energy_source_info"</span><span class="p">)</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span><span class="k">def</span><span class="w"> </span><span class="nf">get_workforce</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">EnergySourceRequest</span><span class="p">):</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span><span class="w"> </span><span class="sd">"""</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span><span class="sd"> Endpoint to get details about energy source</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span><span class="sd"> """</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span> <span class="n">considertion</span> <span class="o">=</span> <span class="s2">"You don't have any specific consideration. Feel free to talk in a more open ended fashion"</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span> <span class="k">if</span> <span class="n">request</span><span class="o">.</span><span class="n">consideration</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span> <span class="n">considertion</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Add specific focus on the following consideration when you summarize the content for the energy source: </span><span class="si">{</span><span class="n">request</span><span class="o">.</span><span class="n">consideration</span><span class="si">}</span><span class="s2">"</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span> <span class="n">response</span> <span class="o">=</span> <span class="p">{</span>
|
||||
</span><span id="line-36"><span class="linenos">36</span> <span class="s2">"energy_source"</span><span class="p">:</span> <span class="n">request</span><span class="o">.</span><span class="n">energy_source</span><span class="p">,</span>
|
||||
</span><span id="line-37"><span class="linenos">37</span> <span class="s2">"consideration"</span><span class="p">:</span> <span class="n">considertion</span><span class="p">,</span>
|
||||
</span><span id="line-38"><span class="linenos">38</span> <span class="p">}</span>
|
||||
</span><span id="line-39"><span class="linenos">39</span> <span class="k">return</span> <span class="n">response</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="demo-app">
|
||||
<h3>Demo App<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#demo-app" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#demo-app'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>For your convenience, we’ve built a <a class="reference external" href="https://github.com/katanemo/archgw/tree/main/demos/samples_python/multi_turn_rag_agent" rel="nofollow noopener">demo app<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>
|
||||
that you can test and modify locally for multi-turn RAG scenarios.</p>
|
||||
<figure class="align-center" id="id6">
|
||||
<a class="reference internal image-reference" href="../_images/mutli-turn-example.png"><img alt="../_images/mutli-turn-example.png" src="../_images/mutli-turn-example.png" style="width: 100%;"/>
|
||||
</a>
|
||||
<figcaption>
|
||||
<p><span class="caption-text">Example multi-turn user conversation showing adjusting retrieval</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id6"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></p>
|
||||
</figcaption>
|
||||
</figure>
|
||||
</section>
|
||||
</section>
|
||||
<section id="summary">
|
||||
<h2>Summary<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#summary" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#summary'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Prompt targets are essential for defining how user prompts are handled within your generative AI applications using Arch.</p>
|
||||
<p>By carefully configuring prompt targets, you can ensure that prompts are accurately routed, necessary parameters are extracted, and backend services are invoked seamlessly. This modular approach not only simplifies your application’s architecture but also enhances scalability, maintainability, and overall user experience.</p>
|
||||
<p>By carefully designing prompt targets as deterministic, task-specific entry points, you ensure that prompts are routed to the right workload, necessary parameters are cleanly extracted and validated, and backend services are invoked with structured inputs. This clear separation between prompt handling and business logic simplifies your architecture, makes behavior more predictable and testable, and improves the scalability and maintainability of your agentic applications.</p>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
|
|
@ -337,8 +438,8 @@ Each parameter can be marked as required or optional. Here is a full list of par
|
|||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../guides/prompt_guard.html">
|
||||
Prompt Guard
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../guides/orchestration.html">
|
||||
Orchestration
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
|
|
@ -347,20 +448,16 @@ Each parameter can be marked as required or optional. Here is a full list of par
|
|||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#what-are-prompt-targets'" class="reference internal" href="#what-are-prompt-targets">What Are Prompt Targets?</a><ul>
|
||||
<li><a :data-current="activeSection === '#key-features'" class="reference internal" href="#key-features">Key Features</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#configuring-prompt-targets'" class="reference internal" href="#configuring-prompt-targets">Configuring Prompt Targets</a><ul>
|
||||
<li><a :data-current="activeSection === '#basic-configuration'" class="reference internal" href="#basic-configuration">Basic Configuration</a></li>
|
||||
<li><a :data-current="activeSection === '#defining-parameters'" class="reference internal" href="#defining-parameters">Defining Parameters</a></li>
|
||||
<li><a :data-current="activeSection === '#example-configuration-for-tools'" class="reference internal" href="#example-configuration-for-tools">Example Configuration For Tools</a></li>
|
||||
<li><a :data-current="activeSection === '#example-configuration-for-agents'" class="reference internal" href="#example-configuration-for-agents">Example Configuration For Agents</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#routing-logic'" class="reference internal" href="#routing-logic">Routing Logic</a><ul>
|
||||
<li><a :data-current="activeSection === '#default-targets'" class="reference internal" href="#default-targets">Default Targets</a></li>
|
||||
<li><a :data-current="activeSection === '#intent-matching'" class="reference internal" href="#intent-matching">Intent Matching</a></li>
|
||||
<li><a :data-current="activeSection === '#multi-turn'" class="reference internal" href="#multi-turn">Multi-Turn</a><ul>
|
||||
<li><a :data-current="activeSection === '#example-2-switching-intent'" class="reference internal" href="#example-2-switching-intent">Example 2: Switching Intent</a></li>
|
||||
<li><a :data-current="activeSection === '#build-multi-turn-rag-apps'" class="reference internal" href="#build-multi-turn-rag-apps">Build Multi-Turn RAG Apps</a></li>
|
||||
<li><a :data-current="activeSection === '#step-1-define-plano-config'" class="reference internal" href="#step-1-define-plano-config">Step 1: Define Plano Config</a></li>
|
||||
<li><a :data-current="activeSection === '#step-2-process-request-in-flask'" class="reference internal" href="#step-2-process-request-in-flask">Step 2: Process Request in Flask</a></li>
|
||||
<li><a :data-current="activeSection === '#demo-app'" class="reference internal" href="#demo-app">Demo App</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#summary'" class="reference internal" href="#summary">Summary</a></li>
|
||||
|
|
@ -372,12 +469,12 @@ Each parameter can be marked as required or optional. Here is a full list of par
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -1,253 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Error Target | Arch Docs v0.3.22</title>
|
||||
<meta content="Error Target | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Error Target | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/tech_overview/error_target.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../../search.html" rel="search" title="Search"/>
|
||||
<link href="../llm_providers/llm_providers.html" rel="next" title="LLM Providers"/>
|
||||
<link href="request_lifecycle.html" rel="prev" title="Request Lifecycle"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
let mode;
|
||||
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
||||
mode = 'dark';
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
mode = 'light';
|
||||
}
|
||||
if (!userPreference) {localStorage.setItem('darkMode', mode)}
|
||||
</script>
|
||||
</head>
|
||||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
K
|
||||
</kbd>
|
||||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
</a>
|
||||
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
|
||||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||||
</svg>
|
||||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</aside>
|
||||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="tech_overview.html">Tech Overview</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Error Target</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="error-target">
|
||||
<span id="id1"></span><h1>Error Target<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#error-target"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p><strong>Error targets</strong> are designed to capture and manage specific issues or exceptions that occur during Arch’s function or system’s execution.</p>
|
||||
<p>These endpoints receive errors forwarded from Arch when issues arise, such as improper function/API calls, guardrail violations, or other processing errors.
|
||||
The errors are communicated to the application via headers like <code class="docutils literal notranslate"><span class="pre">X-Arch-[ERROR-TYPE]</span></code>, enabling you to respond appropriately and handle errors gracefully.</p>
|
||||
<section id="key-concepts">
|
||||
<h2>Key Concepts<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#key-concepts" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#key-concepts'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Error Type</strong>: Categorizes the nature of the error, such as “ValidationError” or “RuntimeError.” These error types help in identifying what kind of issue occurred and provide context for troubleshooting.</p></li>
|
||||
<li><p><strong>Error Message</strong>: A clear, human-readable message describing the error. This should provide enough detail to inform users or developers of the root cause or required action.</p></li>
|
||||
<li><p><strong>Parameter-Specific Errors</strong>: Errors that arise due to invalid or missing parameters when invoking a function. These errors are critical for ensuring the correctness of inputs.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="error-header-example">
|
||||
<h2>Error Header Example<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#error-header-example" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#error-header-example'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<div class="literal-block-wrapper docutils container" id="id2">
|
||||
<div class="code-block-caption"><span class="caption-text">Error Header Example</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="w"> </span>HTTP/1.1<span class="w"> </span><span class="m">400</span><span class="w"> </span>Bad<span class="w"> </span>Request
|
||||
</span><span id="line-2"><span class="w"> </span>X-Arch-Error-Type:<span class="w"> </span>FunctionValidationError
|
||||
</span><span id="line-3"><span class="w"> </span>X-Arch-Error-Message:<span class="w"> </span>Tools<span class="w"> </span>call<span class="w"> </span>parsing<span class="w"> </span>failure
|
||||
</span><span id="line-4"><span class="w"> </span>X-Arch-Target-Prompt:<span class="w"> </span>createUser
|
||||
</span><span id="line-5"><span class="w"> </span>Content-Type:<span class="w"> </span>application/json
|
||||
</span><span id="line-6">
|
||||
</span><span id="line-7"><span class="w"> </span><span class="s2">"messages"</span>:<span class="w"> </span><span class="o">[</span>
|
||||
</span><span id="line-8"><span class="w"> </span><span class="o">{</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="s2">"role"</span>:<span class="w"> </span><span class="s2">"user"</span>,
|
||||
</span><span id="line-10"><span class="w"> </span><span class="s2">"content"</span>:<span class="w"> </span><span class="s2">"Please create a user with the following ID: 1234"</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="o">}</span>,
|
||||
</span><span id="line-12"><span class="w"> </span><span class="o">{</span>
|
||||
</span><span id="line-13"><span class="w"> </span><span class="s2">"role"</span>:<span class="w"> </span><span class="s2">"system"</span>,
|
||||
</span><span id="line-14"><span class="w"> </span><span class="s2">"content"</span>:<span class="w"> </span><span class="s2">"Expected a string for 'user_id', but got an integer."</span>
|
||||
</span><span id="line-15"><span class="w"> </span><span class="o">}</span>
|
||||
</span><span id="line-16"><span class="w"> </span><span class="o">]</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="best-practices-and-tips">
|
||||
<h2>Best Practices and Tips<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#best-practices-and-tips" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#best-practices-and-tips'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Graceful Degradation</strong>: If an error occurs, fail gracefully by providing fallback logic or alternative flows when possible.</p></li>
|
||||
<li><p><strong>Log Errors</strong>: Always log errors on the server side for later analysis.</p></li>
|
||||
<li><p><strong>Client-Side Handling</strong>: Make sure the client can interpret error responses and provide meaningful feedback to the user. Clients should not display raw error codes or stack traces but rather handle them gracefully.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="request_lifecycle.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Request Lifecycle
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../llm_providers/llm_providers.html">
|
||||
LLM Providers
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#key-concepts'" class="reference internal" href="#key-concepts">Key Concepts</a></li>
|
||||
<li><a :data-current="activeSection === '#error-header-example'" class="reference internal" href="#error-header-example">Error Header Example</a></li>
|
||||
<li><a :data-current="activeSection === '#best-practices-and-tips'" class="reference internal" href="#best-practices-and-tips">Best Practices and Tips</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,416 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Prompts | Arch Docs v0.3.22</title>
|
||||
<meta content="Prompts | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Prompts | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/tech_overview/prompt.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../../search.html" rel="search" title="Search"/>
|
||||
<link href="model_serving.html" rel="next" title="Model Serving"/>
|
||||
<link href="listener.html" rel="prev" title="Listener"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
let mode;
|
||||
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
||||
mode = 'dark';
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
mode = 'light';
|
||||
}
|
||||
if (!userPreference) {localStorage.setItem('darkMode', mode)}
|
||||
</script>
|
||||
</head>
|
||||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
K
|
||||
</kbd>
|
||||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
</a>
|
||||
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
|
||||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||||
</svg>
|
||||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="listener.html">Listener</a></li>
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</aside>
|
||||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="tech_overview.html">Tech Overview</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Prompts</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="prompts">
|
||||
<span id="arch-overview-prompt-handling"></span><h1>Prompts<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#prompts"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Arch’s primary design point is to securely accept, process and handle prompts. To do that effectively,
|
||||
Arch relies on Envoy’s HTTP <a class="reference external" href="https://www.envoyproxy.io/docs/envoy/v1.31.2/intro/arch_overview/http/http_connection_management" rel="nofollow noopener">connection management<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>,
|
||||
subsystem and its <strong>prompt handler</strong> subsystem engineered with purpose-built LLMs to
|
||||
implement critical functionality on behalf of developers so that you can stay focused on business logic.</p>
|
||||
<p>Arch’s <strong>prompt handler</strong> subsystem interacts with the <strong>model subsystem</strong> through Envoy’s cluster manager system to ensure robust, resilient and fault-tolerant experience in managing incoming prompts.</p>
|
||||
<div class="admonition seealso">
|
||||
<p class="admonition-title">See also</p>
|
||||
<p>Read more about the <a class="reference internal" href="model_serving.html#model-serving"><span class="std std-ref">model subsystem</span></a> and how the LLMs are hosted in Arch.</p>
|
||||
</div>
|
||||
<section id="messages">
|
||||
<h2>Messages<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#messages" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#messages'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Arch accepts messages directly from the body of the HTTP request in a format that follows the <a class="reference external" href="https://huggingface.co/docs/text-generation-inference/en/messages_api" rel="nofollow noopener">Hugging Face Messages API<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>.
|
||||
This design allows developers to pass a list of messages, where each message is represented as a dictionary
|
||||
containing two key-value pairs:</p>
|
||||
<blockquote>
|
||||
<div><ul class="simple">
|
||||
<li><p><strong>Role</strong>: Defines the role of the message sender, such as “user” or “assistant”.</p></li>
|
||||
<li><p><strong>Content</strong>: Contains the actual text of the message.</p></li>
|
||||
</ul>
|
||||
</div></blockquote>
|
||||
</section>
|
||||
<section id="prompt-guard">
|
||||
<h2>Prompt Guard<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#prompt-guard" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#prompt-guard'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Arch is engineered with <a class="reference external" href="https://huggingface.co/collections/katanemo/arch-guard-6702bdc08b889e4bce8f446d" rel="nofollow noopener">Arch-Guard<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, an industry leading safety layer, powered by a
|
||||
compact and high-performing LLM that monitors incoming prompts to detect and reject jailbreak attempts -
|
||||
ensuring that unauthorized or harmful behaviors are intercepted early in the process.</p>
|
||||
<p>To add jailbreak guardrails, see example below:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id1">
|
||||
<div class="code-block-caption"><span class="caption-text">Example Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id1"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1.0</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="w"> </span><span class="nt">ingress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10000</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="c1"># Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span><span class="c1"># default system prompt used by all prompt targets</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">You are a network assistant that just offers facts; not advice on manufacturers or purchasing decisions.</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span><span class="nt">prompt_guards</span><span class="p">:</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span><span class="w"> </span><span class="nt">input_guards</span><span class="p">:</span>
|
||||
</span><span id="line-21"><mark><span class="linenos">21</span><span class="w"> </span><span class="nt">jailbreak</span><span class="p">:</span>
|
||||
</mark></span><span id="line-22"><mark><span class="linenos">22</span><span class="w"> </span><span class="nt">on_exception</span><span class="p">:</span>
|
||||
</mark></span><span id="line-23"><mark><span class="linenos">23</span><span class="w"> </span><span class="nt">message</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Looks like you're curious about my abilities, but I can only provide assistance within my programmed parameters.</span>
|
||||
</mark></span><span id="line-24"><mark><span class="linenos">24</span>
|
||||
</mark></span><span id="line-25"><mark><span class="linenos">25</span><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</mark></span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>As a roadmap item, Arch will expose the ability for developers to define custom guardrails via Arch-Guard,
|
||||
and add support for additional safety checks defined by developers and hazardous categories like, violent crimes, privacy, hate,
|
||||
etc. To offer feedback on our roadmap, please visit our <a class="reference external" href="https://github.com/orgs/katanemo/projects/1" rel="nofollow noopener">github page<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="prompt-targets">
|
||||
<h2>Prompt Targets<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#prompt-targets" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#prompt-targets'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Once a prompt passes any configured guardrail checks, Arch processes the contents of the incoming conversation
|
||||
and identifies where to forward the conversation to via its <code class="docutils literal notranslate"><span class="pre">prompt</span> <span class="pre">target</span></code> primitive. Prompt targets are endpoints
|
||||
that receive prompts that are processed by Arch. For example, Arch enriches incoming prompts with metadata like knowing
|
||||
when a user’s intent has changed so that you can build faster, more accurate RAG apps.</p>
|
||||
<p>Configuring <code class="docutils literal notranslate"><span class="pre">prompt_targets</span></code> is simple. See example below:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id2">
|
||||
<div class="code-block-caption"><span class="caption-text">Example Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1.0</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="w"> </span><span class="nt">ingress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10000</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="c1"># Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span><span class="c1"># default system prompt used by all prompt targets</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">You are a network assistant that just offers facts; not advice on manufacturers or purchasing decisions.</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span><span class="nt">prompt_guards</span><span class="p">:</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span><span class="w"> </span><span class="nt">input_guards</span><span class="p">:</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span><span class="w"> </span><span class="nt">jailbreak</span><span class="p">:</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span><span class="w"> </span><span class="nt">on_exception</span><span class="p">:</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span><span class="w"> </span><span class="nt">message</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Looks like you're curious about my abilities, but I can only provide assistance within my programmed parameters.</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">information_extraction</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">handel all scenarios that are question and answer in nature. Like summarization, information extraction, etc.</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/summary</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span><span class="w"> </span><span class="c1"># Arch uses the default LLM and treats the response from the endpoint as the prompt to send to the LLM</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span><span class="w"> </span><span class="nt">auto_llm_dispatch_on_response</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span><span class="w"> </span><span class="c1"># override system prompt for this prompt target</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span><span class="w"> </span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">You are a helpful information extraction assistant. Use the information that is provided to you.</span>
|
||||
</span><span id="line-36"><span class="linenos">36</span>
|
||||
</span><span id="line-37"><span class="linenos">37</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">reboot_network_device</span>
|
||||
</span><span id="line-38"><span class="linenos">38</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Reboot a specific network device</span>
|
||||
</span><span id="line-39"><mark><span class="linenos">39</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</mark></span><span id="line-40"><mark><span class="linenos">40</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</mark></span><span id="line-41"><mark><span class="linenos">41</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/action</span>
|
||||
</mark></span><span id="line-42"><mark><span class="linenos">42</span><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</mark></span><span id="line-43"><mark><span class="linenos">43</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">device_id</span>
|
||||
</mark></span><span id="line-44"><mark><span class="linenos">44</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">str</span>
|
||||
</mark></span><span id="line-45"><mark><span class="linenos">45</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Identifier of the network device to reboot.</span>
|
||||
</mark></span><span id="line-46"><mark><span class="linenos">46</span><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</mark></span><span id="line-47"><mark><span class="linenos">47</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">confirmation</span>
|
||||
</mark></span><span id="line-48"><mark><span class="linenos">48</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">bool</span>
|
||||
</mark></span><span id="line-49"><mark><span class="linenos">49</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Confirmation flag to proceed with reboot.</span>
|
||||
</mark></span><span id="line-50"><mark><span class="linenos">50</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">false</span>
|
||||
</mark></span><span id="line-51"><mark><span class="linenos">51</span><span class="w"> </span><span class="nt">enum</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">true</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">false</span><span class="p p-Indicator">]</span>
|
||||
</mark></span><span id="line-52"><mark><span class="linenos">52</span>
|
||||
</mark></span><span id="line-53"><mark><span class="linenos">53</span><span class="c1"># Arch creates a round-robin load balancing between different endpoints, managed via the cluster subsystem.</span>
|
||||
</mark></span><span id="line-54"><span class="linenos">54</span><span class="nt">endpoints</span><span class="p">:</span>
|
||||
</span><span id="line-55"><span class="linenos">55</span><span class="w"> </span><span class="nt">app_server</span><span class="p">:</span>
|
||||
</span><span id="line-56"><span class="linenos">56</span><span class="w"> </span><span class="c1"># value could be ip address or a hostname with port</span>
|
||||
</span><span id="line-57"><span class="linenos">57</span><span class="w"> </span><span class="c1"># this could also be a list of endpoints for load balancing</span>
|
||||
</span><span id="line-58"><span class="linenos">58</span><span class="w"> </span><span class="c1"># for example endpoint: [ ip1:port, ip2:port ]</span>
|
||||
</span><span id="line-59"><span class="linenos">59</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">127.0.0.1:80</span>
|
||||
</span><span id="line-60"><span class="linenos">60</span><span class="w"> </span><span class="c1"># max time to wait for a connection to be established</span>
|
||||
</span><span id="line-61"><span class="linenos">61</span><span class="w"> </span><span class="nt">connect_timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.005s</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="admonition seealso">
|
||||
<p class="admonition-title">See also</p>
|
||||
<p>Check <a class="reference internal" href="../prompt_target.html#prompt-target"><span class="std std-ref">Prompt Target</span></a> for more details!</p>
|
||||
</div>
|
||||
<section id="intent-matching">
|
||||
<h3>Intent Matching<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#intent-matching" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#intent-matching'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Arch uses fast text embedding and intent recognition approaches to first detect the intent of each incoming prompt.
|
||||
This intent matching phase analyzes the prompt’s content and matches it against predefined prompt targets, ensuring that each prompt is forwarded to the most appropriate endpoint.
|
||||
Arch’s intent matching framework considers both the name and description of each prompt target, and uses a composite matching score between embedding similarity and intent classification scores to enhance accuracy in forwarding decisions.</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Intent Recognition</strong>: NLI techniques further refine the matching process by evaluating the semantic alignment between the prompt and potential targets.</p></li>
|
||||
<li><p><strong>Text Embedding</strong>: By embedding the prompt and comparing it to known target vectors, Arch effectively identifies the closest match, ensuring that the prompt is handled by the correct downstream service.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="agentic-apps-via-prompt-targets">
|
||||
<h3>Agentic Apps via Prompt Targets<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#agentic-apps-via-prompt-targets" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#agentic-apps-via-prompt-targets'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>To support agentic apps, like scheduling travel plans or sharing comments on a document - via prompts, Arch uses its function calling abilities to extract critical information from the incoming prompt (or a set of prompts) needed by a downstream backend API or function call before calling it directly.
|
||||
For more details on how you can build agentic applications using Arch, see our full guide <a class="reference internal" href="../../build_with_arch/agent.html#arch-agent-guide"><span class="std std-ref">here</span></a>:</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p><a class="reference external" href="https://huggingface.co/collections/katanemo/arch-function-66f209a693ea8df14317ad68" rel="nofollow noopener">Arch-Function<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> is a collection of dedicated agentic models engineered in Arch to extract information from a (set of) prompts and executes necessary backend API calls.
|
||||
This allows for efficient handling of agentic tasks, such as scheduling data retrieval, by dynamically interacting with backend services.
|
||||
Arch-Function achieves state-of-the-art performance, comparable with frontier models like Claude Sonnet 3.5 ang GPT-4, while being 44x cheaper ($0.10M/token hosted) and 10x faster (p50 latencies of 200ms).</p>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section id="prompting-llms">
|
||||
<h2>Prompting LLMs<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#prompting-llms" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#prompting-llms'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Arch is a single piece of software that is designed to manage both ingress and egress prompt traffic, drawing its distributed proxy nature from the robust <a class="reference external" href="https://envoyproxy.io" rel="nofollow noopener">Envoy<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>.
|
||||
This makes it extremely efficient and capable of handling upstream connections to LLMs.
|
||||
If your application is originating code to an API-based LLM, simply use the OpenAI client and configure it with Arch.
|
||||
By sending traffic through Arch, you can propagate traces, manage and monitor traffic, apply rate limits, and utilize a large set of traffic management capabilities in a centralized way.</p>
|
||||
<div class="admonition attention">
|
||||
<p class="admonition-title">Attention</p>
|
||||
<p>When you start Arch, it automatically creates a listener port for egress calls to upstream LLMs. This is based on the
|
||||
<code class="docutils literal notranslate"><span class="pre">llm_providers</span></code> configuration section in the <code class="docutils literal notranslate"><span class="pre">arch_config.yml</span></code> file. Arch binds itself to a local address such as
|
||||
<code class="docutils literal notranslate"><span class="pre">127.0.0.1:12000</span></code>.</p>
|
||||
</div>
|
||||
<section id="example-using-openai-client-with-arch-as-an-egress-gateway">
|
||||
<h3>Example: Using OpenAI Client with Arch as an Egress Gateway<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-using-openai-client-with-arch-as-an-egress-gateway" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#example-using-openai-client-with-arch-as-an-egress-gateway'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">import</span><span class="w"> </span><span class="nn">openai</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="c1"># Set the OpenAI API base URL to the Arch gateway endpoint</span>
|
||||
</span><span id="line-4"><span class="n">openai</span><span class="o">.</span><span class="n">api_base</span> <span class="o">=</span> <span class="s2">"http://127.0.0.1:12000"</span>
|
||||
</span><span id="line-5">
|
||||
</span><span id="line-6"><span class="c1"># No need to set openai.api_key since it's configured in Arch's gateway</span>
|
||||
</span><span id="line-7">
|
||||
</span><span id="line-8"><span class="c1"># Use the OpenAI client as usual</span>
|
||||
</span><span id="line-9"><span class="n">response</span> <span class="o">=</span> <span class="n">openai</span><span class="o">.</span><span class="n">Completion</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-10"> <span class="n">model</span><span class="o">=</span><span class="s2">"text-davinci-003"</span><span class="p">,</span>
|
||||
</span><span id="line-11"> <span class="n">prompt</span><span class="o">=</span><span class="s2">"What is the capital of France?"</span>
|
||||
</span><span id="line-12"><span class="p">)</span>
|
||||
</span><span id="line-13">
|
||||
</span><span id="line-14"><span class="nb">print</span><span class="p">(</span><span class="s2">"OpenAI Response:"</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">choices</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">strip</span><span class="p">())</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p>In these examples, the OpenAI client is used to send traffic directly through the Arch egress proxy to the LLM of your choice, such as OpenAI.
|
||||
The OpenAI client is configured to route traffic via Arch by setting the proxy to <code class="docutils literal notranslate"><span class="pre">127.0.0.1:12000</span></code>, assuming Arch is running locally and bound to that address and port.
|
||||
This setup allows you to take advantage of Arch’s advanced traffic management features while interacting with LLM APIs like OpenAI.</p>
|
||||
</section>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="listener.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Listener
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="model_serving.html">
|
||||
Model Serving
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#messages'" class="reference internal" href="#messages">Messages</a></li>
|
||||
<li><a :data-current="activeSection === '#prompt-guard'" class="reference internal" href="#prompt-guard">Prompt Guard</a></li>
|
||||
<li><a :data-current="activeSection === '#prompt-targets'" class="reference internal" href="#prompt-targets">Prompt Targets</a><ul>
|
||||
<li><a :data-current="activeSection === '#intent-matching'" class="reference internal" href="#intent-matching">Intent Matching</a></li>
|
||||
<li><a :data-current="activeSection === '#agentic-apps-via-prompt-targets'" class="reference internal" href="#agentic-apps-via-prompt-targets">Agentic Apps via Prompt Targets</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#prompting-llms'" class="reference internal" href="#prompting-llms">Prompting LLMs</a><ul>
|
||||
<li><a :data-current="activeSection === '#example-using-openai-client-with-arch-as-an-egress-gateway'" class="reference internal" href="#example-using-openai-client-with-arch-as-an-egress-gateway">Example: Using OpenAI Client with Arch as an Egress Gateway</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,237 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Terminology | Arch Docs v0.3.22</title>
|
||||
<meta content="Terminology | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Terminology | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/tech_overview/terminology.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../../search.html" rel="search" title="Search"/>
|
||||
<link href="threading_model.html" rel="next" title="Threading Model"/>
|
||||
<link href="tech_overview.html" rel="prev" title="Tech Overview"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
let mode;
|
||||
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
||||
mode = 'dark';
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
mode = 'light';
|
||||
}
|
||||
if (!userPreference) {localStorage.setItem('darkMode', mode)}
|
||||
</script>
|
||||
</head>
|
||||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
K
|
||||
</kbd>
|
||||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
</a>
|
||||
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
|
||||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||||
</svg>
|
||||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</aside>
|
||||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="tech_overview.html">Tech Overview</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Terminology</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="terminology">
|
||||
<span id="arch-terminology"></span><h1>Terminology<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#terminology"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>A few definitions before we dive into the main architecture documentation. Also note, Arch borrows from Envoy’s terminology
|
||||
to keep things consistent in logs and traces, and introduces and clarifies concepts are is relates to LLM applications.</p>
|
||||
<p><strong>Agent</strong>: An application that uses LLMs to handle wide-ranging tasks from users via prompts. This could be as simple
|
||||
as retrieving or summarizing data from an API, or being able to trigger complex actions like adjusting ad campaigns, or
|
||||
changing travel plans via prompts.</p>
|
||||
<p><strong>Arch Config</strong>: Arch operates based on a configuration that controls the behavior of a single instance of the Arch gateway.
|
||||
This where you enable capabilities like LLM routing, fast function calling (via prompt_targets), applying guardrails, and enabling critical
|
||||
features like metrics and tracing. For the full configuration reference of <cite>arch_config.yaml</cite> see <a class="reference internal" href="../../resources/configuration_reference.html#configuration-reference"><span class="std std-ref">here</span></a>.</p>
|
||||
<p><strong>Downstream(Ingress)</strong>: An downstream client (web application, etc.) connects to Arch, sends prompts, and receives responses.</p>
|
||||
<p><strong>Upstream(Egress)</strong>: An upstream host that receives connections and prompts from Arch, and returns context or responses for a prompt</p>
|
||||
<a class="reference internal image-reference" href="../../_images/network-topology-ingress-egress.jpg"><img alt="../../_images/network-topology-ingress-egress.jpg" class="align-center" src="../../_images/network-topology-ingress-egress.jpg" style="width: 100%;"/>
|
||||
</a>
|
||||
<p><strong>Listener</strong>: A <a class="reference internal" href="listener.html#arch-overview-listeners"><span class="std std-ref">listener</span></a> is a named network location (e.g., port, address, path etc.) that Arch
|
||||
listens on to process prompts before forwarding them to your application server endpoints. rch enables you to configure one listener
|
||||
for downstream connections (like port 80, 443) and creates a separate internal listener for calls that initiate from your application
|
||||
code to LLMs.</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>When you start Arch, you specify a listener address/port that you want to bind downstream. But, Arch uses are predefined port
|
||||
that you can use (<code class="docutils literal notranslate"><span class="pre">127.0.0.1:12000</span></code>) to proxy egress calls originating from your application to LLMs (API-based or hosted).
|
||||
For more details, check out <a class="reference internal" href="../llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">LLM providers</span></a>.</p>
|
||||
</div>
|
||||
<p><strong>Prompt Target</strong>: Arch offers a primitive called <a class="reference internal" href="../prompt_target.html#prompt-target"><span class="std std-ref">prompt target</span></a> to help separate business logic from
|
||||
undifferentiated work in building generative AI apps. Prompt targets are endpoints that receive prompts that are processed by Arch.
|
||||
For example, Arch enriches incoming prompts with metadata like knowing when a request is a follow-up or clarifying prompt so that you
|
||||
can build faster, more accurate retrieval (RAG) apps. To support agentic apps, like scheduling travel plans or sharing comments on a
|
||||
document - via prompts, Arch uses its function calling abilities to extract critical information from the incoming prompt (or a set of
|
||||
prompts) needed by a downstream backend API or function call before calling it directly.</p>
|
||||
<p><strong>Model Serving</strong>: Arch is a set of <cite>two</cite> self-contained processes that are designed to run alongside your application servers
|
||||
(or on a separate host connected via a network).The <a class="reference internal" href="model_serving.html#model-serving"><span class="std std-ref">model serving</span></a> process helps Arch make intelligent decisions
|
||||
about the incoming prompts. The model server is designed to call the (fast) purpose-built LLMs in Arch.</p>
|
||||
<p><strong>Error Target</strong>: <a class="reference internal" href="error_target.html#error-target"><span class="std std-ref">Error targets</span></a> are those endpoints that receive forwarded errors from Arch when issues arise,
|
||||
such as failing to properly call a function/API, detecting violations of guardrails, or encountering other processing errors.
|
||||
These errors are communicated to the application via headers <code class="docutils literal notranslate"><span class="pre">X-Arch-[ERROR-TYPE]</span></code>, allowing it to handle the errors gracefully
|
||||
and take appropriate actions.</p>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="tech_overview.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Tech Overview
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="threading_model.html">
|
||||
Threading Model
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -7,15 +7,15 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Intro to Arch | Arch Docs v0.3.22</title>
|
||||
<meta content="Intro to Arch | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Intro to Arch | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Intro to Plano | Plano Docs v0.4</title>
|
||||
<meta content="Intro to Plano | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Intro to Plano | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/get_started/intro_to_arch.html" rel="canonical"/>
|
||||
<link href="./docs/get_started/intro_to_plano.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="quickstart.html" rel="next" title="Quickstart"/>
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="overview.html">Overview</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Intro to Arch</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,74 +148,56 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Intro to Arch</span>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Intro to Plano</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="intro-to-arch">
|
||||
<span id="id1"></span><h1>Intro to Arch<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#intro-to-arch"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>AI demos are easy to build. But past the thrill of a quick hack, you are left building, maintaining and scaling low-level plumbing code for agents that slows down AI innovation.
|
||||
For example:</p>
|
||||
<section id="intro-to-plano">
|
||||
<span id="id1"></span><h1>Intro to Plano<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#intro-to-plano"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Building agentic demos is easy. Delivering agentic applications safely, reliably, and repeatably to production is hard. After a quick hack, you end up building the “hidden AI middleware” to reach production: routing logic to reach the right agent, guardrail hooks for safety and moderation, evaluation and observability glue for continuous learning, and model/provider quirks — scattered across frameworks and application code.</p>
|
||||
<p>Plano solves this by moving core delivery concerns into a unified, out-of-process dataplane. Core capabilities:</p>
|
||||
<ul class="simple">
|
||||
<li><p>You want to build specialized agents, but get stuck writing <strong>routing and handoff</strong> code.</p></li>
|
||||
<li><p>You bogged down with prompt engineering work to <strong>clarify user intent and validate inputs</strong>.</p></li>
|
||||
<li><p>You want to <strong>quickly and safely use new LLMs</strong> but get stuck writing integration code.</p></li>
|
||||
<li><p>You waste cycles writing and maintaining <strong>observability</strong> code, when it can be transparent.</p></li>
|
||||
<li><p>You want to <strong>apply guardrails</strong>, but have to write custom code for each prompt and LLM.</p></li>
|
||||
<li><p><strong>🚦 Orchestration:</strong> Low-latency orchestration between agents, and add new agents without changing app code. When routing lives inside app code, it becomes hard to evolve and easy to duplicate. Moving orchestration into a centrally managed dataplane lets you change strategies without touching your agents, improving performance and reducing maintenance burden while avoiding tight coupling.</p></li>
|
||||
<li><p><strong>🛡️ Guardrails & Memory Hooks:</strong> Apply jailbreak protection, content policies, and context workflows (e.g., rewriting, retrieval, redaction) once via <a class="reference internal" href="../concepts/filter_chain.html#filter-chain"><span class="std std-ref">Filter Chains</span></a> at the dataplane. Instead of re-implementing these in every agentic service, you get centralized governance, reduced code duplication, and consistent behavior across your stack.</p></li>
|
||||
<li><p><strong>🔗 Model Agility:</strong> Route by model, alias (semantic names), or automatically via preferences so agents stay decoupled from specific providers. Swap or add models without refactoring prompts, tool-calling, or streaming handlers throughout your codebase by using Plano’s smart routing and unified API.</p></li>
|
||||
<li><p><strong>🕵 Agentic Signals™:</strong> Zero-code capture of behavior signals, traces, and metrics consistently across every agent. Rather than stitching together logging and metrics per framework, Plano surfaces traces, token usage, and learning signals in one place so you can iterate safely.</p></li>
|
||||
</ul>
|
||||
<p>Arch is designed to solve these problems by providing a unified, out-of-process architecture that integrates with your existing application stack, enabling you to focus on building high-level features rather than plumbing — all without locking you into a framework.</p>
|
||||
<p>Built by core contributors to the widely adopted Envoy Proxy <<a class="reference external" href="https://www.envoyproxy.io/" rel="nofollow noopener">https://www.envoyproxy.io/<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>>_, Plano gives you a production‑grade foundation for agentic applications. It helps <strong>developers</strong> stay focused on the core logic of their agents, helps <strong>product teams</strong> shorten feedback loops for learning, and helps <strong>engineering teams</strong> standardize policy and safety across agents and LLMs. Plano is grounded in open protocols (de facto: OpenAI‑style v1/responses, de jure: MCP) and proven patterns like sidecar deployments, so it plugs in cleanly while remaining robust, scalable, and flexible.</p>
|
||||
<p>In practice, achieving the above goal is incredibly difficult. Plano attempts to do so by providing the following high level features:</p>
|
||||
<figure class="align-center" id="id2">
|
||||
<a class="reference internal image-reference" href="../_images/arch_network_diagram_high_level.png"><img alt="../_images/arch_network_diagram_high_level.png" src="../_images/arch_network_diagram_high_level.png" style="width: 100%;"/>
|
||||
<a class="reference internal image-reference" href="../_images/plano_network_diagram_high_level.png"><img alt="../_images/plano_network_diagram_high_level.png" src="../_images/plano_network_diagram_high_level.png" style="width: 100%;"/>
|
||||
</a>
|
||||
<figcaption>
|
||||
<p><span class="caption-text">High-level network flow of where Arch Gateway sits in your agentic stack. Designed for both ingress and egress prompt traffic.</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></p>
|
||||
<p><span class="caption-text">High-level network flow of where Plano sits in your agentic stack. Designed for both ingress and egress prompt traffic.</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></p>
|
||||
</figcaption>
|
||||
</figure>
|
||||
<p><a class="reference external" href="https://github.com/katanemo/arch" rel="nofollow noopener">Arch<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> is a smart edge and AI gateway for AI-native apps - built by the contributors of Envoy Proxy with the belief that:</p>
|
||||
<blockquote>
|
||||
<div><p><em>Prompts are nuanced and opaque user requests, which require the same capabilities as traditional HTTP requests
|
||||
including secure handling, intelligent routing, robust observability, and integration with backend (API)
|
||||
systems for personalization - all outside business logic.</em></p>
|
||||
</div></blockquote>
|
||||
<p>In practice, achieving the above goal is incredibly difficult. Arch attempts to do so by providing the following high level features:</p>
|
||||
<p><strong>Out-of-process architecture, built on</strong> <a class="reference external" href="http://envoyproxy.io/" rel="nofollow noopener">Envoy<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>:
|
||||
Arch takes a dependency on Envoy and is a self-contained process that is designed to run alongside your application servers.
|
||||
Arch uses Envoy’s HTTP connection management subsystem, HTTP L7 filtering and telemetry capabilities to extend the functionality exclusively for prompts and LLMs.
|
||||
This gives Arch several advantages:</p>
|
||||
<ul class="simple">
|
||||
<li><p>Arch builds on Envoy’s proven success. Envoy is used at massive scale by the leading technology companies of our time including <a class="reference external" href="https://www.airbnb.com" rel="nofollow noopener">AirBnB<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, <a class="reference external" href="https://www.dropbox.com" rel="nofollow noopener">Dropbox<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, <a class="reference external" href="https://www.google.com" rel="nofollow noopener">Google<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, <a class="reference external" href="https://www.reddit.com" rel="nofollow noopener">Reddit<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, <a class="reference external" href="https://www.stripe.com" rel="nofollow noopener">Stripe<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, etc. Its battle tested and scales linearly with usage and enables developers to focus on what really matters: application features and business logic.</p></li>
|
||||
<li><p>Arch works with any application language. A single Arch deployment can act as gateway for AI applications written in Python, Java, C++, Go, Php, etc.</p></li>
|
||||
<li><p>Arch can be deployed and upgraded quickly across your infrastructure transparently without the horrid pain of deploying library upgrades in your applications.</p></li>
|
||||
</ul>
|
||||
<p><strong>Engineered with Fast Task-Specific LLMs (TLMs):</strong> Arch is engineered with specialized LLMs that are designed for the fast, cost-effective and accurate handling of prompts.
|
||||
<p><strong>Engineered with Task-Specific LLMs (TLMs):</strong> Plano is engineered with specialized LLMs that are designed for fast, cost-effective and accurate handling of prompts.
|
||||
These LLMs are designed to be best-in-class for critical tasks like:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Function Calling:</strong> Arch helps you easily personalize your applications by enabling calls to application-specific (API) operations via user prompts.
|
||||
This involves any predefined functions or APIs you want to expose to users to perform tasks, gather information, or manipulate data.
|
||||
With function calling, you have flexibility to support “agentic” experiences tailored to specific use cases - from updating insurance claims to creating ad campaigns - via prompts.
|
||||
Arch analyzes prompts, extracts critical information from prompts, engages in lightweight conversation to gather any missing parameters and makes API calls so that you can focus on writing business logic.
|
||||
For more details, read <a class="reference internal" href="../guides/function_calling.html#function-calling"><span class="std std-ref">Function Calling</span></a>.</p></li>
|
||||
<li><p><strong>Prompt Guard:</strong> Arch helps you improve the safety of your application by applying prompt guardrails in a centralized way for better governance hygiene.
|
||||
<li><p><strong>Agent Orchestration:</strong> <a class="reference external" href="https://huggingface.co/collections/katanemo/plano-orchestrator" rel="nofollow noopener">Plano-Orchestrator<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> is a family of state-of-the-art routing and orchestration models that decide which agent(s) or LLM(s) should handle each request, and in what sequence. Built for real-world multi-agent deployments, it analyzes user intent and conversation context to make precise routing and orchestration decisions while remaining efficient enough for low-latency production use across general chat, coding, and long-context multi-turn conversations.</p></li>
|
||||
<li><p><strong>Function Calling:</strong> Plano lets you expose application-specific (API) operations as tools so that your agents can update records, fetch data, or trigger determininistic workflows via prompts. Under the hood this is backed by Arch-Function-Chat; for more details, read <a class="reference internal" href="../guides/function_calling.html#function-calling"><span class="std std-ref">Function Calling</span></a>.</p></li>
|
||||
<li><p><strong>Guardrails:</strong> Plano helps you improve the safety of your application by applying prompt guardrails in a centralized way for better governance hygiene.
|
||||
With prompt guardrails you can prevent <code class="docutils literal notranslate"><span class="pre">jailbreak</span> <span class="pre">attempts</span></code> present in user’s prompts without having to write a single line of code.
|
||||
To learn more about how to configure guardrails available in Arch, read <a class="reference internal" href="../guides/prompt_guard.html#prompt-guard"><span class="std std-ref">Prompt Guard</span></a>.</p></li>
|
||||
To learn more about how to configure guardrails available in Plano, read <a class="reference internal" href="../guides/prompt_guard.html#prompt-guard"><span class="std std-ref">Prompt Guard</span></a>.</p></li>
|
||||
</ul>
|
||||
<p><strong>Model Proxy:</strong> Plano offers several capabilities for LLM calls originating from your applications, including smart retries on errors from upstream LLMs and automatic cut-over to other LLMs configured in Plano for continuous availability and disaster recovery scenarios. From your application’s perspective you keep using an OpenAI-compatible API, while Plano owns resiliency and failover policies in one place.
|
||||
Plano extends Envoy’s <a class="reference external" href="https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/cluster_manager" rel="nofollow noopener">cluster subsystem<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> to manage upstream connections to LLMs so that you can build resilient, provider-agnostic AI applications.</p>
|
||||
<p><strong>Edge Proxy:</strong> There is substantial benefit in using the same software at the edge (observability, traffic shaping algorithms, applying guardrails, etc.) as for outbound LLM inference use cases. Plano has the feature set that makes it exceptionally well suited as an edge gateway for AI applications.
|
||||
This includes TLS termination, applying guardrails early in the request flow, and intelligently deciding which agent(s) or LLM(s) should handle each request and in what sequence. In practice, you configure listeners and policies once, and every inbound and outbound call flows through the same hardened gateway.</p>
|
||||
<p><strong>Zero-Code Agent Signals™ & Tracing:</strong> Zero-code capture of behavior signals, traces, and metrics consistently across every agent. Plano propagates trace context using the W3C Trace Context standard, specifically through the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header. This allows each component in the system to record its part of the request flow, enabling end-to-end tracing across the entire application. By using OpenTelemetry, Plano ensures that developers can capture this trace data consistently and in a format compatible with various observability tools.</p>
|
||||
<p><strong>Best-In Class Monitoring:</strong> Plano offers several monitoring metrics that help you understand three critical aspects of your application: latency, token usage, and error rates by an upstream LLM provider. Latency measures the speed at which your application is responding to users, which includes metrics like time to first token (TFT), time per output token (TOT) metrics, and the total latency as perceived by users.</p>
|
||||
<p><strong>Out-of-process architecture, built on</strong> <a class="reference external" href="http://envoyproxy.io/" rel="nofollow noopener">Envoy<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>:
|
||||
Plano takes a dependency on Envoy and is a self-contained process that is designed to run alongside your application servers. Plano uses Envoy’s HTTP connection management subsystem, HTTP L7 filtering and telemetry capabilities to extend the functionality exclusively for prompts and LLMs.
|
||||
This gives Plano several advantages:</p>
|
||||
<ul class="simple">
|
||||
<li><p>Plano builds on Envoy’s proven success. Envoy is used at massive scale by the leading technology companies of our time including <a class="reference external" href="https://www.airbnb.com" rel="nofollow noopener">AirBnB<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, <a class="reference external" href="https://www.dropbox.com" rel="nofollow noopener">Dropbox<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, <a class="reference external" href="https://www.google.com" rel="nofollow noopener">Google<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, <a class="reference external" href="https://www.reddit.com" rel="nofollow noopener">Reddit<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, <a class="reference external" href="https://www.stripe.com" rel="nofollow noopener">Stripe<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, etc. Its battle tested and scales linearly with usage and enables developers to focus on what really matters: application features and business logic.</p></li>
|
||||
<li><p>Plano works with any application language. A single Plano deployment can act as gateway for AI applications written in Python, Java, C++, Go, Php, etc.</p></li>
|
||||
<li><p>Plano can be deployed and upgraded quickly across your infrastructure transparently without the horrid pain of deploying library upgrades in your applications.</p></li>
|
||||
</ul>
|
||||
<p><strong>Traffic Management:</strong> Arch offers several capabilities for LLM calls originating from your applications, including smart retries on errors from upstream LLMs, and automatic cut-over to other LLMs configured in Arch for continuous availability and disaster recovery scenarios.
|
||||
Arch extends Envoy’s <a class="reference external" href="https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/cluster_manager" rel="nofollow noopener">cluster subsystem<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> to manage upstream connections to LLMs so that you can build resilient AI applications.</p>
|
||||
<p><strong>Front/edge Gateway:</strong> There is substantial benefit in using the same software at the edge (observability, traffic shaping algorithms, applying guardrails, etc.) as for outbound LLM inference use cases.
|
||||
Arch has the feature set that makes it exceptionally well suited as an edge gateway for AI applications.
|
||||
This includes TLS termination, applying guardrail early in the process, intelligent parameter gathering from prompts, and prompt-based routing to backend APIs.</p>
|
||||
<p><strong>Best-In Class Monitoring:</strong> Arch offers several monitoring metrics that help you understand three critical aspects of
|
||||
your application: latency, token usage, and error rates by an upstream LLM provider. Latency measures the speed at which
|
||||
your application is responding to users, which includes metrics like time to first token (TFT), time per output token (TOT)
|
||||
metrics, and the total latency as perceived by users.</p>
|
||||
<p><strong>End-to-End Tracing:</strong> Arch propagates trace context using the W3C Trace Context standard, specifically through the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header.
|
||||
This allows each component in the system to record its part of the request flow, enabling end-to-end tracing across the entire application.
|
||||
By using OpenTelemetry, Arch ensures that developers can capture this trace data consistently and in a format compatible with various observability tools.
|
||||
For more details, read <a class="reference internal" href="../guides/observability/tracing.html#arch-overview-tracing"><span class="std std-ref">Tracing</span></a>.</p>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
|
|
@ -245,12 +222,12 @@ For more details, read <a class="reference internal" href="../guides/observabili
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Overview | Arch Docs v0.3.22</title>
|
||||
<meta content="Overview | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Overview | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Overview | Plano Docs v0.4</title>
|
||||
<meta content="Overview | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Overview | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/get_started/overview.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="intro_to_arch.html" rel="next" title="Intro to Arch"/>
|
||||
<link href="../index.html" rel="prev" title="Welcome to Arch!"/>
|
||||
<link href="intro_to_plano.html" rel="next" title="Intro to Plano"/>
|
||||
<link href="../index.html" rel="prev" title="Welcome to Plano!"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -163,20 +158,20 @@
|
|||
<div id="content" role="main">
|
||||
<section id="overview">
|
||||
<span id="id1"></span><h1>Overview<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#overview"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p><a class="reference external" href="https://github.com/katanemo/arch" rel="nofollow noopener">Arch<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> is a smart edge and AI gateway for AI agents - one that is natively designed to handle and process prompts, not just network traffic.</p>
|
||||
<p>Built by contributors to the widely adopted <a class="reference external" href="https://www.envoyproxy.io/" rel="nofollow noopener">Envoy Proxy<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, Arch handles the <em>pesky low-level work</em> in building agentic apps — like applying guardrails, clarifying vague user input, routing prompts to the right agent, and unifying access to any LLM. It’s a protocol-friendly and framework-agnostic infrastructure layer designed to help you build and ship agentic apps faster.</p>
|
||||
<p>In this documentation, you will learn how to quickly set up Arch to trigger API calls via prompts, apply prompt guardrails without writing any application-level logic,
|
||||
simplify the interaction with upstream LLMs, and improve observability all while simplifying your application development process.</p>
|
||||
<p><a class="reference external" href="https://github.com/katanemo/plano" rel="nofollow noopener">Plano<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> is delivery infrastructure for agentic apps. A models-native proxy server and data plane designed to help you build agents faster, and deliver them reliably to production.</p>
|
||||
<p>Plano pulls out the rote plumbing work (the “hidden AI middleware”) and decouples you from brittle, ever‑changing framework abstractions. It centralizes what shouldn’t be bespoke in every codebase like agent routing and orchestration, rich agentic signals and traces for continuous improvement, guardrail filters for safety and moderation, and smart LLM routing APIs for UX and DX agility. Use any language or AI framework, and ship agents to production faster with Plano.</p>
|
||||
<p>Built by core contributors to the widely adopted <a class="reference external" href="https://www.envoyproxy.io/" rel="nofollow noopener">Envoy Proxy<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, Plano gives you a production‑grade foundation for agentic applications. It helps <strong>developers</strong> stay focused on the core logic of their agents, helps <strong>product teams</strong> shorten feedback loops for learning, and helps <strong>engineering teams</strong> standardize policy and safety across agents and LLMs. Plano is grounded in open protocols (de facto: OpenAI‑style v1/responses, de jure: MCP) and proven patterns like sidecar deployments, so it plugs in cleanly while remaining robust, scalable, and flexible.</p>
|
||||
<p>In this documentation, you’ll learn how to set up Plano quickly, trigger API calls via prompts, apply guardrails without tight coupling with application code, simplify model and provider integration, and improve observability — so that you can focus on what matters most: the core product logic of your agents.</p>
|
||||
<figure class="align-center" id="id2">
|
||||
<a class="reference internal image-reference" href="../_images/arch_network_diagram_high_level.png"><img alt="../_images/arch_network_diagram_high_level.png" src="../_images/arch_network_diagram_high_level.png" style="width: 100%;"/>
|
||||
<a class="reference internal image-reference" href="../_images/plano_network_diagram_high_level.png"><img alt="../_images/plano_network_diagram_high_level.png" src="../_images/plano_network_diagram_high_level.png" style="width: 100%;"/>
|
||||
</a>
|
||||
<figcaption>
|
||||
<p><span class="caption-text">High-level network flow of where Arch Gateway sits in your agentic stack. Designed for both ingress and egress prompt traffic.</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></p>
|
||||
<p><span class="caption-text">High-level network flow of where Plano sits in your agentic stack. Designed for both ingress and egress traffic.</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></p>
|
||||
</figcaption>
|
||||
</figure>
|
||||
<section id="get-started">
|
||||
<h2>Get Started<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#get-started" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#get-started'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>This section introduces you to Arch and helps you get set up quickly:</p>
|
||||
<p>This section introduces you to Plano and helps you get set up quickly:</p>
|
||||
<div class="sd-container-fluid sd-sphinx-override sd-mb-4 docutils">
|
||||
<div class="sd-row sd-row-cols-3 sd-row-cols-xs-3 sd-row-cols-sm-3 sd-row-cols-md-3 sd-row-cols-lg-3 docutils">
|
||||
<div class="sd-col sd-d-flex-row docutils">
|
||||
|
|
@ -184,7 +179,7 @@ simplify the interaction with upstream LLMs, and improve observability all while
|
|||
<div class="sd-card-body docutils">
|
||||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-apps" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M1.5 3.25c0-.966.784-1.75 1.75-1.75h2.5c.966 0 1.75.784 1.75 1.75v2.5A1.75 1.75 0 0 1 5.75 7.5h-2.5A1.75 1.75 0 0 1 1.5 5.75Zm7 0c0-.966.784-1.75 1.75-1.75h2.5c.966 0 1.75.784 1.75 1.75v2.5a1.75 1.75 0 0 1-1.75 1.75h-2.5A1.75 1.75 0 0 1 8.5 5.75Zm-7 7c0-.966.784-1.75 1.75-1.75h2.5c.966 0 1.75.784 1.75 1.75v2.5a1.75 1.75 0 0 1-1.75 1.75h-2.5a1.75 1.75 0 0 1-1.75-1.75Zm7 0c0-.966.784-1.75 1.75-1.75h2.5c.966 0 1.75.784 1.75 1.75v2.5a1.75 1.75 0 0 1-1.75 1.75h-2.5a1.75 1.75 0 0 1-1.75-1.75ZM3.25 3a.25.25 0 0 0-.25.25v2.5c0 .138.112.25.25.25h2.5A.25.25 0 0 0 6 5.75v-2.5A.25.25 0 0 0 5.75 3Zm7 0a.25.25 0 0 0-.25.25v2.5c0 .138.112.25.25.25h2.5a.25.25 0 0 0 .25-.25v-2.5a.25.25 0 0 0-.25-.25Zm-7 7a.25.25 0 0 0-.25.25v2.5c0 .138.112.25.25.25h2.5a.25.25 0 0 0 .25-.25v-2.5a.25.25 0 0 0-.25-.25Zm7 0a.25.25 0 0 0-.25.25v2.5c0 .138.112.25.25.25h2.5a.25.25 0 0 0 .25-.25v-2.5a.25.25 0 0 0-.25-.25Z"></path></svg> Overview</div>
|
||||
<p class="sd-card-text">Overview of Arch and Doc navigation</p>
|
||||
<p class="sd-card-text">Overview of Plano and Doc navigation</p>
|
||||
</div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="overview.html"><span>overview.html</span></a></div>
|
||||
</div>
|
||||
|
|
@ -192,10 +187,10 @@ simplify the interaction with upstream LLMs, and improve observability all while
|
|||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||||
<div class="sd-card-body docutils">
|
||||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-book" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M0 1.75A.75.75 0 0 1 .75 1h4.253c1.227 0 2.317.59 3 1.501A3.743 3.743 0 0 1 11.006 1h4.245a.75.75 0 0 1 .75.75v10.5a.75.75 0 0 1-.75.75h-4.507a2.25 2.25 0 0 0-1.591.659l-.622.621a.75.75 0 0 1-1.06 0l-.622-.621A2.25 2.25 0 0 0 5.258 13H.75a.75.75 0 0 1-.75-.75Zm7.251 10.324.004-5.073-.002-2.253A2.25 2.25 0 0 0 5.003 2.5H1.5v9h3.757a3.75 3.75 0 0 1 1.994.574ZM8.755 4.75l-.004 7.322a3.752 3.752 0 0 1 1.992-.572H14.5v-9h-3.495a2.25 2.25 0 0 0-2.25 2.25Z"></path></svg> Intro to Arch</div>
|
||||
<p class="sd-card-text">Explore Arch’s features and developer workflow</p>
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-book" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M0 1.75A.75.75 0 0 1 .75 1h4.253c1.227 0 2.317.59 3 1.501A3.743 3.743 0 0 1 11.006 1h4.245a.75.75 0 0 1 .75.75v10.5a.75.75 0 0 1-.75.75h-4.507a2.25 2.25 0 0 0-1.591.659l-.622.621a.75.75 0 0 1-1.06 0l-.622-.621A2.25 2.25 0 0 0 5.258 13H.75a.75.75 0 0 1-.75-.75Zm7.251 10.324.004-5.073-.002-2.253A2.25 2.25 0 0 0 5.003 2.5H1.5v9h3.757a3.75 3.75 0 0 1 1.994.574ZM8.755 4.75l-.004 7.322a3.752 3.752 0 0 1 1.992-.572H14.5v-9h-3.495a2.25 2.25 0 0 0-2.25 2.25Z"></path></svg> Intro to Plano</div>
|
||||
<p class="sd-card-text">Explore Plano’s features and developer workflow</p>
|
||||
</div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="intro_to_arch.html"><span>intro_to_arch.html</span></a></div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="intro_to_plano.html"><span>intro_to_plano.html</span></a></div>
|
||||
</div>
|
||||
<div class="sd-col sd-d-flex-row docutils">
|
||||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||||
|
|
@ -211,24 +206,24 @@ simplify the interaction with upstream LLMs, and improve observability all while
|
|||
</section>
|
||||
<section id="concepts">
|
||||
<h2>Concepts<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#concepts" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#concepts'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Deep dive into essential ideas and mechanisms behind Arch:</p>
|
||||
<p>Deep dive into essential ideas and mechanisms behind Plano:</p>
|
||||
<div class="sd-container-fluid sd-sphinx-override sd-mb-4 docutils">
|
||||
<div class="sd-row sd-row-cols-3 sd-row-cols-xs-3 sd-row-cols-sm-3 sd-row-cols-md-3 sd-row-cols-lg-3 docutils">
|
||||
<div class="sd-col sd-d-flex-row docutils">
|
||||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||||
<div class="sd-card-body docutils">
|
||||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-package" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="m8.878.392 5.25 3.045c.54.314.872.89.872 1.514v6.098a1.75 1.75 0 0 1-.872 1.514l-5.25 3.045a1.75 1.75 0 0 1-1.756 0l-5.25-3.045A1.75 1.75 0 0 1 1 11.049V4.951c0-.624.332-1.201.872-1.514L7.122.392a1.75 1.75 0 0 1 1.756 0ZM7.875 1.69l-4.63 2.685L8 7.133l4.755-2.758-4.63-2.685a.248.248 0 0 0-.25 0ZM2.5 5.677v5.372c0 .09.047.171.125.216l4.625 2.683V8.432Zm6.25 8.271 4.625-2.683a.25.25 0 0 0 .125-.216V5.677L8.75 8.432Z"></path></svg> Tech Overview</div>
|
||||
<p class="sd-card-text">Learn about the technology stack</p>
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-package" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="m8.878.392 5.25 3.045c.54.314.872.89.872 1.514v6.098a1.75 1.75 0 0 1-.872 1.514l-5.25 3.045a1.75 1.75 0 0 1-1.756 0l-5.25-3.045A1.75 1.75 0 0 1 1 11.049V4.951c0-.624.332-1.201.872-1.514L7.122.392a1.75 1.75 0 0 1 1.756 0ZM7.875 1.69l-4.63 2.685L8 7.133l4.755-2.758-4.63-2.685a.248.248 0 0 0-.25 0ZM2.5 5.677v5.372c0 .09.047.171.125.216l4.625 2.683V8.432Zm6.25 8.271 4.625-2.683a.25.25 0 0 0 .125-.216V5.677L8.75 8.432Z"></path></svg> Agents</div>
|
||||
<p class="sd-card-text">Learn about how to build and scale agents with Plano</p>
|
||||
</div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../concepts/tech_overview/tech_overview.html"><span>../concepts/tech_overview/tech_overview.html</span></a></div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../concepts/agents.html"><span>../concepts/agents.html</span></a></div>
|
||||
</div>
|
||||
<div class="sd-col sd-d-flex-row docutils">
|
||||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||||
<div class="sd-card-body docutils">
|
||||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-webhook" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M5.5 4.25a2.25 2.25 0 0 1 4.5 0 .75.75 0 0 0 1.5 0 3.75 3.75 0 1 0-6.14 2.889l-2.272 4.258a.75.75 0 0 0 1.324.706L7 7.25a.75.75 0 0 0-.309-1.015A2.25 2.25 0 0 1 5.5 4.25Z"></path><path d="M7.364 3.607a.75.75 0 0 1 1.03.257l2.608 4.349a3.75 3.75 0 1 1-.628 6.785.75.75 0 0 1 .752-1.299 2.25 2.25 0 1 0-.033-3.88.75.75 0 0 1-1.03-.256L7.107 4.636a.75.75 0 0 1 .257-1.03Z"></path><path d="M2.9 8.776A.75.75 0 0 1 2.625 9.8 2.25 2.25 0 1 0 6 11.75a.75.75 0 0 1 .75-.751h5.5a.75.75 0 0 1 0 1.5H7.425a3.751 3.751 0 1 1-5.55-3.998.75.75 0 0 1 1.024.274Z"></path></svg> LLM Providers</div>
|
||||
<p class="sd-card-text">Explore Arch’s LLM integration options</p>
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-webhook" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M5.5 4.25a2.25 2.25 0 0 1 4.5 0 .75.75 0 0 0 1.5 0 3.75 3.75 0 1 0-6.14 2.889l-2.272 4.258a.75.75 0 0 0 1.324.706L7 7.25a.75.75 0 0 0-.309-1.015A2.25 2.25 0 0 1 5.5 4.25Z"></path><path d="M7.364 3.607a.75.75 0 0 1 1.03.257l2.608 4.349a3.75 3.75 0 1 1-.628 6.785.75.75 0 0 1 .752-1.299 2.25 2.25 0 1 0-.033-3.88.75.75 0 0 1-1.03-.256L7.107 4.636a.75.75 0 0 1 .257-1.03Z"></path><path d="M2.9 8.776A.75.75 0 0 1 2.625 9.8 2.25 2.25 0 1 0 6 11.75a.75.75 0 0 1 .75-.751h5.5a.75.75 0 0 1 0 1.5H7.425a3.751 3.751 0 1 1-5.55-3.998.75.75 0 0 1 1.024.274Z"></path></svg> Model Providers</div>
|
||||
<p class="sd-card-text">Explore Plano’s LLM integration options</p>
|
||||
</div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../concepts/llm_providers/llm_providers.html"><span>../concepts/llm_providers/llm_providers.html</span></a></div>
|
||||
</div>
|
||||
|
|
@ -237,7 +232,7 @@ simplify the interaction with upstream LLMs, and improve observability all while
|
|||
<div class="sd-card-body docutils">
|
||||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-workflow" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M0 1.75C0 .784.784 0 1.75 0h3.5C6.216 0 7 .784 7 1.75v3.5A1.75 1.75 0 0 1 5.25 7H4v4a1 1 0 0 0 1 1h4v-1.25C9 9.784 9.784 9 10.75 9h3.5c.966 0 1.75.784 1.75 1.75v3.5A1.75 1.75 0 0 1 14.25 16h-3.5A1.75 1.75 0 0 1 9 14.25v-.75H5A2.5 2.5 0 0 1 2.5 11V7h-.75A1.75 1.75 0 0 1 0 5.25Zm1.75-.25a.25.25 0 0 0-.25.25v3.5c0 .138.112.25.25.25h3.5a.25.25 0 0 0 .25-.25v-3.5a.25.25 0 0 0-.25-.25Zm9 9a.25.25 0 0 0-.25.25v3.5c0 .138.112.25.25.25h3.5a.25.25 0 0 0 .25-.25v-3.5a.25.25 0 0 0-.25-.25Z"></path></svg> Prompt Target</div>
|
||||
<p class="sd-card-text">Understand how Arch handles prompts</p>
|
||||
<p class="sd-card-text">Understand how Plano handles prompts</p>
|
||||
</div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../concepts/prompt_target.html"><span>../concepts/prompt_target.html</span></a></div>
|
||||
</div>
|
||||
|
|
@ -246,14 +241,14 @@ simplify the interaction with upstream LLMs, and improve observability all while
|
|||
</section>
|
||||
<section id="guides">
|
||||
<h2>Guides<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#guides" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#guides'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Step-by-step tutorials for practical Arch use cases and scenarios:</p>
|
||||
<p>Step-by-step tutorials for practical Plano use cases and scenarios:</p>
|
||||
<div class="sd-container-fluid sd-sphinx-override sd-mb-4 docutils">
|
||||
<div class="sd-row sd-row-cols-3 sd-row-cols-xs-3 sd-row-cols-sm-3 sd-row-cols-md-3 sd-row-cols-lg-3 docutils">
|
||||
<div class="sd-col sd-d-flex-row docutils">
|
||||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||||
<div class="sd-card-body docutils">
|
||||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-shield-check" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="m8.533.133 5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667l5.25-1.68a1.748 1.748 0 0 1 1.066 0Zm-.61 1.429.001.001-5.25 1.68a.251.251 0 0 0-.174.237V7c0 1.36.275 2.666 1.057 3.859.784 1.194 2.121 2.342 4.366 3.298a.196.196 0 0 0 .154 0c2.245-.957 3.582-2.103 4.366-3.297C13.225 9.666 13.5 8.358 13.5 7V3.48a.25.25 0 0 0-.174-.238l-5.25-1.68a.25.25 0 0 0-.153 0ZM11.28 6.28l-3.5 3.5a.75.75 0 0 1-1.06 0l-1.5-1.5a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l.97.97 2.97-2.97a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg> Prompt Guard</div>
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-shield-check" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="m8.533.133 5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667l5.25-1.68a1.748 1.748 0 0 1 1.066 0Zm-.61 1.429.001.001-5.25 1.68a.251.251 0 0 0-.174.237V7c0 1.36.275 2.666 1.057 3.859.784 1.194 2.121 2.342 4.366 3.298a.196.196 0 0 0 .154 0c2.245-.957 3.582-2.103 4.366-3.297C13.225 9.666 13.5 8.358 13.5 7V3.48a.25.25 0 0 0-.174-.238l-5.25-1.68a.25.25 0 0 0-.153 0ZM11.28 6.28l-3.5 3.5a.75.75 0 0 1-1.06 0l-1.5-1.5a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l.97.97 2.97-2.97a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg> Guardrails</div>
|
||||
<p class="sd-card-text">Instructions on securing and validating prompts</p>
|
||||
</div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../guides/prompt_guard.html"><span>../guides/prompt_guard.html</span></a></div>
|
||||
|
|
@ -262,45 +257,45 @@ simplify the interaction with upstream LLMs, and improve observability all while
|
|||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||||
<div class="sd-card-body docutils">
|
||||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-code-square" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25Zm7.47 3.97a.75.75 0 0 1 1.06 0l2 2a.75.75 0 0 1 0 1.06l-2 2a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L10.69 8 9.22 6.53a.75.75 0 0 1 0-1.06ZM6.78 6.53 5.31 8l1.47 1.47a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215l-2-2a.75.75 0 0 1 0-1.06l2-2a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg> Function Calling</div>
|
||||
<p class="sd-card-text">A guide to effective function calling</p>
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-code-square" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25Zm7.47 3.97a.75.75 0 0 1 1.06 0l2 2a.75.75 0 0 1 0 1.06l-2 2a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L10.69 8 9.22 6.53a.75.75 0 0 1 0-1.06ZM6.78 6.53 5.31 8l1.47 1.47a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215l-2-2a.75.75 0 0 1 0-1.06l2-2a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path></svg> LLM Routing</div>
|
||||
<p class="sd-card-text">A guide to effective model selection strategies</p>
|
||||
</div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../guides/function_calling.html"><span>../guides/function_calling.html</span></a></div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../guides/llm_router.html"><span>../guides/llm_router.html</span></a></div>
|
||||
</div>
|
||||
<div class="sd-col sd-d-flex-row docutils">
|
||||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||||
<div class="sd-card-body docutils">
|
||||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-issue-opened" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path></svg> Observability</div>
|
||||
<p class="sd-card-text">Learn to monitor and troubleshoot Arch</p>
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-issue-opened" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path></svg> State Management</div>
|
||||
<p class="sd-card-text">Learn to manage conversation and application state</p>
|
||||
</div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../guides/observability/observability.html"><span>../guides/observability/observability.html</span></a></div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../guides/state.html"><span>../guides/state.html</span></a></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="build-with-arch">
|
||||
<h2>Build with Arch<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#build-with-arch" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#build-with-arch'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>For developers extending and customizing Arch for specialized needs:</p>
|
||||
<section id="build-with-plano">
|
||||
<h2>Build with Plano<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#build-with-plano" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#build-with-plano'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>End to end examples demonstrating how to build agentic applications using Plano:</p>
|
||||
<div class="sd-container-fluid sd-sphinx-override sd-mb-4 docutils">
|
||||
<div class="sd-row sd-row-cols-2 sd-row-cols-xs-2 sd-row-cols-sm-2 sd-row-cols-md-2 sd-row-cols-lg-2 docutils">
|
||||
<div class="sd-col sd-d-flex-row docutils">
|
||||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||||
<div class="sd-card-body docutils">
|
||||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-dependabot" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M5.75 7.5a.75.75 0 0 1 .75.75v1.5a.75.75 0 0 1-1.5 0v-1.5a.75.75 0 0 1 .75-.75Zm5.25.75a.75.75 0 0 0-1.5 0v1.5a.75.75 0 0 0 1.5 0v-1.5Z"></path><path d="M6.25 0h2A.75.75 0 0 1 9 .75V3.5h3.25a2.25 2.25 0 0 1 2.25 2.25V8h.75a.75.75 0 0 1 0 1.5h-.75v2.75a2.25 2.25 0 0 1-2.25 2.25h-8.5a2.25 2.25 0 0 1-2.25-2.25V9.5H.75a.75.75 0 0 1 0-1.5h.75V5.75A2.25 2.25 0 0 1 3.75 3.5H7.5v-2H6.25a.75.75 0 0 1 0-1.5ZM3 5.75v6.5c0 .414.336.75.75.75h8.5a.75.75 0 0 0 .75-.75v-6.5a.75.75 0 0 0-.75-.75h-8.5a.75.75 0 0 0-.75.75Z"></path></svg> Agentic Workflow</div>
|
||||
<p class="sd-card-text">Discover how to create and manage custom agents within Arch</p>
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-dependabot" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M5.75 7.5a.75.75 0 0 1 .75.75v1.5a.75.75 0 0 1-1.5 0v-1.5a.75.75 0 0 1 .75-.75Zm5.25.75a.75.75 0 0 0-1.5 0v1.5a.75.75 0 0 0 1.5 0v-1.5Z"></path><path d="M6.25 0h2A.75.75 0 0 1 9 .75V3.5h3.25a2.25 2.25 0 0 1 2.25 2.25V8h.75a.75.75 0 0 1 0 1.5h-.75v2.75a2.25 2.25 0 0 1-2.25 2.25h-8.5a2.25 2.25 0 0 1-2.25-2.25V9.5H.75a.75.75 0 0 1 0-1.5h.75V5.75A2.25 2.25 0 0 1 3.75 3.5H7.5v-2H6.25a.75.75 0 0 1 0-1.5ZM3 5.75v6.5c0 .414.336.75.75.75h8.5a.75.75 0 0 0 .75-.75v-6.5a.75.75 0 0 0-.75-.75h-8.5a.75.75 0 0 0-.75.75Z"></path></svg> Build Agentic Apps</div>
|
||||
<p class="sd-card-text">Discover how to create and manage custom agents within Plano</p>
|
||||
</div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../build_with_arch/agent.html"><span>../build_with_arch/agent.html</span></a></div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../get_started/quickstart.html#build-agentic-apps-with-plano"><span>../get_started/quickstart.html#build-agentic-apps-with-plano</span></a></div>
|
||||
</div>
|
||||
<div class="sd-col sd-d-flex-row docutils">
|
||||
<div class="sd-card sd-sphinx-override sd-w-100 sd-shadow-sm sd-card-hover docutils">
|
||||
<div class="sd-card-body docutils">
|
||||
<div class="sd-card-title sd-font-weight-bold docutils">
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-stack" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M7.122.392a1.75 1.75 0 0 1 1.756 0l5.003 2.902c.83.481.83 1.68 0 2.162L8.878 8.358a1.75 1.75 0 0 1-1.756 0L2.119 5.456a1.251 1.251 0 0 1 0-2.162ZM8.125 1.69a.248.248 0 0 0-.25 0l-4.63 2.685 4.63 2.685a.248.248 0 0 0 .25 0l4.63-2.685ZM1.601 7.789a.75.75 0 0 1 1.025-.273l5.249 3.044a.248.248 0 0 0 .25 0l5.249-3.044a.75.75 0 0 1 .752 1.298l-5.248 3.044a1.75 1.75 0 0 1-1.756 0L1.874 8.814A.75.75 0 0 1 1.6 7.789Zm0 3.5a.75.75 0 0 1 1.025-.273l5.249 3.044a.248.248 0 0 0 .25 0l5.249-3.044a.75.75 0 0 1 .752 1.298l-5.248 3.044a1.75 1.75 0 0 1-1.756 0l-5.248-3.044a.75.75 0 0 1-.273-1.025Z"></path></svg> RAG Application</div>
|
||||
<p class="sd-card-text">Integrate RAG for knowledge-driven responses</p>
|
||||
<svg aria-hidden="true" class="sd-octicon sd-octicon-stack" height="1.0em" version="1.1" viewbox="0 0 16 16" width="1.0em"><path d="M7.122.392a1.75 1.75 0 0 1 1.756 0l5.003 2.902c.83.481.83 1.68 0 2.162L8.878 8.358a1.75 1.75 0 0 1-1.756 0L2.119 5.456a1.251 1.251 0 0 1 0-2.162ZM8.125 1.69a.248.248 0 0 0-.25 0l-4.63 2.685 4.63 2.685a.248.248 0 0 0 .25 0l4.63-2.685ZM1.601 7.789a.75.75 0 0 1 1.025-.273l5.249 3.044a.248.248 0 0 0 .25 0l5.249-3.044a.75.75 0 0 1 .752 1.298l-5.248 3.044a1.75 1.75 0 0 1-1.756 0L1.874 8.814A.75.75 0 0 1 1.6 7.789Zm0 3.5a.75.75 0 0 1 1.025-.273l5.249 3.044a.248.248 0 0 0 .25 0l5.249-3.044a.75.75 0 0 1 .752 1.298l-5.248 3.044a1.75 1.75 0 0 1-1.756 0l-5.248-3.044a.75.75 0 0 1-.273-1.025Z"></path></svg> Build Multi-LLM Apps</div>
|
||||
<p class="sd-card-text">Learn how to route LLM calls through Plano for enhanced control and observability</p>
|
||||
</div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../build_with_arch/rag.html"><span>../build_with_arch/rag.html</span></a></div>
|
||||
<a class="sd-stretched-link sd-hide-link-text reference external" href="../get_started/quickstart.html#use-plano-as-a-model-proxy-gateway"><span>../get_started/quickstart.html#use-plano-as-a-model-proxy-gateway</span></a></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -312,12 +307,12 @@ simplify the interaction with upstream LLMs, and improve observability all while
|
|||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Welcome to Arch!
|
||||
Welcome to Plano!
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="intro_to_arch.html">
|
||||
Intro to Arch
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="intro_to_plano.html">
|
||||
Intro to Plano
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
|
|
@ -329,7 +324,7 @@ simplify the interaction with upstream LLMs, and improve observability all while
|
|||
<li><a :data-current="activeSection === '#get-started'" class="reference internal" href="#get-started">Get Started</a></li>
|
||||
<li><a :data-current="activeSection === '#concepts'" class="reference internal" href="#concepts">Concepts</a></li>
|
||||
<li><a :data-current="activeSection === '#guides'" class="reference internal" href="#guides">Guides</a></li>
|
||||
<li><a :data-current="activeSection === '#build-with-arch'" class="reference internal" href="#build-with-arch">Build with Arch</a></li>
|
||||
<li><a :data-current="activeSection === '#build-with-plano'" class="reference internal" href="#build-with-plano">Build with Plano</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
|
|
@ -338,12 +333,12 @@ simplify the interaction with upstream LLMs, and improve observability all while
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Quickstart | Arch Docs v0.3.22</title>
|
||||
<meta content="Quickstart | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Quickstart | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Quickstart | Plano Docs v0.4</title>
|
||||
<meta content="Quickstart | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Quickstart | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/get_started/quickstart.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="../concepts/tech_overview/tech_overview.html" rel="next" title="Tech Overview"/>
|
||||
<link href="intro_to_arch.html" rel="prev" title="Intro to Arch"/>
|
||||
<link href="../concepts/listeners.html" rel="next" title="Listeners"/>
|
||||
<link href="intro_to_plano.html" rel="prev" title="Intro to Plano"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -163,7 +158,17 @@
|
|||
<div id="content" role="main">
|
||||
<section id="quickstart">
|
||||
<span id="id1"></span><h1>Quickstart<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#quickstart"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Follow this guide to learn how to quickly set up Arch and integrate it into your generative AI applications.</p>
|
||||
<p>Follow this guide to learn how to quickly set up Plano and integrate it into your generative AI applications. You can:</p>
|
||||
<ul class="simple">
|
||||
<li><p><a class="reference internal" href="#quickstart-agents"><span class="std std-ref">Build agents</span></a> for multi-step workflows (e.g., travel assistants with flights and hotels).</p></li>
|
||||
<li><p><a class="reference internal" href="#quickstart-prompt-targets"><span class="std std-ref">Call deterministic APIs via prompt targets</span></a> to turn instructions directly into function calls.</p></li>
|
||||
<li><p><a class="reference internal" href="#llm-routing-quickstart"><span class="std std-ref">Use Plano as a model proxy (Gateway)</span></a> to standardize access to multiple LLM providers.</p></li>
|
||||
</ul>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>This quickstart assumes basic familiarity with agents and prompt targets from the Concepts section. For background, see <a class="reference internal" href="../concepts/agents.html#agents"><span class="std std-ref">Agents</span></a> and <a class="reference internal" href="../concepts/prompt_target.html#prompt-target"><span class="std std-ref">Prompt Target</span></a>.</p>
|
||||
<p>The full agent and backend API implementations used here are available in the <a class="reference external" href="https://github.com/plano-ai/plano-quickstart" rel="nofollow noopener">plano-quickstart repository<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>. This guide focuses on wiring and configuring Plano (orchestration, prompt targets, and the model proxy), not application code.</p>
|
||||
</div>
|
||||
<section id="prerequisites">
|
||||
<h2>Prerequisites<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#prerequisites" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#prerequisites'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Before you begin, ensure you have the following:</p>
|
||||
|
|
@ -172,24 +177,91 @@
|
|||
<li><p><a class="reference external" href="https://docs.docker.com/compose/install/" rel="nofollow noopener">Docker Compose<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> (v2.29)</p></li>
|
||||
<li><p><a class="reference external" href="https://www.python.org/downloads/" rel="nofollow noopener">Python<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> (v3.10+)</p></li>
|
||||
</ol>
|
||||
<p>Arch’s CLI allows you to manage and interact with the Arch gateway efficiently. To install the CLI, simply run the following command:</p>
|
||||
<p>Plano’s CLI allows you to manage and interact with the Plano efficiently. To install the CLI, simply run the following command:</p>
|
||||
<div class="admonition tip">
|
||||
<p class="admonition-title">Tip</p>
|
||||
<p>We recommend that developers create a new Python virtual environment to isolate dependencies before installing Arch. This ensures that <code class="docutils literal notranslate"><span class="pre">archgw</span></code> and its dependencies do not interfere with other packages on your system.</p>
|
||||
<p>We recommend that developers create a new Python virtual environment to isolate dependencies before installing Plano. This ensures that <code class="docutils literal notranslate"><span class="pre">plano</span></code> and its dependencies do not interfere with other packages on your system.</p>
|
||||
</div>
|
||||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>python<span class="w"> </span>-m<span class="w"> </span>venv<span class="w"> </span>venv
|
||||
</span><span id="line-2"><span class="gp">$ </span><span class="nb">source</span><span class="w"> </span>venv/bin/activate<span class="w"> </span><span class="c1"># On Windows, use: venv\Scripts\activate</span>
|
||||
</span><span id="line-3"><span class="gp">$ </span>pip<span class="w"> </span>install<span class="w"> </span><span class="nv">archgw</span><span class="o">==</span><span class="m">0</span>.3.22
|
||||
</span><span id="line-3"><span class="gp">$ </span>pip<span class="w"> </span>install<span class="w"> </span><span class="nv">plano</span><span class="o">==</span><span class="m">0</span>.4.0
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="build-ai-agent-with-arch-gateway">
|
||||
<h2>Build AI Agent with Arch Gateway<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#build-ai-agent-with-arch-gateway" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#build-ai-agent-with-arch-gateway'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>In the following quickstart, we will show you how easy it is to build an AI agent with the Arch gateway. We will build a currency exchange agent using the following simple steps. For this demo, we will use <cite>https://api.frankfurter.dev/</cite> to fetch the latest prices for currencies and assume USD as the base currency.</p>
|
||||
<section id="step-1-create-arch-config-file">
|
||||
<h3>Step 1. Create arch config file<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-1-create-arch-config-file" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-1-create-arch-config-file'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Create <code class="docutils literal notranslate"><span class="pre">arch_config.yaml</span></code> file with the following content:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="w"> </span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1.0</span>
|
||||
<section id="build-agentic-apps-with-plano">
|
||||
<h2>Build Agentic Apps with Plano<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#build-agentic-apps-with-plano" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#build-agentic-apps-with-plano'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Plano helps you build agentic applications in two complementary ways:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Orchestrate agents</strong>: Let Plano decide which agent or LLM should handle each request and in what sequence.</p></li>
|
||||
<li><p><strong>Call deterministic backends</strong>: Use prompt targets to turn natural-language prompts into structured, validated API calls.</p></li>
|
||||
</ul>
|
||||
<section id="building-agents-with-plano-orchestration">
|
||||
<span id="quickstart-agents"></span><h3>Building agents with Plano orchestration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#building-agents-with-plano-orchestration" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#building-agents-with-plano-orchestration'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Agents are where your business logic lives (the “inner loop”). Plano takes care of the “outer loop”—routing, sequencing, and managing calls across agents and LLMs.</p>
|
||||
<p>At a high level, building agents with Plano looks like this:</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p><strong>Implement your agent</strong> in your framework of choice (Python, JS/TS, etc.), exposing it as an HTTP service.</p></li>
|
||||
<li><p><strong>Route LLM calls through Plano’s Model Proxy</strong>, so all models share a consistent interface and observability.</p></li>
|
||||
<li><p><strong>Configure Plano to orchestrate</strong>: define which agent(s) can handle which kinds of prompts, and let Plano decide when to call an agent vs. an LLM.</p></li>
|
||||
</ol>
|
||||
<p>This quickstart uses a simplified version of the Travel Booking Assistant; for the full multi-agent walkthrough, see <a class="reference internal" href="../guides/orchestration.html#agent-routing"><span class="std std-ref">Orchestration</span></a>.</p>
|
||||
<section id="step-1-minimal-orchestration-config">
|
||||
<h4>Step 1. Minimal orchestration config<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-1-minimal-orchestration-config"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<p>Here is a minimal configuration that wires Plano-Orchestrator to two HTTP services: one for flights and one for hotels.</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1.0</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">flight_agent</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://host.docker.internal:10520</span><span class="w"> </span><span class="c1"># your flights service</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">hotel_agent</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://host.docker.internal:10530</span><span class="w"> </span><span class="c1"># your hotels service</span>
|
||||
</span><span id="line-8">
|
||||
</span><span id="line-9"><span class="nt">model_providers</span><span class="p">:</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-12">
|
||||
</span><span id="line-13"><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-14"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">agent</span>
|
||||
</span><span id="line-15"><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">travel_assistant</span>
|
||||
</span><span id="line-16"><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">8001</span>
|
||||
</span><span id="line-17"><span class="w"> </span><span class="nt">router</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">plano_orchestrator_v1</span>
|
||||
</span><span id="line-18"><span class="w"> </span><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-19"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">flight_agent</span>
|
||||
</span><span id="line-20"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Search for flights and provide flight status.</span>
|
||||
</span><span id="line-21"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">hotel_agent</span>
|
||||
</span><span id="line-22"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Find hotels and check availability.</span>
|
||||
</span><span id="line-23">
|
||||
</span><span id="line-24"><span class="nt">tracing</span><span class="p">:</span>
|
||||
</span><span id="line-25"><span class="w"> </span><span class="nt">random_sampling</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">100</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="step-2-start-your-agents-and-plano">
|
||||
<h4>Step 2. Start your agents and Plano<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-2-start-your-agents-and-plano"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<p>Run your <code class="docutils literal notranslate"><span class="pre">flight_agent</span></code> and <code class="docutils literal notranslate"><span class="pre">hotel_agent</span></code> services (see <a class="reference internal" href="../guides/orchestration.html#agent-routing"><span class="std std-ref">Orchestration</span></a> for a full Travel Booking example), then start Plano with the config above:</p>
|
||||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>plano<span class="w"> </span>up<span class="w"> </span>plano_config.yaml
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p>Plano will start the orchestrator and expose an agent listener on port <code class="docutils literal notranslate"><span class="pre">8001</span></code>.</p>
|
||||
</section>
|
||||
<section id="step-3-send-a-prompt-and-let-plano-route">
|
||||
<h4>Step 3. Send a prompt and let Plano route<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-3-send-a-prompt-and-let-plano-route"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<p>Now send a request to Plano using the OpenAI-compatible chat completions API—the orchestrator will analyze the prompt and route it to the right agent based on intent:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">$<span class="w"> </span>curl<span class="w"> </span>--header<span class="w"> </span><span class="s1">'Content-Type: application/json'</span><span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-2"><span class="w"> </span>--data<span class="w"> </span><span class="s1">'{"messages": [{"role": "user","content": "Find me flights from SFO to JFK tomorrow"}], "model": "openai/gpt-4o"}'</span><span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-3"><span class="w"> </span>http://localhost:8001/v1/chat/completions
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p>You can then ask a follow-up like “Also book me a hotel near JFK” and Plano-Orchestrator will route to <code class="docutils literal notranslate"><span class="pre">hotel_agent</span></code>—your agents stay focused on business logic while Plano handles routing.</p>
|
||||
</section>
|
||||
</section>
|
||||
<section id="deterministic-api-calls-with-prompt-targets">
|
||||
<span id="quickstart-prompt-targets"></span><h3>Deterministic API calls with prompt targets<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#deterministic-api-calls-with-prompt-targets" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#deterministic-api-calls-with-prompt-targets'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Next, we’ll show Plano’s deterministic API calling using a single prompt target. We’ll build a currency exchange backend powered by <cite>https://api.frankfurter.dev/</cite>, assuming USD as the base currency.</p>
|
||||
<section id="step-1-create-plano-config-file">
|
||||
<h4>Step 1. Create plano config file<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-1-create-plano-config-file"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<p>Create <code class="docutils literal notranslate"><span class="pre">plano_config.yaml</span></code> file with the following content:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1.0</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">ingress_traffic</span><span class="p">:</span>
|
||||
|
|
@ -198,54 +270,48 @@
|
|||
</span><span id="line-7"><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-8"><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-9">
|
||||
</span><span id="line-10"><span class="w w-Error"> </span><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-10"><span class="w w-Error"> </span><span class="nt">model_providers</span><span class="p">:</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-13">
|
||||
</span><span id="line-14"><span class="w"> </span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">|</span>
|
||||
</span><span id="line-15"><span class="w"> </span><span class="no">You are a helpful assistant.</span>
|
||||
</span><span id="line-16">
|
||||
</span><span id="line-17"><span class="w"> </span><span class="nt">prompt_guards</span><span class="p">:</span>
|
||||
</span><span id="line-18"><span class="w"> </span><span class="nt">input_guards</span><span class="p">:</span>
|
||||
</span><span id="line-19"><span class="w"> </span><span class="nt">jailbreak</span><span class="p">:</span>
|
||||
</span><span id="line-20"><span class="w"> </span><span class="nt">on_exception</span><span class="p">:</span>
|
||||
</span><span id="line-21"><span class="w"> </span><span class="nt">message</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Looks like you're curious about my abilities, but I can only provide assistance for currency exchange.</span>
|
||||
</span><span id="line-22">
|
||||
</span><span id="line-23"><span class="w"> </span><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-24"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">currency_exchange</span>
|
||||
</span><span id="line-25"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Get currency exchange rate from USD to other currencies</span>
|
||||
</span><span id="line-26"><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</span><span id="line-27"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">currency_symbol</span>
|
||||
</span><span id="line-28"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">the currency that needs conversion</span>
|
||||
</span><span id="line-29"><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-30"><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">str</span>
|
||||
</span><span id="line-31"><span class="w"> </span><span class="nt">in_path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-32"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-33"><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">frankfurther_api</span>
|
||||
</span><span id="line-34"><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/v1/latest?base=USD&symbols={currency_symbol}</span>
|
||||
</span><span id="line-35"><span class="w"> </span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">|</span>
|
||||
</span><span id="line-36"><span class="w"> </span><span class="no">You are a helpful assistant. Show me the currency symbol you want to convert from USD.</span>
|
||||
</span><span id="line-17"><span class="w"> </span><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-18"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">currency_exchange</span>
|
||||
</span><span id="line-19"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Get currency exchange rate from USD to other currencies</span>
|
||||
</span><span id="line-20"><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</span><span id="line-21"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">currency_symbol</span>
|
||||
</span><span id="line-22"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">the currency that needs conversion</span>
|
||||
</span><span id="line-23"><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-24"><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">str</span>
|
||||
</span><span id="line-25"><span class="w"> </span><span class="nt">in_path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-26"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-27"><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">frankfurther_api</span>
|
||||
</span><span id="line-28"><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/v1/latest?base=USD&symbols={currency_symbol}</span>
|
||||
</span><span id="line-29"><span class="w"> </span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">|</span>
|
||||
</span><span id="line-30"><span class="w"> </span><span class="no">You are a helpful assistant. Show me the currency symbol you want to convert from USD.</span>
|
||||
</span><span id="line-31">
|
||||
</span><span id="line-32"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">get_supported_currencies</span>
|
||||
</span><span id="line-33"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Get list of supported currencies for conversion</span>
|
||||
</span><span id="line-34"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-35"><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">frankfurther_api</span>
|
||||
</span><span id="line-36"><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/v1/currencies</span>
|
||||
</span><span id="line-37">
|
||||
</span><span id="line-38"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">get_supported_currencies</span>
|
||||
</span><span id="line-39"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Get list of supported currencies for conversion</span>
|
||||
</span><span id="line-40"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-41"><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">frankfurther_api</span>
|
||||
</span><span id="line-42"><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/v1/currencies</span>
|
||||
</span><span id="line-43">
|
||||
</span><span id="line-44"><span class="w"> </span><span class="nt">endpoints</span><span class="p">:</span>
|
||||
</span><span id="line-45"><span class="w"> </span><span class="nt">frankfurther_api</span><span class="p">:</span>
|
||||
</span><span id="line-46"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">api.frankfurter.dev:443</span>
|
||||
</span><span id="line-47"><span class="w"> </span><span class="nt">protocol</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">https</span>
|
||||
</span><span id="line-38"><span class="w"> </span><span class="nt">endpoints</span><span class="p">:</span>
|
||||
</span><span id="line-39"><span class="w"> </span><span class="nt">frankfurther_api</span><span class="p">:</span>
|
||||
</span><span id="line-40"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">api.frankfurter.dev:443</span>
|
||||
</span><span id="line-41"><span class="w"> </span><span class="nt">protocol</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">https</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="step-2-start-arch-gateway-with-currency-conversion-config">
|
||||
<h3>Step 2. Start arch gateway with currency conversion config<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-2-start-arch-gateway-with-currency-conversion-config" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-2-start-arch-gateway-with-currency-conversion-config'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<div class="highlight-sh notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">$<span class="w"> </span>archgw<span class="w"> </span>up<span class="w"> </span>arch_config.yaml
|
||||
</span><span id="line-2"><span class="m">2024</span>-12-05<span class="w"> </span><span class="m">16</span>:56:27,979<span class="w"> </span>-<span class="w"> </span>cli.main<span class="w"> </span>-<span class="w"> </span>INFO<span class="w"> </span>-<span class="w"> </span>Starting<span class="w"> </span>archgw<span class="w"> </span>cli<span class="w"> </span>version:<span class="w"> </span><span class="m">0</span>.1.5
|
||||
<section id="step-2-start-plano-with-currency-conversion-config">
|
||||
<h4>Step 2. Start plano with currency conversion config<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-2-start-plano-with-currency-conversion-config"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<div class="highlight-sh notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">$<span class="w"> </span>plano<span class="w"> </span>up<span class="w"> </span>plano_config.yaml
|
||||
</span><span id="line-2"><span class="m">2024</span>-12-05<span class="w"> </span><span class="m">16</span>:56:27,979<span class="w"> </span>-<span class="w"> </span>cli.main<span class="w"> </span>-<span class="w"> </span>INFO<span class="w"> </span>-<span class="w"> </span>Starting<span class="w"> </span>plano<span class="w"> </span>cli<span class="w"> </span>version:<span class="w"> </span><span class="m">0</span>.1.5
|
||||
</span><span id="line-3">...
|
||||
</span><span id="line-4"><span class="m">2024</span>-12-05<span class="w"> </span><span class="m">16</span>:56:28,485<span class="w"> </span>-<span class="w"> </span>cli.utils<span class="w"> </span>-<span class="w"> </span>INFO<span class="w"> </span>-<span class="w"> </span>Schema<span class="w"> </span>validation<span class="w"> </span>successful!
|
||||
</span><span id="line-5"><span class="m">2024</span>-12-05<span class="w"> </span><span class="m">16</span>:56:28,485<span class="w"> </span>-<span class="w"> </span>cli.main<span class="w"> </span>-<span class="w"> </span>INFO<span class="w"> </span>-<span class="w"> </span>Starting<span class="w"> </span>arch<span class="w"> </span>model<span class="w"> </span>server<span class="w"> </span>and<span class="w"> </span>arch<span class="w"> </span>gateway
|
||||
</span><span id="line-5"><span class="m">2024</span>-12-05<span class="w"> </span><span class="m">16</span>:56:28,485<span class="w"> </span>-<span class="w"> </span>cli.main<span class="w"> </span>-<span class="w"> </span>INFO<span class="w"> </span>-<span class="w"> </span>Starting<span class="w"> </span>plano<span class="w"> </span>model<span class="w"> </span>server<span class="w"> </span>and<span class="w"> </span>plano<span class="w"> </span>gateway
|
||||
</span><span id="line-6">...
|
||||
</span><span id="line-7"><span class="m">2024</span>-12-05<span class="w"> </span><span class="m">16</span>:56:51,647<span class="w"> </span>-<span class="w"> </span>cli.core<span class="w"> </span>-<span class="w"> </span>INFO<span class="w"> </span>-<span class="w"> </span>Container<span class="w"> </span>is<span class="w"> </span>healthy!
|
||||
</span></code></pre></div>
|
||||
|
|
@ -254,7 +320,7 @@
|
|||
<p>Some sample queries you can ask include: <code class="docutils literal notranslate"><span class="pre">what</span> <span class="pre">is</span> <span class="pre">currency</span> <span class="pre">rate</span> <span class="pre">for</span> <span class="pre">gbp?</span></code> or <code class="docutils literal notranslate"><span class="pre">show</span> <span class="pre">me</span> <span class="pre">list</span> <span class="pre">of</span> <span class="pre">currencies</span> <span class="pre">for</span> <span class="pre">conversion</span></code>.</p>
|
||||
</section>
|
||||
<section id="step-3-interacting-with-gateway-using-curl-command">
|
||||
<h3>Step 3. Interacting with gateway using curl command<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-3-interacting-with-gateway-using-curl-command" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-3-interacting-with-gateway-using-curl-command'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<h4>Step 3. Interacting with gateway using curl command<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-3-interacting-with-gateway-using-curl-command"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<p>Here is a sample curl command you can use to interact:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">$<span class="w"> </span>curl<span class="w"> </span>--header<span class="w"> </span><span class="s1">'Content-Type: application/json'</span><span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-2"><span class="w"> </span>--data<span class="w"> </span><span class="s1">'{"messages": [{"role": "user","content": "what is exchange rate for gbp"}], "model": "none"}'</span><span class="w"> </span><span class="se">\</span>
|
||||
|
|
@ -273,12 +339,13 @@
|
|||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section id="use-arch-gateway-as-llm-router">
|
||||
<h2>Use Arch Gateway as LLM Router<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#use-arch-gateway-as-llm-router" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#use-arch-gateway-as-llm-router'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
</section>
|
||||
<section id="use-plano-as-a-model-proxy-gateway">
|
||||
<span id="llm-routing-quickstart"></span><h2>Use Plano as a Model Proxy (Gateway)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#use-plano-as-a-model-proxy-gateway" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#use-plano-as-a-model-proxy-gateway'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<section id="id2">
|
||||
<h3>Step 1. Create arch config file<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#id2'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Arch operates based on a configuration file where you can define LLM providers, prompt targets, guardrails, etc. Below is an example configuration that defines OpenAI and Mistral LLM providers.</p>
|
||||
<p>Create <code class="docutils literal notranslate"><span class="pre">arch_config.yaml</span></code> file with the following content:</p>
|
||||
<h3>Step 1. Create plano config file<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#id2'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Plano operates based on a configuration file where you can define LLM providers, prompt targets, guardrails, etc. Below is an example configuration that defines OpenAI and Mistral LLM providers.</p>
|
||||
<p>Create <code class="docutils literal notranslate"><span class="pre">plano_config.yaml</span></code> file with the following content:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="w"> </span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1.0</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="nt">listeners</span><span class="p">:</span>
|
||||
|
|
@ -288,7 +355,7 @@
|
|||
</span><span id="line-7"><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-8"><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-9">
|
||||
</span><span id="line-10"><span class="w w-Error"> </span><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-10"><span class="w w-Error"> </span><span class="nt">model_providers</span><span class="p">:</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-13"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
|
|
@ -298,14 +365,14 @@
|
|||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="step-2-start-arch-gateway">
|
||||
<h3>Step 2. Start arch gateway<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-2-start-arch-gateway" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-2-start-arch-gateway'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<section id="step-2-start-plano">
|
||||
<h3>Step 2. Start plano<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-2-start-plano" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-2-start-plano'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Once the config file is created, ensure that you have environment variables set up for <code class="docutils literal notranslate"><span class="pre">MISTRAL_API_KEY</span></code> and <code class="docutils literal notranslate"><span class="pre">OPENAI_API_KEY</span></code> (or these are defined in a <code class="docutils literal notranslate"><span class="pre">.env</span></code> file).</p>
|
||||
<p>Start the Arch gateway:</p>
|
||||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>archgw<span class="w"> </span>up<span class="w"> </span>arch_config.yaml
|
||||
</span><span id="line-2"><span class="go">2024-12-05 11:24:51,288 - cli.main - INFO - Starting archgw cli version: 0.1.5</span>
|
||||
<p>Start Plano:</p>
|
||||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>plano<span class="w"> </span>up<span class="w"> </span>plano_config.yaml
|
||||
</span><span id="line-2"><span class="go">2024-12-05 11:24:51,288 - cli.main - INFO - Starting plano cli version: 0.1.5</span>
|
||||
</span><span id="line-3"><span class="go">2024-12-05 11:24:51,825 - cli.utils - INFO - Schema validation successful!</span>
|
||||
</span><span id="line-4"><span class="go">2024-12-05 11:24:51,825 - cli.main - INFO - Starting arch model server and arch gateway</span>
|
||||
</span><span id="line-4"><span class="go">2024-12-05 11:24:51,825 - cli.main - INFO - Starting plano</span>
|
||||
</span><span id="line-5"><span class="go">...</span>
|
||||
</span><span id="line-6"><span class="go">2024-12-05 11:25:16,131 - cli.core - INFO - Container is healthy!</span>
|
||||
</span></code></pre></div>
|
||||
|
|
@ -315,19 +382,19 @@
|
|||
<h3>Step 3: Interact with LLM<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-3-interact-with-llm" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-3-interact-with-llm'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<section id="step-3-1-using-openai-python-client">
|
||||
<h4>Step 3.1: Using OpenAI Python client<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-3-1-using-openai-python-client"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<p>Make outbound calls via the Arch gateway:</p>
|
||||
<p>Make outbound calls via the Plano gateway:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">from</span><span class="w"> </span><span class="nn">openai</span><span class="w"> </span><span class="kn">import</span> <span class="n">OpenAI</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="c1"># Use the OpenAI client as usual</span>
|
||||
</span><span id="line-4"><span class="n">client</span> <span class="o">=</span> <span class="n">OpenAI</span><span class="p">(</span>
|
||||
</span><span id="line-5"> <span class="c1"># No need to set a specific openai.api_key since it's configured in Arch's gateway</span>
|
||||
</span><span id="line-5"> <span class="c1"># No need to set a specific openai.api_key since it's configured in Plano's gateway</span>
|
||||
</span><span id="line-6"> <span class="n">api_key</span><span class="o">=</span><span class="s1">'--'</span><span class="p">,</span>
|
||||
</span><span id="line-7"> <span class="c1"># Set the OpenAI API base URL to the Arch gateway endpoint</span>
|
||||
</span><span id="line-7"> <span class="c1"># Set the OpenAI API base URL to the Plano gateway endpoint</span>
|
||||
</span><span id="line-8"> <span class="n">base_url</span><span class="o">=</span><span class="s2">"http://127.0.0.1:12000/v1"</span>
|
||||
</span><span id="line-9"><span class="p">)</span>
|
||||
</span><span id="line-10">
|
||||
</span><span id="line-11"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-12"> <span class="c1"># we select model from arch_config file</span>
|
||||
</span><span id="line-12"> <span class="c1"># we select model from plano_config file</span>
|
||||
</span><span id="line-13"> <span class="n">model</span><span class="o">=</span><span class="s2">"--"</span><span class="p">,</span>
|
||||
</span><span id="line-14"> <span class="n">messages</span><span class="o">=</span><span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"What is the capital of France?"</span><span class="p">}],</span>
|
||||
</span><span id="line-15"><span class="p">)</span>
|
||||
|
|
@ -357,54 +424,32 @@
|
|||
</span><span id="line-17"><span class="o">}</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p>You can override model selection using the <code class="docutils literal notranslate"><span class="pre">x-arch-llm-provider-hint</span></code> header. For example, to use Mistral, use the following curl command:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">$<span class="w"> </span>curl<span class="w"> </span>--header<span class="w"> </span><span class="s1">'Content-Type: application/json'</span><span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-2"><span class="w"> </span>--header<span class="w"> </span><span class="s1">'x-arch-llm-provider-hint: ministral-3b'</span><span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-3"><span class="w"> </span>--data<span class="w"> </span><span class="s1">'{"messages": [{"role": "user","content": "What is the capital of France?"}], "model": "none"}'</span><span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-4"><span class="w"> </span>http://localhost:12000/v1/chat/completions
|
||||
</span><span id="line-5">
|
||||
</span><span id="line-6"><span class="o">{</span>
|
||||
</span><span id="line-7"><span class="w"> </span>...
|
||||
</span><span id="line-8"><span class="w"> </span><span class="s2">"model"</span>:<span class="w"> </span><span class="s2">"ministral-3b-latest"</span>,
|
||||
</span><span id="line-9"><span class="w"> </span><span class="s2">"choices"</span>:<span class="w"> </span><span class="o">[</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="o">{</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="s2">"messages"</span>:<span class="w"> </span><span class="o">{</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="s2">"role"</span>:<span class="w"> </span><span class="s2">"assistant"</span>,
|
||||
</span><span id="line-13"><span class="w"> </span><span class="s2">"content"</span>:<span class="w"> </span><span class="s2">"The capital of France is Paris. It is the most populous city in France and is known for its iconic landmarks such as the Eiffel Tower, the Louvre Museum, and Notre-Dame Cathedral. Paris is also a major global center for art, fashion, gastronomy, and culture."</span>,
|
||||
</span><span id="line-14"><span class="w"> </span><span class="o">}</span>,
|
||||
</span><span id="line-15"><span class="w"> </span>...
|
||||
</span><span id="line-16"><span class="w"> </span><span class="o">}</span>
|
||||
</span><span id="line-17"><span class="w"> </span><span class="o">]</span>,
|
||||
</span><span id="line-18"><span class="w"> </span>...
|
||||
</span><span id="line-19"><span class="o">}</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
</section>
|
||||
</section>
|
||||
<section id="next-steps">
|
||||
<h1>Next Steps<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#next-steps"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Congratulations! You’ve successfully set up Arch and made your first prompt-based request. To further enhance your GenAI applications, explore the following resources:</p>
|
||||
<p>Congratulations! You’ve successfully set up Plano and made your first prompt-based request. To further enhance your GenAI applications, explore the following resources:</p>
|
||||
<ul class="simple">
|
||||
<li><p><a class="reference internal" href="overview.html#overview"><span class="std std-ref">Full Documentation</span></a>: Comprehensive guides and references.</p></li>
|
||||
<li><p><a class="reference external" href="https://github.com/katanemo/arch" rel="nofollow noopener">GitHub Repository<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>: Access the source code, contribute, and track updates.</p></li>
|
||||
<li><p><a class="reference external" href="https://github.com/katanemo/arch#contact" rel="nofollow noopener">Support<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>: Get help and connect with the Arch community .</p></li>
|
||||
<li><p><a class="reference external" href="https://github.com/katanemo/plano" rel="nofollow noopener">GitHub Repository<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>: Access the source code, contribute, and track updates.</p></li>
|
||||
<li><p><a class="reference external" href="https://github.com/katanemo/plano#contact" rel="nofollow noopener">Support<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>: Get help and connect with the Plano community .</p></li>
|
||||
</ul>
|
||||
<p>With Arch, building scalable, fast, and personalized GenAI applications has never been easier. Dive deeper into Arch’s capabilities and start creating innovative AI-driven experiences today!</p>
|
||||
<p>With Plano, building scalable, fast, and personalized GenAI applications has never been easier. Dive deeper into Plano’s capabilities and start creating innovative AI-driven experiences today!</p>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="intro_to_arch.html">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="intro_to_plano.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Intro to Arch
|
||||
Intro to Plano
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../concepts/tech_overview/tech_overview.html">
|
||||
Tech Overview
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../concepts/listeners.html">
|
||||
Listeners
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
|
|
@ -415,15 +460,24 @@
|
|||
<ul>
|
||||
<li><ul>
|
||||
<li><a :data-current="activeSection === '#prerequisites'" class="reference internal" href="#prerequisites">Prerequisites</a></li>
|
||||
<li><a :data-current="activeSection === '#build-ai-agent-with-arch-gateway'" class="reference internal" href="#build-ai-agent-with-arch-gateway">Build AI Agent with Arch Gateway</a><ul>
|
||||
<li><a :data-current="activeSection === '#step-1-create-arch-config-file'" class="reference internal" href="#step-1-create-arch-config-file">Step 1. Create arch config file</a></li>
|
||||
<li><a :data-current="activeSection === '#step-2-start-arch-gateway-with-currency-conversion-config'" class="reference internal" href="#step-2-start-arch-gateway-with-currency-conversion-config">Step 2. Start arch gateway with currency conversion config</a></li>
|
||||
<li><a :data-current="activeSection === '#build-agentic-apps-with-plano'" class="reference internal" href="#build-agentic-apps-with-plano">Build Agentic Apps with Plano</a><ul>
|
||||
<li><a :data-current="activeSection === '#building-agents-with-plano-orchestration'" class="reference internal" href="#building-agents-with-plano-orchestration">Building agents with Plano orchestration</a><ul>
|
||||
<li><a :data-current="activeSection === '#step-1-minimal-orchestration-config'" class="reference internal" href="#step-1-minimal-orchestration-config">Step 1. Minimal orchestration config</a></li>
|
||||
<li><a :data-current="activeSection === '#step-2-start-your-agents-and-plano'" class="reference internal" href="#step-2-start-your-agents-and-plano">Step 2. Start your agents and Plano</a></li>
|
||||
<li><a :data-current="activeSection === '#step-3-send-a-prompt-and-let-plano-route'" class="reference internal" href="#step-3-send-a-prompt-and-let-plano-route">Step 3. Send a prompt and let Plano route</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#deterministic-api-calls-with-prompt-targets'" class="reference internal" href="#deterministic-api-calls-with-prompt-targets">Deterministic API calls with prompt targets</a><ul>
|
||||
<li><a :data-current="activeSection === '#step-1-create-plano-config-file'" class="reference internal" href="#step-1-create-plano-config-file">Step 1. Create plano config file</a></li>
|
||||
<li><a :data-current="activeSection === '#step-2-start-plano-with-currency-conversion-config'" class="reference internal" href="#step-2-start-plano-with-currency-conversion-config">Step 2. Start plano with currency conversion config</a></li>
|
||||
<li><a :data-current="activeSection === '#step-3-interacting-with-gateway-using-curl-command'" class="reference internal" href="#step-3-interacting-with-gateway-using-curl-command">Step 3. Interacting with gateway using curl command</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#use-arch-gateway-as-llm-router'" class="reference internal" href="#use-arch-gateway-as-llm-router">Use Arch Gateway as LLM Router</a><ul>
|
||||
<li><a :data-current="activeSection === '#id2'" class="reference internal" href="#id2">Step 1. Create arch config file</a></li>
|
||||
<li><a :data-current="activeSection === '#step-2-start-arch-gateway'" class="reference internal" href="#step-2-start-arch-gateway">Step 2. Start arch gateway</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#use-plano-as-a-model-proxy-gateway'" class="reference internal" href="#use-plano-as-a-model-proxy-gateway">Use Plano as a Model Proxy (Gateway)</a><ul>
|
||||
<li><a :data-current="activeSection === '#id2'" class="reference internal" href="#id2">Step 1. Create plano config file</a></li>
|
||||
<li><a :data-current="activeSection === '#step-2-start-plano'" class="reference internal" href="#step-2-start-plano">Step 2. Start plano</a></li>
|
||||
<li><a :data-current="activeSection === '#step-3-interact-with-llm'" class="reference internal" href="#step-3-interact-with-llm">Step 3: Interact with LLM</a><ul>
|
||||
<li><a :data-current="activeSection === '#step-3-1-using-openai-python-client'" class="reference internal" href="#step-3-1-using-openai-python-client">Step 3.1: Using OpenAI Python client</a></li>
|
||||
<li><a :data-current="activeSection === '#step-3-2-using-curl-command'" class="reference internal" href="#step-3-2-using-curl-command">Step 3.2: Using curl command</a></li>
|
||||
|
|
@ -442,12 +496,12 @@
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -1,311 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Agent Routing and Hand Off | Arch Docs v0.3.22</title>
|
||||
<meta content="Agent Routing and Hand Off | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Agent Routing and Hand Off | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/guides/agent_routing.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="function_calling.html" rel="next" title="Function Calling"/>
|
||||
<link href="prompt_guard.html" rel="prev" title="Prompt Guard"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
let mode;
|
||||
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
||||
mode = 'dark';
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
mode = 'light';
|
||||
}
|
||||
if (!userPreference) {localStorage.setItem('darkMode', mode)}
|
||||
</script>
|
||||
</head>
|
||||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
K
|
||||
</kbd>
|
||||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
</a>
|
||||
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
|
||||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||||
</svg>
|
||||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</aside>
|
||||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Agent Routing and Hand Off</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="agent-routing-and-hand-off">
|
||||
<span id="agent-routing"></span><h1>Agent Routing and Hand Off<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#agent-routing-and-hand-off"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Agent Routing and Hand Off is a key feature in Arch that enables intelligent routing of user prompts to specialized AI agents or human agents based on the nature and complexity of the user’s request.</p>
|
||||
<p>This capability significantly enhances the efficiency and personalization of interactions, ensuring each prompt receives the most appropriate and effective handling. The following section describes
|
||||
the workflow, configuration, and implementation of Agent routing and hand off in Arch.</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p><strong>Agent Selection</strong>
|
||||
When a user submits a prompt, Arch analyzes the input to determine the intent and complexity. Based on the analysis, Arch selects the most suitable agent configured within your application to handle the specific category of the user’s request—such as sales inquiries, technical issues, or complex scenarios requiring human attention.</p></li>
|
||||
<li><p><strong>Prompt Routing</strong>
|
||||
After selecting the appropriate agent, Arch routes the user’s prompt to the designated agent’s endpoint and waits for the agent to respond back with the processed output or further instructions.</p></li>
|
||||
<li><p><strong>Hand Off</strong>
|
||||
Based on follow-up queries from the user, Arch repeats the process of analysis, agent selection, and routing to ensure a seamless hand off between AI agents as needed.</p></li>
|
||||
</ol>
|
||||
<div class="literal-block-wrapper docutils container" id="id1">
|
||||
<div class="code-block-caption"><span class="caption-text">Agent Routing and Hand Off Configuration Example</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id1"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">sales_agent</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Handles queries related to sales and purchases</span>
|
||||
</span><span id="line-4">
|
||||
</span><span id="line-5"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">issues_and_repairs</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">handles issues, repairs, or refunds</span>
|
||||
</span><span id="line-7">
|
||||
</span><span id="line-8"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">escalate_to_human</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">escalates to human agent</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="literal-block-wrapper docutils container" id="id2">
|
||||
<div class="code-block-caption"><span class="caption-text">Agent Routing and Hand Off Implementation Example via FastAPI</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="k">class</span><span class="w"> </span><span class="nc">Agent</span><span class="p">:</span>
|
||||
</span><span id="line-2"> <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">role</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">instructions</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||
</span><span id="line-3"> <span class="bp">self</span><span class="o">.</span><span class="n">system_prompt</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"You are a </span><span class="si">{</span><span class="n">role</span><span class="si">}</span><span class="s2">.</span><span class="se">\n</span><span class="si">{</span><span class="n">instructions</span><span class="si">}</span><span class="s2">"</span>
|
||||
</span><span id="line-4">
|
||||
</span><span id="line-5"> <span class="k">def</span><span class="w"> </span><span class="nf">handle</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">req</span><span class="p">:</span> <span class="n">ChatCompletionsRequest</span><span class="p">):</span>
|
||||
</span><span id="line-6"> <span class="n">messages</span> <span class="o">=</span> <span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"system"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_system_prompt</span><span class="p">()}]</span> <span class="o">+</span> <span class="p">[</span>
|
||||
</span><span id="line-7"> <span class="n">message</span><span class="o">.</span><span class="n">model_dump</span><span class="p">()</span> <span class="k">for</span> <span class="n">message</span> <span class="ow">in</span> <span class="n">req</span><span class="o">.</span><span class="n">messages</span>
|
||||
</span><span id="line-8"> <span class="p">]</span>
|
||||
</span><span id="line-9"> <span class="k">return</span> <span class="n">call_openai</span><span class="p">(</span><span class="n">messages</span><span class="p">,</span> <span class="n">req</span><span class="o">.</span><span class="n">stream</span><span class="p">)</span> <span class="c1">#call_openai is a placeholder for the actual API call</span>
|
||||
</span><span id="line-10">
|
||||
</span><span id="line-11"> <span class="k">def</span><span class="w"> </span><span class="nf">get_system_prompt</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||
</span><span id="line-12"> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">system_prompt</span>
|
||||
</span><span id="line-13">
|
||||
</span><span id="line-14"><span class="c1"># Define your agents</span>
|
||||
</span><span id="line-15"><span class="n">AGENTS</span> <span class="o">=</span> <span class="p">{</span>
|
||||
</span><span id="line-16"> <span class="s2">"sales_agent"</span><span class="p">:</span> <span class="n">Agent</span><span class="p">(</span>
|
||||
</span><span id="line-17"> <span class="n">role</span><span class="o">=</span><span class="s2">"sales agent"</span><span class="p">,</span>
|
||||
</span><span id="line-18"> <span class="n">instructions</span><span class="o">=</span><span class="p">(</span>
|
||||
</span><span id="line-19"> <span class="s2">"Always answer in a sentence or less.</span><span class="se">\n</span><span class="s2">"</span>
|
||||
</span><span id="line-20"> <span class="s2">"Follow the following routine with the user:</span><span class="se">\n</span><span class="s2">"</span>
|
||||
</span><span id="line-21"> <span class="s2">"1. Engage</span><span class="se">\n</span><span class="s2">"</span>
|
||||
</span><span id="line-22"> <span class="s2">"2. Quote ridiculous price</span><span class="se">\n</span><span class="s2">"</span>
|
||||
</span><span id="line-23"> <span class="s2">"3. Reveal caveat if user agrees."</span>
|
||||
</span><span id="line-24"> <span class="p">),</span>
|
||||
</span><span id="line-25"> <span class="p">),</span>
|
||||
</span><span id="line-26"> <span class="s2">"issues_and_repairs"</span><span class="p">:</span> <span class="n">Agent</span><span class="p">(</span>
|
||||
</span><span id="line-27"> <span class="n">role</span><span class="o">=</span><span class="s2">"issues and repairs agent"</span><span class="p">,</span>
|
||||
</span><span id="line-28"> <span class="n">instructions</span><span class="o">=</span><span class="s2">"Propose a solution, offer refund if necessary."</span><span class="p">,</span>
|
||||
</span><span id="line-29"> <span class="p">),</span>
|
||||
</span><span id="line-30"> <span class="s2">"escalate_to_human"</span><span class="p">:</span> <span class="n">Agent</span><span class="p">(</span>
|
||||
</span><span id="line-31"> <span class="n">role</span><span class="o">=</span><span class="s2">"human escalation agent"</span><span class="p">,</span> <span class="n">instructions</span><span class="o">=</span><span class="s2">"Escalate issues to a human."</span>
|
||||
</span><span id="line-32"> <span class="p">),</span>
|
||||
</span><span id="line-33"> <span class="s2">"unknown_agent"</span><span class="p">:</span> <span class="n">Agent</span><span class="p">(</span>
|
||||
</span><span id="line-34"> <span class="n">role</span><span class="o">=</span><span class="s2">"general assistant"</span><span class="p">,</span> <span class="n">instructions</span><span class="o">=</span><span class="s2">"Assist the user in general queries."</span>
|
||||
</span><span id="line-35"> <span class="p">),</span>
|
||||
</span><span id="line-36"><span class="p">}</span>
|
||||
</span><span id="line-37">
|
||||
</span><span id="line-38"><span class="c1">#handle the request from arch gateway</span>
|
||||
</span><span id="line-39"><span class="nd">@app</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="s2">"/v1/chat/completions"</span><span class="p">)</span>
|
||||
</span><span id="line-40"><span class="k">def</span><span class="w"> </span><span class="nf">completion_api</span><span class="p">(</span><span class="n">req</span><span class="p">:</span> <span class="n">ChatCompletionsRequest</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">Request</span><span class="p">):</span>
|
||||
</span><span id="line-41">
|
||||
</span><span id="line-42"> <span class="n">agent_name</span> <span class="o">=</span> <span class="n">req</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"agent-name"</span><span class="p">,</span> <span class="s2">"unknown_agent"</span><span class="p">)</span>
|
||||
</span><span id="line-43"> <span class="n">agent</span> <span class="o">=</span> <span class="n">AGENTS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">agent_name</span><span class="p">)</span>
|
||||
</span><span id="line-44"> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Routing to agent: </span><span class="si">{</span><span class="n">agent_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
</span><span id="line-45">
|
||||
</span><span id="line-46"> <span class="k">return</span> <span class="n">agent</span><span class="o">.</span><span class="n">handle</span><span class="p">(</span><span class="n">req</span><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>The above example demonstrates a simple implementation of Agent Routing and Hand Off using FastAPI. For the full implementation of this example
|
||||
please see our <a class="reference external" href="https://github.com/katanemo/archgw/tree/main/demos/use_cases/orchestrating_agents" rel="nofollow noopener">GitHub demo<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>.</p>
|
||||
</div>
|
||||
<section id="example-use-cases">
|
||||
<h2>Example Use Cases<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-use-cases" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#example-use-cases'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Agent Routing and Hand Off is particularly beneficial in scenarios such as:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Customer Support</strong>: Routing common customer queries to automated support agents, while escalating complex or sensitive issues to human support staff.</p></li>
|
||||
<li><p><strong>Sales and Marketing</strong>: Automatically directing potential leads and sales inquiries to specialized sales agents for timely and targeted follow-ups.</p></li>
|
||||
<li><p><strong>Technical Assistance</strong>: Managing user-reported issues, repairs, or refunds by assigning them to the correct technical or support agent efficiently.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="best-practices-and-tips">
|
||||
<h2>Best Practices and Tips<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#best-practices-and-tips" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#best-practices-and-tips'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>When implementing Agent Routing and Hand Off in your applications, consider these best practices:</p>
|
||||
<ul class="simple">
|
||||
<li><p>Clearly define agent responsibilities: Ensure each agent or human endpoint has a clear, specific description of the prompts they handle, reducing mis-routing.</p></li>
|
||||
<li><p>Monitor and optimize routes: Regularly review how prompts are routed to adjust and optimize agent definitions and configurations.</p></li>
|
||||
</ul>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>To observe traffic to and from agents, please read more about <a class="reference internal" href="observability/observability.html#observability"><span class="std std-ref">observability</span></a> in Arch.</p>
|
||||
</div>
|
||||
<p>By carefully configuring and managing your Agent routing and hand off, you can significantly improve your application’s responsiveness, performance, and overall user satisfaction.</p>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="prompt_guard.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Prompt Guard
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="function_calling.html">
|
||||
Function Calling
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#example-use-cases'" class="reference internal" href="#example-use-cases">Example Use Cases</a></li>
|
||||
<li><a :data-current="activeSection === '#best-practices-and-tips'" class="reference internal" href="#best-practices-and-tips">Best Practices and Tips</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Function Calling | Arch Docs v0.3.22</title>
|
||||
<meta content="Function Calling | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Function Calling | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Function Calling | Plano Docs v0.4</title>
|
||||
<meta content="Function Calling | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Function Calling | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/guides/function_calling.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="llm_router.html" rel="next" title="LLM Routing"/>
|
||||
<link href="agent_routing.html" rel="prev" title="Agent Routing and Hand Off"/>
|
||||
<link href="observability/observability.html" rel="next" title="Observability"/>
|
||||
<link href="llm_router.html" rel="prev" title="LLM Routing"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -163,7 +158,7 @@
|
|||
<div id="content" role="main">
|
||||
<section id="function-calling">
|
||||
<span id="id1"></span><h1>Function Calling<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#function-calling"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p><strong>Function Calling</strong> is a powerful feature in Arch that allows your application to dynamically execute backend functions or services based on user prompts.
|
||||
<p><strong>Function Calling</strong> is a powerful feature in Plano that allows your application to dynamically execute backend functions or services based on user prompts.
|
||||
This enables seamless integration between natural language interactions and backend operations, turning user inputs into actionable results.</p>
|
||||
<section id="what-is-function-calling">
|
||||
<h2>What is Function Calling?<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#what-is-function-calling" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#what-is-function-calling'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
|
|
@ -175,17 +170,17 @@ This feature bridges the gap between generative AI systems and functional busine
|
|||
<ol class="arabic">
|
||||
<li><p><strong>Prompt Parsing</strong></p>
|
||||
<blockquote>
|
||||
<div><p>When a user submits a prompt, Arch analyzes it to determine the intent. Based on this intent, the system identifies whether a function needs to be invoked and which parameters should be extracted.</p>
|
||||
<div><p>When a user submits a prompt, Plano analyzes it to determine the intent. Based on this intent, the system identifies whether a function needs to be invoked and which parameters should be extracted.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Parameter Extraction</strong></p>
|
||||
<blockquote>
|
||||
<div><p>Arch’s advanced natural language processing capabilities automatically extract parameters from the prompt that are necessary for executing the function. These parameters can include text, numbers, dates, locations, or other relevant data points.</p>
|
||||
<div><p>Plano’s advanced natural language processing capabilities automatically extract parameters from the prompt that are necessary for executing the function. These parameters can include text, numbers, dates, locations, or other relevant data points.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Function Invocation</strong></p>
|
||||
<blockquote>
|
||||
<div><p>Once the necessary parameters have been extracted, Arch invokes the relevant backend function. This function could be an API, a database query, or any other form of backend logic. The function is executed with the extracted parameters to produce the desired output.</p>
|
||||
<div><p>Once the necessary parameters have been extracted, Plano invokes the relevant backend function. This function could be an API, a database query, or any other form of backend logic. The function is executed with the extracted parameters to produce the desired output.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Response Handling</strong></p>
|
||||
|
|
@ -234,10 +229,10 @@ Achieving performance on par with GPT-4, these models set a new benchmark in the
|
|||
</section>
|
||||
<section id="implementing-function-calling">
|
||||
<h2>Implementing Function Calling<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#implementing-function-calling" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#implementing-function-calling'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Here’s a step-by-step guide to configuring function calling within your Arch setup:</p>
|
||||
<p>Here’s a step-by-step guide to configuring function calling within your Plano setup:</p>
|
||||
<section id="step-1-define-the-function">
|
||||
<h3>Step 1: Define the Function<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-1-define-the-function" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-1-define-the-function'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>First, create or identify the backend function you want Arch to call. This could be an API endpoint, a script, or any other executable backend logic.</p>
|
||||
<p>First, create or identify the backend function you want Plano to call. This could be an API endpoint, a script, or any other executable backend logic.</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">import</span><span class="w"> </span><span class="nn">requests</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="k">def</span><span class="w"> </span><span class="nf">get_weather</span><span class="p">(</span><span class="n">location</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">unit</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"fahrenheit"</span><span class="p">):</span>
|
||||
|
|
@ -263,8 +258,8 @@ Achieving performance on par with GPT-4, these models set a new benchmark in the
|
|||
</section>
|
||||
<section id="step-2-configure-prompt-targets">
|
||||
<h3>Step 2: Configure Prompt Targets<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-2-configure-prompt-targets" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-2-configure-prompt-targets'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Next, map the function to a prompt target, defining the intent and parameters that Arch will extract from the user’s prompt.
|
||||
Specify the parameters your function needs and how Arch should interpret these.</p>
|
||||
<p>Next, map the function to a prompt target, defining the intent and parameters that Plano will extract from the user’s prompt.
|
||||
Specify the parameters your function needs and how Plano should interpret these.</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id3">
|
||||
<div class="code-block-caption"><span class="caption-text">Prompt Target Example Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id3"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
|
|
@ -290,20 +285,20 @@ Specify the parameters your function needs and how Arch should interpret these.<
|
|||
<p>For a complete refernce of attributes that you can configure in a prompt target, see <a class="reference internal" href="../concepts/prompt_target.html#defining-prompt-target-parameters"><span class="std std-ref">here</span></a>.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="step-3-arch-takes-over">
|
||||
<h3>Step 3: Arch Takes Over<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-3-arch-takes-over" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-3-arch-takes-over'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Once you have defined the functions and configured the prompt targets, Arch Gateway takes care of the remaining work.
|
||||
<section id="step-3-plano-takes-over">
|
||||
<h3>Step 3: Plano Takes Over<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#step-3-plano-takes-over" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#step-3-plano-takes-over'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Once you have defined the functions and configured the prompt targets, Plano takes care of the remaining work.
|
||||
It will automatically validate parameters, and ensure that the required parameters (e.g., location) are present in the prompt, and add validation rules if necessary.</p>
|
||||
<figure class="align-center" id="id4">
|
||||
<a class="reference internal image-reference" href="../_images/arch_network_diagram_high_level.png"><img alt="../_images/arch_network_diagram_high_level.png" src="../_images/arch_network_diagram_high_level.png" style="width: 100%;"/>
|
||||
<a class="reference internal image-reference" href="../_images/plano_network_diagram_high_level.png"><img alt="../_images/plano_network_diagram_high_level.png" src="../_images/plano_network_diagram_high_level.png" style="width: 100%;"/>
|
||||
</a>
|
||||
<figcaption>
|
||||
<p><span class="caption-text">High-level network flow of where Arch Gateway sits in your agentic stack. Managing incoming and outgoing prompt traffic</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id4"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></p>
|
||||
<p><span class="caption-text">High-level network flow of where Plano sits in your agentic stack. Managing incoming and outgoing prompt traffic</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id4"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></p>
|
||||
</figcaption>
|
||||
</figure>
|
||||
<p>Once a downstream function (API) is called, Arch Gateway takes the response and sends it an upstream LLM to complete the request (for summarization, Q/A, text generation tasks).
|
||||
For more details on how Arch Gateway enables you to centralize usage of LLMs, please read <a class="reference internal" href="../concepts/llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">LLM providers</span></a>.</p>
|
||||
<p>By completing these steps, you enable Arch to manage the process from validation to response, ensuring users receive consistent, reliable results - and that you are focused
|
||||
<p>Once a downstream function (API) is called, Plano takes the response and sends it an upstream LLM to complete the request (for summarization, Q/A, text generation tasks).
|
||||
For more details on how Plano enables you to centralize usage of LLMs, please read <a class="reference internal" href="../concepts/llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">LLM providers</span></a>.</p>
|
||||
<p>By completing these steps, you enable Plano to manage the process from validation to response, ensuring users receive consistent, reliable results - and that you are focused
|
||||
on the stuff that matters most.</p>
|
||||
</section>
|
||||
</section>
|
||||
|
|
@ -320,7 +315,7 @@ on the stuff that matters most.</p>
|
|||
</section>
|
||||
<section id="best-practices-and-tips">
|
||||
<h2>Best Practices and Tips<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#best-practices-and-tips" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#best-practices-and-tips'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>When integrating function calling into your generative AI applications, keep these tips in mind to get the most out of our Arch-Function models:</p>
|
||||
<p>When integrating function calling into your generative AI applications, keep these tips in mind to get the most out of our Plano-Function models:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Keep it clear and simple</strong>: Your function names and parameters should be straightforward and easy to understand. Think of it like explaining a task to a smart colleague - the clearer you are, the better the results.</p></li>
|
||||
<li><p><strong>Context is king</strong>: Don’t skimp on the descriptions for your functions and parameters. The more context you provide, the better the LLM can understand when and how to use each function.</p></li>
|
||||
|
|
@ -333,16 +328,16 @@ on the stuff that matters most.</p>
|
|||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="agent_routing.html">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="llm_router.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Agent Routing and Hand Off
|
||||
LLM Routing
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="llm_router.html">
|
||||
LLM Routing
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="observability/observability.html">
|
||||
Observability
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
|
|
@ -360,7 +355,7 @@ on the stuff that matters most.</p>
|
|||
<li><a :data-current="activeSection === '#implementing-function-calling'" class="reference internal" href="#implementing-function-calling">Implementing Function Calling</a><ul>
|
||||
<li><a :data-current="activeSection === '#step-1-define-the-function'" class="reference internal" href="#step-1-define-the-function">Step 1: Define the Function</a></li>
|
||||
<li><a :data-current="activeSection === '#step-2-configure-prompt-targets'" class="reference internal" href="#step-2-configure-prompt-targets">Step 2: Configure Prompt Targets</a></li>
|
||||
<li><a :data-current="activeSection === '#step-3-arch-takes-over'" class="reference internal" href="#step-3-arch-takes-over">Step 3: Arch Takes Over</a></li>
|
||||
<li><a :data-current="activeSection === '#step-3-plano-takes-over'" class="reference internal" href="#step-3-plano-takes-over">Step 3: Plano Takes Over</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#example-use-cases'" class="reference internal" href="#example-use-cases">Example Use Cases</a></li>
|
||||
|
|
@ -373,12 +368,12 @@ on the stuff that matters most.</p>
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>LLM Routing | Arch Docs v0.3.22</title>
|
||||
<meta content="LLM Routing | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="LLM Routing | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>LLM Routing | Plano Docs v0.4</title>
|
||||
<meta content="LLM Routing | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="LLM Routing | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/guides/llm_router.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="observability/observability.html" rel="next" title="Observability"/>
|
||||
<link href="function_calling.html" rel="prev" title="Function Calling"/>
|
||||
<link href="function_calling.html" rel="next" title="Function Calling"/>
|
||||
<link href="orchestration.html" rel="prev" title="Orchestration"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -163,133 +158,185 @@
|
|||
<div id="content" role="main">
|
||||
<section id="llm-routing">
|
||||
<span id="llm-router"></span><h1>LLM Routing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#llm-routing"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>With the rapid proliferation of large language models (LLM) — each optimized for different strengths, style, or latency/cost profile — routing has become an essential technique to operationalize the use of different models.</p>
|
||||
<p>Arch provides three distinct routing approaches to meet different use cases:</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p><strong>Model-based Routing</strong>: Direct routing to specific models using provider/model names</p></li>
|
||||
<li><p><strong>Alias-based Routing</strong>: Semantic routing using custom aliases that map to underlying models</p></li>
|
||||
<li><p><strong>Preference-aligned Routing</strong>: Intelligent routing using the Arch-Router model based on context and user-defined preferences</p></li>
|
||||
</ol>
|
||||
<p>This enables optimal performance, cost efficiency, and response quality by matching requests with the most suitable model from your available LLM fleet.</p>
|
||||
<p>With the rapid proliferation of large language models (LLMs) — each optimized for different strengths, style, or latency/cost profile — routing has become an essential technique to operationalize the use of different models. Plano provides three distinct routing approaches to meet different use cases: <a class="reference internal" href="#model-based-routing"><span class="std std-ref">Model-based routing</span></a>, <a class="reference internal" href="#alias-based-routing"><span class="std std-ref">Alias-based routing</span></a>, and <a class="reference internal" href="#preference-aligned-routing"><span class="std std-ref">Preference-aligned routing</span></a>. This enables optimal performance, cost efficiency, and response quality by matching requests with the most suitable model from your available LLM fleet.</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>For details on supported model providers, configuration options, and client libraries, see <a class="reference internal" href="../concepts/llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">LLM Providers</span></a>.</p>
|
||||
</div>
|
||||
<section id="routing-methods">
|
||||
<h2>Routing Methods<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#routing-methods" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#routing-methods'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<section id="model-based-routing">
|
||||
<h3>Model-based Routing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#model-based-routing" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#model-based-routing'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<span id="id1"></span><h3>Model-based routing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#model-based-routing" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#model-based-routing'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Direct routing allows you to specify exact provider and model combinations using the format <code class="docutils literal notranslate"><span class="pre">provider/model-name</span></code>:</p>
|
||||
<ul class="simple">
|
||||
<li><p>Use provider-specific names like <code class="docutils literal notranslate"><span class="pre">openai/gpt-4o</span></code> or <code class="docutils literal notranslate"><span class="pre">anthropic/claude-3-5-sonnet-20241022</span></code></p></li>
|
||||
<li><p>Use provider-specific names like <code class="docutils literal notranslate"><span class="pre">openai/gpt-5.2</span></code> or <code class="docutils literal notranslate"><span class="pre">anthropic/claude-sonnet-4-5</span></code></p></li>
|
||||
<li><p>Provides full control and transparency over which model handles each request</p></li>
|
||||
<li><p>Ideal for production workloads where you want predictable routing behavior</p></li>
|
||||
</ul>
|
||||
<section id="configuration">
|
||||
<h4>Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configuration"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<p>Configure your LLM providers with specific provider/model names:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id9">
|
||||
<div class="code-block-caption"><span class="caption-text">Model-based Routing Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id9"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">egress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">12000</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-7">
|
||||
</span><span id="line-8"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-5.2</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-12">
|
||||
</span><span id="line-13"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-5</span>
|
||||
</span><span id="line-14"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-15">
|
||||
</span><span id="line-16"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-sonnet-4-5</span>
|
||||
</span><span id="line-17"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$ANTHROPIC_API_KEY</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="client-usage">
|
||||
<h4>Client usage<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#client-usage"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<p>Clients specify exact models:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Direct provider/model specification</span>
|
||||
</span><span id="line-2"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-3"> <span class="n">model</span><span class="o">=</span><span class="s2">"openai/gpt-5.2"</span><span class="p">,</span>
|
||||
</span><span id="line-4"> <span class="n">messages</span><span class="o">=</span><span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Hello!"</span><span class="p">}]</span>
|
||||
</span><span id="line-5"><span class="p">)</span>
|
||||
</span><span id="line-6">
|
||||
</span><span id="line-7"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-8"> <span class="n">model</span><span class="o">=</span><span class="s2">"anthropic/claude-sonnet-4-5"</span><span class="p">,</span>
|
||||
</span><span id="line-9"> <span class="n">messages</span><span class="o">=</span><span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Write a story"</span><span class="p">}]</span>
|
||||
</span><span id="line-10"><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section id="alias-based-routing">
|
||||
<h3>Alias-based Routing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#alias-based-routing" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#alias-based-routing'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<span id="id2"></span><h3>Alias-based routing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#alias-based-routing" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#alias-based-routing'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Alias-based routing lets you create semantic model names that decouple your application from specific providers:</p>
|
||||
<ul class="simple">
|
||||
<li><p>Use meaningful names like <code class="docutils literal notranslate"><span class="pre">fast-model</span></code>, <code class="docutils literal notranslate"><span class="pre">reasoning-model</span></code>, or <code class="docutils literal notranslate"><span class="pre">arch.summarize.v1</span></code> (see <a class="reference internal" href="../concepts/llm_providers/model_aliases.html#model-aliases"><span class="std std-ref">Model Aliases</span></a>)</p></li>
|
||||
<li><p>Use meaningful names like <code class="docutils literal notranslate"><span class="pre">fast-model</span></code>, <code class="docutils literal notranslate"><span class="pre">reasoning-model</span></code>, or <code class="docutils literal notranslate"><span class="pre">plano.summarize.v1</span></code> (see <a class="reference internal" href="../concepts/llm_providers/model_aliases.html#model-aliases"><span class="std std-ref">Model Aliases</span></a>)</p></li>
|
||||
<li><p>Maps semantic names to underlying provider models for easier experimentation and provider switching</p></li>
|
||||
<li><p>Ideal for applications that want abstraction from specific model names while maintaining control</p></li>
|
||||
</ul>
|
||||
<section id="id3">
|
||||
<h4>Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id3"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<p>Configure semantic aliases that map to underlying models:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id10">
|
||||
<div class="code-block-caption"><span class="caption-text">Alias-based Routing Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id10"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">egress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">12000</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-7">
|
||||
</span><span id="line-8"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-5.2</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-11">
|
||||
</span><span id="line-12"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-5</span>
|
||||
</span><span id="line-13"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-14">
|
||||
</span><span id="line-15"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-sonnet-4-5</span>
|
||||
</span><span id="line-16"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$ANTHROPIC_API_KEY</span>
|
||||
</span><span id="line-17">
|
||||
</span><span id="line-18"><span class="nt">model_aliases</span><span class="p">:</span>
|
||||
</span><span id="line-19"><span class="w"> </span><span class="c1"># Model aliases - friendly names that map to actual provider names</span>
|
||||
</span><span id="line-20"><span class="w"> </span><span class="nt">fast-model</span><span class="p">:</span>
|
||||
</span><span id="line-21"><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-5.2</span>
|
||||
</span><span id="line-22">
|
||||
</span><span id="line-23"><span class="w"> </span><span class="nt">reasoning-model</span><span class="p">:</span>
|
||||
</span><span id="line-24"><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-5</span>
|
||||
</span><span id="line-25">
|
||||
</span><span id="line-26"><span class="w"> </span><span class="nt">creative-model</span><span class="p">:</span>
|
||||
</span><span id="line-27"><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">claude-sonnet-4-5</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="id4">
|
||||
<h4>Client usage<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id4"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<p>Clients use semantic names:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Using semantic aliases</span>
|
||||
</span><span id="line-2"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-3"> <span class="n">model</span><span class="o">=</span><span class="s2">"fast-model"</span><span class="p">,</span> <span class="c1"># Routes to best available fast model</span>
|
||||
</span><span id="line-4"> <span class="n">messages</span><span class="o">=</span><span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Quick summary please"</span><span class="p">}]</span>
|
||||
</span><span id="line-5"><span class="p">)</span>
|
||||
</span><span id="line-6">
|
||||
</span><span id="line-7"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-8"> <span class="n">model</span><span class="o">=</span><span class="s2">"reasoning-model"</span><span class="p">,</span> <span class="c1"># Routes to best reasoning model</span>
|
||||
</span><span id="line-9"> <span class="n">messages</span><span class="o">=</span><span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Solve this complex problem"</span><span class="p">}]</span>
|
||||
</span><span id="line-10"><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section id="preference-aligned-routing-arch-router">
|
||||
<span id="preference-aligned-routing"></span><h3>Preference-aligned Routing (Arch-Router)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#preference-aligned-routing-arch-router" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#preference-aligned-routing-arch-router'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Traditional LLM routing approaches face significant limitations: they evaluate performance using benchmarks that often fail to capture human preferences, select from fixed model pools, and operate as “black boxes” without practical mechanisms for encoding user preferences.</p>
|
||||
<p>Arch’s preference-aligned routing addresses these challenges by applying a fundamental engineering principle: decoupling. The framework separates route selection (matching queries to human-readable policies) from model assignment (mapping policies to specific LLMs). This separation allows you to define routing policies using descriptive labels like <code class="docutils literal notranslate"><span class="pre">Domain:</span> <span class="pre">'finance',</span> <span class="pre">Action:</span> <span class="pre">'analyze_earnings_report'</span></code> rather than cryptic identifiers, while independently configuring which models handle each policy.</p>
|
||||
<p>The <a class="reference external" href="https://huggingface.co/katanemo/Arch-Router-1.5B" rel="nofollow noopener">Arch-Router<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> model automatically selects the most appropriate LLM based on:</p>
|
||||
<span id="preference-aligned-routing"></span><h3>Preference-aligned routing (Arch-Router)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#preference-aligned-routing-arch-router" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#preference-aligned-routing-arch-router'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Preference-aligned routing uses the <a class="reference external" href="https://huggingface.co/katanemo/Arch-Router-1.5B" rel="nofollow noopener">Arch-Router<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> model to pick the best LLM based on domain, action, and your configured preferences instead of hard-coding a model.</p>
|
||||
<ul class="simple">
|
||||
<li><p>Domain Analysis: Identifies the subject matter (e.g., legal, healthcare, programming)</p></li>
|
||||
<li><p>Action Classification: Determines the type of operation (e.g., summarization, code generation, translation)</p></li>
|
||||
<li><p>User-Defined Preferences: Maps domains and actions to preferred models using transparent, configurable routing decisions</p></li>
|
||||
<li><p>Human Preference Alignment: Uses domain-action mappings that capture subjective evaluation criteria, ensuring routing aligns with real-world user needs rather than just benchmark scores</p></li>
|
||||
<li><p><strong>Domain</strong>: High-level topic of the request (e.g., legal, healthcare, programming).</p></li>
|
||||
<li><p><strong>Action</strong>: What the user wants to do (e.g., summarize, generate code, translate).</p></li>
|
||||
<li><p><strong>Routing preferences</strong>: Your mapping from (domain, action) to preferred models.</p></li>
|
||||
</ul>
|
||||
<p>This approach supports seamlessly adding new models without retraining and is ideal for dynamic, context-aware routing that adapts to request content and intent.</p>
|
||||
<p>Arch-Router analyzes each prompt to infer domain and action, then applies your preferences to select a model. This decouples <strong>routing policy</strong> (how to choose) from <strong>model assignment</strong> (what to run), making routing transparent, controllable, and easy to extend as you add or swap models.</p>
|
||||
<section id="id5">
|
||||
<h4>Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id5"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<p>To configure preference-aligned dynamic routing, define routing preferences that map domains and actions to specific models:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id11">
|
||||
<div class="code-block-caption"><span class="caption-text">Preference-Aligned Dynamic Routing Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id11"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">egress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">12000</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-7">
|
||||
</span><span id="line-8"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-5.2</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-12">
|
||||
</span><span id="line-13"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-5</span>
|
||||
</span><span id="line-14"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-15"><span class="w"> </span><span class="nt">routing_preferences</span><span class="p">:</span>
|
||||
</span><span id="line-16"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">code understanding</span>
|
||||
</span><span id="line-17"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">understand and explain existing code snippets, functions, or libraries</span>
|
||||
</span><span id="line-18"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">complex reasoning</span>
|
||||
</span><span id="line-19"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">deep analysis, mathematical problem solving, and logical reasoning</span>
|
||||
</span><span id="line-20">
|
||||
</span><span id="line-21"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-sonnet-4-5</span>
|
||||
</span><span id="line-22"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$ANTHROPIC_API_KEY</span>
|
||||
</span><span id="line-23"><span class="w"> </span><span class="nt">routing_preferences</span><span class="p">:</span>
|
||||
</span><span id="line-24"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">creative writing</span>
|
||||
</span><span id="line-25"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">creative content generation, storytelling, and writing assistance</span>
|
||||
</span><span id="line-26"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">code generation</span>
|
||||
</span><span id="line-27"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">generating new code snippets, functions, or boilerplate based on user prompts</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="id6">
|
||||
<h4>Client usage<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id6"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<p>Clients can let the router decide or still specify aliases:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Let Arch-Router choose based on content</span>
|
||||
</span><span id="line-2"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-3"> <span class="n">messages</span><span class="o">=</span><span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Write a creative story about space exploration"</span><span class="p">}]</span>
|
||||
</span><span id="line-4"> <span class="c1"># No model specified - router will analyze and choose claude-sonnet-4-5</span>
|
||||
</span><span id="line-5"><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section id="model-based-routing-workflow">
|
||||
<h2>Model-based Routing Workflow<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#model-based-routing-workflow" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#model-based-routing-workflow'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>For direct model routing, the process is straightforward:</p>
|
||||
<ol class="arabic">
|
||||
<li><p><strong>Client Request</strong></p>
|
||||
<blockquote>
|
||||
<div><p>The client specifies the exact model using provider/model format (<code class="docutils literal notranslate"><span class="pre">openai/gpt-4o</span></code>).</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Provider Validation</strong></p>
|
||||
<blockquote>
|
||||
<div><p>Arch validates that the specified provider and model are configured and available.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Direct Routing</strong></p>
|
||||
<blockquote>
|
||||
<div><p>The request is sent directly to the specified model without analysis or decision-making.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Response Handling</strong></p>
|
||||
<blockquote>
|
||||
<div><p>The response is returned to the client with optional metadata about the routing decision.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="alias-based-routing-workflow">
|
||||
<h2>Alias-based Routing Workflow<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#alias-based-routing-workflow" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#alias-based-routing-workflow'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>For alias-based routing, the process includes name resolution:</p>
|
||||
<ol class="arabic">
|
||||
<li><p><strong>Client Request</strong></p>
|
||||
<blockquote>
|
||||
<div><p>The client specifies a semantic alias name (<code class="docutils literal notranslate"><span class="pre">reasoning-model</span></code>).</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Alias Resolution</strong></p>
|
||||
<blockquote>
|
||||
<div><p>Arch resolves the alias to the actual provider/model name based on configuration.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Model Selection</strong></p>
|
||||
<blockquote>
|
||||
<div><p>If the alias maps to multiple models, Arch selects one based on availability and load balancing.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Request Forwarding</strong></p>
|
||||
<blockquote>
|
||||
<div><p>The request is forwarded to the resolved model.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Response Handling</strong></p>
|
||||
<blockquote>
|
||||
<div><p>The response is returned with optional metadata about the alias resolution.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="preference-aligned-routing-workflow-arch-router">
|
||||
<span id="preference-aligned-routing-workflow"></span><h2>Preference-aligned Routing Workflow (Arch-Router)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#preference-aligned-routing-workflow-arch-router" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#preference-aligned-routing-workflow-arch-router'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>For preference-aligned dynamic routing, the process involves intelligent analysis:</p>
|
||||
<ol class="arabic">
|
||||
<li><p><strong>Prompt Analysis</strong></p>
|
||||
<blockquote>
|
||||
<div><p>When a user submits a prompt without specifying a model, the Arch-Router analyzes it to determine the domain (subject matter) and action (type of operation requested).</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Model Selection</strong></p>
|
||||
<blockquote>
|
||||
<div><p>Based on the analyzed intent and your configured routing preferences, the Router selects the most appropriate model from your available LLM fleet.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Request Forwarding</strong></p>
|
||||
<blockquote>
|
||||
<div><p>Once the optimal model is identified, our gateway forwards the original prompt to the selected LLM endpoint. The routing decision is transparent and can be logged for monitoring and optimization purposes.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Response Handling</strong></p>
|
||||
<blockquote>
|
||||
<div><p>After the selected model processes the request, the response is returned through the gateway. The gateway can optionally add routing metadata or performance metrics to help you understand and optimize your routing decisions.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="id1">
|
||||
<h2>Arch-Router<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id1" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#id1'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<section id="id7">
|
||||
<h2>Arch-Router<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id7" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#id7'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>The <a class="reference external" href="https://huggingface.co/katanemo/Arch-Router-1.5B" rel="nofollow noopener">Arch-Router<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> is a state-of-the-art <strong>preference-based routing model</strong> specifically designed to address the limitations of traditional LLM routing. This compact 1.5B model delivers production-ready performance with low latency and high accuracy while solving key routing challenges.</p>
|
||||
<p><strong>Addressing Traditional Routing Limitations:</strong></p>
|
||||
<p><strong>Human Preference Alignment</strong>
|
||||
|
|
@ -312,152 +359,23 @@ Provides a practical mechanism to encode user preferences through domain-action
|
|||
<li><p><strong>Production-Ready Performance</strong>: Optimized for low-latency, high-throughput applications in multi-model environments.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="implementing-routing">
|
||||
<h2>Implementing Routing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#implementing-routing" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#implementing-routing'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p><strong>Model-based Routing</strong></p>
|
||||
<p>For direct model routing, configure your LLM providers with specific provider/model names:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id3">
|
||||
<div class="code-block-caption"><span class="caption-text">Model-based Routing Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id3"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">egress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">12000</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-7">
|
||||
</span><span id="line-8"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o-mini</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-12">
|
||||
</span><span id="line-13"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-14"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-15">
|
||||
</span><span id="line-16"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-3-5-sonnet-20241022</span>
|
||||
</span><span id="line-17"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$ANTHROPIC_API_KEY</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p>Clients specify exact models:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Direct provider/model specification</span>
|
||||
</span><span id="line-2"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-3"> <span class="n">model</span><span class="o">=</span><span class="s2">"openai/gpt-4o-mini"</span><span class="p">,</span>
|
||||
</span><span id="line-4"> <span class="n">messages</span><span class="o">=</span><span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Hello!"</span><span class="p">}]</span>
|
||||
</span><span id="line-5"><span class="p">)</span>
|
||||
</span><span id="line-6">
|
||||
</span><span id="line-7"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-8"> <span class="n">model</span><span class="o">=</span><span class="s2">"anthropic/claude-3-5-sonnet-20241022"</span><span class="p">,</span>
|
||||
</span><span id="line-9"> <span class="n">messages</span><span class="o">=</span><span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Write a story"</span><span class="p">}]</span>
|
||||
</span><span id="line-10"><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Alias-based Routing</strong></p>
|
||||
<p>Configure semantic aliases that map to underlying models:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id4">
|
||||
<div class="code-block-caption"><span class="caption-text">Alias-based Routing Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id4"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">egress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">12000</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-7">
|
||||
</span><span id="line-8"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o-mini</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-11">
|
||||
</span><span id="line-12"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-13"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-14">
|
||||
</span><span id="line-15"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-3-5-sonnet-20241022</span>
|
||||
</span><span id="line-16"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$ANTHROPIC_API_KEY</span>
|
||||
</span><span id="line-17">
|
||||
</span><span id="line-18"><span class="nt">model_aliases</span><span class="p">:</span>
|
||||
</span><span id="line-19"><span class="w"> </span><span class="c1"># Model aliases - friendly names that map to actual provider names</span>
|
||||
</span><span id="line-20"><span class="w"> </span><span class="nt">fast-model</span><span class="p">:</span>
|
||||
</span><span id="line-21"><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-4o-mini</span>
|
||||
</span><span id="line-22">
|
||||
</span><span id="line-23"><span class="w"> </span><span class="nt">reasoning-model</span><span class="p">:</span>
|
||||
</span><span id="line-24"><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-4o</span>
|
||||
</span><span id="line-25">
|
||||
</span><span id="line-26"><span class="w"> </span><span class="nt">creative-model</span><span class="p">:</span>
|
||||
</span><span id="line-27"><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">claude-3-5-sonnet-20241022</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p>Clients use semantic names:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Using semantic aliases</span>
|
||||
</span><span id="line-2"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-3"> <span class="n">model</span><span class="o">=</span><span class="s2">"fast-model"</span><span class="p">,</span> <span class="c1"># Routes to best available fast model</span>
|
||||
</span><span id="line-4"> <span class="n">messages</span><span class="o">=</span><span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Quick summary please"</span><span class="p">}]</span>
|
||||
</span><span id="line-5"><span class="p">)</span>
|
||||
</span><span id="line-6">
|
||||
</span><span id="line-7"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-8"> <span class="n">model</span><span class="o">=</span><span class="s2">"reasoning-model"</span><span class="p">,</span> <span class="c1"># Routes to best reasoning model</span>
|
||||
</span><span id="line-9"> <span class="n">messages</span><span class="o">=</span><span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Solve this complex problem"</span><span class="p">}]</span>
|
||||
</span><span id="line-10"><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Preference-aligned Routing (Arch-Router)</strong></p>
|
||||
<p>To configure preference-aligned dynamic routing, you need to define routing preferences that map domains and actions to specific models:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id5">
|
||||
<div class="code-block-caption"><span class="caption-text">Preference-Aligned Dynamic Routing Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id5"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">egress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">12000</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-7">
|
||||
</span><span id="line-8"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o-mini</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-12">
|
||||
</span><span id="line-13"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-14"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-15"><span class="w"> </span><span class="nt">routing_preferences</span><span class="p">:</span>
|
||||
</span><span id="line-16"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">code understanding</span>
|
||||
</span><span id="line-17"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">understand and explain existing code snippets, functions, or libraries</span>
|
||||
</span><span id="line-18"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">complex reasoning</span>
|
||||
</span><span id="line-19"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">deep analysis, mathematical problem solving, and logical reasoning</span>
|
||||
</span><span id="line-20">
|
||||
</span><span id="line-21"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-3-5-sonnet-20241022</span>
|
||||
</span><span id="line-22"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$ANTHROPIC_API_KEY</span>
|
||||
</span><span id="line-23"><span class="w"> </span><span class="nt">routing_preferences</span><span class="p">:</span>
|
||||
</span><span id="line-24"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">creative writing</span>
|
||||
</span><span id="line-25"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">creative content generation, storytelling, and writing assistance</span>
|
||||
</span><span id="line-26"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">code generation</span>
|
||||
</span><span id="line-27"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">generating new code snippets, functions, or boilerplate based on user prompts</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p>Clients can let the router decide or use aliases:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Let Arch-Router choose based on content</span>
|
||||
</span><span id="line-2"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-3"> <span class="n">messages</span><span class="o">=</span><span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="s2">"Write a creative story about space exploration"</span><span class="p">}]</span>
|
||||
</span><span id="line-4"> <span class="c1"># No model specified - router will analyze and choose claude-3-5-sonnet-20241022</span>
|
||||
</span><span id="line-5"><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="combining-routing-methods">
|
||||
<h2>Combining Routing Methods<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#combining-routing-methods" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#combining-routing-methods'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>You can combine static model selection with dynamic routing preferences for maximum flexibility:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id6">
|
||||
<div class="code-block-caption"><span class="caption-text">Hybrid Routing Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id6"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="literal-block-wrapper docutils container" id="id12">
|
||||
<div class="code-block-caption"><span class="caption-text">Hybrid Routing Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id12"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o-mini</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-5.2</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-5">
|
||||
</span><span id="line-6"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-5</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-8"><span class="w"> </span><span class="nt">routing_preferences</span><span class="p">:</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">complex_reasoning</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">deep analysis and complex problem solving</span>
|
||||
</span><span id="line-11">
|
||||
</span><span id="line-12"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-3-5-sonnet-20241022</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-sonnet-4-5</span>
|
||||
</span><span id="line-13"><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$ANTHROPIC_API_KEY</span>
|
||||
</span><span id="line-14"><span class="w"> </span><span class="nt">routing_preferences</span><span class="p">:</span>
|
||||
</span><span id="line-15"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">creative_tasks</span>
|
||||
|
|
@ -466,14 +384,14 @@ Provides a practical mechanism to encode user preferences through domain-action
|
|||
</span><span id="line-18"><span class="nt">model_aliases</span><span class="p">:</span>
|
||||
</span><span id="line-19"><span class="w"> </span><span class="c1"># Model aliases - friendly names that map to actual provider names</span>
|
||||
</span><span id="line-20"><span class="w"> </span><span class="nt">fast-model</span><span class="p">:</span>
|
||||
</span><span id="line-21"><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-4o-mini</span>
|
||||
</span><span id="line-21"><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-5.2</span>
|
||||
</span><span id="line-22">
|
||||
</span><span id="line-23"><span class="w"> </span><span class="nt">reasoning-model</span><span class="p">:</span>
|
||||
</span><span id="line-24"><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-4o</span>
|
||||
</span><span id="line-24"><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-5</span>
|
||||
</span><span id="line-25">
|
||||
</span><span id="line-26"><span class="w"> </span><span class="c1"># Aliases that can also participate in dynamic routing</span>
|
||||
</span><span id="line-27"><span class="w"> </span><span class="nt">creative-model</span><span class="p">:</span>
|
||||
</span><span id="line-28"><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">claude-3-5-sonnet-20241022</span>
|
||||
</span><span id="line-28"><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">claude-sonnet-4-5</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -493,8 +411,8 @@ Provides a practical mechanism to encode user preferences through domain-action
|
|||
<li><p><strong>Conversational Routing</strong>: Track conversation context to identify when topics shift between domains or when the type of assistance needed changes mid-conversation.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="best-practicesm">
|
||||
<h2>Best practicesm<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#best-practicesm" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#best-practicesm'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<section id="best-practices">
|
||||
<h2>Best practices<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#best-practices" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#best-practices'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p><strong>💡Consistent Naming:</strong> Route names should align with their descriptions.</p>
|
||||
<ul>
|
||||
|
|
@ -521,22 +439,31 @@ Provides a practical mechanism to encode user preferences through domain-action
|
|||
</ul>
|
||||
</li>
|
||||
<li><p><strong>💡Nouns Descriptor:</strong> Preference-based routers perform better with noun-centric descriptors, as they offer more stable and semantically rich signals for matching.</p></li>
|
||||
<li><p><strong>💡Domain Inclusion:</strong> for best user experience, you should always include domain route. This help the router fall back to domain when action is not</p></li>
|
||||
<li><p><strong>💡Domain Inclusion:</strong> for best user experience, you should always include a domain route. This helps the router fall back to domain when action is not confidently inferred.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="unsupported-features">
|
||||
<h2>Unsupported Features<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#unsupported-features" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#unsupported-features'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>The following features are <strong>not supported</strong> by the Arch-Router model:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Multi-modality</strong>: The model is not trained to process raw image or audio inputs. It can handle textual queries <em>about</em> these modalities (e.g., “generate an image of a cat”), but cannot interpret encoded multimedia data directly.</p></li>
|
||||
<li><p><strong>Function calling</strong>: Arch-Router is designed for <strong>semantic preference matching</strong>, not exact intent classification or tool execution. For structured function invocation, use models in the Plano Function Calling collection instead.</p></li>
|
||||
<li><p><strong>System prompt dependency</strong>: Arch-Router routes based solely on the user’s conversation history. It does not use or rely on system prompts for routing decisions.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="function_calling.html">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="orchestration.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Function Calling
|
||||
Orchestration
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="observability/observability.html">
|
||||
Observability
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="function_calling.html">
|
||||
Function Calling
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
|
|
@ -546,19 +473,28 @@ Provides a practical mechanism to encode user preferences through domain-action
|
|||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#routing-methods'" class="reference internal" href="#routing-methods">Routing Methods</a><ul>
|
||||
<li><a :data-current="activeSection === '#model-based-routing'" class="reference internal" href="#model-based-routing">Model-based Routing</a></li>
|
||||
<li><a :data-current="activeSection === '#alias-based-routing'" class="reference internal" href="#alias-based-routing">Alias-based Routing</a></li>
|
||||
<li><a :data-current="activeSection === '#preference-aligned-routing-arch-router'" class="reference internal" href="#preference-aligned-routing-arch-router">Preference-aligned Routing (Arch-Router)</a></li>
|
||||
<li><a :data-current="activeSection === '#model-based-routing'" class="reference internal" href="#model-based-routing">Model-based routing</a><ul>
|
||||
<li><a :data-current="activeSection === '#configuration'" class="reference internal" href="#configuration">Configuration</a></li>
|
||||
<li><a :data-current="activeSection === '#client-usage'" class="reference internal" href="#client-usage">Client usage</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#model-based-routing-workflow'" class="reference internal" href="#model-based-routing-workflow">Model-based Routing Workflow</a></li>
|
||||
<li><a :data-current="activeSection === '#alias-based-routing-workflow'" class="reference internal" href="#alias-based-routing-workflow">Alias-based Routing Workflow</a></li>
|
||||
<li><a :data-current="activeSection === '#preference-aligned-routing-workflow-arch-router'" class="reference internal" href="#preference-aligned-routing-workflow-arch-router">Preference-aligned Routing Workflow (Arch-Router)</a></li>
|
||||
<li><a :data-current="activeSection === '#id1'" class="reference internal" href="#id1">Arch-Router</a></li>
|
||||
<li><a :data-current="activeSection === '#implementing-routing'" class="reference internal" href="#implementing-routing">Implementing Routing</a></li>
|
||||
<li><a :data-current="activeSection === '#alias-based-routing'" class="reference internal" href="#alias-based-routing">Alias-based routing</a><ul>
|
||||
<li><a :data-current="activeSection === '#id3'" class="reference internal" href="#id3">Configuration</a></li>
|
||||
<li><a :data-current="activeSection === '#id4'" class="reference internal" href="#id4">Client usage</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#preference-aligned-routing-arch-router'" class="reference internal" href="#preference-aligned-routing-arch-router">Preference-aligned routing (Arch-Router)</a><ul>
|
||||
<li><a :data-current="activeSection === '#id5'" class="reference internal" href="#id5">Configuration</a></li>
|
||||
<li><a :data-current="activeSection === '#id6'" class="reference internal" href="#id6">Client usage</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#id7'" class="reference internal" href="#id7">Arch-Router</a></li>
|
||||
<li><a :data-current="activeSection === '#combining-routing-methods'" class="reference internal" href="#combining-routing-methods">Combining Routing Methods</a></li>
|
||||
<li><a :data-current="activeSection === '#example-use-cases'" class="reference internal" href="#example-use-cases">Example Use Cases</a></li>
|
||||
<li><a :data-current="activeSection === '#best-practicesm'" class="reference internal" href="#best-practicesm">Best practicesm</a></li>
|
||||
<li><a :data-current="activeSection === '#best-practices'" class="reference internal" href="#best-practices">Best practices</a></li>
|
||||
<li><a :data-current="activeSection === '#unsupported-features'" class="reference internal" href="#unsupported-features">Unsupported Features</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
|
|
@ -567,12 +503,12 @@ Provides a practical mechanism to encode user preferences through domain-action
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -7,18 +7,18 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Access Logging | Arch Docs v0.3.22</title>
|
||||
<meta content="Access Logging | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Access Logging | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Access Logging | Plano Docs v0.4</title>
|
||||
<meta content="Access Logging | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Access Logging | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/guides/observability/access_logging.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../../search.html" rel="search" title="Search"/>
|
||||
<link href="../../build_with_arch/agent.html" rel="next" title="Agentic Apps"/>
|
||||
<link href="../prompt_guard.html" rel="next" title="Guardrails"/>
|
||||
<link href="monitoring.html" rel="prev" title="Monitoring"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -164,14 +159,14 @@
|
|||
<div id="content" role="main">
|
||||
<section id="access-logging">
|
||||
<span id="arch-access-logging"></span><h1>Access Logging<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#access-logging"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Access logging in Arch refers to the logging of detailed information about each request and response that flows through Arch.
|
||||
It provides visibility into the traffic passing through Arch, which is crucial for monitoring, debugging, and analyzing the
|
||||
<p>Access logging in Plano refers to the logging of detailed information about each request and response that flows through Plano.
|
||||
It provides visibility into the traffic passing through Plano, which is crucial for monitoring, debugging, and analyzing the
|
||||
behavior of AI applications and their interactions.</p>
|
||||
<section id="key-features">
|
||||
<h2>Key Features<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#key-features" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#key-features'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Per-Request Logging</strong>:
|
||||
Each request that passes through Arch is logged. This includes important metadata such as HTTP method,
|
||||
Each request that passes through Plano is logged. This includes important metadata such as HTTP method,
|
||||
path, response status code, request duration, upstream host, and more.</p></li>
|
||||
<li><p><strong>Integration with Monitoring Tools</strong>:
|
||||
Access logs can be exported to centralized logging systems (e.g., ELK stack or Fluentd) or used to feed monitoring and alerting systems.</p></li>
|
||||
|
|
@ -180,21 +175,21 @@ Access logs can be exported to centralized logging systems (e.g., ELK stack or F
|
|||
</section>
|
||||
<section id="how-it-works">
|
||||
<h2>How It Works<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#how-it-works" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#how-it-works'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Arch gateway exposes access logs for every call it manages on your behalf. By default these access logs can be found under <code class="docutils literal notranslate"><span class="pre">~/archgw_logs</span></code>. For example:</p>
|
||||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>tail<span class="w"> </span>-F<span class="w"> </span>~/archgw_logs/access_*.log
|
||||
<p>Plano exposes access logs for every call it manages on your behalf. By default these access logs can be found under <code class="docutils literal notranslate"><span class="pre">~/plano_logs</span></code>. For example:</p>
|
||||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>tail<span class="w"> </span>-F<span class="w"> </span>~/plano_logs/access_*.log
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="go">==> /Users/adilhafeez/archgw_logs/access_llm.log <==</span>
|
||||
</span><span id="line-3"><span class="go">==> /Users/username/plano_logs/access_llm.log <==</span>
|
||||
</span><span id="line-4"><span class="go">[2024-10-10T03:55:49.537Z] "POST /v1/chat/completions HTTP/1.1" 0 DC 0 0 770 - "-" "OpenAI/Python 1.51.0" "469793af-b25f-9b57-b265-f376e8d8c586" "api.openai.com" "162.159.140.245:443"</span>
|
||||
</span><span id="line-5">
|
||||
</span><span id="line-6"><span class="go">==> /Users/adilhafeez/archgw_logs/access_internal.log <==</span>
|
||||
</span><span id="line-6"><span class="go">==> /Users/username/plano_logs/access_internal.log <==</span>
|
||||
</span><span id="line-7"><span class="go">[2024-10-10T03:56:03.906Z] "POST /embeddings HTTP/1.1" 200 - 52 21797 54 53 "-" "-" "604197fe-2a5b-95a2-9367-1d6b30cfc845" "model_server" "192.168.65.254:51000"</span>
|
||||
</span><span id="line-8"><span class="go">[2024-10-10T03:56:03.961Z] "POST /zeroshot HTTP/1.1" 200 - 106 218 87 87 "-" "-" "604197fe-2a5b-95a2-9367-1d6b30cfc845" "model_server" "192.168.65.254:51000"</span>
|
||||
</span><span id="line-9"><span class="go">[2024-10-10T03:56:04.050Z] "POST /v1/chat/completions HTTP/1.1" 200 - 1301 614 441 441 "-" "-" "604197fe-2a5b-95a2-9367-1d6b30cfc845" "model_server" "192.168.65.254:51000"</span>
|
||||
</span><span id="line-10"><span class="go">[2024-10-10T03:56:04.492Z] "POST /hallucination HTTP/1.1" 200 - 556 127 104 104 "-" "-" "604197fe-2a5b-95a2-9367-1d6b30cfc845" "model_server" "192.168.65.254:51000"</span>
|
||||
</span><span id="line-11"><span class="go">[2024-10-10T03:56:04.598Z] "POST /insurance_claim_details HTTP/1.1" 200 - 447 125 17 17 "-" "-" "604197fe-2a5b-95a2-9367-1d6b30cfc845" "api_server" "192.168.65.254:18083"</span>
|
||||
</span><span id="line-12">
|
||||
</span><span id="line-13"><span class="go">==> /Users/adilhafeez/archgw_logs/access_ingress.log <==</span>
|
||||
</span><span id="line-14"><span class="go">[2024-10-10T03:56:03.905Z] "POST /v1/chat/completions HTTP/1.1" 200 - 463 1022 1695 984 "-" "OpenAI/Python 1.51.0" "604197fe-2a5b-95a2-9367-1d6b30cfc845" "arch_llm_listener" "0.0.0.0:12000"</span>
|
||||
</span><span id="line-13"><span class="go">==> /Users/username/plano_logs/access_ingress.log <==</span>
|
||||
</span><span id="line-14"><span class="go">[2024-10-10T03:56:03.905Z] "POST /v1/chat/completions HTTP/1.1" 200 - 463 1022 1695 984 "-" "OpenAI/Python 1.51.0" "604197fe-2a5b-95a2-9367-1d6b30cfc845" "plano_llm_listener" "0.0.0.0:12000"</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
|
|
@ -212,7 +207,7 @@ Access logs can be exported to centralized logging systems (e.g., ELK stack or F
|
|||
<li><p>DURATION: The total time taken to process the request.</p></li>
|
||||
</ul>
|
||||
<p>For example for following request:</p>
|
||||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="go">[2024-10-10T03:56:03.905Z] "POST /v1/chat/completions HTTP/1.1" 200 - 463 1022 1695 984 "-" "OpenAI/Python 1.51.0" "604197fe-2a5b-95a2-9367-1d6b30cfc845" "arch_llm_listener" "0.0.0.0:12000"</span>
|
||||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="go">[2024-10-10T03:56:03.905Z] "POST /v1/chat/completions HTTP/1.1" 200 - 463 1022 1695 984 "-" "OpenAI/Python 1.51.0" "604197fe-2a5b-95a2-9367-1d6b30cfc845" "plano_llm_listener" "0.0.0.0:12000"</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p>Total duration was 1695ms, and the upstream service took 984ms to process the request. Bytes received and sent were 463 and 1022 respectively.</p>
|
||||
|
|
@ -228,8 +223,8 @@ Access logs can be exported to centralized logging systems (e.g., ELK stack or F
|
|||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../../build_with_arch/agent.html">
|
||||
Agentic Apps
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../prompt_guard.html">
|
||||
Guardrails
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
|
|
@ -249,12 +244,12 @@ Access logs can be exported to centralized logging systems (e.g., ELK stack or F
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -7,13 +7,13 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Monitoring | Arch Docs v0.3.22</title>
|
||||
<meta content="Monitoring | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Monitoring | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Monitoring | Plano Docs v0.4</title>
|
||||
<meta content="Monitoring | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Monitoring | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/guides/observability/monitoring.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -168,11 +163,11 @@
|
|||
and instrumentation for generating, collecting, processing, and exporting telemetry data, such as traces,
|
||||
metrics, and logs. Its flexible design supports a wide range of backends and seamlessly integrates with
|
||||
modern application tools.</p>
|
||||
<p>Arch acts a <em>source</em> for several monitoring metrics related to <strong>prompts</strong> and <strong>LLMs</strong> natively integrated
|
||||
<p>Plano acts a <em>source</em> for several monitoring metrics related to <strong>agents</strong> and <strong>LLMs</strong> natively integrated
|
||||
via <a class="reference external" href="https://opentelemetry.io/" rel="nofollow noopener">OpenTelemetry<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> to help you understand three critical aspects of your application:
|
||||
latency, token usage, and error rates by an upstream LLM provider. Latency measures the speed at which your application
|
||||
is responding to users, which includes metrics like time to first token (TFT), time per output token (TOT) metrics, and
|
||||
the total latency as perceived by users. Below are some screenshots how Arch integrates natively with tools like
|
||||
the total latency as perceived by users. Below are some screenshots how Plano integrates natively with tools like
|
||||
<a class="reference external" href="https://grafana.com/grafana/dashboards/" rel="nofollow noopener">Grafana<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> via <a class="reference external" href="https://prometheus.io/" rel="nofollow noopener">Promethus<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p>
|
||||
<section id="metrics-dashboard-via-grafana">
|
||||
<h2>Metrics Dashboard (via Grafana)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#metrics-dashboard-via-grafana" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#metrics-dashboard-via-grafana'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
|
|
@ -185,7 +180,7 @@ the total latency as perceived by users. Below are some screenshots how Arch int
|
|||
</section>
|
||||
<section id="configure-monitoring">
|
||||
<h2>Configure Monitoring<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configure-monitoring" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configure-monitoring'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Arch gateway publishes stats endpoint at <a class="reference external" href="http://localhost:19901/stats" rel="nofollow noopener">http://localhost:19901/stats<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>. As noted above, Arch is a source for metrics. To view and manipulate dashbaords, you will
|
||||
<p>Plano publishes stats endpoint at <a class="reference external" href="http://localhost:19901/stats" rel="nofollow noopener">http://localhost:19901/stats<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>. As noted above, Plano is a source for metrics. To view and manipulate dashbaords, you will
|
||||
need to configiure <a class="reference external" href="https://prometheus.io/" rel="nofollow noopener">Promethus<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> (as a metrics store) and <a class="reference external" href="https://grafana.com/grafana/dashboards/" rel="nofollow noopener">Grafana<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> for dashboards. Below
|
||||
are some sample configuration files for both, respectively.</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id5">
|
||||
|
|
@ -202,7 +197,7 @@ are some sample configuration files for both, respectively.</p>
|
|||
</span><span id="line-10"><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10s</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="nt">api_version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v2</span>
|
||||
</span><span id="line-12"><span class="nt">scrape_configs</span><span class="p">:</span>
|
||||
</span><span id="line-13"><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">archgw</span>
|
||||
</span><span id="line-13"><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">job_name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">plano</span>
|
||||
</span><span id="line-14"><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">honor_timestamps</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-15"><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">scrape_interval</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">15s</span>
|
||||
</span><span id="line-16"><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">scrape_timeout</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10s</span>
|
||||
|
|
@ -261,12 +256,12 @@ are some sample configuration files for both, respectively.</p>
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Observability | Arch Docs v0.3.22</title>
|
||||
<meta content="Observability | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Observability | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Observability | Plano Docs v0.4</title>
|
||||
<meta content="Observability | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Observability | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/guides/observability/observability.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../../search.html" rel="search" title="Search"/>
|
||||
<link href="tracing.html" rel="next" title="Tracing"/>
|
||||
<link href="../llm_router.html" rel="prev" title="LLM Routing"/>
|
||||
<link href="../function_calling.html" rel="prev" title="Function Calling"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="current reference internal expandable" href="#">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -192,11 +187,11 @@
|
|||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../llm_router.html">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../function_calling.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
LLM Routing
|
||||
Function Calling
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
|
|
@ -213,12 +208,12 @@
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -7,13 +7,13 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Tracing | Arch Docs v0.3.22</title>
|
||||
<meta content="Tracing | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Tracing | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Tracing | Plano Docs v0.4</title>
|
||||
<meta content="Tracing | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Tracing | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/guides/observability/tracing.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -175,9 +170,9 @@ modern application tools. A key feature of OpenTelemetry is its commitment to st
|
|||
requests in an AI application. With tracing, you can capture a detailed view of how requests propagate
|
||||
through various services and components, which is crucial for <strong>debugging</strong>, <strong>performance optimization</strong>,
|
||||
and understanding complex AI agent architectures like Co-pilots.</p>
|
||||
<p><strong>Arch</strong> propagates trace context using the W3C Trace Context standard, specifically through the
|
||||
<p><strong>Plano</strong> propagates trace context using the W3C Trace Context standard, specifically through the
|
||||
<code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header. This allows each component in the system to record its part of the request
|
||||
flow, enabling <strong>end-to-end tracing</strong> across the entire application. By using OpenTelemetry, Arch ensures
|
||||
flow, enabling <strong>end-to-end tracing</strong> across the entire application. By using OpenTelemetry, Plano ensures
|
||||
that developers can capture this trace data consistently and in a format compatible with various observability
|
||||
tools.</p>
|
||||
<a class="reference internal image-reference" href="../../_images/tracing.png"><img alt="../../_images/tracing.png" class="align-center" src="../../_images/tracing.png" style="width: 100%;"/>
|
||||
|
|
@ -197,8 +192,8 @@ making it easy to visualize traces in the tools you’re already usi</p></li>
|
|||
<section id="how-to-initiate-a-trace">
|
||||
<h2>How to Initiate A Trace<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#how-to-initiate-a-trace" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#how-to-initiate-a-trace'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ol class="arabic simple">
|
||||
<li><p><strong>Enable Tracing Configuration</strong>: Simply add the <code class="docutils literal notranslate"><span class="pre">random_sampling</span></code> in <code class="docutils literal notranslate"><span class="pre">tracing</span></code> section to 100`` flag to in the <a class="reference internal" href="../../concepts/tech_overview/listener.html#arch-overview-listeners"><span class="std std-ref">listener</span></a> config</p></li>
|
||||
<li><p><strong>Trace Context Propagation</strong>: Arch automatically propagates the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header. When a request is received, Arch will:</p>
|
||||
<li><p><strong>Enable Tracing Configuration</strong>: Simply add the <code class="docutils literal notranslate"><span class="pre">random_sampling</span></code> in <code class="docutils literal notranslate"><span class="pre">tracing</span></code> section to 100`` flag to in the <a class="reference internal" href="../../concepts/listeners.html#plano-overview-listeners"><span class="std std-ref">listener</span></a> config</p></li>
|
||||
<li><p><strong>Trace Context Propagation</strong>: Plano automatically propagates the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header. When a request is received, Plano will:</p>
|
||||
<ul class="simple">
|
||||
<li><p>Generate a new <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header if one is not present.</p></li>
|
||||
<li><p>Extract the trace context from the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header if it exists.</p></li>
|
||||
|
|
@ -212,7 +207,7 @@ You can adjust this value from 0-100.</p></li>
|
|||
</section>
|
||||
<section id="trace-propagation">
|
||||
<h2>Trace Propagation<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#trace-propagation" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#trace-propagation'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Arch uses the W3C Trace Context standard for trace propagation, which relies on the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header.
|
||||
<p>Plano uses the W3C Trace Context standard for trace propagation, which relies on the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header.
|
||||
This header carries tracing information in a standardized format, enabling interoperability between different
|
||||
tracing systems.</p>
|
||||
<section id="header-format">
|
||||
|
|
@ -231,7 +226,7 @@ tracing systems.</p>
|
|||
<section id="instrumentation">
|
||||
<h3>Instrumentation<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#instrumentation" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#instrumentation'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>To integrate AI tracing, your application needs to follow a few simple steps. The steps
|
||||
below are very common practice, and not unique to Arch, when you reading tracing headers and export
|
||||
below are very common practice, and not unique to Plano, when you reading tracing headers and export
|
||||
<a class="reference external" href="https://docs.lightstep.com/docs/understand-distributed-tracing" rel="nofollow noopener">spans<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> for distributed tracing.</p>
|
||||
<ul class="simple">
|
||||
<li><p>Read the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header from incoming requests.</p></li>
|
||||
|
|
@ -296,94 +291,6 @@ below are very common practice, and not unique to Arch, when you reading tracing
|
|||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section id="ai-agent-tracing-visualization-example">
|
||||
<h3>AI Agent Tracing Visualization Example<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#ai-agent-tracing-visualization-example" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#ai-agent-tracing-visualization-example'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>The following is an example of tracing for an AI-powered customer support system.
|
||||
A customer interacts with AI agents, which forward their requests through different
|
||||
specialized services and external systems.</p>
|
||||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="o">+--------------------------+</span>
|
||||
</span><span id="line-2"><span class="o">|</span> <span class="n">Customer</span> <span class="n">Interaction</span> <span class="o">|</span>
|
||||
</span><span id="line-3"><span class="o">+--------------------------+</span>
|
||||
</span><span id="line-4"> <span class="o">|</span>
|
||||
</span><span id="line-5"> <span class="n">v</span>
|
||||
</span><span id="line-6"><span class="o">+--------------------------+</span> <span class="o">+--------------------------+</span>
|
||||
</span><span id="line-7"><span class="o">|</span> <span class="n">Agent</span> <span class="mi">1</span> <span class="p">(</span><span class="n">Main</span> <span class="o">-</span> <span class="n">Arch</span><span class="p">)</span> <span class="o">|</span> <span class="o">----></span> <span class="o">|</span> <span class="n">External</span> <span class="n">Payment</span> <span class="n">Service</span> <span class="o">|</span>
|
||||
</span><span id="line-8"><span class="o">+--------------------------+</span> <span class="o">+--------------------------+</span>
|
||||
</span><span id="line-9"> <span class="o">|</span> <span class="o">|</span>
|
||||
</span><span id="line-10"> <span class="n">v</span> <span class="n">v</span>
|
||||
</span><span id="line-11"><span class="o">+--------------------------+</span> <span class="o">+--------------------------+</span>
|
||||
</span><span id="line-12"><span class="o">|</span> <span class="n">Agent</span> <span class="mi">2</span> <span class="p">(</span><span class="n">Support</span> <span class="o">-</span> <span class="n">Arch</span><span class="p">)</span><span class="o">|</span> <span class="o">----></span> <span class="o">|</span> <span class="n">Internal</span> <span class="n">Tech</span> <span class="n">Support</span> <span class="o">|</span>
|
||||
</span><span id="line-13"><span class="o">+--------------------------+</span> <span class="o">+--------------------------+</span>
|
||||
</span><span id="line-14"> <span class="o">|</span> <span class="o">|</span>
|
||||
</span><span id="line-15"> <span class="n">v</span> <span class="n">v</span>
|
||||
</span><span id="line-16"><span class="o">+--------------------------+</span> <span class="o">+--------------------------+</span>
|
||||
</span><span id="line-17"><span class="o">|</span> <span class="n">Agent</span> <span class="mi">3</span> <span class="p">(</span><span class="n">Orders</span><span class="o">-</span> <span class="n">Arch</span><span class="p">)</span> <span class="o">|</span> <span class="o">----></span> <span class="o">|</span> <span class="n">Inventory</span> <span class="n">Management</span> <span class="o">|</span>
|
||||
</span><span id="line-18"><span class="o">+--------------------------+</span> <span class="o">+--------------------------+</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<section id="trace-breakdown">
|
||||
<h4>Trace Breakdown:<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#trace-breakdown"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h4>
|
||||
<ul class="simple">
|
||||
<li><dl class="simple">
|
||||
<dt>Customer Interaction:</dt><dd><ul>
|
||||
<li><p>Span 1: Customer initiates a request via the AI-powered chatbot for billing support (e.g., asking for payment details).</p></li>
|
||||
</ul>
|
||||
</dd>
|
||||
</dl>
|
||||
</li>
|
||||
<li><dl class="simple">
|
||||
<dt>AI Agent 1 (Main - Arch):</dt><dd><ul>
|
||||
<li><p>Span 2: AI Agent 1 (Main) processes the request and identifies it as related to billing, forwarding the request
|
||||
to an external payment service.</p></li>
|
||||
<li><p>Span 3: AI Agent 1 determines that additional technical support is needed for processing and forwards the request
|
||||
to AI Agent 2.</p></li>
|
||||
</ul>
|
||||
</dd>
|
||||
</dl>
|
||||
</li>
|
||||
<li><dl class="simple">
|
||||
<dt>External Payment Service:</dt><dd><ul>
|
||||
<li><p>Span 4: The external payment service processes the payment-related request (e.g., verifying payment status) and sends
|
||||
the response back to AI Agent 1.</p></li>
|
||||
</ul>
|
||||
</dd>
|
||||
</dl>
|
||||
</li>
|
||||
<li><dl class="simple">
|
||||
<dt>AI Agent 2 (Tech - Arch):</dt><dd><ul>
|
||||
<li><p>Span 5: AI Agent 2, responsible for technical queries, processes a request forwarded from AI Agent 1 (e.g., checking for
|
||||
any account issues).</p></li>
|
||||
<li><p>Span 6: AI Agent 2 forwards the query to Internal Tech Support for further investigation.</p></li>
|
||||
</ul>
|
||||
</dd>
|
||||
</dl>
|
||||
</li>
|
||||
<li><dl class="simple">
|
||||
<dt>Internal Tech Support:</dt><dd><ul>
|
||||
<li><p>Span 7: Internal Tech Support processes the request (e.g., resolving account access issues) and responds to AI Agent 2.</p></li>
|
||||
</ul>
|
||||
</dd>
|
||||
</dl>
|
||||
</li>
|
||||
<li><dl class="simple">
|
||||
<dt>AI Agent 3 (Orders - Arch):</dt><dd><ul>
|
||||
<li><p>Span 8: AI Agent 3 handles order-related queries. AI Agent 1 forwards the request to AI Agent 3 after payment verification
|
||||
is completed.</p></li>
|
||||
<li><p>Span 9: AI Agent 3 forwards a request to the Inventory Management system to confirm product availability for a pending order.</p></li>
|
||||
</ul>
|
||||
</dd>
|
||||
</dl>
|
||||
</li>
|
||||
<li><dl class="simple">
|
||||
<dt>Inventory Management:</dt><dd><ul>
|
||||
<li><p>Span 10: The Inventory Management system checks stock and availability and returns the information to AI Agent 3.</p></li>
|
||||
</ul>
|
||||
</dd>
|
||||
</dl>
|
||||
</li>
|
||||
</ul>
|
||||
</section>
|
||||
</section>
|
||||
</section>
|
||||
<section id="integrating-with-tracing-tools">
|
||||
<h2>Integrating with Tracing Tools<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#integrating-with-tracing-tools" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#integrating-with-tracing-tools'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
|
|
@ -461,10 +368,10 @@ is completed.</p></li>
|
|||
</section>
|
||||
<section id="langtrace">
|
||||
<h3>Langtrace<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#langtrace" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#langtrace'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Langtrace is an observability tool designed specifically for large language models (LLMs). It helps you capture, analyze, and understand how LLMs are used in your applications including those built using Arch.</p>
|
||||
<p>Langtrace is an observability tool designed specifically for large language models (LLMs). It helps you capture, analyze, and understand how LLMs are used in your applications including those built using Plano.</p>
|
||||
<p>To send tracing data to <a class="reference external" href="https://docs.langtrace.ai/supported-integrations/llm-tools/arch" rel="nofollow noopener">Langtrace<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>:</p>
|
||||
<ol class="arabic">
|
||||
<li><p><strong>Configure Arch</strong>: Make sure Arch is installed and setup correctly. For more information, refer to the <a class="reference external" href="https://github.com/katanemo/archgw?tab=readme-ov-file#prerequisites" rel="nofollow noopener">installation guide<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>.</p></li>
|
||||
<li><p><strong>Configure Plano</strong>: Make sure Plano is installed and setup correctly. For more information, refer to the <a class="reference external" href="https://github.com/katanemo/archgw?tab=readme-ov-file#prerequisites" rel="nofollow noopener">installation guide<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>.</p></li>
|
||||
<li><p><strong>Install Langtrace</strong>: Install the Langtrace SDK.:</p>
|
||||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>pip<span class="w"> </span>install<span class="w"> </span>langtrace-python-sdk
|
||||
</span></code></pre></div>
|
||||
|
|
@ -512,7 +419,7 @@ is completed.</p></li>
|
|||
</section>
|
||||
<section id="summary">
|
||||
<h2>Summary<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#summary" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#summary'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>By leveraging the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header for trace context propagation, Arch enables developers to implement
|
||||
<p>By leveraging the <code class="docutils literal notranslate"><span class="pre">traceparent</span></code> header for trace context propagation, Plano enables developers to implement
|
||||
tracing efficiently. This approach simplifies the process of collecting and analyzing tracing data in common
|
||||
tools like AWS X-Ray and Datadog, enhancing observability and facilitating faster debugging and optimization.</p>
|
||||
</section>
|
||||
|
|
@ -560,10 +467,6 @@ tools like AWS X-Ray and Datadog, enhancing observability and facilitating faste
|
|||
<li><a :data-current="activeSection === '#example-with-opentelemetry-in-python'" class="reference internal" href="#example-with-opentelemetry-in-python">Example with OpenTelemetry in Python</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#ai-agent-tracing-visualization-example'" class="reference internal" href="#ai-agent-tracing-visualization-example">AI Agent Tracing Visualization Example</a><ul>
|
||||
<li><a :data-current="activeSection === '#trace-breakdown'" class="reference internal" href="#trace-breakdown">Trace Breakdown:</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#integrating-with-tracing-tools'" class="reference internal" href="#integrating-with-tracing-tools">Integrating with Tracing Tools</a><ul>
|
||||
|
|
@ -583,12 +486,12 @@ tools like AWS X-Ray and Datadog, enhancing observability and facilitating faste
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
939
guides/orchestration.html
Executable file
939
guides/orchestration.html
Executable file
|
|
@ -0,0 +1,939 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Orchestration | Plano Docs v0.4</title>
|
||||
<meta content="Orchestration | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Orchestration | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/guides/orchestration.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="llm_router.html" rel="next" title="LLM Routing"/>
|
||||
<link href="../concepts/prompt_target.html" rel="prev" title="Prompt Target"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
let mode;
|
||||
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
||||
mode = 'dark';
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
mode = 'light';
|
||||
}
|
||||
if (!userPreference) {localStorage.setItem('darkMode', mode)}
|
||||
</script>
|
||||
</head>
|
||||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
K
|
||||
</kbd>
|
||||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
</a>
|
||||
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
|
||||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||||
</svg>
|
||||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</aside>
|
||||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Orchestration</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="orchestration">
|
||||
<span id="agent-routing"></span><h1>Orchestration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#orchestration"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Building multi-agent systems allow you to route requests across multiple specialized agents, each designed to handle specific types of tasks.
|
||||
Plano makes it easy to build and scale these systems by managing the orchestration layer—deciding which agent(s) should handle each request—while you focus on implementing individual agent logic.</p>
|
||||
<p>This guide shows you how to configure and implement multi-agent orchestration in Plano using a real-world example: a <strong>Travel Booking Assistant</strong> that routes queries to specialized agents for weather and flights.</p>
|
||||
<section id="how-it-works">
|
||||
<h2>How It Works<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#how-it-works" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#how-it-works'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Plano’s orchestration layer analyzes incoming prompts and routes them to the most appropriate agent based on user intent and conversation context. The workflow is:</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p><strong>User submits a prompt</strong>: The request arrives at Plano’s agent listener.</p></li>
|
||||
<li><p><strong>Agent selection</strong>: Plano uses an LLM to analyze the prompt and determine user intent and complexity. By default, this uses <a class="reference external" href="https://huggingface.co/collections/katanemo/plano-orchestrator" rel="nofollow noopener">Plano-Orchestrator-30B-A3B<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, which offers performance of foundation models at 1/10th the cost. The LLM routes the request to the most suitable agent configured in your system—such as a weather agent or flight agent.</p></li>
|
||||
<li><p><strong>Agent handles request</strong>: Once the selected agent receives the request object from Plano, it manages its own <a class="reference internal" href="../concepts/agents.html#agents"><span class="std std-ref">inner loop</span></a> until the task is complete. This means the agent autonomously calls models, invokes tools, processes data, and reasons about next steps—all within its specialized domain—before returning the final response.</p></li>
|
||||
<li><p><strong>Seamless handoffs</strong>: For multi-turn conversations, Plano repeats the intent analysis for each follow-up query, enabling smooth handoffs between agents as the conversation evolves.</p></li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="example-travel-booking-assistant">
|
||||
<h2>Example: Travel Booking Assistant<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-travel-booking-assistant" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#example-travel-booking-assistant'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Let’s walk through a complete multi-agent system: a Travel Booking Assistant that helps users plan trips by providing weather forecasts and flight information. This system uses two specialized agents:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Weather Agent</strong>: Provides real-time weather conditions and multi-day forecasts</p></li>
|
||||
<li><p><strong>Flight Agent</strong>: Searches for flights between airports with real-time tracking</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="configuration">
|
||||
<h2>Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configuration" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configuration'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Configure your agents in the <code class="docutils literal notranslate"><span class="pre">listeners</span></code> section of your <code class="docutils literal notranslate"><span class="pre">plano_config.yaml</span></code>:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id1">
|
||||
<div class="code-block-caption"><span class="caption-text">Travel Booking Multi-Agent Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id1"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.3.0</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">weather_agent</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://host.docker.internal:10510</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">flight_agent</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://host.docker.internal:10520</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="nt">model_providers</span><span class="p">:</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o-mini</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span><span class="w"> </span><span class="c1"># smaller, faster, cheaper model for extracting entities like location</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">agent</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">travel_booking_service</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">8001</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span><span class="w"> </span><span class="nt">router</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">plano_orchestrator_v1</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span><span class="w"> </span><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">weather_agent</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">|</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span><span class="w"> </span><span class="no">WeatherAgent is a specialized AI assistant for real-time weather information and forecasts. It provides accurate weather data for any city worldwide using the Open-Meteo API, helping travelers plan their trips with up-to-date weather conditions.</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span><span class="w"> </span><span class="no">Capabilities:</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span><span class="w"> </span><span class="no">* Get real-time weather conditions and multi-day forecasts for any city worldwide using Open-Meteo API (free, no API key needed)</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span><span class="w"> </span><span class="no">* Provides current temperature</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span><span class="w"> </span><span class="no">* Provides multi-day forecasts</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span><span class="w"> </span><span class="no">* Provides weather conditions</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span><span class="w"> </span><span class="no">* Provides sunrise/sunset times</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span><span class="w"> </span><span class="no">* Provides detailed weather information</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span><span class="w"> </span><span class="no">* Understands conversation context to resolve location references from previous messages</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span><span class="w"> </span><span class="no">* Handles weather-related questions including "What's the weather in [city]?", "What's the forecast for [city]?", "How's the weather in [city]?"</span>
|
||||
</span><span id="line-36"><span class="linenos">36</span><span class="w"> </span><span class="no">* When queries include both weather and other travel questions (e.g., flights, currency), this agent answers ONLY the weather part</span>
|
||||
</span><span id="line-37"><span class="linenos">37</span>
|
||||
</span><span id="line-38"><span class="linenos">38</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">flight_agent</span>
|
||||
</span><span id="line-39"><span class="linenos">39</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">|</span>
|
||||
</span><span id="line-40"><span class="linenos">40</span>
|
||||
</span><span id="line-41"><span class="linenos">41</span><span class="w"> </span><span class="no">FlightAgent is an AI-powered tool specialized in providing live flight information between airports. It leverages the FlightAware AeroAPI to deliver real-time flight status, gate information, and delay updates.</span>
|
||||
</span><span id="line-42"><span class="linenos">42</span>
|
||||
</span><span id="line-43"><span class="linenos">43</span><span class="w"> </span><span class="no">Capabilities:</span>
|
||||
</span><span id="line-44"><span class="linenos">44</span><span class="w"> </span><span class="no">* Get live flight information between airports using FlightAware AeroAPI</span>
|
||||
</span><span id="line-45"><span class="linenos">45</span><span class="w"> </span><span class="no">* Shows real-time flight status</span>
|
||||
</span><span id="line-46"><span class="linenos">46</span><span class="w"> </span><span class="no">* Shows scheduled/estimated/actual departure and arrival times</span>
|
||||
</span><span id="line-47"><span class="linenos">47</span><span class="w"> </span><span class="no">* Shows gate and terminal information</span>
|
||||
</span><span id="line-48"><span class="linenos">48</span><span class="w"> </span><span class="no">* Shows delays</span>
|
||||
</span><span id="line-49"><span class="linenos">49</span><span class="w"> </span><span class="no">* Shows aircraft type</span>
|
||||
</span><span id="line-50"><span class="linenos">50</span><span class="w"> </span><span class="no">* Shows flight status</span>
|
||||
</span><span id="line-51"><span class="linenos">51</span><span class="w"> </span><span class="no">* Automatically resolves city names to airport codes (IATA/ICAO)</span>
|
||||
</span><span id="line-52"><span class="linenos">52</span><span class="w"> </span><span class="no">* Understands conversation context to infer origin/destination from follow-up questions</span>
|
||||
</span><span id="line-53"><span class="linenos">53</span><span class="w"> </span><span class="no">* Handles flight-related questions including "What flights go from [city] to [city]?", "Do flights go to [city]?", "Are there direct flights from [city]?"</span>
|
||||
</span><span id="line-54"><span class="linenos">54</span><span class="w"> </span><span class="no">* When queries include both flight and other travel questions (e.g., weather, currency), this agent answers ONLY the flight part</span>
|
||||
</span><span id="line-55"><span class="linenos">55</span>
|
||||
</span><span id="line-56"><span class="linenos">56</span><span class="nt">tracing</span><span class="p">:</span>
|
||||
</span><span id="line-57"><span class="linenos">57</span><span class="w"> </span><span class="nt">random_sampling</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">100</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p><strong>Key Configuration Elements:</strong></p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>agent listener</strong>: A listener of <code class="docutils literal notranslate"><span class="pre">type:</span> <span class="pre">agent</span></code> tells Plano to perform intent analysis and routing for incoming requests.</p></li>
|
||||
<li><p><strong>agents list</strong>: Define each agent with an <code class="docutils literal notranslate"><span class="pre">id</span></code>, <code class="docutils literal notranslate"><span class="pre">description</span></code> (used for routing decisions)</p></li>
|
||||
<li><p><strong>router</strong>: The <code class="docutils literal notranslate"><span class="pre">plano_orchestrator_v1</span></code> router uses Plano-Orchestrator to analyze user intent and select the appropriate agent.</p></li>
|
||||
<li><p><strong>filter_chain</strong>: Optionally attach <a class="reference internal" href="../concepts/filter_chain.html#filter-chain"><span class="std std-ref">filter chains</span></a> to agents for guardrails, query rewriting, or context enrichment.</p></li>
|
||||
</ul>
|
||||
<p><strong>Writing Effective Agent Descriptions</strong></p>
|
||||
<p>Agent descriptions are critical—they’re used by Plano-Orchestrator to make routing decisions. Effective descriptions should include:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Clear introduction</strong>: A concise statement explaining what the agent is and its primary purpose</p></li>
|
||||
<li><p><strong>Capabilities section</strong>: A bulleted list of specific capabilities, including:</p>
|
||||
<ul>
|
||||
<li><p>What APIs or data sources it uses (e.g., “Open-Meteo API”, “FlightAware AeroAPI”)</p></li>
|
||||
<li><p>What information it provides (e.g., “current temperature”, “multi-day forecasts”, “gate information”)</p></li>
|
||||
<li><p>How it handles context (e.g., “Understands conversation context to resolve location references”)</p></li>
|
||||
<li><p>What question patterns it handles (e.g., “What’s the weather in [city]?”)</p></li>
|
||||
<li><p>How it handles multi-part queries (e.g., “When queries include both weather and flights, this agent answers ONLY the weather part”)</p></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p>Here’s an example of a well-structured agent description:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">weather_agent</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">|</span>
|
||||
</span><span id="line-3">
|
||||
</span><span id="line-4"><span class="w"> </span><span class="no">WeatherAgent is a specialized AI assistant for real-time weather information</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="no">and forecasts. It provides accurate weather data for any city worldwide using</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="no">the Open-Meteo API, helping travelers plan their trips with up-to-date weather</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="no">conditions.</span>
|
||||
</span><span id="line-8">
|
||||
</span><span id="line-9"><span class="w"> </span><span class="no">Capabilities:</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="no">* Get real-time weather conditions and multi-day forecasts for any city worldwide</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="no">* Provides current temperature, weather conditions, sunrise/sunset times</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="no">* Provides detailed weather information including multi-day forecasts</span>
|
||||
</span><span id="line-13"><span class="w"> </span><span class="no">* Understands conversation context to resolve location references from previous messages</span>
|
||||
</span><span id="line-14"><span class="w"> </span><span class="no">* Handles weather-related questions including "What's the weather in [city]?"</span>
|
||||
</span><span id="line-15"><span class="w"> </span><span class="no">* When queries include both weather and other travel questions (e.g., flights),</span>
|
||||
</span><span id="line-16"><span class="w"> </span><span class="no">this agent answers ONLY the weather part</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>We will soon support “Agents as Tools” via Model Context Protocol (MCP), enabling agents to dynamically discover and invoke other agents as tools. Track progress on <a class="reference external" href="https://github.com/katanemo/archgw/issues/646" rel="nofollow noopener">GitHub Issue #646<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="implementation">
|
||||
<h2>Implementation<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#implementation" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#implementation'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Agents are HTTP services that receive routed requests from Plano. Each agent implements the OpenAI Chat Completions API format, making them compatible with standard LLM clients.</p>
|
||||
<section id="agent-structure">
|
||||
<h3>Agent Structure<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#agent-structure" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#agent-structure'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Let’s examine the Weather Agent implementation:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id2">
|
||||
<div class="code-block-caption"><span class="caption-text">Weather Agent - Core Structure</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nd">@app</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="s2">"/v1/chat/completions"</span><span class="p">)</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span><span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">handle_request</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">Request</span><span class="p">):</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="w"> </span><span class="sd">"""HTTP endpoint for chat completions with streaming support."""</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span> <span class="n">request_body</span> <span class="o">=</span> <span class="k">await</span> <span class="n">request</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span> <span class="n">messages</span> <span class="o">=</span> <span class="n">request_body</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"messages"</span><span class="p">,</span> <span class="p">[])</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span> <span class="s2">"messages detail json dumps: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">messages</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">),</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span> <span class="p">)</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span> <span class="n">traceparent_header</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"traceparent"</span><span class="p">)</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span> <span class="k">return</span> <span class="n">StreamingResponse</span><span class="p">(</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span> <span class="n">invoke_weather_agent</span><span class="p">(</span><span class="n">request</span><span class="p">,</span> <span class="n">request_body</span><span class="p">,</span> <span class="n">traceparent_header</span><span class="p">),</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span> <span class="n">media_type</span><span class="o">=</span><span class="s2">"text/plain"</span><span class="p">,</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span> <span class="n">headers</span><span class="o">=</span><span class="p">{</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span> <span class="s2">"content-type"</span><span class="p">:</span> <span class="s2">"text/event-stream"</span><span class="p">,</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span> <span class="p">},</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span> <span class="p">)</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span><span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">invoke_weather_agent</span><span class="p">(</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p><strong>Key Points:</strong></p>
|
||||
<ul class="simple">
|
||||
<li><p>Agents expose a <code class="docutils literal notranslate"><span class="pre">/v1/chat/completions</span></code> endpoint that matches OpenAI’s API format</p></li>
|
||||
<li><p>They use Plano’s LLM gateway (via <code class="docutils literal notranslate"><span class="pre">LLM_GATEWAY_ENDPOINT</span></code>) for all LLM calls</p></li>
|
||||
<li><p>They receive the full conversation history in <code class="docutils literal notranslate"><span class="pre">request_body.messages</span></code></p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="information-extraction-with-llms">
|
||||
<h3>Information Extraction with LLMs<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#information-extraction-with-llms" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#information-extraction-with-llms'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Agents use LLMs to extract structured information from natural language queries. This enables them to understand user intent and extract parameters needed for API calls.</p>
|
||||
<p>The Weather Agent extracts location information:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id3">
|
||||
<div class="code-block-caption"><span class="caption-text">Weather Agent - Location Extraction</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id3"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span> <span class="n">instructions</span> <span class="o">=</span> <span class="s2">"""Extract the location for WEATHER queries. Return just the city name.</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="s2"> Rules:</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="s2"> 1. For multi-part queries, extract ONLY the location mentioned with weather keywords ("weather in [location]")</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="s2"> 2. If user says "there" or "that city", it typically refers to the DESTINATION city in travel contexts (not the origin)</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="s2"> 3. For flight queries with weather, "there" means the destination city where they're traveling TO</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="s2"> 4. Return plain text (e.g., "London", "New York", "Paris, France")</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="s2"> 5. If no weather location found, return "NOT_FOUND"</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span><span class="s2"> Examples:</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="s2"> - "What's the weather in London?" -> "London"</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="s2"> - "Flights from Seattle to Atlanta, and show me the weather there" -> "Atlanta"</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span><span class="s2"> - "Can you get me flights from Seattle to Atlanta tomorrow, and also please show me the weather there" -> "Atlanta"</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span><span class="s2"> - "What's the weather in Seattle, and what is one flight that goes direct to Atlanta?" -> "Seattle"</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span><span class="s2"> - User asked about flights to Atlanta, then "what's the weather like there?" -> "Atlanta"</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span><span class="s2"> - "I'm going to Seattle" -> "Seattle"</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span><span class="s2"> - "What's happening?" -> "NOT_FOUND"</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span><span class="s2"> Extract location:"""</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span> <span class="k">try</span><span class="p">:</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span> <span class="n">user_messages</span> <span class="o">=</span> <span class="p">[</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span> <span class="n">msg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"content"</span><span class="p">)</span> <span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">messages</span> <span class="k">if</span> <span class="n">msg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"role"</span><span class="p">)</span> <span class="o">==</span> <span class="s2">"user"</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span> <span class="p">]</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">user_messages</span><span class="p">:</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span> <span class="n">location</span> <span class="o">=</span> <span class="s2">"New York"</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span> <span class="k">else</span><span class="p">:</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span> <span class="n">ctx</span> <span class="o">=</span> <span class="n">extract</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span> <span class="n">extra_headers</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span> <span class="n">inject</span><span class="p">(</span><span class="n">extra_headers</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="n">ctx</span><span class="p">)</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span> <span class="c1"># For location extraction, pass full conversation for context (e.g., "there" = previous destination)</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span> <span class="n">response</span> <span class="o">=</span> <span class="k">await</span> <span class="n">openai_client_via_plano</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-36"><span class="linenos">36</span> <span class="n">model</span><span class="o">=</span><span class="n">LOCATION_MODEL</span><span class="p">,</span>
|
||||
</span><span id="line-37"><span class="linenos">37</span> <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
|
||||
</span><span id="line-38"><span class="linenos">38</span> <span class="p">{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"system"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="n">instructions</span><span class="p">},</span>
|
||||
</span><span id="line-39"><span class="linenos">39</span> <span class="o">*</span><span class="p">[</span>
|
||||
</span><span id="line-40"><span class="linenos">40</span> <span class="p">{</span><span class="s2">"role"</span><span class="p">:</span> <span class="n">msg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"role"</span><span class="p">),</span> <span class="s2">"content"</span><span class="p">:</span> <span class="n">msg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"content"</span><span class="p">)}</span>
|
||||
</span><span id="line-41"><span class="linenos">41</span> <span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">messages</span>
|
||||
</span><span id="line-42"><span class="linenos">42</span> <span class="p">],</span>
|
||||
</span><span id="line-43"><span class="linenos">43</span> <span class="p">],</span>
|
||||
</span><span id="line-44"><span class="linenos">44</span> <span class="n">temperature</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
|
||||
</span><span id="line-45"><span class="linenos">45</span> <span class="n">max_tokens</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||
</span><span id="line-46"><span class="linenos">46</span> <span class="n">extra_headers</span><span class="o">=</span><span class="n">extra_headers</span> <span class="k">if</span> <span class="n">extra_headers</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
|
||||
</span><span id="line-47"><span class="linenos">47</span> <span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p>The Flight Agent extracts more complex information—origin, destination, and dates:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id4">
|
||||
<div class="code-block-caption"><span class="caption-text">Flight Agent - Flight Information Extraction</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id4"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">extract_flight_route</span><span class="p">(</span><span class="n">messages</span><span class="p">:</span> <span class="nb">list</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">Request</span><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span><span class="w"> </span><span class="sd">"""Extract origin, destination, and date from conversation using LLM."""</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span> <span class="n">extraction_prompt</span> <span class="o">=</span> <span class="s2">"""Extract flight origin, destination cities, and travel date from the conversation.</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="s2"> Rules:</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="s2"> 1. Look for patterns: "flight from X to Y", "flights to Y", "fly from X"</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="s2"> 2. Extract dates like "tomorrow", "next week", "December 25", "12/25", "on Monday"</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="s2"> 3. Use conversation context to fill in missing details</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="s2"> 4. Return JSON: {"origin": "City" or null, "destination": "City" or null, "date": "YYYY-MM-DD" or null}</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="s2"> Examples:</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span><span class="s2"> - "Flight from Seattle to Atlanta tomorrow" -> {"origin": "Seattle", "destination": "Atlanta", "date": "2025-12-24"}</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span><span class="s2"> - "What flights go to New York?" -> {"origin": null, "destination": "New York", "date": null}</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span><span class="s2"> - "Flights to Miami on Christmas" -> {"origin": null, "destination": "Miami", "date": "2025-12-25"}</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span><span class="s2"> - "Show me flights from LA to NYC next Monday" -> {"origin": "LA", "destination": "NYC", "date": "2025-12-30"}</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span><span class="s2"> Today is December 23, 2025. Extract flight route and date:"""</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span> <span class="k">try</span><span class="p">:</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span> <span class="n">ctx</span> <span class="o">=</span> <span class="n">extract</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span> <span class="n">extra_headers</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span> <span class="n">inject</span><span class="p">(</span><span class="n">extra_headers</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="n">ctx</span><span class="p">)</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span> <span class="n">response</span> <span class="o">=</span> <span class="k">await</span> <span class="n">openai_client_via_plano</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span> <span class="n">model</span><span class="o">=</span><span class="n">EXTRACTION_MODEL</span><span class="p">,</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span> <span class="n">messages</span><span class="o">=</span><span class="p">[</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span> <span class="p">{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"system"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="n">extraction_prompt</span><span class="p">},</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span> <span class="o">*</span><span class="p">[</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span> <span class="p">{</span><span class="s2">"role"</span><span class="p">:</span> <span class="n">msg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"role"</span><span class="p">),</span> <span class="s2">"content"</span><span class="p">:</span> <span class="n">msg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"content"</span><span class="p">)}</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span> <span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">messages</span><span class="p">[</span><span class="o">-</span><span class="mi">5</span><span class="p">:]</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span> <span class="p">],</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span> <span class="p">],</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span> <span class="n">temperature</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span> <span class="n">max_tokens</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span>
|
||||
</span><span id="line-36"><span class="linenos">36</span> <span class="n">extra_headers</span><span class="o">=</span><span class="n">extra_headers</span> <span class="k">if</span> <span class="n">extra_headers</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
|
||||
</span><span id="line-37"><span class="linenos">37</span> <span class="p">)</span>
|
||||
</span><span id="line-38"><span class="linenos">38</span>
|
||||
</span><span id="line-39"><span class="linenos">39</span> <span class="n">result</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">choices</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">content</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
</span><span id="line-40"><span class="linenos">40</span> <span class="k">if</span> <span class="s2">"```json"</span> <span class="ow">in</span> <span class="n">result</span><span class="p">:</span>
|
||||
</span><span id="line-41"><span class="linenos">41</span> <span class="n">result</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"```json"</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"```"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
</span><span id="line-42"><span class="linenos">42</span> <span class="k">elif</span> <span class="s2">"```"</span> <span class="ow">in</span> <span class="n">result</span><span class="p">:</span>
|
||||
</span><span id="line-43"><span class="linenos">43</span> <span class="n">result</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"```"</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"```"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
</span><span id="line-44"><span class="linenos">44</span>
|
||||
</span><span id="line-45"><span class="linenos">45</span> <span class="n">route</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
|
||||
</span><span id="line-46"><span class="linenos">46</span> <span class="k">return</span> <span class="p">{</span>
|
||||
</span><span id="line-47"><span class="linenos">47</span> <span class="s2">"origin"</span><span class="p">:</span> <span class="n">route</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"origin"</span><span class="p">),</span>
|
||||
</span><span id="line-48"><span class="linenos">48</span> <span class="s2">"destination"</span><span class="p">:</span> <span class="n">route</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"destination"</span><span class="p">),</span>
|
||||
</span><span id="line-49"><span class="linenos">49</span> <span class="s2">"date"</span><span class="p">:</span> <span class="n">route</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"date"</span><span class="p">),</span>
|
||||
</span><span id="line-50"><span class="linenos">50</span> <span class="p">}</span>
|
||||
</span><span id="line-51"><span class="linenos">51</span> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
</span><span id="line-52"><span class="linenos">52</span> <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Error extracting flight route: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p><strong>Key Points:</strong></p>
|
||||
<ul class="simple">
|
||||
<li><p>Use smaller, faster models (like <code class="docutils literal notranslate"><span class="pre">gpt-4o-mini</span></code>) for extraction tasks</p></li>
|
||||
<li><p>Include conversation context to handle follow-up questions and pronouns</p></li>
|
||||
<li><p>Use structured prompts with clear output formats (JSON)</p></li>
|
||||
<li><p>Handle edge cases with fallback values</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="calling-external-apis">
|
||||
<h3>Calling External APIs<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#calling-external-apis" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#calling-external-apis'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>After extracting information, agents call external APIs to fetch real-time data:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id5">
|
||||
<div class="code-block-caption"><span class="caption-text">Weather Agent - External API Call</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id5"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span> <span class="c1"># Geocode city to get coordinates</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span> <span class="n">geocode_url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://geocoding-api.open-meteo.com/v1/search?name=</span><span class="si">{</span><span class="n">quote</span><span class="p">(</span><span class="n">location</span><span class="p">)</span><span class="si">}</span><span class="s2">&count=1&language=en&format=json"</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span> <span class="n">geocode_response</span> <span class="o">=</span> <span class="k">await</span> <span class="n">http_client</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">geocode_url</span><span class="p">)</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span> <span class="k">if</span> <span class="n">geocode_response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">!=</span> <span class="mi">200</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">geocode_response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span> <span class="s2">"results"</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span> <span class="p">):</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Could not geocode </span><span class="si">{</span><span class="n">location</span><span class="si">}</span><span class="s2">, using New York"</span><span class="p">)</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span> <span class="n">location</span> <span class="o">=</span> <span class="s2">"New York"</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span> <span class="n">geocode_url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://geocoding-api.open-meteo.com/v1/search?name=</span><span class="si">{</span><span class="n">quote</span><span class="p">(</span><span class="n">location</span><span class="p">)</span><span class="si">}</span><span class="s2">&count=1&language=en&format=json"</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span> <span class="n">geocode_response</span> <span class="o">=</span> <span class="k">await</span> <span class="n">http_client</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">geocode_url</span><span class="p">)</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span> <span class="n">geocode_data</span> <span class="o">=</span> <span class="n">geocode_response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">geocode_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"results"</span><span class="p">):</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span> <span class="k">return</span> <span class="p">{</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span> <span class="s2">"location"</span><span class="p">:</span> <span class="n">location</span><span class="p">,</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span> <span class="s2">"weather"</span><span class="p">:</span> <span class="p">{</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span> <span class="s2">"date"</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">),</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span> <span class="s2">"day_name"</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%A"</span><span class="p">),</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span> <span class="s2">"temperature_c"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span> <span class="s2">"temperature_f"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span> <span class="s2">"weather_code"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span> <span class="s2">"error"</span><span class="p">:</span> <span class="s2">"Could not retrieve weather data"</span><span class="p">,</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span> <span class="p">},</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span> <span class="p">}</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span> <span class="n">result</span> <span class="o">=</span> <span class="n">geocode_data</span><span class="p">[</span><span class="s2">"results"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span> <span class="n">location_name</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"name"</span><span class="p">,</span> <span class="n">location</span><span class="p">)</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span> <span class="n">latitude</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"latitude"</span><span class="p">]</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span> <span class="n">longitude</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"longitude"</span><span class="p">]</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span> <span class="sa">f</span><span class="s2">"Geocoded '</span><span class="si">{</span><span class="n">location</span><span class="si">}</span><span class="s2">' to </span><span class="si">{</span><span class="n">location_name</span><span class="si">}</span><span class="s2"> (</span><span class="si">{</span><span class="n">latitude</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="n">longitude</span><span class="si">}</span><span class="s2">)"</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span> <span class="p">)</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span>
|
||||
</span><span id="line-36"><span class="linenos">36</span> <span class="c1"># Get weather forecast</span>
|
||||
</span><span id="line-37"><span class="linenos">37</span> <span class="n">weather_url</span> <span class="o">=</span> <span class="p">(</span>
|
||||
</span><span id="line-38"><span class="linenos">38</span> <span class="sa">f</span><span class="s2">"https://api.open-meteo.com/v1/forecast?"</span>
|
||||
</span><span id="line-39"><span class="linenos">39</span> <span class="sa">f</span><span class="s2">"latitude=</span><span class="si">{</span><span class="n">latitude</span><span class="si">}</span><span class="s2">&longitude=</span><span class="si">{</span><span class="n">longitude</span><span class="si">}</span><span class="s2">&"</span>
|
||||
</span><span id="line-40"><span class="linenos">40</span> <span class="sa">f</span><span class="s2">"current=temperature_2m&"</span>
|
||||
</span><span id="line-41"><span class="linenos">41</span> <span class="sa">f</span><span class="s2">"daily=sunrise,sunset,temperature_2m_max,temperature_2m_min,weather_code&"</span>
|
||||
</span><span id="line-42"><span class="linenos">42</span> <span class="sa">f</span><span class="s2">"forecast_days=</span><span class="si">{</span><span class="n">days</span><span class="si">}</span><span class="s2">&timezone=auto"</span>
|
||||
</span><span id="line-43"><span class="linenos">43</span> <span class="p">)</span>
|
||||
</span><span id="line-44"><span class="linenos">44</span>
|
||||
</span><span id="line-45"><span class="linenos">45</span> <span class="n">weather_response</span> <span class="o">=</span> <span class="k">await</span> <span class="n">http_client</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">weather_url</span><span class="p">)</span>
|
||||
</span><span id="line-46"><span class="linenos">46</span> <span class="k">if</span> <span class="n">weather_response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">!=</span> <span class="mi">200</span><span class="p">:</span>
|
||||
</span><span id="line-47"><span class="linenos">47</span> <span class="k">return</span> <span class="p">{</span>
|
||||
</span><span id="line-48"><span class="linenos">48</span> <span class="s2">"location"</span><span class="p">:</span> <span class="n">location_name</span><span class="p">,</span>
|
||||
</span><span id="line-49"><span class="linenos">49</span> <span class="s2">"weather"</span><span class="p">:</span> <span class="p">{</span>
|
||||
</span><span id="line-50"><span class="linenos">50</span> <span class="s2">"date"</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">),</span>
|
||||
</span><span id="line-51"><span class="linenos">51</span> <span class="s2">"day_name"</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%A"</span><span class="p">),</span>
|
||||
</span><span id="line-52"><span class="linenos">52</span> <span class="s2">"temperature_c"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||
</span><span id="line-53"><span class="linenos">53</span> <span class="s2">"temperature_f"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||
</span><span id="line-54"><span class="linenos">54</span> <span class="s2">"weather_code"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||
</span><span id="line-55"><span class="linenos">55</span> <span class="s2">"error"</span><span class="p">:</span> <span class="s2">"Could not retrieve weather data"</span><span class="p">,</span>
|
||||
</span><span id="line-56"><span class="linenos">56</span> <span class="p">},</span>
|
||||
</span><span id="line-57"><span class="linenos">57</span> <span class="p">}</span>
|
||||
</span><span id="line-58"><span class="linenos">58</span>
|
||||
</span><span id="line-59"><span class="linenos">59</span> <span class="n">weather_data</span> <span class="o">=</span> <span class="n">weather_response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
</span><span id="line-60"><span class="linenos">60</span> <span class="n">current_temp</span> <span class="o">=</span> <span class="n">weather_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"current"</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"temperature_2m"</span><span class="p">)</span>
|
||||
</span><span id="line-61"><span class="linenos">61</span> <span class="n">daily</span> <span class="o">=</span> <span class="n">weather_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"daily"</span><span class="p">,</span> <span class="p">{})</span>
|
||||
</span><span id="line-62"><span class="linenos">62</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p>The Flight Agent calls FlightAware’s AeroAPI:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id6">
|
||||
<div class="code-block-caption"><span class="caption-text">Flight Agent - External API Call</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id6"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">get_flights</span><span class="p">(</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span> <span class="n">origin_code</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">dest_code</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">travel_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="nb">dict</span><span class="p">]:</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="w"> </span><span class="sd">"""Get flights between two airports using FlightAware API.</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="sd"> Args:</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="sd"> origin_code: Origin airport IATA code</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="sd"> dest_code: Destination airport IATA code</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="sd"> travel_date: Travel date in YYYY-MM-DD format, defaults to today</span>
|
||||
</span><span id="line-10"><span class="linenos"> 10</span>
|
||||
</span><span id="line-11"><span class="linenos"> 11</span><span class="sd"> Note: FlightAware API limits searches to 2 days in the future.</span>
|
||||
</span><span id="line-12"><span class="linenos"> 12</span><span class="sd"> """</span>
|
||||
</span><span id="line-13"><span class="linenos"> 13</span> <span class="k">try</span><span class="p">:</span>
|
||||
</span><span id="line-14"><span class="linenos"> 14</span> <span class="c1"># Use provided date or default to today</span>
|
||||
</span><span id="line-15"><span class="linenos"> 15</span> <span class="k">if</span> <span class="n">travel_date</span><span class="p">:</span>
|
||||
</span><span id="line-16"><span class="linenos"> 16</span> <span class="n">search_date</span> <span class="o">=</span> <span class="n">travel_date</span>
|
||||
</span><span id="line-17"><span class="linenos"> 17</span> <span class="k">else</span><span class="p">:</span>
|
||||
</span><span id="line-18"><span class="linenos"> 18</span> <span class="n">search_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
</span><span id="line-19"><span class="linenos"> 19</span>
|
||||
</span><span id="line-20"><span class="linenos"> 20</span> <span class="c1"># Validate date is not too far in the future (FlightAware limit: 2 days)</span>
|
||||
</span><span id="line-21"><span class="linenos"> 21</span> <span class="n">search_date_obj</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">search_date</span><span class="p">,</span> <span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
</span><span id="line-22"><span class="linenos"> 22</span> <span class="n">today</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">hour</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">minute</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">second</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">microsecond</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
</span><span id="line-23"><span class="linenos"> 23</span> <span class="n">days_ahead</span> <span class="o">=</span> <span class="p">(</span><span class="n">search_date_obj</span> <span class="o">-</span> <span class="n">today</span><span class="p">)</span><span class="o">.</span><span class="n">days</span>
|
||||
</span><span id="line-24"><span class="linenos"> 24</span>
|
||||
</span><span id="line-25"><span class="linenos"> 25</span> <span class="k">if</span> <span class="n">days_ahead</span> <span class="o">></span> <span class="mi">2</span><span class="p">:</span>
|
||||
</span><span id="line-26"><span class="linenos"> 26</span> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
</span><span id="line-27"><span class="linenos"> 27</span> <span class="sa">f</span><span class="s2">"Requested date </span><span class="si">{</span><span class="n">search_date</span><span class="si">}</span><span class="s2"> is </span><span class="si">{</span><span class="n">days_ahead</span><span class="si">}</span><span class="s2"> days ahead, exceeds FlightAware 2-day limit"</span>
|
||||
</span><span id="line-28"><span class="linenos"> 28</span> <span class="p">)</span>
|
||||
</span><span id="line-29"><span class="linenos"> 29</span> <span class="k">return</span> <span class="p">{</span>
|
||||
</span><span id="line-30"><span class="linenos"> 30</span> <span class="s2">"origin_code"</span><span class="p">:</span> <span class="n">origin_code</span><span class="p">,</span>
|
||||
</span><span id="line-31"><span class="linenos"> 31</span> <span class="s2">"destination_code"</span><span class="p">:</span> <span class="n">dest_code</span><span class="p">,</span>
|
||||
</span><span id="line-32"><span class="linenos"> 32</span> <span class="s2">"flights"</span><span class="p">:</span> <span class="p">[],</span>
|
||||
</span><span id="line-33"><span class="linenos"> 33</span> <span class="s2">"count"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
|
||||
</span><span id="line-34"><span class="linenos"> 34</span> <span class="s2">"error"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"FlightAware API only provides flight data up to 2 days in the future. The requested date (</span><span class="si">{</span><span class="n">search_date</span><span class="si">}</span><span class="s2">) is </span><span class="si">{</span><span class="n">days_ahead</span><span class="si">}</span><span class="s2"> days ahead. Please search for today, tomorrow, or the day after."</span><span class="p">,</span>
|
||||
</span><span id="line-35"><span class="linenos"> 35</span> <span class="p">}</span>
|
||||
</span><span id="line-36"><span class="linenos"> 36</span>
|
||||
</span><span id="line-37"><span class="linenos"> 37</span> <span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">AEROAPI_BASE_URL</span><span class="si">}</span><span class="s2">/airports/</span><span class="si">{</span><span class="n">origin_code</span><span class="si">}</span><span class="s2">/flights/to/</span><span class="si">{</span><span class="n">dest_code</span><span class="si">}</span><span class="s2">"</span>
|
||||
</span><span id="line-38"><span class="linenos"> 38</span> <span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"x-apikey"</span><span class="p">:</span> <span class="n">AEROAPI_KEY</span><span class="p">}</span>
|
||||
</span><span id="line-39"><span class="linenos"> 39</span> <span class="n">params</span> <span class="o">=</span> <span class="p">{</span>
|
||||
</span><span id="line-40"><span class="linenos"> 40</span> <span class="s2">"start"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">search_date</span><span class="si">}</span><span class="s2">T00:00:00Z"</span><span class="p">,</span>
|
||||
</span><span id="line-41"><span class="linenos"> 41</span> <span class="s2">"end"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">search_date</span><span class="si">}</span><span class="s2">T23:59:59Z"</span><span class="p">,</span>
|
||||
</span><span id="line-42"><span class="linenos"> 42</span> <span class="s2">"connection"</span><span class="p">:</span> <span class="s2">"nonstop"</span><span class="p">,</span>
|
||||
</span><span id="line-43"><span class="linenos"> 43</span> <span class="s2">"max_pages"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
</span><span id="line-44"><span class="linenos"> 44</span> <span class="p">}</span>
|
||||
</span><span id="line-45"><span class="linenos"> 45</span>
|
||||
</span><span id="line-46"><span class="linenos"> 46</span> <span class="n">response</span> <span class="o">=</span> <span class="k">await</span> <span class="n">http_client</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span> <span class="n">params</span><span class="o">=</span><span class="n">params</span><span class="p">)</span>
|
||||
</span><span id="line-47"><span class="linenos"> 47</span>
|
||||
</span><span id="line-48"><span class="linenos"> 48</span> <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">!=</span> <span class="mi">200</span><span class="p">:</span>
|
||||
</span><span id="line-49"><span class="linenos"> 49</span> <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
|
||||
</span><span id="line-50"><span class="linenos"> 50</span> <span class="sa">f</span><span class="s2">"FlightAware API error </span><span class="si">{</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="si">}</span><span class="s2">"</span>
|
||||
</span><span id="line-51"><span class="linenos"> 51</span> <span class="p">)</span>
|
||||
</span><span id="line-52"><span class="linenos"> 52</span> <span class="k">return</span> <span class="kc">None</span>
|
||||
</span><span id="line-53"><span class="linenos"> 53</span>
|
||||
</span><span id="line-54"><span class="linenos"> 54</span> <span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
</span><span id="line-55"><span class="linenos"> 55</span> <span class="n">flights</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
</span><span id="line-56"><span class="linenos"> 56</span>
|
||||
</span><span id="line-57"><span class="linenos"> 57</span> <span class="c1"># Log raw API response for debugging</span>
|
||||
</span><span id="line-58"><span class="linenos"> 58</span> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"FlightAware API returned </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'flights'</span><span class="p">,</span><span class="w"> </span><span class="p">[]))</span><span class="si">}</span><span class="s2"> flights"</span><span class="p">)</span>
|
||||
</span><span id="line-59"><span class="linenos"> 59</span>
|
||||
</span><span id="line-60"><span class="linenos"> 60</span> <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">flight_group</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span>
|
||||
</span><span id="line-61"><span class="linenos"> 61</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"flights"</span><span class="p">,</span> <span class="p">[])[:</span><span class="mi">5</span><span class="p">]</span>
|
||||
</span><span id="line-62"><span class="linenos"> 62</span> <span class="p">):</span> <span class="c1"># Limit to 5 flights</span>
|
||||
</span><span id="line-63"><span class="linenos"> 63</span> <span class="c1"># FlightAware API nests data in segments array</span>
|
||||
</span><span id="line-64"><span class="linenos"> 64</span> <span class="n">segments</span> <span class="o">=</span> <span class="n">flight_group</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"segments"</span><span class="p">,</span> <span class="p">[])</span>
|
||||
</span><span id="line-65"><span class="linenos"> 65</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">segments</span><span class="p">:</span>
|
||||
</span><span id="line-66"><span class="linenos"> 66</span> <span class="k">continue</span>
|
||||
</span><span id="line-67"><span class="linenos"> 67</span>
|
||||
</span><span id="line-68"><span class="linenos"> 68</span> <span class="n">flight</span> <span class="o">=</span> <span class="n">segments</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="c1"># Get first segment (direct flights only have one)</span>
|
||||
</span><span id="line-69"><span class="linenos"> 69</span>
|
||||
</span><span id="line-70"><span class="linenos"> 70</span> <span class="c1"># Extract airport codes from nested objects</span>
|
||||
</span><span id="line-71"><span class="linenos"> 71</span> <span class="n">flight_origin</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
</span><span id="line-72"><span class="linenos"> 72</span> <span class="n">flight_dest</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
</span><span id="line-73"><span class="linenos"> 73</span>
|
||||
</span><span id="line-74"><span class="linenos"> 74</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">flight</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"origin"</span><span class="p">),</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
</span><span id="line-75"><span class="linenos"> 75</span> <span class="n">flight_origin</span> <span class="o">=</span> <span class="n">flight</span><span class="p">[</span><span class="s2">"origin"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"code_iata"</span><span class="p">)</span>
|
||||
</span><span id="line-76"><span class="linenos"> 76</span>
|
||||
</span><span id="line-77"><span class="linenos"> 77</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">flight</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"destination"</span><span class="p">),</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
</span><span id="line-78"><span class="linenos"> 78</span> <span class="n">flight_dest</span> <span class="o">=</span> <span class="n">flight</span><span class="p">[</span><span class="s2">"destination"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"code_iata"</span><span class="p">)</span>
|
||||
</span><span id="line-79"><span class="linenos"> 79</span>
|
||||
</span><span id="line-80"><span class="linenos"> 80</span> <span class="c1"># Build flight object</span>
|
||||
</span><span id="line-81"><span class="linenos"> 81</span> <span class="n">flights</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
</span><span id="line-82"><span class="linenos"> 82</span> <span class="p">{</span>
|
||||
</span><span id="line-83"><span class="linenos"> 83</span> <span class="s2">"airline"</span><span class="p">:</span> <span class="n">flight</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"operator"</span><span class="p">),</span>
|
||||
</span><span id="line-84"><span class="linenos"> 84</span> <span class="s2">"flight_number"</span><span class="p">:</span> <span class="n">flight</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"ident_iata"</span><span class="p">)</span> <span class="ow">or</span> <span class="n">flight</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"ident"</span><span class="p">),</span>
|
||||
</span><span id="line-85"><span class="linenos"> 85</span> <span class="s2">"departure_time"</span><span class="p">:</span> <span class="n">flight</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"scheduled_out"</span><span class="p">),</span>
|
||||
</span><span id="line-86"><span class="linenos"> 86</span> <span class="s2">"arrival_time"</span><span class="p">:</span> <span class="n">flight</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"scheduled_in"</span><span class="p">),</span>
|
||||
</span><span id="line-87"><span class="linenos"> 87</span> <span class="s2">"origin"</span><span class="p">:</span> <span class="n">flight_origin</span><span class="p">,</span>
|
||||
</span><span id="line-88"><span class="linenos"> 88</span> <span class="s2">"destination"</span><span class="p">:</span> <span class="n">flight_dest</span><span class="p">,</span>
|
||||
</span><span id="line-89"><span class="linenos"> 89</span> <span class="s2">"aircraft_type"</span><span class="p">:</span> <span class="n">flight</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"aircraft_type"</span><span class="p">),</span>
|
||||
</span><span id="line-90"><span class="linenos"> 90</span> <span class="s2">"status"</span><span class="p">:</span> <span class="n">flight</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"status"</span><span class="p">),</span>
|
||||
</span><span id="line-91"><span class="linenos"> 91</span> <span class="s2">"terminal_origin"</span><span class="p">:</span> <span class="n">flight</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"terminal_origin"</span><span class="p">),</span>
|
||||
</span><span id="line-92"><span class="linenos"> 92</span> <span class="s2">"gate_origin"</span><span class="p">:</span> <span class="n">flight</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"gate_origin"</span><span class="p">),</span>
|
||||
</span><span id="line-93"><span class="linenos"> 93</span> <span class="p">}</span>
|
||||
</span><span id="line-94"><span class="linenos"> 94</span> <span class="p">)</span>
|
||||
</span><span id="line-95"><span class="linenos"> 95</span>
|
||||
</span><span id="line-96"><span class="linenos"> 96</span> <span class="k">return</span> <span class="p">{</span>
|
||||
</span><span id="line-97"><span class="linenos"> 97</span> <span class="s2">"origin_code"</span><span class="p">:</span> <span class="n">origin_code</span><span class="p">,</span>
|
||||
</span><span id="line-98"><span class="linenos"> 98</span> <span class="s2">"destination_code"</span><span class="p">:</span> <span class="n">dest_code</span><span class="p">,</span>
|
||||
</span><span id="line-99"><span class="linenos"> 99</span> <span class="s2">"flights"</span><span class="p">:</span> <span class="n">flights</span><span class="p">,</span>
|
||||
</span><span id="line-100"><span class="linenos">100</span> <span class="s2">"count"</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">flights</span><span class="p">),</span>
|
||||
</span><span id="line-101"><span class="linenos">101</span> <span class="p">}</span>
|
||||
</span><span id="line-102"><span class="linenos">102</span> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
</span><span id="line-103"><span class="linenos">103</span> <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Error fetching flights: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
</span><span id="line-104"><span class="linenos">104</span> <span class="k">return</span> <span class="kc">None</span>
|
||||
</span><span id="line-105"><span class="linenos">105</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p><strong>Key Points:</strong></p>
|
||||
<ul class="simple">
|
||||
<li><p>Use async HTTP clients (like <code class="docutils literal notranslate"><span class="pre">httpx.AsyncClient</span></code>) for non-blocking API calls</p></li>
|
||||
<li><p>Transform external API responses into consistent, structured formats</p></li>
|
||||
<li><p>Handle errors gracefully with fallback values</p></li>
|
||||
<li><p>Cache or validate data when appropriate (e.g., airport code validation)</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="preparing-context-and-generating-responses">
|
||||
<h3>Preparing Context and Generating Responses<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#preparing-context-and-generating-responses" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#preparing-context-and-generating-responses'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Agents combine extracted information, API data, and conversation history to generate responses:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id7">
|
||||
<div class="code-block-caption"><span class="caption-text">Weather Agent - Context Preparation and Response Generation</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id7"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span> <span class="n">last_user_msg</span> <span class="o">=</span> <span class="n">get_last_user_content</span><span class="p">(</span><span class="n">messages</span><span class="p">)</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span> <span class="n">days</span> <span class="o">=</span> <span class="mi">1</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span> <span class="k">if</span> <span class="s2">"forecast"</span> <span class="ow">in</span> <span class="n">last_user_msg</span> <span class="ow">or</span> <span class="s2">"week"</span> <span class="ow">in</span> <span class="n">last_user_msg</span><span class="p">:</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span> <span class="n">days</span> <span class="o">=</span> <span class="mi">7</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span> <span class="k">elif</span> <span class="s2">"tomorrow"</span> <span class="ow">in</span> <span class="n">last_user_msg</span><span class="p">:</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span> <span class="n">days</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span> <span class="c1"># Extract specific number of days if mentioned (e.g., "5 day forecast")</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span> <span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span> <span class="n">day_match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s2">"(\d{1,2})\s+day"</span><span class="p">,</span> <span class="n">last_user_msg</span><span class="p">)</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span> <span class="k">if</span> <span class="n">day_match</span><span class="p">:</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span> <span class="n">requested_days</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">day_match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span> <span class="n">days</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">requested_days</span><span class="p">,</span> <span class="mi">16</span><span class="p">)</span> <span class="c1"># API supports max 16 days</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span> <span class="c1"># Get live weather data (location extraction happens inside this function)</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span> <span class="n">weather_data</span> <span class="o">=</span> <span class="k">await</span> <span class="n">get_weather_data</span><span class="p">(</span><span class="n">request</span><span class="p">,</span> <span class="n">messages</span><span class="p">,</span> <span class="n">days</span><span class="p">)</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span> <span class="c1"># Create weather context to append to user message</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span> <span class="n">forecast_type</span> <span class="o">=</span> <span class="s2">"forecast"</span> <span class="k">if</span> <span class="n">days</span> <span class="o">></span> <span class="mi">1</span> <span class="k">else</span> <span class="s2">"current weather"</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span> <span class="n">weather_context</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"""</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span><span class="s2">Weather data for </span><span class="si">{</span><span class="n">weather_data</span><span class="p">[</span><span class="s1">'location'</span><span class="p">]</span><span class="si">}</span><span class="s2"> (</span><span class="si">{</span><span class="n">forecast_type</span><span class="si">}</span><span class="s2">):</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span><span class="si">{</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">weather_data</span><span class="p">,</span><span class="w"> </span><span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span><span class="si">}</span><span class="s2">"""</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span> <span class="c1"># System prompt for weather agent</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span> <span class="n">instructions</span> <span class="o">=</span> <span class="s2">"""You are a weather assistant in a multi-agent system. You will receive weather data in JSON format with these fields:</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span><span class="s2"> - "location": City name</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span><span class="s2"> - "forecast": Array of weather objects, each with date, day_name, temperature_c, temperature_f, temperature_max_c, temperature_min_c, weather_code, sunrise, sunset</span>
|
||||
</span><span id="line-32"><span class="linenos">32</span><span class="s2"> - weather_code: WMO code (0=clear, 1-3=partly cloudy, 45-48=fog, 51-67=rain, 71-86=snow, 95-99=thunderstorm)</span>
|
||||
</span><span id="line-33"><span class="linenos">33</span>
|
||||
</span><span id="line-34"><span class="linenos">34</span><span class="s2"> Your task:</span>
|
||||
</span><span id="line-35"><span class="linenos">35</span><span class="s2"> 1. Present the weather/forecast clearly for the location</span>
|
||||
</span><span id="line-36"><span class="linenos">36</span><span class="s2"> 2. For single day: show current conditions</span>
|
||||
</span><span id="line-37"><span class="linenos">37</span><span class="s2"> 3. For multi-day: show each day with date and conditions</span>
|
||||
</span><span id="line-38"><span class="linenos">38</span><span class="s2"> 4. Include temperature in both Celsius and Fahrenheit</span>
|
||||
</span><span id="line-39"><span class="linenos">39</span><span class="s2"> 5. Describe conditions naturally based on weather_code</span>
|
||||
</span><span id="line-40"><span class="linenos">40</span><span class="s2"> 6. Use conversational language</span>
|
||||
</span><span id="line-41"><span class="linenos">41</span>
|
||||
</span><span id="line-42"><span class="linenos">42</span><span class="s2"> Important: If the conversation includes information from other agents (like flight details), acknowledge and build upon that context naturally. Your primary focus is weather, but maintain awareness of the full conversation.</span>
|
||||
</span><span id="line-43"><span class="linenos">43</span>
|
||||
</span><span id="line-44"><span class="linenos">44</span><span class="s2"> Remember: Only use the provided data. If fields are null, mention data is unavailable."""</span>
|
||||
</span><span id="line-45"><span class="linenos">45</span>
|
||||
</span><span id="line-46"><span class="linenos">46</span> <span class="c1"># Build message history with weather data appended to the last user message</span>
|
||||
</span><span id="line-47"><span class="linenos">47</span> <span class="n">response_messages</span> <span class="o">=</span> <span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"system"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="n">instructions</span><span class="p">}]</span>
|
||||
</span><span id="line-48"><span class="linenos">48</span>
|
||||
</span><span id="line-49"><span class="linenos">49</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">msg</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">messages</span><span class="p">):</span>
|
||||
</span><span id="line-50"><span class="linenos">50</span> <span class="c1"># Append weather data to the last user message</span>
|
||||
</span><span id="line-51"><span class="linenos">51</span> <span class="k">if</span> <span class="n">i</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">messages</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">msg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"role"</span><span class="p">)</span> <span class="o">==</span> <span class="s2">"user"</span><span class="p">:</span>
|
||||
</span><span id="line-52"><span class="linenos">52</span> <span class="n">response_messages</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
</span><span id="line-53"><span class="linenos">53</span> <span class="p">{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="n">msg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"content"</span><span class="p">)</span> <span class="o">+</span> <span class="n">weather_context</span><span class="p">}</span>
|
||||
</span><span id="line-54"><span class="linenos">54</span> <span class="p">)</span>
|
||||
</span><span id="line-55"><span class="linenos">55</span> <span class="k">else</span><span class="p">:</span>
|
||||
</span><span id="line-56"><span class="linenos">56</span> <span class="n">response_messages</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
</span><span id="line-57"><span class="linenos">57</span> <span class="p">{</span><span class="s2">"role"</span><span class="p">:</span> <span class="n">msg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"role"</span><span class="p">),</span> <span class="s2">"content"</span><span class="p">:</span> <span class="n">msg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"content"</span><span class="p">)}</span>
|
||||
</span><span id="line-58"><span class="linenos">58</span> <span class="p">)</span>
|
||||
</span><span id="line-59"><span class="linenos">59</span>
|
||||
</span><span id="line-60"><span class="linenos">60</span> <span class="k">try</span><span class="p">:</span>
|
||||
</span><span id="line-61"><span class="linenos">61</span> <span class="n">ctx</span> <span class="o">=</span> <span class="n">extract</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span>
|
||||
</span><span id="line-62"><span class="linenos">62</span> <span class="n">extra_headers</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"x-envoy-max-retries"</span><span class="p">:</span> <span class="s2">"3"</span><span class="p">}</span>
|
||||
</span><span id="line-63"><span class="linenos">63</span> <span class="n">inject</span><span class="p">(</span><span class="n">extra_headers</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="n">ctx</span><span class="p">)</span>
|
||||
</span><span id="line-64"><span class="linenos">64</span>
|
||||
</span><span id="line-65"><span class="linenos">65</span> <span class="n">stream</span> <span class="o">=</span> <span class="k">await</span> <span class="n">openai_client_via_plano</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-66"><span class="linenos">66</span> <span class="n">model</span><span class="o">=</span><span class="n">WEATHER_MODEL</span><span class="p">,</span>
|
||||
</span><span id="line-67"><span class="linenos">67</span> <span class="n">messages</span><span class="o">=</span><span class="n">response_messages</span><span class="p">,</span>
|
||||
</span><span id="line-68"><span class="linenos">68</span> <span class="n">temperature</span><span class="o">=</span><span class="n">request_body</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"temperature"</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">),</span>
|
||||
</span><span id="line-69"><span class="linenos">69</span> <span class="n">max_tokens</span><span class="o">=</span><span class="n">request_body</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"max_tokens"</span><span class="p">,</span> <span class="mi">1000</span><span class="p">),</span>
|
||||
</span><span id="line-70"><span class="linenos">70</span> <span class="n">stream</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
</span><span id="line-71"><span class="linenos">71</span> <span class="n">extra_headers</span><span class="o">=</span><span class="n">extra_headers</span><span class="p">,</span>
|
||||
</span><span id="line-72"><span class="linenos">72</span> <span class="p">)</span>
|
||||
</span><span id="line-73"><span class="linenos">73</span>
|
||||
</span><span id="line-74"><span class="linenos">74</span> <span class="k">async</span> <span class="k">for</span> <span class="n">chunk</span> <span class="ow">in</span> <span class="n">stream</span><span class="p">:</span>
|
||||
</span><span id="line-75"><span class="linenos">75</span> <span class="k">if</span> <span class="n">chunk</span><span class="o">.</span><span class="n">choices</span><span class="p">:</span>
|
||||
</span><span id="line-76"><span class="linenos">76</span> <span class="k">yield</span> <span class="sa">f</span><span class="s2">"data: </span><span class="si">{</span><span class="n">chunk</span><span class="o">.</span><span class="n">model_dump_json</span><span class="p">()</span><span class="si">}</span><span class="se">\n\n</span><span class="s2">"</span>
|
||||
</span><span id="line-77"><span class="linenos">77</span>
|
||||
</span><span id="line-78"><span class="linenos">78</span> <span class="k">yield</span> <span class="s2">"data: [DONE]</span><span class="se">\n\n</span><span class="s2">"</span>
|
||||
</span><span id="line-79"><span class="linenos">79</span>
|
||||
</span><span id="line-80"><span class="linenos">80</span> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
</span><span id="line-81"><span class="linenos">81</span> <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Error generating weather response: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p><strong>Key Points:</strong></p>
|
||||
<ul class="simple">
|
||||
<li><p>Use system messages to provide structured data to the LLM</p></li>
|
||||
<li><p>Include full conversation history for context-aware responses</p></li>
|
||||
<li><p>Stream responses for better user experience</p></li>
|
||||
<li><p>Route all LLM calls through Plano’s gateway for consistent behavior and observability</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
</section>
|
||||
<section id="best-practices">
|
||||
<h2>Best Practices<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#best-practices" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#best-practices'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p><strong>Write Clear Agent Descriptions</strong></p>
|
||||
<p>Agent descriptions are used by Plano-Orchestrator to make routing decisions. Be specific about what each agent handles:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Good - specific and actionable</span>
|
||||
</span><span id="line-2"><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">flight_agent</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Get live flight information between airports using FlightAware AeroAPI. Shows real-time flight status, scheduled/estimated/actual departure and arrival times, gate and terminal information, delays, aircraft type, and flight status. Automatically resolves city names to airport codes (IATA/ICAO). Understands conversation context to infer origin/destination from follow-up questions.</span>
|
||||
</span><span id="line-4">
|
||||
</span><span id="line-5"><span class="c1"># Less ideal - too vague</span>
|
||||
</span><span id="line-6"><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">flight_agent</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Handles flight queries</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Use Conversation Context Effectively</strong></p>
|
||||
<p>Include conversation history in your extraction and response generation:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Include conversation context for extraction</span>
|
||||
</span><span id="line-2"><span class="n">conversation_context</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
</span><span id="line-3"><span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">messages</span><span class="p">:</span>
|
||||
</span><span id="line-4"> <span class="n">conversation_context</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s2">"role"</span><span class="p">:</span> <span class="n">msg</span><span class="o">.</span><span class="n">role</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="n">msg</span><span class="o">.</span><span class="n">content</span><span class="p">})</span>
|
||||
</span><span id="line-5">
|
||||
</span><span id="line-6"><span class="c1"># Use recent context (last 10 messages)</span>
|
||||
</span><span id="line-7"><span class="n">context_messages</span> <span class="o">=</span> <span class="n">conversation_context</span><span class="p">[</span><span class="o">-</span><span class="mi">10</span><span class="p">:]</span> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">conversation_context</span><span class="p">)</span> <span class="o">></span> <span class="mi">10</span> <span class="k">else</span> <span class="n">conversation_context</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Route LLM Calls Through Plano’s Model Proxy</strong></p>
|
||||
<p>Always route LLM calls through Plano’s <a class="reference internal" href="../concepts/llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">Model Proxy</span></a> for consistent responses, smart routing, and rich observability:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="n">openai_client_via_plano</span> <span class="o">=</span> <span class="n">AsyncOpenAI</span><span class="p">(</span>
|
||||
</span><span id="line-2"> <span class="n">base_url</span><span class="o">=</span><span class="n">LLM_GATEWAY_ENDPOINT</span><span class="p">,</span> <span class="c1"># Plano's LLM gateway</span>
|
||||
</span><span id="line-3"> <span class="n">api_key</span><span class="o">=</span><span class="s2">"EMPTY"</span><span class="p">,</span>
|
||||
</span><span id="line-4"><span class="p">)</span>
|
||||
</span><span id="line-5">
|
||||
</span><span id="line-6"><span class="n">response</span> <span class="o">=</span> <span class="k">await</span> <span class="n">openai_client_via_plano</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-7"> <span class="n">model</span><span class="o">=</span><span class="s2">"openai/gpt-4o"</span><span class="p">,</span>
|
||||
</span><span id="line-8"> <span class="n">messages</span><span class="o">=</span><span class="n">messages</span><span class="p">,</span>
|
||||
</span><span id="line-9"> <span class="n">stream</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
</span><span id="line-10"><span class="p">)</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Handle Errors Gracefully</strong></p>
|
||||
<p>Provide fallback values and clear error messages:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">get_weather_data</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">Request</span><span class="p">,</span> <span class="n">messages</span><span class="p">:</span> <span class="nb">list</span><span class="p">,</span> <span class="n">days</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">):</span>
|
||||
</span><span id="line-2"> <span class="k">try</span><span class="p">:</span>
|
||||
</span><span id="line-3"> <span class="c1"># ... extraction and API logic ...</span>
|
||||
</span><span id="line-4"> <span class="n">location</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">choices</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">content</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">"</span><span class="se">\"</span><span class="s2">'`.,!?"</span><span class="p">)</span>
|
||||
</span><span id="line-5"> <span class="k">if</span> <span class="ow">not</span> <span class="n">location</span> <span class="ow">or</span> <span class="n">location</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"NOT_FOUND"</span><span class="p">:</span>
|
||||
</span><span id="line-6"> <span class="n">location</span> <span class="o">=</span> <span class="s2">"New York"</span> <span class="c1"># Fallback to default</span>
|
||||
</span><span id="line-7"> <span class="k">return</span> <span class="n">weather_data</span>
|
||||
</span><span id="line-8"> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
</span><span id="line-9"> <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Error getting weather data: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
</span><span id="line-10"> <span class="k">return</span> <span class="p">{</span><span class="s2">"location"</span><span class="p">:</span> <span class="s2">"New York"</span><span class="p">,</span> <span class="s2">"weather"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"error"</span><span class="p">:</span> <span class="s2">"Could not retrieve weather data"</span><span class="p">}}</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Use Appropriate Models for Tasks</strong></p>
|
||||
<p>Use smaller, faster models for extraction tasks and larger models for final responses:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Extraction: Use smaller, faster model</span>
|
||||
</span><span id="line-2"><span class="n">LOCATION_MODEL</span> <span class="o">=</span> <span class="s2">"openai/gpt-4o-mini"</span>
|
||||
</span><span id="line-3">
|
||||
</span><span id="line-4"><span class="c1"># Final response: Use larger, more capable model</span>
|
||||
</span><span id="line-5"><span class="n">WEATHER_MODEL</span> <span class="o">=</span> <span class="s2">"openai/gpt-4o"</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Stream Responses</strong></p>
|
||||
<p>Stream responses for better user experience:</p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">invoke_weather_agent</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">Request</span><span class="p">,</span> <span class="n">request_body</span><span class="p">:</span> <span class="nb">dict</span><span class="p">,</span> <span class="n">traceparent_header</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
||||
</span><span id="line-2"> <span class="c1"># ... prepare messages with weather data ...</span>
|
||||
</span><span id="line-3">
|
||||
</span><span id="line-4"> <span class="n">stream</span> <span class="o">=</span> <span class="k">await</span> <span class="n">openai_client_via_plano</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-5"> <span class="n">model</span><span class="o">=</span><span class="n">WEATHER_MODEL</span><span class="p">,</span>
|
||||
</span><span id="line-6"> <span class="n">messages</span><span class="o">=</span><span class="n">response_messages</span><span class="p">,</span>
|
||||
</span><span id="line-7"> <span class="n">temperature</span><span class="o">=</span><span class="n">request_body</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"temperature"</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">),</span>
|
||||
</span><span id="line-8"> <span class="n">max_tokens</span><span class="o">=</span><span class="n">request_body</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"max_tokens"</span><span class="p">,</span> <span class="mi">1000</span><span class="p">),</span>
|
||||
</span><span id="line-9"> <span class="n">stream</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
</span><span id="line-10"> <span class="n">extra_headers</span><span class="o">=</span><span class="n">extra_headers</span><span class="p">,</span>
|
||||
</span><span id="line-11"> <span class="p">)</span>
|
||||
</span><span id="line-12">
|
||||
</span><span id="line-13"> <span class="k">async</span> <span class="k">for</span> <span class="n">chunk</span> <span class="ow">in</span> <span class="n">stream</span><span class="p">:</span>
|
||||
</span><span id="line-14"> <span class="k">if</span> <span class="n">chunk</span><span class="o">.</span><span class="n">choices</span><span class="p">:</span>
|
||||
</span><span id="line-15"> <span class="k">yield</span> <span class="sa">f</span><span class="s2">"data: </span><span class="si">{</span><span class="n">chunk</span><span class="o">.</span><span class="n">model_dump_json</span><span class="p">()</span><span class="si">}</span><span class="se">\n\n</span><span class="s2">"</span>
|
||||
</span><span id="line-16">
|
||||
</span><span id="line-17"> <span class="k">yield</span> <span class="s2">"data: [DONE]</span><span class="se">\n\n</span><span class="s2">"</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="common-use-cases">
|
||||
<h2>Common Use Cases<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#common-use-cases" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#common-use-cases'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Multi-agent orchestration is particularly powerful for:</p>
|
||||
<p><strong>Travel and Booking Systems</strong></p>
|
||||
<p>Route queries to specialized agents for weather and flights:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">weather_agent</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Get real-time weather conditions and forecasts</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">flight_agent</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Search for flights and provide flight status</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Customer Support</strong></p>
|
||||
<p>Route common queries to automated support agents while escalating complex issues:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">tier1_support</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Handles common FAQs, password resets, and basic troubleshooting</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">tier2_support</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Handles complex technical issues requiring deep product knowledge</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">human_escalation</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Escalates sensitive issues or unresolved problems to human agents</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Sales and Marketing</strong></p>
|
||||
<p>Direct leads and inquiries to specialized sales agents:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">product_recommendation</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Recommends products based on user needs and preferences</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">pricing_agent</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Provides pricing information and quotes</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">sales_closer</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Handles final negotiations and closes deals</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Technical Documentation and Support</strong></p>
|
||||
<p>Combine RAG agents for documentation lookup with specialized troubleshooting agents:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">docs_agent</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Retrieves relevant documentation and guides</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">filter_chain</span><span class="p">:</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">query_rewriter</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">context_builder</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">troubleshoot_agent</span>
|
||||
</span><span id="line-8"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Diagnoses and resolves technical issues step by step</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="next-steps">
|
||||
<h2>Next Steps<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#next-steps" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#next-steps'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p>Learn more about <a class="reference internal" href="../concepts/agents.html#agents"><span class="std std-ref">agents</span></a> and the inner vs. outer loop model</p></li>
|
||||
<li><p>Explore <a class="reference internal" href="../concepts/filter_chain.html#filter-chain"><span class="std std-ref">filter chains</span></a> for adding guardrails and context enrichment</p></li>
|
||||
<li><p>See <a class="reference internal" href="observability/observability.html#observability"><span class="std std-ref">observability</span></a> for monitoring multi-agent workflows</p></li>
|
||||
<li><p>Review the <a class="reference internal" href="../concepts/llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">LLM Providers</span></a> guide for model routing within agents</p></li>
|
||||
<li><p>Check out the complete <a class="reference external" href="https://github.com/katanemo/plano/tree/main/demos/use_cases/travel_booking" rel="nofollow noopener">Travel Booking demo<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> on GitHub</p></li>
|
||||
</ul>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>To observe traffic to and from agents, please read more about <a class="reference internal" href="observability/observability.html#observability"><span class="std std-ref">observability</span></a> in Plano.</p>
|
||||
</div>
|
||||
<p>By carefully configuring and managing your Agent routing and hand off, you can significantly improve your application’s responsiveness, performance, and overall user satisfaction.</p>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../concepts/prompt_target.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Prompt Target
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="llm_router.html">
|
||||
LLM Routing
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#how-it-works'" class="reference internal" href="#how-it-works">How It Works</a></li>
|
||||
<li><a :data-current="activeSection === '#example-travel-booking-assistant'" class="reference internal" href="#example-travel-booking-assistant">Example: Travel Booking Assistant</a></li>
|
||||
<li><a :data-current="activeSection === '#configuration'" class="reference internal" href="#configuration">Configuration</a></li>
|
||||
<li><a :data-current="activeSection === '#implementation'" class="reference internal" href="#implementation">Implementation</a><ul>
|
||||
<li><a :data-current="activeSection === '#agent-structure'" class="reference internal" href="#agent-structure">Agent Structure</a></li>
|
||||
<li><a :data-current="activeSection === '#information-extraction-with-llms'" class="reference internal" href="#information-extraction-with-llms">Information Extraction with LLMs</a></li>
|
||||
<li><a :data-current="activeSection === '#calling-external-apis'" class="reference internal" href="#calling-external-apis">Calling External APIs</a></li>
|
||||
<li><a :data-current="activeSection === '#preparing-context-and-generating-responses'" class="reference internal" href="#preparing-context-and-generating-responses">Preparing Context and Generating Responses</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#best-practices'" class="reference internal" href="#best-practices">Best Practices</a></li>
|
||||
<li><a :data-current="activeSection === '#common-use-cases'" class="reference internal" href="#common-use-cases">Common Use Cases</a></li>
|
||||
<li><a :data-current="activeSection === '#next-steps'" class="reference internal" href="#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Prompt Guard | Arch Docs v0.3.22</title>
|
||||
<meta content="Prompt Guard | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Prompt Guard | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Guardrails | Plano Docs v0.4</title>
|
||||
<meta content="Guardrails | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Guardrails | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/guides/prompt_guard.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="agent_routing.html" rel="next" title="Agent Routing and Hand Off"/>
|
||||
<link href="../concepts/prompt_target.html" rel="prev" title="Prompt Target"/>
|
||||
<link href="state.html" rel="next" title="Conversational State"/>
|
||||
<link href="observability/access_logging.html" rel="prev" title="Access Logging"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,103 +148,133 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Prompt Guard</span>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Guardrails</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="prompt-guard">
|
||||
<span id="id1"></span><h1>Prompt Guard<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#prompt-guard"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p><strong>Prompt guard</strong> is a security and validation feature offered in Arch to protect agents, by filtering and analyzing prompts before they reach your application logic.
|
||||
In applications where prompts generate responses or execute specific actions based on user inputs, prompt guard minimizes risks like malicious inputs (or misaligned outputs).
|
||||
By adding a layer of input scrutiny, prompt guards ensures safer, more reliable, and accurate interactions with agents.</p>
|
||||
<section id="why-prompt-guard">
|
||||
<h2>Why Prompt Guard<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#why-prompt-guard" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#why-prompt-guard'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<section id="guardrails">
|
||||
<span id="prompt-guard"></span><h1>Guardrails<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#guardrails"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p><strong>Guardrails</strong> are Plano’s way of applying safety and validation checks to prompts before they reach your application logic. They are typically implemented as
|
||||
filters in a <a class="reference internal" href="../concepts/filter_chain.html#filter-chain"><span class="std std-ref">Filter Chain</span></a> attached to an agent, so every request passes through a consistent processing layer.</p>
|
||||
<section id="why-guardrails">
|
||||
<h2>Why Guardrails<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#why-guardrails" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#why-guardrails'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Guardrails are essential for maintaining control over AI-driven applications. They help enforce organizational policies, ensure compliance with regulations
|
||||
(like GDPR or HIPAA), and protect users from harmful or inappropriate content. In applications where prompts generate responses or trigger actions, guardrails
|
||||
minimize risks like malicious inputs, off-topic queries, or misaligned outputs—adding a consistent layer of input scrutiny that makes interactions safer,
|
||||
more reliable, and easier to reason about.</p>
|
||||
<ul class="simple">
|
||||
<li><dl class="simple">
|
||||
<dt><strong>Prompt Sanitization via Arch-Guard</strong></dt><dd><ul>
|
||||
<li><p><strong>Jailbreak Prevention</strong>: Detects and filters inputs that might attempt jailbreak attacks, like alternating LLM intended behavior, exposing the system prompt, or bypassing ethnics safety.</p></li>
|
||||
<li><p><strong>Jailbreak Prevention</strong>: Detect and filter inputs that attempt to change LLM behavior, expose system prompts, or bypass safety policies.</p></li>
|
||||
<li><p><strong>Domain and Topicality Enforcement</strong>: Ensure that agents only respond to prompts within an approved domain (for example, finance-only or healthcare-only use cases) and reject unrelated queries.</p></li>
|
||||
<li><p><strong>Dynamic Error Handling</strong>: Provide clear error messages when requests violate policy, helping users correct their inputs.</p></li>
|
||||
</ul>
|
||||
</dd>
|
||||
</dl>
|
||||
</li>
|
||||
<li><dl class="simple">
|
||||
<dt><strong>Dynamic Error Handling</strong></dt><dd><ul>
|
||||
<li><p><strong>Automatic Correction</strong>: Applies error-handling techniques to suggest corrections for minor input errors, such as typos or misformatted data.</p></li>
|
||||
<li><p><strong>Feedback Mechanism</strong>: Provides informative error messages to users, helping them understand how to correct input mistakes or adhere to guidelines.</p></li>
|
||||
</ul>
|
||||
</dd>
|
||||
</dl>
|
||||
</li>
|
||||
</ul>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>Today, Arch offers support for jailbreak via Arch-Guard. We will be adding support for additional guards in Q1, 2025 (including response guardrails)</p>
|
||||
</div>
|
||||
<section id="what-is-arch-guard">
|
||||
<h3>What Is Arch-Guard<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#what-is-arch-guard" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#what-is-arch-guard'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p><a class="reference external" href="https://huggingface.co/collections/katanemo/arch-guard-6702bdc08b889e4bce8f446d" rel="nofollow noopener">Arch-Guard<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> is a robust classifier model specifically trained on a diverse corpus of prompt attacks.
|
||||
It excels at detecting explicitly malicious prompts, providing an essential layer of security for LLM applications.</p>
|
||||
<p>By embedding Arch-Guard within the Arch architecture, we empower developers to build robust, LLM-powered applications while prioritizing security and safety. With Arch-Guard, you can navigate the complexities of prompt management with confidence, knowing you have a reliable defense against malicious input.</p>
|
||||
</section>
|
||||
<section id="example-configuration">
|
||||
<h3>Example Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#example-configuration" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#example-configuration'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Here is an example of using Arch-Guard in Arch:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id2">
|
||||
<div class="code-block-caption"><span class="caption-text">Arch-Guard Example Configuration</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos">1</span><span class="w"> </span><span class="nt">on_exception</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="linenos">2</span><span class="w"> </span><span class="nt">message</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Looks like you're curious about my abilities, but I can only provide assistance within my programmed parameters.</span>
|
||||
</span><span id="line-3"><span class="linenos">3</span>
|
||||
</span><span id="line-4"><span class="linenos">4</span><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-5"><span class="linenos">5</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">information_extraction</span>
|
||||
<section id="how-guardrails-work">
|
||||
<h2>How Guardrails Work<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#how-guardrails-work" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#how-guardrails-work'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Guardrails can be implemented as either in-process MCP filters or as HTTP-based filters. HTTP filters are external services that receive the request over HTTP, validate it, and return a response to allow or reject the request. This makes it easy to use filters written in any language or run them as independent services.</p>
|
||||
<p>Each filter receives the chat messages, evaluates them against policy, and either lets the request continue or raises a <code class="docutils literal notranslate"><span class="pre">ToolError</span></code> (or returns an error response) to reject it with a helpful error message.</p>
|
||||
<p>The example below shows an input guard for TechCorp’s customer support system that validates queries are within the company’s domain:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id1">
|
||||
<div class="code-block-caption"><span class="caption-text">Example domain validation guard using FastMCP</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id1"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">List</span>
|
||||
</span><span id="line-2"><span class="kn">from</span><span class="w"> </span><span class="nn">fastmcp.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">ToolError</span>
|
||||
</span><span id="line-3"><span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">mcp</span>
|
||||
</span><span id="line-4">
|
||||
</span><span id="line-5"><span class="nd">@mcp</span><span class="o">.</span><span class="n">tool</span>
|
||||
</span><span id="line-6"><span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">input_guards</span><span class="p">(</span><span class="n">messages</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">ChatMessage</span><span class="p">])</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">ChatMessage</span><span class="p">]:</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="sd">"""Validates queries are within TechCorp's domain."""</span>
|
||||
</span><span id="line-8">
|
||||
</span><span id="line-9"> <span class="c1"># Get the user's query</span>
|
||||
</span><span id="line-10"> <span class="n">user_query</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span>
|
||||
</span><span id="line-11"> <span class="p">(</span><span class="n">msg</span><span class="o">.</span><span class="n">content</span> <span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="n">messages</span><span class="p">)</span> <span class="k">if</span> <span class="n">msg</span><span class="o">.</span><span class="n">role</span> <span class="o">==</span> <span class="s2">"user"</span><span class="p">),</span>
|
||||
</span><span id="line-12"> <span class="s2">""</span>
|
||||
</span><span id="line-13"> <span class="p">)</span>
|
||||
</span><span id="line-14">
|
||||
</span><span id="line-15"> <span class="c1"># Use an LLM to validate the query scope (simplified)</span>
|
||||
</span><span id="line-16"> <span class="n">is_valid</span> <span class="o">=</span> <span class="k">await</span> <span class="n">validate_with_llm</span><span class="p">(</span><span class="n">user_query</span><span class="p">)</span>
|
||||
</span><span id="line-17">
|
||||
</span><span id="line-18"> <span class="k">if</span> <span class="ow">not</span> <span class="n">is_valid</span><span class="p">:</span>
|
||||
</span><span id="line-19"> <span class="k">raise</span> <span class="n">ToolError</span><span class="p">(</span>
|
||||
</span><span id="line-20"> <span class="s2">"I can only assist with questions related to TechCorp and its services. "</span>
|
||||
</span><span id="line-21"> <span class="s2">"Please ask about TechCorp's products, pricing, SLAs, or technical support."</span>
|
||||
</span><span id="line-22"> <span class="p">)</span>
|
||||
</span><span id="line-23">
|
||||
</span><span id="line-24"> <span class="k">return</span> <span class="n">messages</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p>To wire this guardrail into Plano, define the filter and add it to your agent’s filter chain:</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id2">
|
||||
<div class="code-block-caption"><span class="caption-text">Plano configuration with input guard filter</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">filters</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">input_guards</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://localhost:10500</span>
|
||||
</span><span id="line-4">
|
||||
</span><span id="line-5"><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">agent</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">agent_1</span>
|
||||
</span><span id="line-8"><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">8001</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="nt">router</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">plano_orchestrator_v1</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">rag_agent</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">virtual assistant for retrieval augmented generation tasks</span>
|
||||
</span><span id="line-13"><span class="w"> </span><span class="nt">filter_chain</span><span class="p">:</span>
|
||||
</span><span id="line-14"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">input_guards</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p>When a request arrives at <code class="docutils literal notranslate"><span class="pre">agent_1</span></code>, Plano invokes the <code class="docutils literal notranslate"><span class="pre">input_guards</span></code> filter first. If validation passes, the request continues to
|
||||
the agent. If validation fails (<code class="docutils literal notranslate"><span class="pre">ToolError</span></code> raised), Plano returns an error response to the caller.</p>
|
||||
</section>
|
||||
</section>
|
||||
<section id="how-arch-guard-works">
|
||||
<h2>How Arch-Guard Works<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#how-arch-guard-works" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#how-arch-guard-works'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ol class="arabic">
|
||||
<li><p><strong>Pre-Processing Stage</strong></p>
|
||||
<blockquote>
|
||||
<div><p>As a request or prompt is received, Arch Guard first performs validation. If any violations are detected, the input is flagged, and a tailored error message may be returned.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
<li><p><strong>Error Handling and Feedback</strong></p>
|
||||
<blockquote>
|
||||
<div><p>If the prompt contains errors or does not meet certain criteria, the user receives immediate feedback or correction suggestions, enhancing usability and reducing the chance of repeated input mistakes.</p>
|
||||
</div></blockquote>
|
||||
</li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="benefits-of-using-arch-guard">
|
||||
<h2>Benefits of Using Arch Guard<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#benefits-of-using-arch-guard" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#benefits-of-using-arch-guard'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Enhanced Security</strong>: Protects against injection attacks, harmful content, and misuse, securing both system and user data.</p></li>
|
||||
<li><p><strong>Better User Experience</strong>: Clear feedback and error correction improve user interactions by guiding them to correct input formats and constraints.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="summary">
|
||||
<h2>Summary<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#summary" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#summary'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Prompt guard is an essential tool for any prompt-based system that values security, accuracy, and compliance.
|
||||
By implementing Prompt Guard, developers can provide a robust layer of input validation and security, leading to better-performing, reliable, and safer applications.</p>
|
||||
<section id="testing-the-guardrail">
|
||||
<h2>Testing the Guardrail<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#testing-the-guardrail" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#testing-the-guardrail'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Here’s an example of the guardrail in action, rejecting a query about Apple Corporation (outside TechCorp’s domain):</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id3">
|
||||
<div class="code-block-caption"><span class="caption-text">Request that violates the guardrail policy</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id3"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">curl<span class="w"> </span>-X<span class="w"> </span>POST<span class="w"> </span>http://localhost:8001/v1/chat/completions<span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-2"><span class="w"> </span>-H<span class="w"> </span><span class="s2">"Content-Type: application/json"</span><span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-3"><span class="w"> </span>-d<span class="w"> </span><span class="s1">'{</span>
|
||||
</span><span id="line-4"><span class="s1"> "model": "gpt-4",</span>
|
||||
</span><span id="line-5"><span class="s1"> "messages": [</span>
|
||||
</span><span id="line-6"><span class="s1"> {</span>
|
||||
</span><span id="line-7"><span class="s1"> "role": "user",</span>
|
||||
</span><span id="line-8"><span class="s1"> "content": "what is sla for apple corporation?"</span>
|
||||
</span><span id="line-9"><span class="s1"> }</span>
|
||||
</span><span id="line-10"><span class="s1"> ],</span>
|
||||
</span><span id="line-11"><span class="s1"> "stream": false</span>
|
||||
</span><span id="line-12"><span class="s1"> }'</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="literal-block-wrapper docutils container" id="id4">
|
||||
<div class="code-block-caption"><span class="caption-text">Error response from the guardrail</span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id4"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="p">{</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">"error"</span><span class="p">:</span><span class="w"> </span><span class="s2">"ClientError"</span><span class="p">,</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">"agent"</span><span class="p">:</span><span class="w"> </span><span class="s2">"input_guards"</span><span class="p">,</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">"status"</span><span class="p">:</span><span class="w"> </span><span class="mi">400</span><span class="p">,</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">"agent_response"</span><span class="p">:</span><span class="w"> </span><span class="s2">"I apologize, but I can only assist with questions related to TechCorp and its services. Your query appears to be outside this scope. The query is about SLA for Apple Corporation, which is unrelated to TechCorp.\n\nPlease ask me about TechCorp's products, services, pricing, SLAs, or technical support."</span>
|
||||
</span><span id="line-6"><span class="p">}</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<p>This prevents out-of-scope queries from reaching your agent while providing clear feedback to users about why their request was rejected.</p>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../concepts/prompt_target.html">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="observability/access_logging.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Prompt Target
|
||||
Access Logging
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="agent_routing.html">
|
||||
Agent Routing and Hand Off
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="state.html">
|
||||
Conversational State
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
|
|
@ -258,14 +283,9 @@ By implementing Prompt Guard, developers can provide a robust layer of input val
|
|||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#why-prompt-guard'" class="reference internal" href="#why-prompt-guard">Why Prompt Guard</a><ul>
|
||||
<li><a :data-current="activeSection === '#what-is-arch-guard'" class="reference internal" href="#what-is-arch-guard">What Is Arch-Guard</a></li>
|
||||
<li><a :data-current="activeSection === '#example-configuration'" class="reference internal" href="#example-configuration">Example Configuration</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#how-arch-guard-works'" class="reference internal" href="#how-arch-guard-works">How Arch-Guard Works</a></li>
|
||||
<li><a :data-current="activeSection === '#benefits-of-using-arch-guard'" class="reference internal" href="#benefits-of-using-arch-guard">Benefits of Using Arch Guard</a></li>
|
||||
<li><a :data-current="activeSection === '#summary'" class="reference internal" href="#summary">Summary</a></li>
|
||||
<li><a :data-current="activeSection === '#why-guardrails'" class="reference internal" href="#why-guardrails">Why Guardrails</a></li>
|
||||
<li><a :data-current="activeSection === '#how-guardrails-work'" class="reference internal" href="#how-guardrails-work">How Guardrails Work</a></li>
|
||||
<li><a :data-current="activeSection === '#testing-the-guardrail'" class="reference internal" href="#testing-the-guardrail">Testing the Guardrail</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
|
|
@ -274,12 +294,12 @@ By implementing Prompt Guard, developers can provide a robust layer of input val
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
463
guides/state.html
Executable file
463
guides/state.html
Executable file
|
|
@ -0,0 +1,463 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Conversational State | Plano Docs v0.4</title>
|
||||
<meta content="Conversational State | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Conversational State | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/guides/state.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="../resources/tech_overview/tech_overview.html" rel="next" title="Tech Overview"/>
|
||||
<link href="prompt_guard.html" rel="prev" title="Guardrails"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
let mode;
|
||||
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
||||
mode = 'dark';
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
mode = 'light';
|
||||
}
|
||||
if (!userPreference) {localStorage.setItem('darkMode', mode)}
|
||||
</script>
|
||||
</head>
|
||||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
K
|
||||
</kbd>
|
||||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
</a>
|
||||
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
|
||||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||||
</svg>
|
||||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</aside>
|
||||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Conversational State</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="conversational-state">
|
||||
<span id="managing-conversational-state"></span><h1>Conversational State<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#conversational-state"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>The OpenAI Responses API (<code class="docutils literal notranslate"><span class="pre">v1/responses</span></code>) is designed for multi-turn conversations where context needs to persist across requests. Plano provides a unified <code class="docutils literal notranslate"><span class="pre">v1/responses</span></code> API that works with <strong>any LLM provider</strong>—OpenAI, Anthropic, Azure OpenAI, DeepSeek, or any OpenAI-compatible provider—while automatically managing conversational state for you.</p>
|
||||
<p>Unlike the traditional Chat Completions API where you manually manage conversation history by including all previous messages in each request, Plano handles state management behind the scenes. This means you can use the Responses API with any model provider, and Plano will persist conversation context across requests—making it ideal for building conversational agents that remember context without bloating every request with full message history.</p>
|
||||
<section id="how-it-works">
|
||||
<h2>How It Works<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#how-it-works" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#how-it-works'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>When a client calls the Responses API:</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p><strong>First request</strong>: Plano generates a unique <code class="docutils literal notranslate"><span class="pre">resp_id</span></code> and stores the conversation state (messages, model, provider, timestamp).</p></li>
|
||||
<li><p><strong>Subsequent requests</strong>: The client includes the <code class="docutils literal notranslate"><span class="pre">previous_resp_id</span></code> from the previous response. Plano retrieves the stored conversation state, merges it with the new input, and sends the combined context to the LLM.</p></li>
|
||||
<li><p><strong>Response</strong>: The LLM sees the full conversation history without the client needing to resend all previous messages.</p></li>
|
||||
</ol>
|
||||
<p>This pattern dramatically reduces bandwidth and makes it easier to build multi-turn agents—Plano handles the state plumbing so you can focus on agent logic.</p>
|
||||
<p><strong>Example Using OpenAI Python SDK:</strong></p>
|
||||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="kn">from</span><span class="w"> </span><span class="nn">openai</span><span class="w"> </span><span class="kn">import</span> <span class="n">OpenAI</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="c1"># Point to Plano's Model Proxy endpoint</span>
|
||||
</span><span id="line-4"><span class="n">client</span> <span class="o">=</span> <span class="n">OpenAI</span><span class="p">(</span>
|
||||
</span><span id="line-5"> <span class="n">api_key</span><span class="o">=</span><span class="s2">"test-key"</span><span class="p">,</span>
|
||||
</span><span id="line-6"> <span class="n">base_url</span><span class="o">=</span><span class="s2">"http://127.0.0.1:12000/v1"</span>
|
||||
</span><span id="line-7"><span class="p">)</span>
|
||||
</span><span id="line-8">
|
||||
</span><span id="line-9"><span class="c1"># First turn - Plano creates a new conversation state</span>
|
||||
</span><span id="line-10"><span class="n">response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">responses</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-11"> <span class="n">model</span><span class="o">=</span><span class="s2">"claude-sonnet-4-5"</span><span class="p">,</span> <span class="c1"># Works with any configured provider</span>
|
||||
</span><span id="line-12"> <span class="nb">input</span><span class="o">=</span><span class="s2">"My name is Alice and I like Python"</span>
|
||||
</span><span id="line-13"><span class="p">)</span>
|
||||
</span><span id="line-14">
|
||||
</span><span id="line-15"><span class="c1"># Save the response_id for conversation continuity</span>
|
||||
</span><span id="line-16"><span class="n">resp_id</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">id</span>
|
||||
</span><span id="line-17"><span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Assistant: </span><span class="si">{</span><span class="n">response</span><span class="o">.</span><span class="n">output_text</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
</span><span id="line-18">
|
||||
</span><span id="line-19"><span class="c1"># Second turn - Plano automatically retrieves previous context</span>
|
||||
</span><span id="line-20"><span class="n">resp2</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">responses</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
|
||||
</span><span id="line-21"> <span class="n">model</span><span class="o">=</span><span class="s2">"claude-sonnet-4-5"</span><span class="p">,</span> <span class="c1"># Make sure its configured in plano_config.yaml</span>
|
||||
</span><span id="line-22"> <span class="nb">input</span><span class="o">=</span><span class="s2">"Please list all the messages you have received in our conversation, numbering each one."</span><span class="p">,</span>
|
||||
</span><span id="line-23"> <span class="n">previous_response_id</span><span class="o">=</span><span class="n">resp_id</span><span class="p">,</span>
|
||||
</span><span id="line-24"><span class="p">)</span>
|
||||
</span><span id="line-25">
|
||||
</span><span id="line-26"><span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Assistant: </span><span class="si">{</span><span class="n">resp2</span><span class="o">.</span><span class="n">output_text</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
</span><span id="line-27"><span class="c1"># Output: "Your name is Alice and your favorite language is Python"</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p>Notice how the second request only includes the new user message—Plano automatically merges it with the stored conversation history before sending to the LLM.</p>
|
||||
</section>
|
||||
<section id="configuration-overview">
|
||||
<h2>Configuration Overview<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configuration-overview" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configuration-overview'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>State storage is configured in the <code class="docutils literal notranslate"><span class="pre">state_storage</span></code> section of your <code class="docutils literal notranslate"><span class="pre">plano_config.yaml</span></code>:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">state_storage</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span><span class="w"> </span><span class="c1"># Type: memory | postgres</span>
|
||||
</span><span id="line-3"><mark><span class="linenos"> 3</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">postgres</span>
|
||||
</mark></span><span id="line-4"><span class="linenos"> 4</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="w"> </span><span class="c1"># Connection string for postgres type</span>
|
||||
</span><span id="line-6"><mark><span class="linenos"> 6</span><span class="w"> </span><span class="c1"># Environment variables are supported using $VAR_NAME or ${VAR_NAME} syntax</span>
|
||||
</mark></span><span id="line-7"><mark><span class="linenos"> 7</span><span class="w"> </span><span class="c1"># Replace [USER] and [HOST] with your actual database credentials</span>
|
||||
</mark></span><span id="line-8"><mark><span class="linenos"> 8</span><span class="w"> </span><span class="c1"># Variables like $DB_PASSWORD MUST be set before running config validation/rendering</span>
|
||||
</mark></span><span id="line-9"><mark><span class="linenos"> 9</span><span class="w"> </span><span class="c1"># Example: Replace [USER] with 'myuser' and [HOST] with 'db.example.com:5432'</span>
|
||||
</mark></span><span id="line-10"><mark><span class="linenos">10</span><span class="w"> </span><span class="nt">connection_string</span><span class="p">:</span><span class="w"> </span><span class="s">"postgresql://[USER]:$DB_PASSWORD@[HOST]:5432/postgres"</span>
|
||||
</mark></span></code></pre></div>
|
||||
</div>
|
||||
<p>Plano supports two storage backends:</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Memory</strong>: Fast, ephemeral storage for development and testing. State is lost when Plano restarts.</p></li>
|
||||
<li><p><strong>PostgreSQL</strong>: Durable, production-ready storage with support for Supabase and self-hosted PostgreSQL instances.</p></li>
|
||||
</ul>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>If you don’t configure <code class="docutils literal notranslate"><span class="pre">state_storage</span></code>, conversation state management is <strong>disabled</strong>. The Responses API will still work, but clients must manually include full conversation history in each request (similar to the Chat Completions API behavior).</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="memory-storage-development">
|
||||
<h2>Memory Storage (Development)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#memory-storage-development" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#memory-storage-development'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Memory storage keeps conversation state in-memory using a thread-safe <code class="docutils literal notranslate"><span class="pre">HashMap</span></code>. It’s perfect for local development, demos, and testing, but all state is lost when Plano restarts.</p>
|
||||
<p><strong>Configuration</strong></p>
|
||||
<p>Add this to your <code class="docutils literal notranslate"><span class="pre">plano_config.yaml</span></code>:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">state_storage</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">memory</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p>That’s it. No additional setup required.</p>
|
||||
<p><strong>When to Use Memory Storage</strong></p>
|
||||
<ul class="simple">
|
||||
<li><p>Local development and debugging</p></li>
|
||||
<li><p>Demos and proof-of-concepts</p></li>
|
||||
<li><p>Automated testing environments</p></li>
|
||||
<li><p>Single-instance deployments where persistence isn’t critical</p></li>
|
||||
</ul>
|
||||
<p><strong>Limitations</strong></p>
|
||||
<ul class="simple">
|
||||
<li><p>State is lost on restart</p></li>
|
||||
<li><p>Not suitable for production workloads</p></li>
|
||||
<li><p>Cannot scale across multiple Plano instances</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="postgresql-storage-production">
|
||||
<h2>PostgreSQL Storage (Production)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#postgresql-storage-production" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#postgresql-storage-production'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>PostgreSQL storage provides durable, production-grade conversation state management. It works with both self-hosted PostgreSQL and Supabase (PostgreSQL-as-a-service), making it ideal for scaling multi-agent systems in production.</p>
|
||||
<section id="prerequisites">
|
||||
<h3>Prerequisites<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#prerequisites" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#prerequisites'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Before configuring PostgreSQL storage, you need:</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p>A PostgreSQL database (version 12 or later)</p></li>
|
||||
<li><p>Database credentials (host, user, password)</p></li>
|
||||
<li><p>The <code class="docutils literal notranslate"><span class="pre">conversation_states</span></code> table created in your database</p></li>
|
||||
</ol>
|
||||
<p><strong>Setting Up the Database</strong></p>
|
||||
<p>Run the SQL schema to create the required table:</p>
|
||||
<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="c1">-- Conversation State Storage Table</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span><span class="c1">-- This table stores conversational context for the OpenAI Responses API</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="c1">-- Run this SQL against your PostgreSQL/Supabase database before enabling conversation state storage</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="k">CREATE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="k">IF</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">EXISTS</span><span class="w"> </span><span class="n">conversation_states</span><span class="w"> </span><span class="p">(</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="w"> </span><span class="n">response_id</span><span class="w"> </span><span class="nb">TEXT</span><span class="w"> </span><span class="k">PRIMARY</span><span class="w"> </span><span class="k">KEY</span><span class="p">,</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="w"> </span><span class="n">input_items</span><span class="w"> </span><span class="n">JSONB</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="w"> </span><span class="n">created_at</span><span class="w"> </span><span class="nb">BIGINT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="w"> </span><span class="n">model</span><span class="w"> </span><span class="nb">TEXT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span>
|
||||
</span><span id="line-10"><span class="linenos">10</span><span class="w"> </span><span class="n">provider</span><span class="w"> </span><span class="nb">TEXT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span>
|
||||
</span><span id="line-11"><span class="linenos">11</span><span class="w"> </span><span class="n">updated_at</span><span class="w"> </span><span class="k">TIMESTAMP</span><span class="w"> </span><span class="k">DEFAULT</span><span class="w"> </span><span class="k">CURRENT_TIMESTAMP</span>
|
||||
</span><span id="line-12"><span class="linenos">12</span><span class="p">);</span>
|
||||
</span><span id="line-13"><span class="linenos">13</span>
|
||||
</span><span id="line-14"><span class="linenos">14</span><span class="c1">-- Indexes for common query patterns</span>
|
||||
</span><span id="line-15"><span class="linenos">15</span><span class="k">CREATE</span><span class="w"> </span><span class="k">INDEX</span><span class="w"> </span><span class="k">IF</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">EXISTS</span><span class="w"> </span><span class="n">idx_conversation_states_created_at</span>
|
||||
</span><span id="line-16"><span class="linenos">16</span><span class="w"> </span><span class="k">ON</span><span class="w"> </span><span class="n">conversation_states</span><span class="p">(</span><span class="n">created_at</span><span class="p">);</span>
|
||||
</span><span id="line-17"><span class="linenos">17</span>
|
||||
</span><span id="line-18"><span class="linenos">18</span><span class="k">CREATE</span><span class="w"> </span><span class="k">INDEX</span><span class="w"> </span><span class="k">IF</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">EXISTS</span><span class="w"> </span><span class="n">idx_conversation_states_provider</span>
|
||||
</span><span id="line-19"><span class="linenos">19</span><span class="w"> </span><span class="k">ON</span><span class="w"> </span><span class="n">conversation_states</span><span class="p">(</span><span class="n">provider</span><span class="p">);</span>
|
||||
</span><span id="line-20"><span class="linenos">20</span>
|
||||
</span><span id="line-21"><span class="linenos">21</span><span class="c1">-- Optional: Add a policy for automatic cleanup of old conversations</span>
|
||||
</span><span id="line-22"><span class="linenos">22</span><span class="c1">-- Uncomment and adjust the retention period as needed</span>
|
||||
</span><span id="line-23"><span class="linenos">23</span><span class="c1">-- CREATE INDEX IF NOT EXISTS idx_conversation_states_updated_at</span>
|
||||
</span><span id="line-24"><span class="linenos">24</span><span class="c1">-- ON conversation_states(updated_at);</span>
|
||||
</span><span id="line-25"><span class="linenos">25</span>
|
||||
</span><span id="line-26"><span class="linenos">26</span><span class="k">COMMENT</span><span class="w"> </span><span class="k">ON</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">conversation_states</span><span class="w"> </span><span class="k">IS</span><span class="w"> </span><span class="s1">'Stores conversation history for OpenAI Responses API continuity'</span><span class="p">;</span>
|
||||
</span><span id="line-27"><span class="linenos">27</span><span class="k">COMMENT</span><span class="w"> </span><span class="k">ON</span><span class="w"> </span><span class="k">COLUMN</span><span class="w"> </span><span class="n">conversation_states</span><span class="p">.</span><span class="n">response_id</span><span class="w"> </span><span class="k">IS</span><span class="w"> </span><span class="s1">'Unique identifier for the conversation state'</span><span class="p">;</span>
|
||||
</span><span id="line-28"><span class="linenos">28</span><span class="k">COMMENT</span><span class="w"> </span><span class="k">ON</span><span class="w"> </span><span class="k">COLUMN</span><span class="w"> </span><span class="n">conversation_states</span><span class="p">.</span><span class="n">input_items</span><span class="w"> </span><span class="k">IS</span><span class="w"> </span><span class="s1">'JSONB array of conversation messages and context'</span><span class="p">;</span>
|
||||
</span><span id="line-29"><span class="linenos">29</span><span class="k">COMMENT</span><span class="w"> </span><span class="k">ON</span><span class="w"> </span><span class="k">COLUMN</span><span class="w"> </span><span class="n">conversation_states</span><span class="p">.</span><span class="n">created_at</span><span class="w"> </span><span class="k">IS</span><span class="w"> </span><span class="s1">'Unix timestamp (seconds) when the conversation started'</span><span class="p">;</span>
|
||||
</span><span id="line-30"><span class="linenos">30</span><span class="k">COMMENT</span><span class="w"> </span><span class="k">ON</span><span class="w"> </span><span class="k">COLUMN</span><span class="w"> </span><span class="n">conversation_states</span><span class="p">.</span><span class="n">model</span><span class="w"> </span><span class="k">IS</span><span class="w"> </span><span class="s1">'Model name used for this conversation'</span><span class="p">;</span>
|
||||
</span><span id="line-31"><span class="linenos">31</span><span class="k">COMMENT</span><span class="w"> </span><span class="k">ON</span><span class="w"> </span><span class="k">COLUMN</span><span class="w"> </span><span class="n">conversation_states</span><span class="p">.</span><span class="n">provider</span><span class="w"> </span><span class="k">IS</span><span class="w"> </span><span class="s1">'LLM provider (e.g., openai, anthropic, bedrock)'</span><span class="p">;</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Using psql:</strong></p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">psql<span class="w"> </span><span class="nv">$DATABASE_URL</span><span class="w"> </span>-f<span class="w"> </span>docs/db_setup/conversation_states.sql
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Using Supabase Dashboard:</strong></p>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Log in to your Supabase project</p></li>
|
||||
<li><p>Navigate to the SQL Editor</p></li>
|
||||
<li><p>Copy and paste the SQL from <code class="docutils literal notranslate"><span class="pre">docs/db_setup/conversation_states.sql</span></code></p></li>
|
||||
<li><p>Run the query</p></li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="configuration">
|
||||
<h3>Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configuration" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configuration'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Once the database table is created, configure Plano to use PostgreSQL storage:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">state_storage</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">postgres</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">connection_string</span><span class="p">:</span><span class="w"> </span><span class="s">"postgresql://user:password@host:5432/database"</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Using Environment Variables</strong></p>
|
||||
<p>You should <strong>never</strong> hardcode credentials. Use environment variables instead:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">state_storage</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">postgres</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">connection_string</span><span class="p">:</span><span class="w"> </span><span class="s">"postgresql://myuser:$DB_PASSWORD@db.example.com:5432/postgres"</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p>Then set the environment variable before running Plano:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nb">export</span><span class="w"> </span><span class="nv">DB_PASSWORD</span><span class="o">=</span><span class="s2">"your-secure-password"</span>
|
||||
</span><span id="line-2"><span class="c1"># Run Plano or config validation</span>
|
||||
</span><span id="line-3">./plano
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<div class="admonition warning">
|
||||
<p class="admonition-title">Warning</p>
|
||||
<p><strong>Special Characters in Passwords</strong>: If your password contains special characters like <code class="docutils literal notranslate"><span class="pre">#</span></code>, <code class="docutils literal notranslate"><span class="pre">@</span></code>, or <code class="docutils literal notranslate"><span class="pre">&</span></code>, you must URL-encode them in the connection string. For example, <code class="docutils literal notranslate"><span class="pre">MyPass#123</span></code> becomes <code class="docutils literal notranslate"><span class="pre">MyPass%23123</span></code>.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="supabase-connection-strings">
|
||||
<h3>Supabase Connection Strings<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#supabase-connection-strings" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#supabase-connection-strings'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Supabase requires different connection strings depending on your network setup. Most users should use the <strong>Session Pooler</strong> connection string.</p>
|
||||
<p><strong>IPv4 Networks (Most Common)</strong></p>
|
||||
<p>Use the Session Pooler connection string (port 5432):</p>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">postgresql://postgres.[PROJECT-REF]:[PASSWORD]@aws-0-[REGION].pooler.supabase.com:5432/postgres
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>IPv6 Networks</strong></p>
|
||||
<p>Use the direct connection (port 5432):</p>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">postgresql://postgres:[PASSWORD]@db.[PROJECT-REF].supabase.co:5432/postgres
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>Finding Your Connection String</strong></p>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Go to your Supabase project dashboard</p></li>
|
||||
<li><p>Navigate to <strong>Settings → Database → Connection Pooling</strong></p></li>
|
||||
<li><p>Copy the <strong>Session mode</strong> connection string</p></li>
|
||||
<li><p>Replace <code class="docutils literal notranslate"><span class="pre">[YOUR-PASSWORD]</span></code> with your actual database password</p></li>
|
||||
<li><p>URL-encode special characters in the password</p></li>
|
||||
</ol>
|
||||
<p><strong>Example Configuration</strong></p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">state_storage</span><span class="p">:</span>
|
||||
</span><span id="line-2"><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">postgres</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">connection_string</span><span class="p">:</span><span class="w"> </span><span class="s">"postgresql://postgres.myproject:$DB_PASSWORD@aws-0-us-west-2.pooler.supabase.com:5432/postgres"</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p>Then set the environment variable:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># If your password is "MyPass#123", encode it as "MyPass%23123"</span>
|
||||
</span><span id="line-2"><span class="nb">export</span><span class="w"> </span><span class="nv">DB_PASSWORD</span><span class="o">=</span><span class="s2">"MyPass%23123"</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section id="troubleshooting">
|
||||
<h2>Troubleshooting<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#troubleshooting" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#troubleshooting'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p><strong>“Table ‘conversation_states’ does not exist”</strong></p>
|
||||
<p>Run the SQL schema from <code class="docutils literal notranslate"><span class="pre">docs/db_setup/conversation_states.sql</span></code> against your database.</p>
|
||||
<p><strong>Connection errors with Supabase</strong></p>
|
||||
<ul class="simple">
|
||||
<li><p>Verify you’re using the correct connection string format (Session Pooler for IPv4)</p></li>
|
||||
<li><p>Check that your password is URL-encoded if it contains special characters</p></li>
|
||||
<li><p>Ensure your Supabase project hasn’t paused due to inactivity (free tier)</p></li>
|
||||
</ul>
|
||||
<p><strong>Permission errors</strong></p>
|
||||
<p>Ensure your database user has the following permissions:</p>
|
||||
<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="k">GRANT</span><span class="w"> </span><span class="k">SELECT</span><span class="p">,</span><span class="w"> </span><span class="k">INSERT</span><span class="p">,</span><span class="w"> </span><span class="k">UPDATE</span><span class="p">,</span><span class="w"> </span><span class="k">DELETE</span><span class="w"> </span><span class="k">ON</span><span class="w"> </span><span class="n">conversation_states</span><span class="w"> </span><span class="k">TO</span><span class="w"> </span><span class="n">your_user</span><span class="p">;</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p><strong>State not persisting across requests</strong></p>
|
||||
<ul class="simple">
|
||||
<li><p>Verify <code class="docutils literal notranslate"><span class="pre">state_storage</span></code> is configured in your <code class="docutils literal notranslate"><span class="pre">plano_config.yaml</span></code></p></li>
|
||||
<li><p>Check Plano logs for state storage initialization messages</p></li>
|
||||
<li><p>Ensure the client is sending the <code class="docutils literal notranslate"><span class="pre">prev_response_id={$response_id}</span></code> from previous responses</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="best-practices">
|
||||
<h2>Best Practices<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#best-practices" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#best-practices'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ol class="arabic simple">
|
||||
<li><p><strong>Use environment variables for credentials</strong>: Never hardcode database passwords in configuration files.</p></li>
|
||||
<li><p><strong>Start with memory storage for development</strong>: Switch to PostgreSQL when moving to production.</p></li>
|
||||
<li><p><strong>Implement cleanup policies</strong>: Prevent unbounded growth by regularly archiving or deleting old conversations.</p></li>
|
||||
<li><p><strong>Monitor storage usage</strong>: Track conversation state table size and query performance in production.</p></li>
|
||||
<li><p><strong>Test failover scenarios</strong>: Ensure your application handles storage backend failures gracefully.</p></li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="next-steps">
|
||||
<h2>Next Steps<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#next-steps" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#next-steps'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p>Learn more about building <a class="reference internal" href="../concepts/agents.html#agents"><span class="std std-ref">agents</span></a> that leverage conversational state</p></li>
|
||||
<li><p>Explore <a class="reference internal" href="../concepts/filter_chain.html#filter-chain"><span class="std std-ref">filter chains</span></a> for enriching conversation context</p></li>
|
||||
<li><p>See the <a class="reference internal" href="../concepts/llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">LLM Providers</span></a> guide for configuring model routing</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="prompt_guard.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Guardrails
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../resources/tech_overview/tech_overview.html">
|
||||
Tech Overview
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#how-it-works'" class="reference internal" href="#how-it-works">How It Works</a></li>
|
||||
<li><a :data-current="activeSection === '#configuration-overview'" class="reference internal" href="#configuration-overview">Configuration Overview</a></li>
|
||||
<li><a :data-current="activeSection === '#memory-storage-development'" class="reference internal" href="#memory-storage-development">Memory Storage (Development)</a></li>
|
||||
<li><a :data-current="activeSection === '#postgresql-storage-production'" class="reference internal" href="#postgresql-storage-production">PostgreSQL Storage (Production)</a><ul>
|
||||
<li><a :data-current="activeSection === '#prerequisites'" class="reference internal" href="#prerequisites">Prerequisites</a></li>
|
||||
<li><a :data-current="activeSection === '#configuration'" class="reference internal" href="#configuration">Configuration</a></li>
|
||||
<li><a :data-current="activeSection === '#supabase-connection-strings'" class="reference internal" href="#supabase-connection-strings">Supabase Connection Strings</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#troubleshooting'" class="reference internal" href="#troubleshooting">Troubleshooting</a></li>
|
||||
<li><a :data-current="activeSection === '#best-practices'" class="reference internal" href="#best-practices">Best Practices</a></li>
|
||||
<li><a :data-current="activeSection === '#next-steps'" class="reference internal" href="#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
5327
includes/llms.txt
Executable file
5327
includes/llms.txt
Executable file
File diff suppressed because it is too large
Load diff
113
index.html
113
index.html
|
|
@ -7,13 +7,13 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Welcome to Arch! | Arch Docs v0.3.22</title>
|
||||
<meta content="Welcome to Arch! | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Welcome to Arch! | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Welcome to Plano! | Plano Docs v0.4</title>
|
||||
<meta content="Welcome to Plano! | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Welcome to Plano! | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/index.html" rel="canonical"/>
|
||||
<link href="_static/favicon.ico" rel="icon"/>
|
||||
|
|
@ -38,7 +38,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="#">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -55,7 +55,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -74,40 +74,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="#">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -117,27 +110,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="resources/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -151,12 +146,13 @@
|
|||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<div id="content" role="main">
|
||||
<section id="welcome-to-arch">
|
||||
<h1>Welcome to Arch!<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#welcome-to-arch"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<a class="reference internal image-reference" href="_images/arch-logo.png"><img alt="_images/arch-logo.png" class="align-center" src="_images/arch-logo.png" style="width: 100%;"/>
|
||||
<section id="welcome-to-plano">
|
||||
<h1>Welcome to Plano!<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#welcome-to-plano"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<a class="reference internal image-reference" href="_images/PlanoTagline.svg"><img alt="_images/PlanoTagline.svg" class="align-center" src="_images/PlanoTagline.svg" style="width: 100%;"/>
|
||||
</a>
|
||||
<a href="https://www.producthunt.com/posts/arch-3?embed=true&utm_source=badge-top-post-badge&utm_medium=badge&utm_souce=badge-arch-3" target="_blank"><img alt="Arch - Build fast, hyper-personalized agents with intelligent infra | Product Hunt" height="54" src="https://api.producthunt.com/widgets/embed-image/v1/top-post-badge.svg?post_id=565761&theme=dark&period=daily&t=1742433071161" style="width: 250px; height: 54px;" width="250"/></a><p><a class="reference external" href="https://github.com/katanemo/arch" rel="nofollow noopener">Arch<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> is a models-native edge and LLM proxy/gateway for AI agents - one that is natively designed to handle and process prompts, not just network traffic.</p>
|
||||
<p>Built by contributors to the widely adopted <a class="reference external" href="https://www.envoyproxy.io/" rel="nofollow noopener">Envoy Proxy<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, Arch handles the <em>pesky low-level work</em> in building agentic apps — like applying guardrails, clarifying vague user input, routing prompts to the right agent, and unifying access to any LLM. It’s a language and framework friendly infrastructure layer designed to help you build and ship agentic apps faster.</p>
|
||||
<p><a class="reference external" href="https://github.com/katanemo/plano" rel="nofollow noopener">Plano<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a> is delivery infrastructure for agentic apps. A models-native proxy server and data plane designed to help you build agents faster, and deliver them reliably to production.</p>
|
||||
<p>Plano pulls out the rote plumbing work (aka “hidden AI middleware”) and decouples you from brittle, ever‑changing framework abstractions. It centralizes what shouldn’t be bespoke in every codebase like agent routing and orchestration, rich agentic signals and traces for continuous improvement, guardrail filters for safety and moderation, and smart LLM routing APIs for UX and DX agility. Use any language or AI framework, and ship agents to production faster with Plano.</p>
|
||||
<p>Built by contributors to the widely adopted <a class="reference external" href="https://www.envoyproxy.io/" rel="nofollow noopener">Envoy Proxy<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>, Plano <strong>helps developers</strong> focus more on the core product logic of agents, <strong>product teams</strong> accelerate feedback loops for reinforcement learning, and <strong>engineering teams</strong> standardize policies and access controls across every agent and LLM for safer, more reliable scaling.</p>
|
||||
<div class="sd-tab-set docutils">
|
||||
<input checked="checked" id="sd-tab-item-0" name="sd-tab-set-0" type="radio"/>
|
||||
<label class="sd-tab-label" for="sd-tab-item-0">
|
||||
|
|
@ -165,7 +161,7 @@ Get Started</label><div class="sd-tab-content docutils">
|
|||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
|
|
@ -177,17 +173,10 @@ Concepts</label><div class="sd-tab-content docutils">
|
|||
<div class="toctree-wrapper compound">
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/tech_overview/tech_overview.html">Tech Overview</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/llm_providers/llm_providers.html">LLM Providers</a><ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/llm_providers/llm_providers.html">Model (LLM) Providers</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -203,39 +192,35 @@ Guides</label><div class="sd-tab-content docutils">
|
|||
<div class="toctree-wrapper compound">
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/observability/observability.html">Observability</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
<input id="sd-tab-item-3" name="sd-tab-set-0" type="radio"/>
|
||||
<label class="sd-tab-label" for="sd-tab-item-3">
|
||||
Build with Arch</label><div class="sd-tab-content docutils">
|
||||
<div class="toctree-wrapper compound">
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
<input id="sd-tab-item-4" name="sd-tab-set-0" type="radio"/>
|
||||
<label class="sd-tab-label" for="sd-tab-item-4">
|
||||
Resources</label><div class="sd-tab-content docutils">
|
||||
<div class="toctree-wrapper compound">
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="resources/tech_overview/tech_overview.html">Tech Overview</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -256,12 +241,12 @@ Resources</label><div class="sd-tab-content docutils">
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
BIN
objects.inv
BIN
objects.inv
Binary file not shown.
|
|
@ -7,17 +7,18 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Configuration Reference | Arch Docs v0.3.22</title>
|
||||
<meta content="Configuration Reference | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Configuration Reference | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Configuration Reference | Plano Docs v0.4</title>
|
||||
<meta content="Configuration Reference | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Configuration Reference | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/resources/configuration_reference.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="llms_txt.html" rel="next" title="llms.txt"/>
|
||||
<link href="deployment.html" rel="prev" title="Deployment"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
|
|
@ -38,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -55,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -74,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -117,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -152,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -162,111 +158,120 @@
|
|||
<div id="content" role="main">
|
||||
<section id="configuration-reference">
|
||||
<span id="id1"></span><h1>Configuration Reference<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configuration-reference"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>The following is a complete reference of the <code class="docutils literal notranslate"><span class="pre">arch_config.yml</span></code> that controls the behavior of a single instance of
|
||||
<p>The following is a complete reference of the <code class="docutils literal notranslate"><span class="pre">plano_config.yml</span></code> that controls the behavior of a single instance of
|
||||
the Arch gateway. This where you enable capabilities like routing to upstream LLm providers, defining prompt_targets
|
||||
where prompts get routed to, apply guardrails, and enable critical agent observability features.</p>
|
||||
<div class="literal-block-wrapper docutils container" id="id2">
|
||||
<div class="code-block-caption"><span class="caption-text"><a class="reference download internal" download="" href="../_downloads/ca9d3b7116524473d8adbde7cf15d167/arch_config_full_reference.yaml"><code class="xref download docutils literal notranslate"><span class="pre">Arch</span> <span class="pre">Configuration</span> <span class="pre">-</span> <span class="pre">Full</span> <span class="pre">Reference</span></code></a></span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span><span class="w"> </span><span class="nt">ingress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10000</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">5s</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span><span class="w"> </span><span class="nt">egress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-10"><span class="linenos"> 10</span><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-11"><span class="linenos"> 11</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">12000</span>
|
||||
</span><span id="line-12"><span class="linenos"> 12</span><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-13"><span class="linenos"> 13</span><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">5s</span>
|
||||
</span><span id="line-14"><span class="linenos"> 14</span>
|
||||
</span><span id="line-15"><span class="linenos"> 15</span><span class="c1"># Arch creates a round-robin load balancing between different endpoints, managed via the cluster subsystem.</span>
|
||||
</span><span id="line-16"><span class="linenos"> 16</span><span class="nt">endpoints</span><span class="p">:</span>
|
||||
</span><span id="line-17"><span class="linenos"> 17</span><span class="w"> </span><span class="nt">app_server</span><span class="p">:</span>
|
||||
</span><span id="line-18"><span class="linenos"> 18</span><span class="w"> </span><span class="c1"># value could be ip address or a hostname with port</span>
|
||||
</span><span id="line-19"><span class="linenos"> 19</span><span class="w"> </span><span class="c1"># this could also be a list of endpoints for load balancing</span>
|
||||
</span><span id="line-20"><span class="linenos"> 20</span><span class="w"> </span><span class="c1"># for example endpoint: [ ip1:port, ip2:port ]</span>
|
||||
</span><span id="line-21"><span class="linenos"> 21</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">127.0.0.1:80</span>
|
||||
</span><span id="line-22"><span class="linenos"> 22</span><span class="w"> </span><span class="c1"># max time to wait for a connection to be established</span>
|
||||
</span><span id="line-23"><span class="linenos"> 23</span><span class="w"> </span><span class="nt">connect_timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.005s</span>
|
||||
</span><span id="line-24"><span class="linenos"> 24</span>
|
||||
</span><span id="line-25"><span class="linenos"> 25</span><span class="w"> </span><span class="nt">mistral_local</span><span class="p">:</span>
|
||||
</span><span id="line-26"><span class="linenos"> 26</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">127.0.0.1:8001</span>
|
||||
</span><span id="line-27"><span class="linenos"> 27</span>
|
||||
</span><span id="line-28"><span class="linenos"> 28</span><span class="w"> </span><span class="nt">error_target</span><span class="p">:</span>
|
||||
</span><span id="line-29"><span class="linenos"> 29</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">error_target_1</span>
|
||||
</span><span id="line-30"><span class="linenos"> 30</span>
|
||||
</span><span id="line-31"><span class="linenos"> 31</span><span class="c1"># Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way</span>
|
||||
</span><span id="line-32"><span class="linenos"> 32</span><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-33"><span class="linenos"> 33</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-34"><span class="linenos"> 34</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-35"><span class="linenos"> 35</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-36"><span class="linenos"> 36</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
<div class="code-block-caption"><span class="caption-text"><a class="reference download internal" download="" href="../_downloads/ca9d3b7116524473d8adbde7cf15d167/arch_config_full_reference.yaml"><code class="xref download docutils literal notranslate"><span class="pre">Plano</span> <span class="pre">Configuration</span> <span class="pre">-</span> <span class="pre">Full</span> <span class="pre">Reference</span></code></a></span><a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#id2"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></div>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="linenos"> 1</span><span class="c1"># Arch Gateway configuration version</span>
|
||||
</span><span id="line-2"><span class="linenos"> 2</span><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.3.0</span>
|
||||
</span><span id="line-3"><span class="linenos"> 3</span>
|
||||
</span><span id="line-4"><span class="linenos"> 4</span>
|
||||
</span><span id="line-5"><span class="linenos"> 5</span><span class="c1"># External HTTP agents - API type is controlled by request path (/v1/responses, /v1/messages, /v1/chat/completions)</span>
|
||||
</span><span id="line-6"><span class="linenos"> 6</span><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-7"><span class="linenos"> 7</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">weather_agent</span><span class="w"> </span><span class="c1"># Example agent for weather</span>
|
||||
</span><span id="line-8"><span class="linenos"> 8</span><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://host.docker.internal:10510</span>
|
||||
</span><span id="line-9"><span class="linenos"> 9</span>
|
||||
</span><span id="line-10"><span class="linenos"> 10</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">flight_agent</span><span class="w"> </span><span class="c1"># Example agent for flights</span>
|
||||
</span><span id="line-11"><span class="linenos"> 11</span><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://host.docker.internal:10520</span>
|
||||
</span><span id="line-12"><span class="linenos"> 12</span>
|
||||
</span><span id="line-13"><span class="linenos"> 13</span>
|
||||
</span><span id="line-14"><span class="linenos"> 14</span><span class="c1"># MCP filters applied to requests/responses (e.g., input validation, query rewriting)</span>
|
||||
</span><span id="line-15"><span class="linenos"> 15</span><span class="nt">filters</span><span class="p">:</span>
|
||||
</span><span id="line-16"><span class="linenos"> 16</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">input_guards</span><span class="w"> </span><span class="c1"># Example filter for input validation</span>
|
||||
</span><span id="line-17"><span class="linenos"> 17</span><span class="w"> </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://host.docker.internal:10500</span>
|
||||
</span><span id="line-18"><span class="linenos"> 18</span><span class="w"> </span><span class="c1"># type: mcp (default)</span>
|
||||
</span><span id="line-19"><span class="linenos"> 19</span><span class="w"> </span><span class="c1"># transport: streamable-http (default)</span>
|
||||
</span><span id="line-20"><span class="linenos"> 20</span><span class="w"> </span><span class="c1"># tool: input_guards (default - same as filter id)</span>
|
||||
</span><span id="line-21"><span class="linenos"> 21</span>
|
||||
</span><span id="line-22"><span class="linenos"> 22</span>
|
||||
</span><span id="line-23"><span class="linenos"> 23</span><span class="c1"># LLM provider configurations with API keys and model routing</span>
|
||||
</span><span id="line-24"><span class="linenos"> 24</span><span class="nt">model_providers</span><span class="p">:</span>
|
||||
</span><span id="line-25"><span class="linenos"> 25</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-26"><span class="linenos"> 26</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-27"><span class="linenos"> 27</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-28"><span class="linenos"> 28</span>
|
||||
</span><span id="line-29"><span class="linenos"> 29</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o-mini</span>
|
||||
</span><span id="line-30"><span class="linenos"> 30</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-31"><span class="linenos"> 31</span>
|
||||
</span><span id="line-32"><span class="linenos"> 32</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">anthropic/claude-sonnet-4-0</span>
|
||||
</span><span id="line-33"><span class="linenos"> 33</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$ANTHROPIC_API_KEY</span>
|
||||
</span><span id="line-34"><span class="linenos"> 34</span>
|
||||
</span><span id="line-35"><span class="linenos"> 35</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">mistral/ministral-3b-latest</span>
|
||||
</span><span id="line-36"><span class="linenos"> 36</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$MISTRAL_API_KEY</span>
|
||||
</span><span id="line-37"><span class="linenos"> 37</span>
|
||||
</span><span id="line-38"><span class="linenos"> 38</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$MISTRAL_API_KEY</span>
|
||||
</span><span id="line-39"><span class="linenos"> 39</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">mistral/mistral-8x7b</span>
|
||||
</span><span id="line-40"><span class="linenos"> 40</span>
|
||||
</span><span id="line-41"><span class="linenos"> 41</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">mistral/mistral-7b-instruct</span>
|
||||
</span><span id="line-42"><span class="linenos"> 42</span><span class="w"> </span><span class="nt">base_url</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://mistral_local</span>
|
||||
</span><span id="line-38"><span class="linenos"> 38</span>
|
||||
</span><span id="line-39"><span class="linenos"> 39</span><span class="c1"># Model aliases - use friendly names instead of full provider model names</span>
|
||||
</span><span id="line-40"><span class="linenos"> 40</span><span class="nt">model_aliases</span><span class="p">:</span>
|
||||
</span><span id="line-41"><span class="linenos"> 41</span><span class="w"> </span><span class="nt">fast-llm</span><span class="p">:</span>
|
||||
</span><span id="line-42"><span class="linenos"> 42</span><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-4o-mini</span>
|
||||
</span><span id="line-43"><span class="linenos"> 43</span>
|
||||
</span><span id="line-44"><span class="linenos"> 44</span><span class="c1"># Model aliases - friendly names that map to actual provider names</span>
|
||||
</span><span id="line-45"><span class="linenos"> 45</span><span class="nt">model_aliases</span><span class="p">:</span>
|
||||
</span><span id="line-46"><span class="linenos"> 46</span><span class="w"> </span><span class="c1"># Alias for summarization tasks -> fast/cheap model</span>
|
||||
</span><span id="line-47"><span class="linenos"> 47</span><span class="w"> </span><span class="nt">arch.summarize.v1</span><span class="p">:</span>
|
||||
</span><span id="line-48"><span class="linenos"> 48</span><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-4o</span>
|
||||
</span><span id="line-49"><span class="linenos"> 49</span>
|
||||
</span><span id="line-50"><span class="linenos"> 50</span><span class="w"> </span><span class="c1"># Alias for general purpose tasks -> latest model</span>
|
||||
</span><span id="line-51"><span class="linenos"> 51</span><span class="w"> </span><span class="nt">arch.v1</span><span class="p">:</span>
|
||||
</span><span id="line-52"><span class="linenos"> 52</span><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">mistral-8x7b</span>
|
||||
</span><span id="line-53"><span class="linenos"> 53</span>
|
||||
</span><span id="line-54"><span class="linenos"> 54</span><span class="c1"># provides a way to override default settings for the arch system</span>
|
||||
</span><span id="line-55"><span class="linenos"> 55</span><span class="nt">overrides</span><span class="p">:</span>
|
||||
</span><span id="line-56"><span class="linenos"> 56</span><span class="w"> </span><span class="c1"># By default Arch uses an NLI + embedding approach to match an incoming prompt to a prompt target.</span>
|
||||
</span><span id="line-57"><span class="linenos"> 57</span><span class="w"> </span><span class="c1"># The intent matching threshold is kept at 0.80, you can override this behavior if you would like</span>
|
||||
</span><span id="line-58"><span class="linenos"> 58</span><span class="w"> </span><span class="nt">prompt_target_intent_matching_threshold</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.60</span>
|
||||
</span><span id="line-59"><span class="linenos"> 59</span>
|
||||
</span><span id="line-60"><span class="linenos"> 60</span><span class="c1"># default system prompt used by all prompt targets</span>
|
||||
</span><span id="line-61"><span class="linenos"> 61</span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">You are a network assistant that just offers facts; not advice on manufacturers or purchasing decisions.</span>
|
||||
</span><span id="line-62"><span class="linenos"> 62</span>
|
||||
</span><span id="line-63"><span class="linenos"> 63</span><span class="nt">prompt_guards</span><span class="p">:</span>
|
||||
</span><span id="line-64"><span class="linenos"> 64</span><span class="w"> </span><span class="nt">input_guards</span><span class="p">:</span>
|
||||
</span><span id="line-65"><span class="linenos"> 65</span><span class="w"> </span><span class="nt">jailbreak</span><span class="p">:</span>
|
||||
</span><span id="line-66"><span class="linenos"> 66</span><span class="w"> </span><span class="nt">on_exception</span><span class="p">:</span>
|
||||
</span><span id="line-67"><span class="linenos"> 67</span><span class="w"> </span><span class="nt">message</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Looks like you're curious about my abilities, but I can only provide assistance within my programmed parameters.</span>
|
||||
</span><span id="line-68"><span class="linenos"> 68</span>
|
||||
</span><span id="line-69"><span class="linenos"> 69</span><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-70"><span class="linenos"> 70</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">information_extraction</span>
|
||||
</span><span id="line-71"><span class="linenos"> 71</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-72"><span class="linenos"> 72</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">handel all scenarios that are question and answer in nature. Like summarization, information extraction, etc.</span>
|
||||
</span><span id="line-73"><span class="linenos"> 73</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-74"><span class="linenos"> 74</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</span><span id="line-75"><span class="linenos"> 75</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/summary</span>
|
||||
</span><span id="line-76"><span class="linenos"> 76</span><span class="w"> </span><span class="nt">http_method</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">POST</span>
|
||||
</span><span id="line-77"><span class="linenos"> 77</span><span class="w"> </span><span class="c1"># Arch uses the default LLM and treats the response from the endpoint as the prompt to send to the LLM</span>
|
||||
</span><span id="line-78"><span class="linenos"> 78</span><span class="w"> </span><span class="nt">auto_llm_dispatch_on_response</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-79"><span class="linenos"> 79</span><span class="w"> </span><span class="c1"># override system prompt for this prompt target</span>
|
||||
</span><span id="line-80"><span class="linenos"> 80</span><span class="w"> </span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">You are a helpful information extraction assistant. Use the information that is provided to you.</span>
|
||||
</span><span id="line-44"><span class="linenos"> 44</span><span class="w"> </span><span class="nt">smart-llm</span><span class="p">:</span>
|
||||
</span><span id="line-45"><span class="linenos"> 45</span><span class="w"> </span><span class="nt">target</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">gpt-4o</span>
|
||||
</span><span id="line-46"><span class="linenos"> 46</span>
|
||||
</span><span id="line-47"><span class="linenos"> 47</span>
|
||||
</span><span id="line-48"><span class="linenos"> 48</span><span class="c1"># HTTP listeners - entry points for agent routing, prompt targets, and direct LLM access</span>
|
||||
</span><span id="line-49"><span class="linenos"> 49</span><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-50"><span class="linenos"> 50</span><span class="w"> </span><span class="c1"># Agent listener for routing requests to multiple agents</span>
|
||||
</span><span id="line-51"><span class="linenos"> 51</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">agent</span>
|
||||
</span><span id="line-52"><span class="linenos"> 52</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">travel_booking_service</span>
|
||||
</span><span id="line-53"><span class="linenos"> 53</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">8001</span>
|
||||
</span><span id="line-54"><span class="linenos"> 54</span><span class="w"> </span><span class="nt">router</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">plano_orchestrator_v1</span>
|
||||
</span><span id="line-55"><span class="linenos"> 55</span><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-56"><span class="linenos"> 56</span><span class="w"> </span><span class="nt">agents</span><span class="p">:</span>
|
||||
</span><span id="line-57"><span class="linenos"> 57</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">id</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">rag_agent</span>
|
||||
</span><span id="line-58"><span class="linenos"> 58</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">virtual assistant for retrieval augmented generation tasks</span>
|
||||
</span><span id="line-59"><span class="linenos"> 59</span><span class="w"> </span><span class="nt">filter_chain</span><span class="p">:</span>
|
||||
</span><span id="line-60"><span class="linenos"> 60</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">input_guards</span>
|
||||
</span><span id="line-61"><span class="linenos"> 61</span>
|
||||
</span><span id="line-62"><span class="linenos"> 62</span><span class="w"> </span><span class="c1"># Model listener for direct LLM access</span>
|
||||
</span><span id="line-63"><span class="linenos"> 63</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">model</span>
|
||||
</span><span id="line-64"><span class="linenos"> 64</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">model_1</span>
|
||||
</span><span id="line-65"><span class="linenos"> 65</span><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-66"><span class="linenos"> 66</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">12000</span>
|
||||
</span><span id="line-67"><span class="linenos"> 67</span>
|
||||
</span><span id="line-68"><span class="linenos"> 68</span><span class="w"> </span><span class="c1"># Prompt listener for function calling (for prompt_targets)</span>
|
||||
</span><span id="line-69"><span class="linenos"> 69</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">prompt</span>
|
||||
</span><span id="line-70"><span class="linenos"> 70</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">prompt_function_listener</span>
|
||||
</span><span id="line-71"><span class="linenos"> 71</span><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-72"><span class="linenos"> 72</span><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10000</span>
|
||||
</span><span id="line-73"><span class="linenos"> 73</span><span class="w"> </span><span class="c1"># This listener is used for prompt_targets and function calling</span>
|
||||
</span><span id="line-74"><span class="linenos"> 74</span>
|
||||
</span><span id="line-75"><span class="linenos"> 75</span>
|
||||
</span><span id="line-76"><span class="linenos"> 76</span><span class="c1"># Reusable service endpoints</span>
|
||||
</span><span id="line-77"><span class="linenos"> 77</span><span class="nt">endpoints</span><span class="p">:</span>
|
||||
</span><span id="line-78"><span class="linenos"> 78</span><span class="w"> </span><span class="nt">app_server</span><span class="p">:</span>
|
||||
</span><span id="line-79"><span class="linenos"> 79</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">127.0.0.1:80</span>
|
||||
</span><span id="line-80"><span class="linenos"> 80</span><span class="w"> </span><span class="nt">connect_timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.005s</span>
|
||||
</span><span id="line-81"><span class="linenos"> 81</span>
|
||||
</span><span id="line-82"><span class="linenos"> 82</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">reboot_network_device</span>
|
||||
</span><span id="line-83"><span class="linenos"> 83</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Reboot a specific network device</span>
|
||||
</span><span id="line-84"><span class="linenos"> 84</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-85"><span class="linenos"> 85</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</span><span id="line-86"><span class="linenos"> 86</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/action</span>
|
||||
</span><span id="line-87"><span class="linenos"> 87</span><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</span><span id="line-88"><span class="linenos"> 88</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">device_id</span>
|
||||
</span><span id="line-89"><span class="linenos"> 89</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">str</span>
|
||||
</span><span id="line-90"><span class="linenos"> 90</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Identifier of the network device to reboot.</span>
|
||||
</span><span id="line-91"><span class="linenos"> 91</span><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-92"><span class="linenos"> 92</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">confirmation</span>
|
||||
</span><span id="line-93"><span class="linenos"> 93</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">bool</span>
|
||||
</span><span id="line-94"><span class="linenos"> 94</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Confirmation flag to proceed with reboot.</span>
|
||||
</span><span id="line-95"><span class="linenos"> 95</span><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">false</span>
|
||||
</span><span id="line-96"><span class="linenos"> 96</span><span class="w"> </span><span class="nt">enum</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">true</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">false</span><span class="p p-Indicator">]</span>
|
||||
</span><span id="line-97"><span class="linenos"> 97</span>
|
||||
</span><span id="line-98"><span class="linenos"> 98</span><span class="nt">tracing</span><span class="p">:</span>
|
||||
</span><span id="line-99"><span class="linenos"> 99</span><span class="w"> </span><span class="c1"># sampling rate. Note by default Arch works on OpenTelemetry compatible tracing.</span>
|
||||
</span><span id="line-100"><span class="linenos">100</span><span class="w"> </span><span class="nt">sampling_rate</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.1</span>
|
||||
</span><span id="line-82"><span class="linenos"> 82</span><span class="w"> </span><span class="nt">mistral_local</span><span class="p">:</span>
|
||||
</span><span id="line-83"><span class="linenos"> 83</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">127.0.0.1:8001</span>
|
||||
</span><span id="line-84"><span class="linenos"> 84</span>
|
||||
</span><span id="line-85"><span class="linenos"> 85</span>
|
||||
</span><span id="line-86"><span class="linenos"> 86</span><span class="c1"># Prompt targets for function calling and API orchestration</span>
|
||||
</span><span id="line-87"><span class="linenos"> 87</span><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-88"><span class="linenos"> 88</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">get_current_weather</span>
|
||||
</span><span id="line-89"><span class="linenos"> 89</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Get current weather at a location.</span>
|
||||
</span><span id="line-90"><span class="linenos"> 90</span><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</span><span id="line-91"><span class="linenos"> 91</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">location</span>
|
||||
</span><span id="line-92"><span class="linenos"> 92</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">The location to get the weather for</span>
|
||||
</span><span id="line-93"><span class="linenos"> 93</span><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-94"><span class="linenos"> 94</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">string</span>
|
||||
</span><span id="line-95"><span class="linenos"> 95</span><span class="w"> </span><span class="nt">format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">City, State</span>
|
||||
</span><span id="line-96"><span class="linenos"> 96</span><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">days</span>
|
||||
</span><span id="line-97"><span class="linenos"> 97</span><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">the number of days for the request</span>
|
||||
</span><span id="line-98"><span class="linenos"> 98</span><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-99"><span class="linenos"> 99</span><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">int</span>
|
||||
</span><span id="line-100"><span class="linenos">100</span><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-101"><span class="linenos">101</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</span><span id="line-102"><span class="linenos">102</span><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/weather</span>
|
||||
</span><span id="line-103"><span class="linenos">103</span><span class="w"> </span><span class="nt">http_method</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">POST</span>
|
||||
</span><span id="line-104"><span class="linenos">104</span>
|
||||
</span><span id="line-105"><span class="linenos">105</span>
|
||||
</span><span id="line-106"><span class="linenos">106</span><span class="c1"># OpenTelemetry tracing configuration</span>
|
||||
</span><span id="line-107"><span class="linenos">107</span><span class="nt">tracing</span><span class="p">:</span>
|
||||
</span><span id="line-108"><span class="linenos">108</span><span class="w"> </span><span class="c1"># Random sampling percentage (1-100)</span>
|
||||
</span><span id="line-109"><span class="linenos">109</span><span class="w"> </span><span class="nt">random_sampling</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">100</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -280,18 +285,26 @@ where prompts get routed to, apply guardrails, and enable critical agent observa
|
|||
Deployment
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="llms_txt.html">
|
||||
llms.txt
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Deployment | Arch Docs v0.3.22</title>
|
||||
<meta content="Deployment | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Deployment | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Deployment | Plano Docs v0.4</title>
|
||||
<meta content="Deployment | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Deployment | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/resources/deployment.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="configuration_reference.html" rel="next" title="Configuration Reference"/>
|
||||
<link href="../build_with_arch/multi_turn.html" rel="prev" title="Multi-Turn"/>
|
||||
<link href="tech_overview/threading_model.html" rel="prev" title="Threading Model"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,40 +75,33 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -118,27 +111,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -163,28 +158,28 @@
|
|||
<div id="content" role="main">
|
||||
<section id="deployment">
|
||||
<span id="id1"></span><h1>Deployment<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#deployment"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>This guide shows how to deploy Arch directly using Docker without the archgw CLI, including basic runtime checks for routing and health monitoring.</p>
|
||||
<p>This guide shows how to deploy Plano directly using Docker without the <code class="docutils literal notranslate"><span class="pre">plano</span></code> CLI, including basic runtime checks for routing and health monitoring.</p>
|
||||
<section id="docker-deployment">
|
||||
<h2>Docker Deployment<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#docker-deployment" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#docker-deployment'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Below is a minimal, production-ready example showing how to deploy the Arch Docker image directly and run basic runtime checks. Adjust image names, tags, and the <code class="docutils literal notranslate"><span class="pre">arch_config.yaml</span></code> path to match your environment.</p>
|
||||
<p>Below is a minimal, production-ready example showing how to deploy the Plano Docker image directly and run basic runtime checks. Adjust image names, tags, and the <code class="docutils literal notranslate"><span class="pre">plano_config.yaml</span></code> path to match your environment.</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>You will need to pass all required environment variables that are referenced in your <code class="docutils literal notranslate"><span class="pre">arch_config.yaml</span></code> file.</p>
|
||||
<p>You will need to pass all required environment variables that are referenced in your <code class="docutils literal notranslate"><span class="pre">plano_config.yaml</span></code> file.</p>
|
||||
</div>
|
||||
<p>For <code class="docutils literal notranslate"><span class="pre">arch_config.yaml</span></code>, you can use any sample configuration defined earlier in the documentation. For example, you can try the <a class="reference internal" href="../guides/llm_router.html#llm-router"><span class="std std-ref">LLM Routing</span></a> sample config.</p>
|
||||
<p>For <code class="docutils literal notranslate"><span class="pre">plano_config.yaml</span></code>, you can use any sample configuration defined earlier in the documentation. For example, you can try the <a class="reference internal" href="../guides/llm_router.html#llm-router"><span class="std std-ref">LLM Routing</span></a> sample config.</p>
|
||||
<section id="docker-compose-setup">
|
||||
<h3>Docker Compose Setup<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#docker-compose-setup" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#docker-compose-setup'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Create a <code class="docutils literal notranslate"><span class="pre">docker-compose.yml</span></code> file with the following configuration:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># docker-compose.yml</span>
|
||||
</span><span id="line-2"><span class="nt">services</span><span class="p">:</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">archgw</span><span class="p">:</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">katanemo/archgw:0.3.22</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">container_name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">archgw</span>
|
||||
</span><span id="line-3"><span class="w"> </span><span class="nt">plano</span><span class="p">:</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">katanemo/plano:0.4.0</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">container_name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">plano</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">ports</span><span class="p">:</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">"10000:10000"</span><span class="w"> </span><span class="c1"># ingress (client -> arch)</span>
|
||||
</span><span id="line-8"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">"12000:12000"</span><span class="w"> </span><span class="c1"># egress (arch -> upstream/llm proxy)</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">"10000:10000"</span><span class="w"> </span><span class="c1"># ingress (client -> plano)</span>
|
||||
</span><span id="line-8"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">"12000:12000"</span><span class="w"> </span><span class="c1"># egress (plano -> upstream/llm proxy)</span>
|
||||
</span><span id="line-9"><span class="w"> </span><span class="nt">volumes</span><span class="p">:</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">./arch_config.yaml:/app/arch_config.yaml:ro</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">./plano_config.yaml:/app/plano_config.yaml:ro</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="nt">environment</span><span class="p">:</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">OPENAI_API_KEY=${OPENAI_API_KEY:?error}</span>
|
||||
</span><span id="line-13"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:?error}</span>
|
||||
|
|
@ -193,14 +188,14 @@
|
|||
</section>
|
||||
<section id="starting-the-stack">
|
||||
<h3>Starting the Stack<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#starting-the-stack" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#starting-the-stack'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Start the services from the directory containing <code class="docutils literal notranslate"><span class="pre">docker-compose.yml</span></code> and <code class="docutils literal notranslate"><span class="pre">arch_config.yaml</span></code>:</p>
|
||||
<p>Start the services from the directory containing <code class="docutils literal notranslate"><span class="pre">docker-compose.yml</span></code> and <code class="docutils literal notranslate"><span class="pre">plano_config.yaml</span></code>:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Set required environment variables and start services</span>
|
||||
</span><span id="line-2"><span class="nv">OPENAI_API_KEY</span><span class="o">=</span>xxx<span class="w"> </span><span class="nv">ANTHROPIC_API_KEY</span><span class="o">=</span>yyy<span class="w"> </span>docker<span class="w"> </span>compose<span class="w"> </span>up<span class="w"> </span>-d
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p>Check container health and logs:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">docker<span class="w"> </span>compose<span class="w"> </span>ps
|
||||
</span><span id="line-2">docker<span class="w"> </span>compose<span class="w"> </span>logs<span class="w"> </span>-f<span class="w"> </span>archgw
|
||||
</span><span id="line-2">docker<span class="w"> </span>compose<span class="w"> </span>logs<span class="w"> </span>-f<span class="w"> </span>plano
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
|
|
@ -211,14 +206,14 @@
|
|||
<section id="gateway-smoke-test">
|
||||
<h3>Gateway Smoke Test<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#gateway-smoke-test" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#gateway-smoke-test'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Test the chat completion endpoint with automatic routing:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Request handled by the gateway. 'model: "none"' lets Arch decide routing</span>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="c1"># Request handled by the gateway. 'model: "none"' lets Plano decide routing</span>
|
||||
</span><span id="line-2">curl<span class="w"> </span>--header<span class="w"> </span><span class="s1">'Content-Type: application/json'</span><span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-3"><span class="w"> </span>--data<span class="w"> </span><span class="s1">'{"messages":[{"role":"user","content":"tell me a joke"}], "model":"none"}'</span><span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-4"><span class="w"> </span>http://localhost:12000/v1/chat/completions<span class="w"> </span><span class="p">|</span><span class="w"> </span>jq<span class="w"> </span>.model
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p>Expected output:</p>
|
||||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="s2">"gpt-4o-2024-08-06"</span>
|
||||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="s2">"gpt-5.2"</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
|
|
@ -226,12 +221,12 @@
|
|||
<h3>Model-Based Routing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#model-based-routing" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#model-based-routing'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<p>Test explicit provider and model routing:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><code><span id="line-1">curl<span class="w"> </span>-s<span class="w"> </span>-H<span class="w"> </span><span class="s2">"Content-Type: application/json"</span><span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-2"><span class="w"> </span>-d<span class="w"> </span><span class="s1">'{"messages":[{"role":"user","content":"Explain quantum computing"}], "model":"anthropic/claude-3-5-sonnet-20241022"}'</span><span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-2"><span class="w"> </span>-d<span class="w"> </span><span class="s1">'{"messages":[{"role":"user","content":"Explain quantum computing"}], "model":"anthropic/claude-sonnet-4-5"}'</span><span class="w"> </span><span class="se">\</span>
|
||||
</span><span id="line-3"><span class="w"> </span>http://localhost:12000/v1/chat/completions<span class="w"> </span><span class="p">|</span><span class="w"> </span>jq<span class="w"> </span>.model
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<p>Expected output:</p>
|
||||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="s2">"claude-3-5-sonnet-20241022"</span>
|
||||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="s2">"claude-sonnet-4-5"</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
|
|
@ -241,18 +236,18 @@
|
|||
<section id="common-issues-and-solutions">
|
||||
<h3>Common Issues and Solutions<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#common-issues-and-solutions" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#common-issues-and-solutions'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
<dl class="simple">
|
||||
<dt><strong>Environment Variables</strong></dt><dd><p>Ensure all environment variables (<code class="docutils literal notranslate"><span class="pre">OPENAI_API_KEY</span></code>, <code class="docutils literal notranslate"><span class="pre">ANTHROPIC_API_KEY</span></code>, etc.) used by <code class="docutils literal notranslate"><span class="pre">arch_config.yaml</span></code> are set before starting services.</p>
|
||||
<dt><strong>Environment Variables</strong></dt><dd><p>Ensure all environment variables (<code class="docutils literal notranslate"><span class="pre">OPENAI_API_KEY</span></code>, <code class="docutils literal notranslate"><span class="pre">ANTHROPIC_API_KEY</span></code>, etc.) used by <code class="docutils literal notranslate"><span class="pre">plano_config.yaml</span></code> are set before starting services.</p>
|
||||
</dd>
|
||||
<dt><strong>TLS/Connection Errors</strong></dt><dd><p>If you encounter TLS or connection errors to upstream providers:</p>
|
||||
<ul class="simple">
|
||||
<li><p>Check DNS resolution</p></li>
|
||||
<li><p>Verify proxy settings</p></li>
|
||||
<li><p>Confirm correct protocol and port in your <code class="docutils literal notranslate"><span class="pre">arch_config</span></code> endpoints</p></li>
|
||||
<li><p>Confirm correct protocol and port in your <code class="docutils literal notranslate"><span class="pre">plano_config</span></code> endpoints</p></li>
|
||||
</ul>
|
||||
</dd>
|
||||
<dt><strong>Verbose Logging</strong></dt><dd><p>To enable more detailed logs for debugging:</p>
|
||||
<ul class="simple">
|
||||
<li><p>Run archgw with a higher component log level</p></li>
|
||||
<li><p>Run plano with a higher component log level</p></li>
|
||||
<li><p>See the <a class="reference internal" href="../guides/observability/observability.html#observability"><span class="std std-ref">Observability</span></a> guide for logging and monitoring details</p></li>
|
||||
<li><p>Rebuild the image if required with updated log configuration</p></li>
|
||||
</ul>
|
||||
|
|
@ -265,11 +260,11 @@
|
|||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../build_with_arch/multi_turn.html">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="tech_overview/threading_model.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Multi-Turn
|
||||
Threading Model
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
|
|
@ -305,12 +300,12 @@
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
189
resources/llms_txt.html
Executable file
189
resources/llms_txt.html
Executable file
|
|
@ -0,0 +1,189 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html :class="{'dark': darkMode === 'dark' || (darkMode === 'system' && window.matchMedia('(prefers-color-scheme: dark)').matches)}" class="scroll-smooth" data-content_root="../" lang="en" x-data="{ darkMode: localStorage.getItem('darkMode') || localStorage.setItem('darkMode', 'system'), activeSection: '' }" x-init="$watch('darkMode', val => localStorage.setItem('darkMode', val))">
|
||||
<head>
|
||||
<meta content="width=device-width, initial-scale=1.0" name="viewport"/>
|
||||
<meta charset="utf-8"/>
|
||||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>llms.txt | Plano Docs v0.4</title>
|
||||
<meta content="llms.txt | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="llms.txt | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/resources/llms_txt.html" rel="canonical"/>
|
||||
<link href="../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../search.html" rel="search" title="Search"/>
|
||||
<link href="configuration_reference.html" rel="prev" title="Configuration Reference"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
let mode;
|
||||
if (userPreference === 'dark' || window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
||||
mode = 'dark';
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
mode = 'light';
|
||||
}
|
||||
if (!userPreference) {localStorage.setItem('darkMode', mode)}
|
||||
</script>
|
||||
</head>
|
||||
<body :class="{ 'overflow-hidden': showSidebar }" class="min-h-screen font-sans antialiased bg-background text-foreground" x-data="{ showSidebar: false, showScrollTop: false }">
|
||||
<div @click.self="showSidebar = false" class="fixed inset-0 z-50 overflow-hidden bg-background/80 backdrop-blur-sm md:hidden" x-cloak="" x-show="showSidebar"></div><div class="relative flex flex-col min-h-screen" id="page"><a class="absolute top-0 left-0 z-[100] block bg-background p-4 text-xl transition -translate-x-full opacity-0 focus:translate-x-0 focus:opacity-100" href="#content">
|
||||
Skip to content
|
||||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle navigation menu</span>
|
||||
</button>
|
||||
<div class="flex items-center justify-between flex-1 space-x-2 sm:space-x-4 md:justify-end">
|
||||
<div class="flex-1 w-full md:w-auto md:flex-none"><form @keydown.k.window.meta="$refs.search.focus()" action="../search.html" class="relative flex items-center group" id="searchbox" method="get">
|
||||
<input aria-label="Search the docs" class="inline-flex items-center font-medium transition-colors bg-transparent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 ring-offset-background border border-input hover:bg-accent focus:bg-accent hover:text-accent-foreground focus:text-accent-foreground hover:placeholder-accent-foreground py-2 px-4 relative h-9 w-full justify-start rounded-[0.5rem] text-sm text-muted-foreground sm:pr-12 md:w-40 lg:w-64" id="search-input" name="q" placeholder="Search ..." type="search" x-ref="search"/>
|
||||
<kbd class="pointer-events-none absolute right-1.5 top-2 hidden h-5 select-none text-muted-foreground items-center gap-1 rounded border border-border bg-muted px-1.5 font-mono text-[10px] font-medium opacity-100 sm:flex group-hover:bg-accent group-hover:text-accent-foreground">
|
||||
<span class="text-xs">⌘</span>
|
||||
K
|
||||
</kbd>
|
||||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
</a>
|
||||
<button @click="darkMode = darkMode === 'light' ? 'dark' : 'light'" aria-label="Color theme switcher" class="relative inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md hover:bg-accent hover:text-accent-foreground h-9 w-9" type="button">
|
||||
<svg class="absolute transition-all scale-100 rotate-0 dark:-rotate-90 dark:scale-0" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 685q45.456 0 77.228-31.772Q589 621.456 589 576q0-45.456-31.772-77.228Q525.456 467 480 467q-45.456 0-77.228 31.772Q371 530.544 371 576q0 45.456 31.772 77.228Q434.544 685 480 685Zm0 91q-83 0-141.5-58.5T280 576q0-83 58.5-141.5T480 376q83 0 141.5 58.5T680 576q0 83-58.5 141.5T480 776ZM80 621.5q-19.152 0-32.326-13.174T34.5 576q0-19.152 13.174-32.326T80 530.5h80q19.152 0 32.326 13.174T205.5 576q0 19.152-13.174 32.326T160 621.5H80Zm720 0q-19.152 0-32.326-13.174T754.5 576q0-19.152 13.174-32.326T800 530.5h80q19.152 0 32.326 13.174T925.5 576q0 19.152-13.174 32.326T880 621.5h-80Zm-320-320q-19.152 0-32.326-13.174T434.5 256v-80q0-19.152 13.174-32.326T480 130.5q19.152 0 32.326 13.174T525.5 176v80q0 19.152-13.174 32.326T480 301.5Zm0 720q-19.152 0-32.326-13.17Q434.5 995.152 434.5 976v-80q0-19.152 13.174-32.326T480 850.5q19.152 0 32.326 13.174T525.5 896v80q0 19.152-13.174 32.33-13.174 13.17-32.326 13.17ZM222.174 382.065l-43-42Q165.5 327.391 166 308.239t13.174-33.065q13.435-13.674 32.587-13.674t32.065 13.674l42.239 43q12.674 13.435 12.555 31.706-.12 18.272-12.555 31.946-12.674 13.674-31.445 13.413-18.772-.261-32.446-13.174Zm494 494.761-42.239-43q-12.674-13.435-12.674-32.087t12.674-31.565Q686.609 756.5 705.38 757q18.772.5 32.446 13.174l43 41.761Q794.5 824.609 794 843.761t-13.174 33.065Q767.391 890.5 748.239 890.5t-32.065-13.674Zm-42-494.761Q660.5 369.391 661 350.62q.5-18.772 13.174-32.446l41.761-43Q728.609 261.5 747.761 262t33.065 13.174q13.674 13.435 13.674 32.587t-13.674 32.065l-43 42.239q-13.435 12.674-31.706 12.555-18.272-.12-31.946-12.555Zm-495 494.761Q165.5 863.391 165.5 844.239t13.674-32.065l43-42.239q13.435-12.674 32.087-12.674t31.565 12.674Q299.5 782.609 299 801.38q-.5 18.772-13.174 32.446l-41.761 43Q231.391 890.5 212.239 890t-33.065-13.174ZM480 576Z"></path>
|
||||
</svg>
|
||||
<svg class="absolute transition-all scale-0 rotate-90 dark:rotate-0 dark:scale-100" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 936q-151 0-255.5-104.5T120 576q0-138 90-239.5T440 218q25-3 39 18t-1 44q-17 26-25.5 55t-8.5 61q0 90 63 153t153 63q31 0 61.5-9t54.5-25q21-14 43-1.5t19 39.5q-14 138-117.5 229T480 936Zm0-80q88 0 158-48.5T740 681q-20 5-40 8t-40 3q-123 0-209.5-86.5T364 396q0-20 3-40t8-40q-78 32-126.5 102T200 576q0 116 82 198t198 82Zm-10-270Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../concepts/prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview/tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<button @click="showSidebar = false" class="absolute md:hidden right-4 top-4 rounded-sm opacity-70 transition-opacity hover:opacity-100" type="button">
|
||||
<svg class="h-4 w-4" fill="currentColor" height="24" stroke="none" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M480 632 284 828q-11 11-28 11t-28-11q-11-11-11-28t11-28l196-196-196-196q-11-11-11-28t11-28q11-11 28-11t28 11l196 196 196-196q11-11 28-11t28 11q11 11 11 28t-11 28L536 576l196 196q11 11 11 28t-11 28q-11 11-28 11t-28-11L480 632Z"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</aside>
|
||||
<main class="relative py-6 lg:gap-10 lg:py-8 xl:grid xl:grid-cols-[1fr_300px]">
|
||||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../index.html">
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">llms.txt</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="llms-txt">
|
||||
<h1>llms.txt<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#llms-txt"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>This project generates a single plaintext file containing the compiled text of all documentation pages, useful for large context models to reference Plano documentation.</p>
|
||||
<p>Open it here: <a class="reference external" href="../includes/llms.txt" rel="nofollow noopener">llms.txt<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a></p>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="configuration_reference.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Configuration Reference
|
||||
</a>
|
||||
</div>
|
||||
</div></div>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../_static/theme.js?v=073f68d9"></script>
|
||||
<script src="../_static/design-tabs.js?v=f930bc37"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Model Serving | Arch Docs v0.3.22</title>
|
||||
<meta content="Model Serving | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Model Serving | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Bright Staff | Plano Docs v0.4</title>
|
||||
<meta content="Bright Staff | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Bright Staff | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/tech_overview/model_serving.html" rel="canonical"/>
|
||||
<link href="./docs/resources/tech_overview/model_serving.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../../search.html" rel="search" title="Search"/>
|
||||
<link href="request_lifecycle.html" rel="next" title="Request Lifecycle"/>
|
||||
<link href="prompt.html" rel="prev" title="Prompts"/>
|
||||
<link href="threading_model.html" rel="next" title="Threading Model"/>
|
||||
<link href="request_lifecycle.html" rel="prev" title="Request Lifecycle"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,70 +75,65 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="error_target.html">Error Target</a></li>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_target.html">Prompt Target</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,89 +148,59 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="mr-1">/</div><a class="hover:text-foreground overflow-hidden text-ellipsis whitespace-nowrap" href="tech_overview.html">Tech Overview</a>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Model Serving</span>
|
||||
<div class="mr-1">/</div><span aria-current="page" class="font-medium text-foreground overflow-hidden text-ellipsis whitespace-nowrap">Bright Staff</span>
|
||||
</nav>
|
||||
<div id="content" role="main">
|
||||
<section id="model-serving">
|
||||
<span id="id1"></span><h1>Model Serving<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#model-serving"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Arch is a set of <cite>two</cite> self-contained processes that are designed to run alongside your application
|
||||
servers (or on a separate host connected via a network). The first process is designated to manage low-level
|
||||
networking and HTTP related concerns, and the other process is for model serving, which helps Arch make
|
||||
intelligent decisions about the incoming prompts. The model server is designed to call the purpose-built
|
||||
LLMs in Arch.</p>
|
||||
<a class="reference internal image-reference" href="../../_images/arch-system-architecture.jpg"><img alt="../../_images/arch-system-architecture.jpg" class="align-center" src="../../_images/arch-system-architecture.jpg" style="width: 40%;"/>
|
||||
<section id="bright-staff">
|
||||
<span id="id1"></span><h1>Bright Staff<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#bright-staff"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Bright Staff is Plano’s memory-efficient, lightweight controller for agentic traffic. It sits inside the Plano
|
||||
data plane and makes real-time decisions about how prompts are handled, forwarded, and processed.</p>
|
||||
<p>Rather than running a separate “model server” subsystem, Plano relies on Envoy’s HTTP connection management
|
||||
and cluster subsystem to talk to different models and backends over HTTP(S). Bright Staff uses these primitives to:
|
||||
* Inspect prompts, conversation state, and metadata.
|
||||
* Decide which upstream model(s), tool backends, or APIs to call, and in what order.
|
||||
* Coordinate retries, fallbacks, and traffic splitting across providers and models.</p>
|
||||
<p>Plano is designed to run alongside your application servers in your cloud VPC, on-premises, or in local
|
||||
development. It does not require a GPU itself; GPUs live where your models are hosted (third-party APIs or your
|
||||
own deployments), and Plano reaches them via HTTP.</p>
|
||||
<a class="reference internal image-reference" href="../../_images/plano-system-architecture.png"><img alt="../../_images/plano-system-architecture.png" class="align-center" src="../../_images/plano-system-architecture.png" style="width: 40%;"/>
|
||||
</a>
|
||||
<p>Arch’ is designed to be deployed in your cloud VPC, on a on-premises host, and can work on devices that don’t
|
||||
have a GPU. Note, GPU devices are need for fast and cost-efficient use, so that Arch (model server, specifically)
|
||||
can process prompts quickly and forward control back to the application host. There are three modes in which Arch
|
||||
can be configured to run its <strong>model server</strong> subsystem:</p>
|
||||
<section id="local-serving-cpu-moderate">
|
||||
<h2>Local Serving (CPU - Moderate)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#local-serving-cpu-moderate" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#local-serving-cpu-moderate'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>The following bash commands enable you to configure the model server subsystem in Arch to run local on device
|
||||
and only use CPU devices. This will be the slowest option but can be useful in dev/test scenarios where GPUs
|
||||
might not be available.</p>
|
||||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>archgw<span class="w"> </span>up<span class="w"> </span>--local-cpu
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="cloud-serving-gpu-blazing-fast">
|
||||
<h2>Cloud Serving (GPU - Blazing Fast)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#cloud-serving-gpu-blazing-fast" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#cloud-serving-gpu-blazing-fast'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>The command below instructs Arch to intelligently use GPUs locally for fast intent detection, but default to
|
||||
cloud serving for function calling and guardrails scenarios to dramatically improve the speed and overall performance
|
||||
of your applications.</p>
|
||||
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="gp">$ </span>archgw<span class="w"> </span>up
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>Arch’s model serving in the cloud is priced at $0.05M/token (156x cheaper than GPT-4o) with average latency
|
||||
of 200ms (10x faster than GPT-4o). Please refer to our <a class="reference internal" href="../../get_started/quickstart.html#quickstart"><span class="std std-ref">Get Started</span></a> to know
|
||||
how to generate API keys for model serving</p>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="prompt.html">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="request_lifecycle.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Prompts
|
||||
Request Lifecycle
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="request_lifecycle.html">
|
||||
Request Lifecycle
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="threading_model.html">
|
||||
Threading Model
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
</a>
|
||||
</div>
|
||||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#local-serving-cpu-moderate'" class="reference internal" href="#local-serving-cpu-moderate">Local Serving (CPU - Moderate)</a></li>
|
||||
<li><a :data-current="activeSection === '#cloud-serving-gpu-blazing-fast'" class="reference internal" href="#cloud-serving-gpu-blazing-fast">Cloud Serving (GPU - Blazing Fast)</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
</div></div>
|
||||
</main>
|
||||
</div>
|
||||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Request Lifecycle | Arch Docs v0.3.22</title>
|
||||
<meta content="Request Lifecycle | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Request Lifecycle | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Request Lifecycle | Plano Docs v0.4</title>
|
||||
<meta content="Request Lifecycle | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Request Lifecycle | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/tech_overview/request_lifecycle.html" rel="canonical"/>
|
||||
<link href="./docs/resources/tech_overview/request_lifecycle.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../../search.html" rel="search" title="Search"/>
|
||||
<link href="error_target.html" rel="next" title="Error Target"/>
|
||||
<link href="model_serving.html" rel="prev" title="Model Serving"/>
|
||||
<link href="model_serving.html" rel="next" title="Bright Staff"/>
|
||||
<link href="tech_overview.html" rel="prev" title="Tech Overview"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,70 +75,65 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="error_target.html">Error Target</a></li>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_target.html">Prompt Target</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -164,155 +159,80 @@
|
|||
<div id="content" role="main">
|
||||
<section id="request-lifecycle">
|
||||
<span id="lifecycle-of-a-request"></span><h1>Request Lifecycle<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#request-lifecycle"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Below we describe the events in the lifecycle of a request passing through an Arch gateway instance. We first
|
||||
describe how Arch fits into the request path and then the internal events that take place following
|
||||
the arrival of a request at Arch from downstream clients. We follow the request until the corresponding
|
||||
<p>Below we describe the events in the lifecycle of a request passing through a Plano instance. We first
|
||||
describe how Plano fits into the request path and then the internal events that take place following
|
||||
the arrival of a request at Plano from downstream clients. We follow the request until the corresponding
|
||||
dispatch upstream and the response path.</p>
|
||||
<a class="reference internal image-reference" href="../../_images/network-topology-ingress-egress.jpg"><img alt="../../_images/network-topology-ingress-egress.jpg" class="align-center" src="../../_images/network-topology-ingress-egress.jpg" style="width: 100%;"/>
|
||||
<a class="reference internal image-reference" href="../../_images/network-topology-ingress-egress.png"><img alt="../../_images/network-topology-ingress-egress.png" class="align-center" src="../../_images/network-topology-ingress-egress.png" style="width: 100%;"/>
|
||||
</a>
|
||||
<section id="terminology">
|
||||
<h2>Terminology<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#terminology" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#terminology'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>We recommend that you get familiar with some of the <a class="reference internal" href="terminology.html#arch-terminology"><span class="std std-ref">terminology</span></a> used in Arch
|
||||
before reading this section.</p>
|
||||
</section>
|
||||
<section id="network-topology">
|
||||
<h2>Network topology<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#network-topology" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#network-topology'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>How a request flows through the components in a network (including Arch) depends on the network’s topology.
|
||||
Arch can be used in a wide variety of networking topologies. We focus on the inner operation of Arch below,
|
||||
but briefly we address how Arch relates to the rest of the network in this section.</p>
|
||||
<p>How a request flows through the components in a network (including Plano) depends on the network’s topology.
|
||||
Plano can be used in a wide variety of networking topologies. We focus on the inner operations of Plano below,
|
||||
but briefly we address how Plano relates to the rest of the network in this section.</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Downstream(Ingress)</strong> listeners take requests from upstream clients like a web UI or clients that forward
|
||||
prompts to you local application responses from the application flow back through Arch to the downstream.</p></li>
|
||||
prompts to you local application responses from the application flow back through Plano to the downstream.</p></li>
|
||||
<li><p><strong>Upstream(Egress)</strong> listeners take requests from the application and forward them to LLMs.</p></li>
|
||||
</ul>
|
||||
<a class="reference internal image-reference" href="../../_images/network-topology-ingress-egress.jpg"><img alt="../../_images/network-topology-ingress-egress.jpg" class="align-center" src="../../_images/network-topology-ingress-egress.jpg" style="width: 100%;"/>
|
||||
</a>
|
||||
<p>In practice, Arch can be deployed on the edge and as an internal load balancer between AI agents. A request path may
|
||||
traverse multiple Arch gateways:</p>
|
||||
<a class="reference internal image-reference" href="../../_images/network-topology-agent.jpg"><img alt="../../_images/network-topology-agent.jpg" class="align-center" src="../../_images/network-topology-agent.jpg" style="width: 100%;"/>
|
||||
</a>
|
||||
</section>
|
||||
<section id="high-level-architecture">
|
||||
<h2>High level architecture<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#high-level-architecture" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#high-level-architecture'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Arch is a set of <strong>two</strong> self-contained processes that are designed to run alongside your application servers
|
||||
(or on a separate server connected to your application servers via a network). The first process is designated
|
||||
to manage HTTP-level networking and connection management concerns (protocol management, request id generation,
|
||||
header sanitization, etc.), and the other process is for <strong>model serving</strong>, which helps Arch make intelligent
|
||||
decisions about the incoming prompts. The model server hosts the purpose-built LLMs to
|
||||
manage several critical, but undifferentiated, prompt related tasks on behalf of developers.</p>
|
||||
<p>The request processing path in Arch has three main parts:</p>
|
||||
<p>Plano is a set of <strong>two</strong> self-contained processes that are designed to run alongside your application servers
|
||||
(or on a separate server connected to your application servers via a network).</p>
|
||||
<p>The first process is designated to manage HTTP-level networking and connection management concerns (protocol management, request id generation, header sanitization, etc.), and the other process is a <strong>controller</strong>, which helps Plano make intelligent decisions about the incoming prompts. The controller hosts the purpose-built LLMs to manage several critical, but undifferentiated, prompt related tasks on behalf of developers.</p>
|
||||
<p>The request processing path in Plano has three main parts:</p>
|
||||
<ul class="simple">
|
||||
<li><p><a class="reference internal" href="listener.html#arch-overview-listeners"><span class="std std-ref">Listener subsystem</span></a> which handles <strong>downstream</strong> and <strong>upstream</strong> request
|
||||
processing. It is responsible for managing the downstream (ingress) and the upstream (egress) request
|
||||
lifecycle. The downstream and upstream HTTP/2 codec lives here.</p></li>
|
||||
<li><p><a class="reference internal" href="prompt.html#arch-overview-prompt-handling"><span class="std std-ref">Prompt handler subsystem</span></a> which is responsible for selecting and
|
||||
forwarding prompts <code class="docutils literal notranslate"><span class="pre">prompt_targets</span></code> and establishes the lifecycle of any <strong>upstream</strong> connection to a
|
||||
hosted endpoint that implements domain-specific business logic for incoming prompts. This is where knowledge
|
||||
of targets and endpoint health, load balancing and connection pooling exists.</p></li>
|
||||
<li><p><a class="reference internal" href="model_serving.html#model-serving"><span class="std std-ref">Model serving subsystem</span></a> which helps Arch make intelligent decisions about the
|
||||
incoming prompts. The model server is designed to call the purpose-built LLMs in Arch.</p></li>
|
||||
<li><p><a class="reference internal" href="../../concepts/listeners.html#plano-overview-listeners"><span class="std std-ref">Listener subsystem</span></a> which handles <strong>downstream</strong> and <strong>upstream</strong> request
|
||||
processing. It is responsible for managing the inbound(edge) and outbound(egress) request lifecycle. The downstream and upstream HTTP/2 codec lives here. This also includes the lifecycle of any <strong>upstream</strong> connection to an LLM provider or tool backend. The listenser subsystmem manages connection pools, load balancing, retries, and failover.</p></li>
|
||||
<li><p><a class="reference internal" href="model_serving.html#bright-staff"><span class="std std-ref">Bright Staff controller subsystem</span></a> is Plano’s memory-efficient, lightweight controller for agentic traffic. It sits inside the Plano data plane and makes real-time decisions about how prompts are handled, forwarded, and processed.</p></li>
|
||||
</ul>
|
||||
<p>The three subsystems are bridged with either the HTTP router filter, and the cluster manager subsystems of Envoy.</p>
|
||||
<p>Also, Arch utilizes <a class="reference external" href="https://blog.envoyproxy.io/envoy-threading-model-a8d44b922310" rel="nofollow noopener">Envoy event-based thread model<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>.
|
||||
A main thread is responsible for the server lifecycle, configuration processing, stats, etc. and some number of
|
||||
<a class="reference internal" href="threading_model.html#arch-overview-threading"><span class="std std-ref">worker threads</span></a> process requests. All threads operate around an event loop (<a class="reference external" href="https://libevent.org/" rel="nofollow noopener">libevent<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>)
|
||||
and any given downstream TCP connection will be handled by exactly one worker thread for its lifetime. Each worker
|
||||
thread maintains its own pool of TCP connections to upstream endpoints.</p>
|
||||
<p>These two subsystems are bridged with either the HTTP router filter, and the cluster manager subsystems of Envoy.</p>
|
||||
<p>Also, Plano utilizes <a class="reference external" href="https://blog.envoyproxy.io/envoy-threading-model-a8d44b922310" rel="nofollow noopener">Envoy event-based thread model<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>. A main thread is responsible for the server lifecycle, configuration processing, stats, etc. and some number of <a class="reference internal" href="threading_model.html#arch-overview-threading"><span class="std std-ref">worker threads</span></a> process requests. All threads operate around an event loop (<a class="reference external" href="https://libevent.org/" rel="nofollow noopener">libevent<svg fill="currentColor" height="1em" stroke="none" viewbox="0 96 960 960" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M188 868q-11-11-11-28t11-28l436-436H400q-17 0-28.5-11.5T360 336q0-17 11.5-28.5T400 296h320q17 0 28.5 11.5T760 336v320q0 17-11.5 28.5T720 696q-17 0-28.5-11.5T680 656V432L244 868q-11 11-28 11t-28-11Z"></path></svg></a>) and any given downstream TCP connection will be handled by exactly one worker thread for its lifetime. Each worker thread maintains its own pool of TCP connections to upstream endpoints.</p>
|
||||
<p>Worker threads rarely share state and operate in a trivially parallel fashion. This threading model
|
||||
enables scaling to very high core count CPUs.</p>
|
||||
</section>
|
||||
<section id="configuration">
|
||||
<h2>Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configuration" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configuration'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Today, only support a static bootstrap configuration file for simplicity today:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.1.0</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">ingress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10000</span>
|
||||
</span><span id="line-7"><span class="w"> </span><span class="nt">message_format</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai</span>
|
||||
</span><span id="line-8"><span class="w"> </span><span class="nt">timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
|
||||
</span><span id="line-9">
|
||||
</span><span id="line-10"><span class="c1"># Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way</span>
|
||||
</span><span id="line-11"><span class="nt">llm_providers</span><span class="p">:</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-13"><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-14"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-15">
|
||||
</span><span id="line-16"><span class="c1"># default system prompt used by all prompt targets</span>
|
||||
</span><span id="line-17"><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">You are a network assistant that just offers facts; not advice on manufacturers or purchasing decisions.</span>
|
||||
</span><span id="line-18">
|
||||
</span><span id="line-19"><span class="nt">prompt_guards</span><span class="p">:</span>
|
||||
</span><span id="line-20"><span class="w"> </span><span class="nt">input_guards</span><span class="p">:</span>
|
||||
</span><span id="line-21"><span class="w"> </span><span class="nt">jailbreak</span><span class="p">:</span>
|
||||
</span><span id="line-22"><span class="w"> </span><span class="nt">on_exception</span><span class="p">:</span>
|
||||
</span><span id="line-23"><span class="w"> </span><span class="nt">message</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Looks like you're curious about my abilities, but I can only provide assistance within my programmed parameters.</span>
|
||||
</span><span id="line-24">
|
||||
</span><span id="line-25"><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-26"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">information_extraction</span>
|
||||
</span><span id="line-27"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-28"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">handel all scenarios that are question and answer in nature. Like summarization, information extraction, etc.</span>
|
||||
</span><span id="line-29"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-30"><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</span><span id="line-31"><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/summary</span>
|
||||
</span><span id="line-32"><span class="w"> </span><span class="c1"># Arch uses the default LLM and treats the response from the endpoint as the prompt to send to the LLM</span>
|
||||
</span><span id="line-33"><span class="w"> </span><span class="nt">auto_llm_dispatch_on_response</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-34"><span class="w"> </span><span class="c1"># override system prompt for this prompt target</span>
|
||||
</span><span id="line-35"><span class="w"> </span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">You are a helpful information extraction assistant. Use the information that is provided to you.</span>
|
||||
</span><span id="line-36">
|
||||
</span><span id="line-37"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">reboot_network_device</span>
|
||||
</span><span id="line-38"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Reboot a specific network device</span>
|
||||
</span><span id="line-39"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-40"><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</span><span id="line-41"><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/action</span>
|
||||
</span><span id="line-42"><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</span><span id="line-43"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">device_id</span>
|
||||
</span><span id="line-44"><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">str</span>
|
||||
</span><span id="line-45"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Identifier of the network device to reboot.</span>
|
||||
</span><span id="line-46"><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-47"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">confirmation</span>
|
||||
</span><span id="line-48"><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">bool</span>
|
||||
</span><span id="line-49"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Confirmation flag to proceed with reboot.</span>
|
||||
</span><span id="line-50"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">false</span>
|
||||
</span><span id="line-51"><span class="w"> </span><span class="nt">enum</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">true</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">false</span><span class="p p-Indicator">]</span>
|
||||
</span><span id="line-52">
|
||||
</span><span id="line-53"><span class="c1"># Arch creates a round-robin load balancing between different endpoints, managed via the cluster subsystem.</span>
|
||||
</span><span id="line-54"><span class="nt">endpoints</span><span class="p">:</span>
|
||||
</span><span id="line-55"><span class="w"> </span><span class="nt">app_server</span><span class="p">:</span>
|
||||
</span><span id="line-56"><span class="w"> </span><span class="c1"># value could be ip address or a hostname with port</span>
|
||||
</span><span id="line-57"><span class="w"> </span><span class="c1"># this could also be a list of endpoints for load balancing</span>
|
||||
</span><span id="line-58"><span class="w"> </span><span class="c1"># for example endpoint: [ ip1:port, ip2:port ]</span>
|
||||
</span><span id="line-59"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">127.0.0.1:80</span>
|
||||
</span><span id="line-60"><span class="w"> </span><span class="c1"># max time to wait for a connection to be established</span>
|
||||
</span><span id="line-61"><span class="w"> </span><span class="nt">connect_timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.005s</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="request-flow-ingress">
|
||||
<h2>Request Flow (Ingress)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#request-flow-ingress" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#request-flow-ingress'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>A brief outline of the lifecycle of a request and response using the example configuration above:</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p><strong>TCP Connection Establishment</strong>:
|
||||
A TCP connection from downstream is accepted by an Arch listener running on a worker thread.
|
||||
A TCP connection from downstream is accepted by an Plano listener running on a worker thread.
|
||||
The listener filter chain provides SNI and other pre-TLS information. The transport socket, typically TLS,
|
||||
decrypts incoming data for processing.</p></li>
|
||||
<li><p><strong>Prompt Guardrails Check</strong>:
|
||||
Arch first checks the incoming prompts for guardrails such as jailbreak attempts. This ensures
|
||||
that harmful or unwanted behaviors are detected early in the request processing pipeline.</p></li>
|
||||
<li><p><strong>Intent Matching</strong>:
|
||||
The decrypted data stream is de-framed by the HTTP/2 codec in Arch’s HTTP connection manager. Arch performs
|
||||
intent matching via is <strong>prompt-handler</strong> subsystem using the name and description of the defined prompt targets,
|
||||
determining which endpoint should handle the prompt.</p></li>
|
||||
<li><p><strong>Parameter Gathering with Arch-Function</strong>:
|
||||
If a prompt target requires specific parameters, Arch engages Arch-FC to extract the necessary details
|
||||
from the incoming prompt(s). This process gathers the critical information needed for downstream API calls.</p></li>
|
||||
<li><p><strong>API Call Execution</strong>:
|
||||
Arch routes the prompt to the appropriate backend API or function call. If an endpoint cluster is identified,
|
||||
load balancing is performed, circuit breakers are checked, and the request is proxied to the upstream endpoint.</p></li>
|
||||
<li><p><strong>Default Summarization by Upstream LLM</strong>:
|
||||
By default, if no specific endpoint processing is needed, the prompt is sent to an upstream LLM for summarization.
|
||||
This ensures that responses are concise and relevant, enhancing user experience in RAG (Retrieval Augmented Generation)
|
||||
and agentic applications.</p></li>
|
||||
</ol>
|
||||
<ol class="arabic simple" start="3">
|
||||
<li><p><strong>Routing Decision (Agent vs Prompt Target)</strong>:
|
||||
The decrypted data stream is de-framed by the HTTP/2 codec in Plano’s HTTP connection manager. Plano performs
|
||||
intent matching (via the Bright Staff controller and prompt-handling logic) using the configured agents and
|
||||
<a class="reference internal" href="../../concepts/prompt_target.html#prompt-target"><span class="std std-ref">prompt targets</span></a>, determining whether this request should be handled by an agent workflow
|
||||
(with optional <a class="reference internal" href="../../concepts/filter_chain.html#filter-chain"><span class="std std-ref">Filter Chains</span></a>) or by a deterministic prompt target.</p></li>
|
||||
</ol>
|
||||
<p>4a. <strong>Agent Path: Orchestration and Filter Chains</strong></p>
|
||||
<blockquote>
|
||||
<div><p>If the request is routed to an <strong>agent</strong>, Plano executes any attached <a class="reference internal" href="../../concepts/filter_chain.html#filter-chain"><span class="std std-ref">Filter Chains</span></a> first. These filters can apply guardrails, rewrite prompts, or enrich context (for example, RAG retrieval) before the agent runs. Once filters complete, the Bright Staff controller orchestrates which downstream tools, APIs, or LLMs the agent should call and in what sequence.</p>
|
||||
<ul class="simple">
|
||||
<li><p>Plano may call one or more backend APIs or tools on behalf of the agent.</p></li>
|
||||
<li><p>If an endpoint cluster is identified, load balancing is performed, circuit breakers are checked, and the request is proxied to the appropriate upstream endpoint.</p></li>
|
||||
<li><p>If no specific endpoint is required, the prompt is sent to an upstream LLM using Plano’s model proxy for
|
||||
completion or summarization.</p></li>
|
||||
</ul>
|
||||
<p>For more on agent workflows and orchestration, see <a class="reference internal" href="../../concepts/prompt_target.html#prompt-target"><span class="std std-ref">Prompt Targets and Agents</span></a> and
|
||||
<a class="reference internal" href="../../concepts/filter_chain.html#filter-chain"><span class="std std-ref">Agent Filter Chains</span></a>.</p>
|
||||
</div></blockquote>
|
||||
<p>4b. <strong>Prompt Target Path: Deterministic Tool/API Calls</strong></p>
|
||||
<blockquote>
|
||||
<div><p>If the request is routed to a <strong>prompt target</strong>, Plano treats it as a deterministic, task-specific call.
|
||||
Plano engages its function-calling and parameter-gathering capabilities to extract the necessary details
|
||||
from the incoming prompt(s) and produce the structured inputs your backend expects.</p>
|
||||
<ul class="simple">
|
||||
<li><p><strong>Parameter Gathering</strong>: Plano extracts and validates parameters defined on the prompt target (for example,
|
||||
currency symbols, dates, or entity identifiers) so your backend does not need to parse natural language.</p></li>
|
||||
<li><p><strong>API Call Execution</strong>: Plano then routes the call to the configured backend endpoint. If an endpoint cluster is identified, load balancing and circuit-breaker checks are applied before proxying the request upstream.</p></li>
|
||||
</ul>
|
||||
<p>For more on how to design and configure prompt targets, see <a class="reference internal" href="../../concepts/prompt_target.html#prompt-target"><span class="std std-ref">Prompt Target</span></a>.</p>
|
||||
</div></blockquote>
|
||||
<ol class="arabic simple" start="5">
|
||||
<li><p><strong>Error Handling and Forwarding</strong>:
|
||||
Errors encountered during processing, such as failed function calls or guardrail detections, are forwarded to
|
||||
designated error targets. Error details are communicated through specific headers to the application:</p>
|
||||
|
|
@ -329,26 +249,21 @@ Responses pass through HTTP filters in reverse order, ensuring any necessary pro
|
|||
</section>
|
||||
<section id="request-flow-egress">
|
||||
<h2>Request Flow (Egress)<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#request-flow-egress" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#request-flow-egress'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>A brief outline of the lifecycle of a request and response in the context of egress traffic from an application to Large Language Models (LLMs) via Arch:</p>
|
||||
<p>A brief outline of the lifecycle of a request and response in the context of egress traffic from an application to Large Language Models (LLMs) via Plano:</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p><strong>HTTP Connection Establishment to LLM</strong>:
|
||||
Arch initiates an HTTP connection to the upstream LLM service. This connection is handled by Arch’s egress listener
|
||||
running on a worker thread. The connection typically uses a secure transport protocol such as HTTPS, ensuring the
|
||||
prompt data is encrypted before being sent to the LLM service.</p></li>
|
||||
Plano initiates an HTTP connection to the upstream LLM service. This connection is handled by Plano’s egress listener running on a worker thread. The connection typically uses a secure transport protocol such as HTTPS, ensuring the prompt data is encrypted before being sent to the LLM service.</p></li>
|
||||
<li><p><strong>Rate Limiting</strong>:
|
||||
Before sending the request to the LLM, Arch applies rate-limiting policies to ensure that the upstream LLM service
|
||||
is not overwhelmed by excessive traffic. Rate limits are enforced per client or service, ensuring fair usage and
|
||||
preventing accidental or malicious overload. If the rate limit is exceeded, Arch may return an appropriate HTTP
|
||||
error (e.g., 429 Too Many Requests) without sending the prompt to the LLM.</p></li>
|
||||
Before sending the request to the LLM, Plano applies rate-limiting policies to ensure that the upstream LLM service is not overwhelmed by excessive traffic. Rate limits are enforced per client or service, ensuring fair usage and preventing accidental or malicious overload. If the rate limit is exceeded, Plano may return an appropriate HTTP error (e.g., 429 Too Many Requests) without sending the prompt to the LLM.</p></li>
|
||||
<li><p><strong>Seamless Request Transformation and Smart Routing</strong>:
|
||||
After rate limiting, Plano normalizes the outgoing request into a provider-agnostic shape and applies smart routing decisions using the configured <a class="reference internal" href="../../concepts/llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">LLM Providers</span></a>. This includes translating client-specific conventions into a unified OpenAI-style contract, enriching or overriding parameters (for example, temperature or max tokens) based on policy, and choosing the best target model or provider using <a class="reference internal" href="../../concepts/llm_providers/llm_providers.html#llm-providers"><span class="std std-ref">model-based, alias-based, or preference-aligned routing</span></a>.</p></li>
|
||||
<li><p><strong>Load Balancing to (hosted) LLM Endpoints</strong>:
|
||||
After passing the rate-limiting checks, Arch routes the prompt to the appropriate LLM endpoint.
|
||||
If multiple LLM providers instances are available, load balancing is performed to distribute traffic evenly
|
||||
across the instances. Arch checks the health of the LLM endpoints using circuit breakers and health checks,
|
||||
After smart routing selects the target provider/model, Plano routes the prompt to the appropriate LLM endpoint.
|
||||
If multiple LLM provider instances are available, load balancing is performed to distribute traffic evenly
|
||||
across the instances. Plano checks the health of the LLM endpoints using circuit breakers and health checks,
|
||||
ensuring that the prompt is only routed to a healthy, responsive instance.</p></li>
|
||||
<li><p><strong>Response Reception and Forwarding</strong>:
|
||||
Once the LLM processes the prompt, Arch receives the response from the LLM service. The response is typically a
|
||||
generated text, completion, or summarization. Upon reception, Arch decrypts (if necessary) and handles the response,
|
||||
passing it through any egress processing pipeline defined by the application, such as logging or additional response filtering.</p></li>
|
||||
Once the LLM processes the prompt, Plano receives the response from the LLM service. The response is typically a generated text, completion, or summarization. Upon reception, Plano decrypts (if necessary) and handles the response, passing it through any egress processing pipeline defined by the application, such as logging or additional response filtering.</p></li>
|
||||
</ol>
|
||||
<section id="post-request-processing">
|
||||
<h3>Post-request processing<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#post-request-processing" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#post-request-processing'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h3>
|
||||
|
|
@ -364,19 +279,75 @@ processing request headers and then finalized by the HCM during post-request pro
|
|||
</ul>
|
||||
</section>
|
||||
</section>
|
||||
<section id="configuration">
|
||||
<h2>Configuration<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#configuration" x-intersect.margin.0%.0%.-70%.0%="activeSection = '#configuration'"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h2>
|
||||
<p>Today, only support a static bootstrap configuration file for simplicity today:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><code><span id="line-1"><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v0.2.0</span>
|
||||
</span><span id="line-2">
|
||||
</span><span id="line-3"><span class="nt">listeners</span><span class="p">:</span>
|
||||
</span><span id="line-4"><span class="w"> </span><span class="nt">ingress_traffic</span><span class="p">:</span>
|
||||
</span><span id="line-5"><span class="w"> </span><span class="nt">address</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.0.0.0</span>
|
||||
</span><span id="line-6"><span class="w"> </span><span class="nt">port</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10000</span>
|
||||
</span><span id="line-7">
|
||||
</span><span id="line-8"><span class="c1"># Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way</span>
|
||||
</span><span id="line-9"><span class="nt">model_providers</span><span class="p">:</span>
|
||||
</span><span id="line-10"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">access_key</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">$OPENAI_API_KEY</span>
|
||||
</span><span id="line-11"><span class="w"> </span><span class="nt">model</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openai/gpt-4o</span>
|
||||
</span><span id="line-12"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-13">
|
||||
</span><span id="line-14"><span class="nt">prompt_targets</span><span class="p">:</span>
|
||||
</span><span id="line-15"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">information_extraction</span>
|
||||
</span><span id="line-16"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-17"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">handel all scenarios that are question and answer in nature. Like summarization, information extraction, etc.</span>
|
||||
</span><span id="line-18"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-19"><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</span><span id="line-20"><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/summary</span>
|
||||
</span><span id="line-21"><span class="w"> </span><span class="c1"># Arch uses the default LLM and treats the response from the endpoint as the prompt to send to the LLM</span>
|
||||
</span><span id="line-22"><span class="w"> </span><span class="nt">auto_llm_dispatch_on_response</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-23"><span class="w"> </span><span class="c1"># override system prompt for this prompt target</span>
|
||||
</span><span id="line-24"><span class="w"> </span><span class="nt">system_prompt</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">You are a helpful information extraction assistant. Use the information that is provided to you.</span>
|
||||
</span><span id="line-25">
|
||||
</span><span id="line-26"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">reboot_network_device</span>
|
||||
</span><span id="line-27"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Reboot a specific network device</span>
|
||||
</span><span id="line-28"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span>
|
||||
</span><span id="line-29"><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">app_server</span>
|
||||
</span><span id="line-30"><span class="w"> </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/agent/action</span>
|
||||
</span><span id="line-31"><span class="w"> </span><span class="nt">parameters</span><span class="p">:</span>
|
||||
</span><span id="line-32"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">device_id</span>
|
||||
</span><span id="line-33"><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">str</span>
|
||||
</span><span id="line-34"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Identifier of the network device to reboot.</span>
|
||||
</span><span id="line-35"><span class="w"> </span><span class="nt">required</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
|
||||
</span><span id="line-36"><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">confirmation</span>
|
||||
</span><span id="line-37"><span class="w"> </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">bool</span>
|
||||
</span><span id="line-38"><span class="w"> </span><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Confirmation flag to proceed with reboot.</span>
|
||||
</span><span id="line-39"><span class="w"> </span><span class="nt">default</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">false</span>
|
||||
</span><span id="line-40"><span class="w"> </span><span class="nt">enum</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">true</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">false</span><span class="p p-Indicator">]</span>
|
||||
</span><span id="line-41">
|
||||
</span><span id="line-42"><span class="c1"># Arch creates a round-robin load balancing between different endpoints, managed via the cluster subsystem.</span>
|
||||
</span><span id="line-43"><span class="nt">endpoints</span><span class="p">:</span>
|
||||
</span><span id="line-44"><span class="w"> </span><span class="nt">app_server</span><span class="p">:</span>
|
||||
</span><span id="line-45"><span class="w"> </span><span class="c1"># value could be ip address or a hostname with port</span>
|
||||
</span><span id="line-46"><span class="w"> </span><span class="c1"># this could also be a list of endpoints for load balancing</span>
|
||||
</span><span id="line-47"><span class="w"> </span><span class="c1"># for example endpoint: [ ip1:port, ip2:port ]</span>
|
||||
</span><span id="line-48"><span class="w"> </span><span class="nt">endpoint</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">127.0.0.1:80</span>
|
||||
</span><span id="line-49"><span class="w"> </span><span class="c1"># max time to wait for a connection to be established</span>
|
||||
</span><span id="line-50"><span class="w"> </span><span class="nt">connect_timeout</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0.005s</span>
|
||||
</span></code></pre></div>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="model_serving.html">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="tech_overview.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Model Serving
|
||||
Tech Overview
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="error_target.html">
|
||||
Error Target
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="model_serving.html">
|
||||
Bright Staff
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
|
|
@ -385,15 +356,14 @@ processing request headers and then finalized by the HCM during post-request pro
|
|||
</div></div><aside class="hidden text-sm xl:block" id="right-sidebar">
|
||||
<div class="sticky top-16 -mt-10 max-h-[calc(100vh-5rem)] overflow-y-auto pt-6 space-y-2"><p class="font-medium">On this page</p>
|
||||
<ul>
|
||||
<li><a :data-current="activeSection === '#terminology'" class="reference internal" href="#terminology">Terminology</a></li>
|
||||
<li><a :data-current="activeSection === '#network-topology'" class="reference internal" href="#network-topology">Network topology</a></li>
|
||||
<li><a :data-current="activeSection === '#high-level-architecture'" class="reference internal" href="#high-level-architecture">High level architecture</a></li>
|
||||
<li><a :data-current="activeSection === '#configuration'" class="reference internal" href="#configuration">Configuration</a></li>
|
||||
<li><a :data-current="activeSection === '#request-flow-ingress'" class="reference internal" href="#request-flow-ingress">Request Flow (Ingress)</a></li>
|
||||
<li><a :data-current="activeSection === '#request-flow-egress'" class="reference internal" href="#request-flow-egress">Request Flow (Egress)</a><ul>
|
||||
<li><a :data-current="activeSection === '#post-request-processing'" class="reference internal" href="#post-request-processing">Post-request processing</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a :data-current="activeSection === '#configuration'" class="reference internal" href="#configuration">Configuration</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
|
|
@ -402,12 +372,12 @@ processing request headers and then finalized by the HCM during post-request pro
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Tech Overview | Arch Docs v0.3.22</title>
|
||||
<meta content="Tech Overview | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Tech Overview | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Tech Overview | Plano Docs v0.4</title>
|
||||
<meta content="Tech Overview | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Tech Overview | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/tech_overview/tech_overview.html" rel="canonical"/>
|
||||
<link href="./docs/resources/tech_overview/tech_overview.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../../search.html" rel="search" title="Search"/>
|
||||
<link href="terminology.html" rel="next" title="Terminology"/>
|
||||
<link href="../../get_started/quickstart.html" rel="prev" title="Quickstart"/>
|
||||
<link href="request_lifecycle.html" rel="next" title="Request Lifecycle"/>
|
||||
<link href="../../guides/state.html" rel="prev" title="Conversational State"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,70 +75,65 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="current reference internal expandable" href="#">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="error_target.html">Error Target</a></li>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_target.html">Prompt Target</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="current reference internal expandable" href="#">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -165,56 +160,31 @@
|
|||
<span id="id1"></span><h1>Tech Overview<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#tech-overview"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<div class="toctree-wrapper compound">
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="listener.html">Listener</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="listener.html#downstream-ingress">Downstream (Ingress)</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="listener.html#upstream-egress">Upstream (Egress)</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="listener.html#configure-listener">Configure Listener</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="prompt.html">Prompts</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="prompt.html#messages">Messages</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="prompt.html#prompt-guard">Prompt Guard</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="prompt.html#prompt-targets">Prompt Targets</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="prompt.html#prompting-llms">Prompting LLMs</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="model_serving.html">Model Serving</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html#local-serving-cpu-moderate">Local Serving (CPU - Moderate)</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html#cloud-serving-gpu-blazing-fast">Cloud Serving (GPU - Blazing Fast)</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="request_lifecycle.html">Request Lifecycle</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html#terminology">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html#network-topology">Network topology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html#high-level-architecture">High level architecture</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html#configuration">Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html#request-flow-ingress">Request Flow (Ingress)</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html#request-flow-egress">Request Flow (Egress)</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html#configuration">Configuration</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="error_target.html">Error Target</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="error_target.html#key-concepts">Key Concepts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="error_target.html#error-header-example">Error Header Example</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="error_target.html#best-practices-and-tips">Best Practices and Tips</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../../get_started/quickstart.html">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../../guides/state.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Quickstart
|
||||
Conversational State
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="terminology.html">
|
||||
Terminology
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="request_lifecycle.html">
|
||||
Request Lifecycle
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
|
|
@ -226,12 +196,12 @@
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
|
|
@ -7,19 +7,19 @@
|
|||
<meta content="white" media="(prefers-color-scheme: light)" name="theme-color"/>
|
||||
<meta content="black" media="(prefers-color-scheme: dark)" name="theme-color"/>
|
||||
<meta content="width=device-width, initial-scale=1" name="viewport"/>
|
||||
<title>Threading Model | Arch Docs v0.3.22</title>
|
||||
<meta content="Threading Model | Arch Docs v0.3.22" property="og:title"/>
|
||||
<meta content="Threading Model | Arch Docs v0.3.22" name="twitter:title"/>
|
||||
<title>Threading Model | Plano Docs v0.4</title>
|
||||
<meta content="Threading Model | Plano Docs v0.4" property="og:title"/>
|
||||
<meta content="Threading Model | Plano Docs v0.4" name="twitter:title"/>
|
||||
<link href="../../_static/pygments.css?v=466e7b45" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/theme.css?v=42baaae4" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/_static/custom.css" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/sphinx-design.min.css?v=95c83b7e" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/css/custom.css?v=2929376a" rel="stylesheet" type="text/css"/>
|
||||
<link href="../../_static/awesome-sphinx-design.css?v=15e0fffa" rel="stylesheet" type="text/css"/>
|
||||
<link href="./docs/concepts/tech_overview/threading_model.html" rel="canonical"/>
|
||||
<link href="./docs/resources/tech_overview/threading_model.html" rel="canonical"/>
|
||||
<link href="../../_static/favicon.ico" rel="icon"/>
|
||||
<link href="../../search.html" rel="search" title="Search"/>
|
||||
<link href="listener.html" rel="next" title="Listener"/>
|
||||
<link href="terminology.html" rel="prev" title="Terminology"/>
|
||||
<link href="../deployment.html" rel="next" title="Deployment"/>
|
||||
<link href="model_serving.html" rel="prev" title="Bright Staff"/>
|
||||
<script>
|
||||
<!-- Prevent Flash of wrong theme -->
|
||||
const userPreference = localStorage.getItem('darkMode');
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
</a><header class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a class="flex items-center mr-6" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="24" src="../../_static/favicon.ico" width="24"/><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button @click="showSidebar = true" class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden" type="button">
|
||||
<svg aria-hidden="true" fill="currentColor" height="24" viewbox="0 96 960 960" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M152.587 825.087q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440Zm0-203.587q-19.152 0-32.326-13.174T107.087 576q0-19.152 13.174-32.326t32.326-13.174h320q19.152 0 32.326 13.174T518.087 576q0 19.152-13.174 32.326T472.587 621.5h-320Zm0-203.587q-19.152 0-32.326-13.174t-13.174-32.326q0-19.152 13.174-32.326t32.326-13.174h440q19.152 0 32.326 13.174t13.174 32.326q0 19.152-13.174 32.326t-32.326 13.174h-440ZM708.913 576l112.174 112.174q12.674 12.674 12.674 31.826t-12.674 31.826Q808.413 764.5 789.261 764.5t-31.826-12.674l-144-144Q600 594.391 600 576t13.435-31.826l144-144q12.674-12.674 31.826-12.674t31.826 12.674q12.674 12.674 12.674 31.826t-12.674 31.826L708.913 576Z"></path>
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<a href="https://github.com/katanemo/plano" rel="noopener nofollow" title="Visit repository on GitHub">
|
||||
<div class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg fill="currentColor" height="26px" style="margin-top:-2px;display:inline" viewbox="0 0 45 44" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor" fill-rule="evenodd"></path></svg>
|
||||
</div>
|
||||
|
|
@ -75,70 +75,65 @@
|
|||
</header>
|
||||
<div class="flex-1"><div class="container flex-1 items-start md:grid md:grid-cols-[220px_minmax(0,1fr)] md:gap-6 lg:grid-cols-[240px_minmax(0,1fr)] lg:gap-10"><aside :aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }" class="fixed inset-y-0 left-0 md:top-14 z-50 md:z-30 bg-background md:bg-transparent transition-all duration-100 -translate-x-full md:translate-x-0 ml-0 p-6 md:p-0 md:-ml-2 md:h-[calc(100vh-3.5rem)] w-5/6 md:w-full shrink-0 overflow-y-auto border-r border-border md:sticky" id="left-sidebar">
|
||||
<a class="!justify-start text-sm md:!hidden bg-background" href="../../index.html">
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img alt="Logo" class="mr-2 dark:invert" height="16" src="../../_static/favicon.ico" width="16"/><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6">
|
||||
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async="" src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="error_target.html">Error Target</a></li>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../concepts/llm_providers/llm_providers.html">Model (LLM) Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../llm_providers/llm_providers.html">LLM Providers<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../prompt_target.html">Prompt Target</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../concepts/prompt_target.html">Prompt Target</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="../../guides/observability/observability.html">Observability<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current" x-data="{ expanded: $el.classList.contains('current') ? true : false }"><a :class="{ 'expanded' : expanded }" @click="expanded = !expanded" class="reference internal expandable" href="tech_overview.html">Tech Overview<button @click.prevent.stop="expanded = !expanded" type="button"><span class="sr-only"></span><svg fill="currentColor" height="18px" stroke="none" viewbox="0 0 24 24" width="18px" xmlns="http://www.w3.org/2000/svg"><path d="M10 6L8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"></path></svg></button></a><ul class="current" x-show="expanded">
|
||||
<li class="toctree-l2"><a class="reference internal" href="request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2 current"><a class="current reference internal" href="#">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
@ -153,7 +148,7 @@
|
|||
<div class="w-full min-w-0 mx-auto">
|
||||
<nav aria-label="breadcrumbs" class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground" href="../../index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg aria-label="Home" class="md:hidden" fill="currentColor" height="18" stroke="none" viewbox="0 96 960 960" width="18" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M240 856h120V616h240v240h120V496L480 316 240 496v360Zm-80 80V456l320-240 320 240v480H520V696h-80v240H160Zm320-350Z"></path>
|
||||
</svg>
|
||||
|
|
@ -164,14 +159,14 @@
|
|||
<div id="content" role="main">
|
||||
<section id="threading-model">
|
||||
<span id="arch-overview-threading"></span><h1>Threading Model<a @click.prevent="window.navigator.clipboard.writeText($el.href); $el.setAttribute('data-tooltip', 'Copied!'); setTimeout(() => $el.setAttribute('data-tooltip', 'Copy link to this element'), 2000)" aria-label="Copy link to this element" class="headerlink" data-tooltip="Copy link to this element" href="#threading-model"><svg height="1em" viewbox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"></path></svg></a></h1>
|
||||
<p>Arch builds on top of Envoy’s single process with multiple threads architecture.</p>
|
||||
<p>Plano builds on top of Envoy’s single process with multiple threads architecture.</p>
|
||||
<p>A single <em>primary</em> thread controls various sporadic coordination tasks while some number of <em>worker</em>
|
||||
threads perform filtering, and forwarding.</p>
|
||||
<p>Once a connection is accepted, the connection spends the rest of its lifetime bound to a single worker
|
||||
thread. All the functionality around prompt handling from a downstream client is handled in a separate worker thread.
|
||||
This allows the majority of Arch to be largely single threaded (embarrassingly parallel) with a small amount
|
||||
This allows the majority of Plano to be largely single threaded (embarrassingly parallel) with a small amount
|
||||
of more complex code handling coordination between the worker threads.</p>
|
||||
<p>Generally, Arch is written to be 100% non-blocking.</p>
|
||||
<p>Generally, Plano is written to be 100% non-blocking.</p>
|
||||
<div class="admonition tip">
|
||||
<p class="admonition-title">Tip</p>
|
||||
<p>For most workloads we recommend configuring the number of worker threads to be equal to the number of
|
||||
|
|
@ -180,16 +175,16 @@ hardware threads on the machine.</p>
|
|||
</section>
|
||||
</div><div class="flex justify-between items-center pt-6 mt-12 border-t border-border gap-4">
|
||||
<div class="mr-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="terminology.html">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="model_serving.html">
|
||||
<svg class="mr-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="15 18 9 12 15 6"></polyline>
|
||||
</svg>
|
||||
Terminology
|
||||
Bright Staff
|
||||
</a>
|
||||
</div>
|
||||
<div class="ml-auto">
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="listener.html">
|
||||
Listener
|
||||
<a class="inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors border border-input hover:bg-accent hover:text-accent-foreground py-2 px-4" href="../deployment.html">
|
||||
Deployment
|
||||
<svg class="ml-2 h-4 w-4" fill="none" height="24" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" viewbox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||||
<polyline points="9 18 15 12 9 6"></polyline>
|
||||
</svg>
|
||||
|
|
@ -201,12 +196,12 @@ hardware threads on the machine.</p>
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
<script src="../../_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="../../_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="../../_static/theme.js?v=073f68d9"></script>
|
||||
61
search.html
61
search.html
|
|
@ -12,13 +12,13 @@
|
|||
<meta name="theme-color" media="(prefers-color-scheme: light)" content="white" />
|
||||
<meta name="theme-color" media="(prefers-color-scheme: dark)" content="black" />
|
||||
|
||||
<title>Search | Arch Docs v0.3.22</title>
|
||||
<meta property="og:title" content="Search | Arch Docs v0.3.22" />
|
||||
<meta name="twitter:title" content="Search | Arch Docs v0.3.22" />
|
||||
<title>Search | Plano Docs v0.4</title>
|
||||
<meta property="og:title" content="Search | Plano Docs v0.4" />
|
||||
<meta name="twitter:title" content="Search | Plano Docs v0.4" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=466e7b45" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/theme.css?v=42baaae4" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/_static/custom.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/custom.css?v=2929376a" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/awesome-sphinx-design.css?v=15e0fffa" />
|
||||
<link rel="canonical" href="./docs/search.html" />
|
||||
<link rel="icon" href="_static/favicon.ico" />
|
||||
|
|
@ -44,7 +44,7 @@
|
|||
class="sticky top-0 z-40 w-full border-b shadow-sm border-border supports-backdrop-blur:bg-background/60 bg-background/95 backdrop-blur"><div class="container flex items-center h-14">
|
||||
<div class="hidden mr-4 md:flex">
|
||||
<a href="index.html" class="flex items-center mr-6">
|
||||
<img height="24" width="24" class="mr-2 dark:invert" src="_static/favicon.ico" alt="Logo" /><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img height="24" width="24" class="mr-2 dark:invert" src="_static/favicon.ico" alt="Logo" /><span class="hidden font-bold sm:inline-block text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a></div><button
|
||||
class="inline-flex items-center justify-center h-10 px-0 py-2 mr-2 text-base font-medium transition-colors rounded-md hover:text-accent-foreground hover:bg-transparent md:hidden"
|
||||
type="button" @click="showSidebar = true">
|
||||
|
|
@ -75,7 +75,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<nav class="flex items-center space-x-1">
|
||||
<a href="https://github.com/katanemo/arch" title="Visit repository on GitHub" rel="noopener nofollow">
|
||||
<a href="https://github.com/katanemo/plano" title="Visit repository on GitHub" rel="noopener nofollow">
|
||||
<div
|
||||
class="inline-flex items-center justify-center px-0 text-sm font-medium transition-colors rounded-md disabled:opacity-50 disabled:pointer-events-none hover:bg-accent hover:text-accent-foreground h-9 w-9">
|
||||
<svg height="26px" style="margin-top:-2px;display:inline" viewBox="0 0 45 44" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M22.477.927C10.485.927.76 10.65.76 22.647c0 9.596 6.223 17.736 14.853 20.608 1.087.2 1.483-.47 1.483-1.047 0-.516-.019-1.881-.03-3.693-6.04 1.312-7.315-2.912-7.315-2.912-.988-2.51-2.412-3.178-2.412-3.178-1.972-1.346.149-1.32.149-1.32 2.18.154 3.327 2.24 3.327 2.24 1.937 3.318 5.084 2.36 6.321 1.803.197-1.403.759-2.36 1.379-2.903-4.823-.548-9.894-2.412-9.894-10.734 0-2.37.847-4.31 2.236-5.828-.224-.55-.969-2.759.214-5.748 0 0 1.822-.584 5.972 2.226 1.732-.482 3.59-.722 5.437-.732 1.845.01 3.703.25 5.437.732 4.147-2.81 5.967-2.226 5.967-2.226 1.185 2.99.44 5.198.217 5.748 1.392 1.517 2.232 3.457 2.232 5.828 0 8.344-5.078 10.18-9.916 10.717.779.67 1.474 1.996 1.474 4.021 0 2.904-.027 5.247-.027 5.96 0 .58.392 1.256 1.493 1.044C37.981 40.375 44.2 32.24 44.2 22.647c0-11.996-9.726-21.72-21.722-21.72" fill="currentColor"/></svg>
|
||||
|
|
@ -107,42 +107,35 @@
|
|||
:aria-hidden="!showSidebar" :class="{ 'translate-x-0': showSidebar }">
|
||||
|
||||
<a href="index.html" class="!justify-start text-sm md:!hidden bg-background">
|
||||
<img height="16" width="16" class="mr-2 dark:invert" src="_static/favicon.ico" alt="Logo" /><span class="font-bold text-clip whitespace-nowrap">Arch Docs v0.3.22</span>
|
||||
<img height="16" width="16" class="mr-2 dark:invert" src="_static/favicon.ico" alt="Logo" /><span class="font-bold text-clip whitespace-nowrap">Plano Docs v0.4</span>
|
||||
</a>
|
||||
|
||||
<div class="relative overflow-hidden md:overflow-auto my-4 md:my-0 h-[calc(100vh-8rem)] md:h-auto">
|
||||
<div class="overflow-y-auto h-full w-full relative pr-6"><!-- _templates/analytics.html -->
|
||||
|
||||
<!-- Google tag (gtag.js) -->
|
||||
<script async src="https://www.googletagmanager.com/gtag/js?id=G-K2LXXSX6HB"></script>
|
||||
<script async src="https://www.googletagmanager.com/gtag/js?id=G-EH2VW19FXE"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-K2LXXSX6HB');
|
||||
gtag('config', 'G-EH2VW19FXE');
|
||||
</script>
|
||||
<nav class="table w-full min-w-full my-6 lg:my-8">
|
||||
<p class="caption" role="heading"><span class="caption-text">Get Started</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/intro_to_arch.html">Intro to Arch</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/intro_to_plano.html">Intro to Plano</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/quickstart.html">Quickstart</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="get_started/quickstart.html#next-steps">Next Steps</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Concepts</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/tech_overview/tech_overview.html">Tech Overview</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/terminology.html">Terminology</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/listener.html">Listener</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/prompt.html">Prompts</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/model_serving.html">Model Serving</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/tech_overview/error_target.html">Error Target</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/llm_providers/llm_providers.html">LLM Providers</a><ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/listeners.html">Listeners</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/agents.html">Agents</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/filter_chain.html">Filter Chains</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="concepts/llm_providers/llm_providers.html">Model (LLM) Providers</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/llm_providers/supported_providers.html">Supported Providers & Configuration</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/llm_providers/client_libraries.html">Client Libraries</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="concepts/llm_providers/model_aliases.html">Model Aliases</a></li>
|
||||
|
|
@ -152,27 +145,29 @@
|
|||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Guides</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/prompt_guard.html">Prompt Guard</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/agent_routing.html">Agent Routing and Hand Off</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/orchestration.html">Orchestration</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/llm_router.html">LLM Routing</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/function_calling.html">Function Calling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/observability/observability.html">Observability</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="guides/observability/tracing.html">Tracing</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="guides/observability/monitoring.html">Monitoring</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="guides/observability/access_logging.html">Access Logging</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Build with Arch</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="build_with_arch/agent.html">Agentic Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="build_with_arch/rag.html">RAG Apps</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="build_with_arch/multi_turn.html">Multi-Turn</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/prompt_guard.html">Guardrails</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="guides/state.html">Conversational State</a></li>
|
||||
</ul>
|
||||
<p class="caption" role="heading"><span class="caption-text">Resources</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="resources/tech_overview/tech_overview.html">Tech Overview</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="resources/tech_overview/request_lifecycle.html">Request Lifecycle</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="resources/tech_overview/model_serving.html">Bright Staff</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="resources/tech_overview/threading_model.html">Threading Model</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="resources/deployment.html">Deployment</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="resources/configuration_reference.html">Configuration Reference</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="resources/llms_txt.html">llms.txt</a></li>
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
|
|
@ -193,7 +188,7 @@
|
|||
class="flex items-center mb-4 space-x-1 text-sm text-muted-foreground">
|
||||
<a class="overflow-hidden text-ellipsis whitespace-nowrap hover:text-foreground"
|
||||
href="index.html">
|
||||
<span class="hidden md:inline">Arch Docs v0.3.22</span>
|
||||
<span class="hidden md:inline">Plano Docs v0.4</span>
|
||||
<svg xmlns="http://www.w3.org/2000/svg"
|
||||
height="18"
|
||||
width="18"
|
||||
|
|
@ -221,14 +216,14 @@
|
|||
</div><footer class="py-6 border-t border-border md:py-0">
|
||||
<div class="container flex flex-col items-center justify-between gap-4 md:h-24 md:flex-row">
|
||||
<div class="flex flex-col items-center gap-4 px-8 md:flex-row md:gap-2 md:px-0">
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 23, 2025. </p>
|
||||
<p class="text-sm leading-loose text-center text-muted-foreground md:text-left">© 2025, Katanemo Labs, Inc Last updated: Dec 24, 2025. </p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
|
||||
|
||||
<script src="_static/documentation_options.js?v=3bad885e"></script>
|
||||
<script src="_static/documentation_options.js?v=8cf1ab6b"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script defer="defer" src="_static/theme.js?v=073f68d9"></script>
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,2 +1,2 @@
|
|||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>./docsbuild_with_arch/agent.html</loc></url><url><loc>./docsbuild_with_arch/multi_turn.html</loc></url><url><loc>./docsbuild_with_arch/rag.html</loc></url><url><loc>./docsconcepts/llm_providers/client_libraries.html</loc></url><url><loc>./docsconcepts/llm_providers/llm_providers.html</loc></url><url><loc>./docsconcepts/llm_providers/model_aliases.html</loc></url><url><loc>./docsconcepts/llm_providers/supported_providers.html</loc></url><url><loc>./docsconcepts/prompt_target.html</loc></url><url><loc>./docsconcepts/tech_overview/error_target.html</loc></url><url><loc>./docsconcepts/tech_overview/listener.html</loc></url><url><loc>./docsconcepts/tech_overview/model_serving.html</loc></url><url><loc>./docsconcepts/tech_overview/prompt.html</loc></url><url><loc>./docsconcepts/tech_overview/request_lifecycle.html</loc></url><url><loc>./docsconcepts/tech_overview/tech_overview.html</loc></url><url><loc>./docsconcepts/tech_overview/terminology.html</loc></url><url><loc>./docsconcepts/tech_overview/threading_model.html</loc></url><url><loc>./docsget_started/intro_to_arch.html</loc></url><url><loc>./docsget_started/overview.html</loc></url><url><loc>./docsget_started/quickstart.html</loc></url><url><loc>./docsguides/agent_routing.html</loc></url><url><loc>./docsguides/function_calling.html</loc></url><url><loc>./docsguides/llm_router.html</loc></url><url><loc>./docsguides/observability/access_logging.html</loc></url><url><loc>./docsguides/observability/monitoring.html</loc></url><url><loc>./docsguides/observability/observability.html</loc></url><url><loc>./docsguides/observability/tracing.html</loc></url><url><loc>./docsguides/prompt_guard.html</loc></url><url><loc>./docsindex.html</loc></url><url><loc>./docsresources/configuration_reference.html</loc></url><url><loc>./docsresources/deployment.html</loc></url><url><loc>./docssearch.html</loc></url></urlset>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>./docsconcepts/agents.html</loc></url><url><loc>./docsconcepts/filter_chain.html</loc></url><url><loc>./docsconcepts/listeners.html</loc></url><url><loc>./docsconcepts/llm_providers/client_libraries.html</loc></url><url><loc>./docsconcepts/llm_providers/llm_providers.html</loc></url><url><loc>./docsconcepts/llm_providers/model_aliases.html</loc></url><url><loc>./docsconcepts/llm_providers/supported_providers.html</loc></url><url><loc>./docsconcepts/prompt_target.html</loc></url><url><loc>./docsget_started/intro_to_plano.html</loc></url><url><loc>./docsget_started/overview.html</loc></url><url><loc>./docsget_started/quickstart.html</loc></url><url><loc>./docsguides/function_calling.html</loc></url><url><loc>./docsguides/llm_router.html</loc></url><url><loc>./docsguides/observability/access_logging.html</loc></url><url><loc>./docsguides/observability/monitoring.html</loc></url><url><loc>./docsguides/observability/observability.html</loc></url><url><loc>./docsguides/observability/tracing.html</loc></url><url><loc>./docsguides/orchestration.html</loc></url><url><loc>./docsguides/prompt_guard.html</loc></url><url><loc>./docsguides/state.html</loc></url><url><loc>./docsindex.html</loc></url><url><loc>./docsresources/configuration_reference.html</loc></url><url><loc>./docsresources/deployment.html</loc></url><url><loc>./docsresources/llms_txt.html</loc></url><url><loc>./docsresources/tech_overview/model_serving.html</loc></url><url><loc>./docsresources/tech_overview/request_lifecycle.html</loc></url><url><loc>./docsresources/tech_overview/tech_overview.html</loc></url><url><loc>./docsresources/tech_overview/threading_model.html</loc></url><url><loc>./docssearch.html</loc></url></urlset>
|
||||
Loading…
Add table
Add a link
Reference in a new issue