mirror of
https://github.com/katanemo/plano.git
synced 2026-06-17 15:25:17 +02:00
update readme
This commit is contained in:
Adil Hafeez
parent
0a69fc2398
commit
0f93905fae
7 changed files with 77 additions and 23 deletions
|
|
@ -1,4 +1,4 @@
|
|||
# build docker iamge for arch gateway
|
||||
# build docker image for arch gateway
|
||||
FROM rust:1.82.0 as builder
|
||||
RUN rustup -v target add wasm32-wasip1
|
||||
WORKDIR /arch
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ stats_config:
|
|||
- 180000
|
||||
static_resources:
|
||||
listeners:
|
||||
|
||||
- name: ingress_traffic
|
||||
address:
|
||||
socket_address:
|
||||
|
|
|
|||
|
|
@ -57,8 +57,6 @@ def docker_start_archgw_detached(
|
|||
port_mappings_args = [item for port in port_mappings for item in ("-p", port)]
|
||||
|
||||
volume_mappings = [
|
||||
# TODO: fix log path
|
||||
# f"{logs_path_abs}:/var/log:rw",
|
||||
f"{arch_config_file}:/app/arch_config.yaml:ro",
|
||||
# "/Users/adilhafeez/src/intelligent-prompt-gateway/crates/target/wasm32-wasip1/release:/etc/envoy/proxy-wasm-plugins:ro",
|
||||
]
|
||||
|
|
|
|||
84
crates/Cargo.lock
generated
84
crates/Cargo.lock
generated
|
|
@ -651,7 +651,7 @@ dependencies = [
|
|||
"futures",
|
||||
"hyper 0.14.32",
|
||||
"hyper-rustls 0.24.2",
|
||||
"hyper-timeout",
|
||||
"hyper-timeout 0.4.1",
|
||||
"log",
|
||||
"pin-project",
|
||||
"rand 0.8.5",
|
||||
|
|
@ -878,7 +878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
|
||||
dependencies = [
|
||||
"fallible-iterator",
|
||||
"indexmap",
|
||||
"indexmap 2.6.0",
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
|
|
@ -920,7 +920,7 @@ dependencies = [
|
|||
"futures-sink",
|
||||
"futures-util",
|
||||
"http 0.2.12",
|
||||
"indexmap",
|
||||
"indexmap 2.6.0",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
|
|
@ -939,7 +939,7 @@ dependencies = [
|
|||
"futures-core",
|
||||
"futures-sink",
|
||||
"http 1.1.0",
|
||||
"indexmap",
|
||||
"indexmap 2.6.0",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
|
|
@ -956,6 +956,12 @@ dependencies = [
|
|||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.13.2"
|
||||
|
|
@ -1173,6 +1179,19 @@ dependencies = [
|
|||
"tokio-io-timeout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-timeout"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
|
||||
dependencies = [
|
||||
"hyper 1.6.0",
|
||||
"hyper-util",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-tls"
|
||||
version = "0.6.0"
|
||||
|
|
@ -1354,6 +1373,16 @@ dependencies = [
|
|||
"icu_properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "1.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"hashbrown 0.12.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.6.0"
|
||||
|
|
@ -1655,7 +1684,7 @@ checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e"
|
|||
dependencies = [
|
||||
"crc32fast",
|
||||
"hashbrown 0.15.0",
|
||||
"indexmap",
|
||||
"indexmap 2.6.0",
|
||||
"memchr",
|
||||
]
|
||||
|
||||
|
|
@ -1752,6 +1781,8 @@ dependencies = [
|
|||
"prost",
|
||||
"reqwest",
|
||||
"thiserror 2.0.12",
|
||||
"tokio",
|
||||
"tonic",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
|
|
@ -2233,7 +2264,7 @@ dependencies = [
|
|||
"tokio",
|
||||
"tokio-native-tls",
|
||||
"tokio-util",
|
||||
"tower",
|
||||
"tower 0.5.2",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
|
|
@ -2507,7 +2538,7 @@ version = "0.9.34+deprecated"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"indexmap 2.6.0",
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
|
|
@ -2960,7 +2991,7 @@ version = "0.22.22"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"indexmap 2.6.0",
|
||||
"serde",
|
||||
"serde_spanned",
|
||||
"toml_datetime",
|
||||
|
|
@ -2979,10 +3010,35 @@ dependencies = [
|
|||
"http 1.1.0",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"hyper 1.6.0",
|
||||
"hyper-timeout 0.5.2",
|
||||
"hyper-util",
|
||||
"percent-encoding",
|
||||
"pin-project",
|
||||
"prost",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tower 0.4.13",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tower"
|
||||
version = "0.4.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"indexmap 1.9.3",
|
||||
"pin-project",
|
||||
"pin-project-lite",
|
||||
"rand 0.8.5",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
|
|
@ -3337,7 +3393,7 @@ dependencies = [
|
|||
"ahash 0.8.11",
|
||||
"bitflags 2.6.0",
|
||||
"hashbrown 0.14.5",
|
||||
"indexmap",
|
||||
"indexmap 2.6.0",
|
||||
"semver",
|
||||
"serde",
|
||||
]
|
||||
|
|
@ -3349,7 +3405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "5c771866898879073c53b565a6c7b49953795159836714ac56a5befb581227c5"
|
||||
dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
"indexmap",
|
||||
"indexmap 2.6.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3380,7 +3436,7 @@ dependencies = [
|
|||
"fxprof-processed-profile",
|
||||
"gimli 0.28.1",
|
||||
"hashbrown 0.14.5",
|
||||
"indexmap",
|
||||
"indexmap 2.6.0",
|
||||
"ittapi",
|
||||
"libc",
|
||||
"libm",
|
||||
|
|
@ -3504,7 +3560,7 @@ dependencies = [
|
|||
"cranelift-bitset",
|
||||
"cranelift-entity",
|
||||
"gimli 0.28.1",
|
||||
"indexmap",
|
||||
"indexmap 2.6.0",
|
||||
"log",
|
||||
"object",
|
||||
"postcard",
|
||||
|
|
@ -3615,7 +3671,7 @@ checksum = "8f88e49a9b81746ec0cede5505e40a4012c92cb5054cd7ef4300dc57c36f26b1"
|
|||
dependencies = [
|
||||
"anyhow",
|
||||
"heck 0.4.1",
|
||||
"indexmap",
|
||||
"indexmap 2.6.0",
|
||||
"wit-parser",
|
||||
]
|
||||
|
||||
|
|
@ -3916,7 +3972,7 @@ checksum = "ceeb0424aa8679f3fcf2d6e3cfa381f3d6fa6179976a2c05a6249dd2bb426716"
|
|||
dependencies = [
|
||||
"anyhow",
|
||||
"id-arena",
|
||||
"indexmap",
|
||||
"indexmap 2.6.0",
|
||||
"log",
|
||||
"semver",
|
||||
"serde",
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ hyper = { version = "1.6.0", features = ["full"] }
|
|||
hyper-util = "0.1.11"
|
||||
opentelemetry = "0.29.1"
|
||||
opentelemetry-http = "0.29.0"
|
||||
opentelemetry-otlp = "0.29.0"
|
||||
opentelemetry-otlp = {version="0.29.0", features=["trace", "tonic", "grpc-tonic"]}
|
||||
opentelemetry-stdout = "0.29.0"
|
||||
opentelemetry_sdk = "0.29.0"
|
||||
pretty_assertions = "1.4.1"
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ use std::sync::Arc;
|
|||
use bytes::Bytes;
|
||||
use common::api::open_ai::ChatCompletionsRequest;
|
||||
use common::consts::ARCH_PROVIDER_HINT_HEADER;
|
||||
use common::utils::shorten_string;
|
||||
use http_body_util::combinators::BoxBody;
|
||||
use http_body_util::{BodyExt, Full, StreamBody};
|
||||
use hyper::body::Frame;
|
||||
|
|
@ -26,7 +27,6 @@ pub async fn chat_completions(
|
|||
router_service: Arc<RouterService>,
|
||||
llm_provider_endpoint: String,
|
||||
) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
|
||||
|
||||
let mut request_headers = request.headers().clone();
|
||||
|
||||
let chat_request_bytes = request.collect().await?.to_bytes();
|
||||
|
|
@ -42,8 +42,8 @@ pub async fn chat_completions(
|
|||
};
|
||||
|
||||
info!(
|
||||
"request body: {}",
|
||||
&serde_json::to_string(&chat_completion_request).unwrap()
|
||||
"request body received: {}",
|
||||
shorten_string(&serde_json::to_string(&chat_completion_request).unwrap())
|
||||
);
|
||||
|
||||
let trace_parent = request_headers
|
||||
|
|
@ -108,7 +108,7 @@ pub async fn chat_completions(
|
|||
}
|
||||
|
||||
if chat_completion_request.stream {
|
||||
// Create a channel to send data
|
||||
// channel to create async stream
|
||||
let (tx, rx) = mpsc::channel::<Bytes>(16);
|
||||
|
||||
// Spawn a task to send data as it becomes available
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
# Usage based LLM Routing
|
||||
This demo shows how you can use user preferences to route user prompts to appropriate llm. See [arch_config.yaml](arch_config.yaml) for details on how you can define user preferences.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue