From 0f93905fae932939c68148bcae5ce1ddb5fc6a90 Mon Sep 17 00:00:00 2001 From: Adil Hafeez Date: Mon, 19 May 2025 09:10:00 -0700 Subject: [PATCH] update readme --- arch/Dockerfile | 2 +- arch/envoy.template.yaml | 1 - arch/tools/cli/docker_cli.py | 2 - crates/Cargo.lock | 84 +++++++++++++++---- crates/brightstaff/Cargo.toml | 2 +- .../src/handlers/chat_completions.rs | 8 +- .../preference_based_routing/README.md | 1 + 7 files changed, 77 insertions(+), 23 deletions(-) diff --git a/arch/Dockerfile b/arch/Dockerfile index 42e20755..b3117630 100644 --- a/arch/Dockerfile +++ b/arch/Dockerfile @@ -1,4 +1,4 @@ -# build docker iamge for arch gateway +# build docker image for arch gateway FROM rust:1.82.0 as builder RUN rustup -v target add wasm32-wasip1 WORKDIR /arch diff --git a/arch/envoy.template.yaml b/arch/envoy.template.yaml index 3e696e2e..3a2856b6 100644 --- a/arch/envoy.template.yaml +++ b/arch/envoy.template.yaml @@ -29,7 +29,6 @@ stats_config: - 180000 static_resources: listeners: - - name: ingress_traffic address: socket_address: diff --git a/arch/tools/cli/docker_cli.py b/arch/tools/cli/docker_cli.py index 4e7b88ef..b5a31040 100644 --- a/arch/tools/cli/docker_cli.py +++ b/arch/tools/cli/docker_cli.py @@ -57,8 +57,6 @@ def docker_start_archgw_detached( port_mappings_args = [item for port in port_mappings for item in ("-p", port)] volume_mappings = [ - # TODO: fix log path - # f"{logs_path_abs}:/var/log:rw", f"{arch_config_file}:/app/arch_config.yaml:ro", # "/Users/adilhafeez/src/intelligent-prompt-gateway/crates/target/wasm32-wasip1/release:/etc/envoy/proxy-wasm-plugins:ro", ] diff --git a/crates/Cargo.lock b/crates/Cargo.lock index 545cd2fe..8ce8097e 100644 --- a/crates/Cargo.lock +++ b/crates/Cargo.lock @@ -651,7 +651,7 @@ dependencies = [ "futures", "hyper 0.14.32", "hyper-rustls 0.24.2", - "hyper-timeout", + "hyper-timeout 0.4.1", "log", "pin-project", "rand 0.8.5", @@ -878,7 +878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" dependencies = [ "fallible-iterator", - "indexmap", + "indexmap 2.6.0", "stable_deref_trait", ] @@ -920,7 +920,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -939,7 +939,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -956,6 +956,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.13.2" @@ -1173,6 +1179,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper 1.6.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.6.0" @@ -1354,6 +1373,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.6.0" @@ -1655,7 +1684,7 @@ checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "crc32fast", "hashbrown 0.15.0", - "indexmap", + "indexmap 2.6.0", "memchr", ] @@ -1752,6 +1781,8 @@ dependencies = [ "prost", "reqwest", "thiserror 2.0.12", + "tokio", + "tonic", "tracing", ] @@ -2233,7 +2264,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-util", - "tower", + "tower 0.5.2", "tower-service", "url", "wasm-bindgen", @@ -2507,7 +2538,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -2960,7 +2991,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -2979,10 +3010,35 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "http-body-util", + "hyper 1.6.0", + "hyper-timeout 0.5.2", + "hyper-util", "percent-encoding", "pin-project", "prost", + "tokio", "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -3337,7 +3393,7 @@ dependencies = [ "ahash 0.8.11", "bitflags 2.6.0", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.6.0", "semver", "serde", ] @@ -3349,7 +3405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c771866898879073c53b565a6c7b49953795159836714ac56a5befb581227c5" dependencies = [ "bitflags 2.6.0", - "indexmap", + "indexmap 2.6.0", ] [[package]] @@ -3380,7 +3436,7 @@ dependencies = [ "fxprof-processed-profile", "gimli 0.28.1", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.6.0", "ittapi", "libc", "libm", @@ -3504,7 +3560,7 @@ dependencies = [ "cranelift-bitset", "cranelift-entity", "gimli 0.28.1", - "indexmap", + "indexmap 2.6.0", "log", "object", "postcard", @@ -3615,7 +3671,7 @@ checksum = "8f88e49a9b81746ec0cede5505e40a4012c92cb5054cd7ef4300dc57c36f26b1" dependencies = [ "anyhow", "heck 0.4.1", - "indexmap", + "indexmap 2.6.0", "wit-parser", ] @@ -3916,7 +3972,7 @@ checksum = "ceeb0424aa8679f3fcf2d6e3cfa381f3d6fa6179976a2c05a6249dd2bb426716" dependencies = [ "anyhow", "id-arena", - "indexmap", + "indexmap 2.6.0", "log", "semver", "serde", diff --git a/crates/brightstaff/Cargo.toml b/crates/brightstaff/Cargo.toml index 1bccb230..3f51b6a0 100644 --- a/crates/brightstaff/Cargo.toml +++ b/crates/brightstaff/Cargo.toml @@ -16,7 +16,7 @@ hyper = { version = "1.6.0", features = ["full"] } hyper-util = "0.1.11" opentelemetry = "0.29.1" opentelemetry-http = "0.29.0" -opentelemetry-otlp = "0.29.0" +opentelemetry-otlp = {version="0.29.0", features=["trace", "tonic", "grpc-tonic"]} opentelemetry-stdout = "0.29.0" opentelemetry_sdk = "0.29.0" pretty_assertions = "1.4.1" diff --git a/crates/brightstaff/src/handlers/chat_completions.rs b/crates/brightstaff/src/handlers/chat_completions.rs index 08e3fd7e..d1610d61 100644 --- a/crates/brightstaff/src/handlers/chat_completions.rs +++ b/crates/brightstaff/src/handlers/chat_completions.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use bytes::Bytes; use common::api::open_ai::ChatCompletionsRequest; use common::consts::ARCH_PROVIDER_HINT_HEADER; +use common::utils::shorten_string; use http_body_util::combinators::BoxBody; use http_body_util::{BodyExt, Full, StreamBody}; use hyper::body::Frame; @@ -26,7 +27,6 @@ pub async fn chat_completions( router_service: Arc, llm_provider_endpoint: String, ) -> Result>, hyper::Error> { - let mut request_headers = request.headers().clone(); let chat_request_bytes = request.collect().await?.to_bytes(); @@ -42,8 +42,8 @@ pub async fn chat_completions( }; info!( - "request body: {}", - &serde_json::to_string(&chat_completion_request).unwrap() + "request body received: {}", + shorten_string(&serde_json::to_string(&chat_completion_request).unwrap()) ); let trace_parent = request_headers @@ -108,7 +108,7 @@ pub async fn chat_completions( } if chat_completion_request.stream { - // Create a channel to send data + // channel to create async stream let (tx, rx) = mpsc::channel::(16); // Spawn a task to send data as it becomes available diff --git a/demos/use_cases/preference_based_routing/README.md b/demos/use_cases/preference_based_routing/README.md index 470f6eaf..4f703afe 100644 --- a/demos/use_cases/preference_based_routing/README.md +++ b/demos/use_cases/preference_based_routing/README.md @@ -1 +1,2 @@ # Usage based LLM Routing +This demo shows how you can use user preferences to route user prompts to appropriate llm. See [arch_config.yaml](arch_config.yaml) for details on how you can define user preferences.