From 2ccbcb3ff5eee889cfd963d40c4fa7ca09374c68 Mon Sep 17 00:00:00 2001 From: Adil Hafeez Date: Tue, 27 May 2025 11:03:30 -0700 Subject: [PATCH] add support for openwebui --- arch/envoy.template.yaml | 2 +- arch/tools/cli/docker_cli.py | 1 + crates/brightstaff/src/handlers/mod.rs | 1 + crates/brightstaff/src/main.rs | 34 ++++++++++ crates/common/src/api/open_ai.rs | 63 ++++++++++++++++--- .../use_cases/llm_routing/docker-compose.yaml | 18 +++--- 6 files changed, 100 insertions(+), 19 deletions(-) diff --git a/arch/envoy.template.yaml b/arch/envoy.template.yaml index c1373335..962f43dd 100644 --- a/arch/envoy.template.yaml +++ b/arch/envoy.template.yaml @@ -728,7 +728,7 @@ static_resources: - endpoint: address: socket_address: - address: 0.0.0.0 + address: host.docker.internal port_value: 9091 hostname: localhost diff --git a/arch/tools/cli/docker_cli.py b/arch/tools/cli/docker_cli.py index 0cb058ad..e8a12a13 100644 --- a/arch/tools/cli/docker_cli.py +++ b/arch/tools/cli/docker_cli.py @@ -52,6 +52,7 @@ def docker_start_archgw_detached( port_mappings = [ f"{prompt_gateway_port}:{prompt_gateway_port}", f"{llm_gateway_port}:{llm_gateway_port}", + f"{llm_gateway_port+1}:{llm_gateway_port+1}", "19901:9901", ] port_mappings_args = [item for port in port_mappings for item in ("-p", port)] diff --git a/crates/brightstaff/src/handlers/mod.rs b/crates/brightstaff/src/handlers/mod.rs index dcf38982..6de38b5b 100644 --- a/crates/brightstaff/src/handlers/mod.rs +++ b/crates/brightstaff/src/handlers/mod.rs @@ -1 +1,2 @@ pub mod chat_completions; +pub mod models; diff --git a/crates/brightstaff/src/main.rs b/crates/brightstaff/src/main.rs index fd14a88a..0c922485 100644 --- a/crates/brightstaff/src/main.rs +++ b/crates/brightstaff/src/main.rs @@ -1,4 +1,5 @@ use brightstaff::handlers::chat_completions::chat_completions; +use brightstaff::handlers::models::list_models; use brightstaff::router::llm_router::RouterService; use bytes::Bytes; use common::configuration::Configuration; @@ -72,6 +73,8 @@ async fn main() -> Result<(), Box> { let arch_config = Arc::new(config); + let llm_providers = Arc::new(arch_config.llm_providers.clone()); + debug!( "arch_config: {:?}", &serde_json::to_string(arch_config.as_ref()).unwrap() @@ -104,10 +107,12 @@ async fn main() -> Result<(), Box> { let router_service = Arc::clone(&router_service); let llm_provider_endpoint = llm_provider_endpoint.clone(); + let llm_providers = llm_providers.clone(); let service = service_fn(move |req| { let router_service = Arc::clone(&router_service); let parent_cx = extract_context_from_request(&req); let llm_provider_endpoint = llm_provider_endpoint.clone(); + let llm_providers = llm_providers.clone(); async move { match (req.method(), req.uri().path()) { @@ -116,6 +121,35 @@ async fn main() -> Result<(), Box> { .with_context(parent_cx) .await } + (&Method::GET, "/v1/models") => { + Ok(list_models(llm_providers).await) + } + (&Method::OPTIONS, "/v1/models") => { + let mut response = Response::new(empty()); + *response.status_mut() = StatusCode::NO_CONTENT; + response.headers_mut().insert( + "Allow", + "GET, OPTIONS".parse().unwrap(), + ); + response.headers_mut().insert( + "Access-Control-Allow-Origin", + "*".parse().unwrap(), + ); + response.headers_mut().insert( + "Access-Control-Allow-Headers", + "Authorization, Content-Type".parse().unwrap(), + ); + response.headers_mut().insert( + "Access-Control-Allow-Methods", + "GET, POST, OPTIONS".parse().unwrap(), + ); + response.headers_mut().insert( + "Content-Type", + "application/json".parse().unwrap(), + ); + + Ok(response) + } _ => { let mut not_found = Response::new(empty()); *not_found.status_mut() = StatusCode::NOT_FOUND; diff --git a/crates/common/src/api/open_ai.rs b/crates/common/src/api/open_ai.rs index b059ecad..7c492798 100644 --- a/crates/common/src/api/open_ai.rs +++ b/crates/common/src/api/open_ai.rs @@ -1,7 +1,10 @@ -use crate::consts::{ARCH_FC_MODEL_NAME, ASSISTANT_ROLE}; +use crate::{ + configuration::LlmProvider, + consts::{ARCH_FC_MODEL_NAME, ASSISTANT_ROLE}, +}; +use core::{panic, str}; use serde::{ser::SerializeMap, Deserialize, Serialize}; use serde_yaml::Value; -use core::panic; use std::{ collections::{HashMap, VecDeque}, fmt::Display, @@ -420,6 +423,45 @@ pub fn to_server_events(chunks: Vec) -> String { response_str } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelDetail { + pub id: String, + pub object: String, + pub created: usize, + pub owned_by: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ModelObject { + #[serde(rename = "list")] + List, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Models { + pub object: ModelObject, + pub data: Vec, +} + +impl From> for Models { + fn from(llm_providers: Vec) -> Self { + let data = llm_providers + .iter() + .map(|provider| ModelDetail { + id: provider.model.as_ref().unwrap().clone(), + object: "model".to_string(), + created: 1721172741, + owned_by: "system".to_string(), + }) + .collect(); + + Models { + object: ModelObject::List, + data, + } + } +} + #[cfg(test)] mod test { use crate::api::open_ai::{ChatCompletionsRequest, ContentType, MultiPartContentType}; @@ -775,7 +817,10 @@ data: [DONE] if let Some(ContentType::MultiPart(multi_part_content)) = chat_completions_request.messages[0].content.as_ref() { - assert_eq!(multi_part_content[0].content_type, MultiPartContentType::Text); + assert_eq!( + multi_part_content[0].content_type, + MultiPartContentType::Text + ); assert_eq!( multi_part_content[0].text, Some("What city do you want to know the weather for?".to_string()) @@ -815,22 +860,24 @@ data: [DONE] chat_completions_request.messages[0].content.as_ref() { assert_eq!(multi_part_content.len(), 2); - assert_eq!(multi_part_content[0].content_type, MultiPartContentType::Text); + assert_eq!( + multi_part_content[0].content_type, + MultiPartContentType::Text + ); assert_eq!( multi_part_content[0].text, Some("What city do you want to know the weather for?".to_string()) ); - assert_eq!(multi_part_content[1].content_type, MultiPartContentType::Text); assert_eq!( - multi_part_content[1].text, - Some("hello world".to_string()) + multi_part_content[1].content_type, + MultiPartContentType::Text ); + assert_eq!(multi_part_content[1].text, Some("hello world".to_string())); } else { panic!("Expected MultiPartContent"); } } - #[test] fn stream_chunk_parse_claude() { const CHUNK_RESPONSE: &str = r#"data: {"id":"msg_01DZDMxYSgq8aPQxMQoBv6Kb","choices":[{"index":0,"delta":{"role":"assistant"}}],"created":1747685264,"model":"claude-3-7-sonnet-latest","object":"chat.completion.chunk"} diff --git a/demos/use_cases/llm_routing/docker-compose.yaml b/demos/use_cases/llm_routing/docker-compose.yaml index c2d794c6..fad02191 100644 --- a/demos/use_cases/llm_routing/docker-compose.yaml +++ b/demos/use_cases/llm_routing/docker-compose.yaml @@ -1,17 +1,15 @@ services: - chatbot_ui: - build: - context: ../../shared/chatbot_ui - dockerfile: Dockerfile + + open-web-ui: + image: ghcr.io/open-webui/open-webui:main + container_name: open-webui + restart: always ports: - - "18080:8080" + - "8080:8080" environment: - - CHAT_COMPLETION_ENDPOINT=http://host.docker.internal:12000/v1 - extra_hosts: - - "host.docker.internal:host-gateway" - volumes: - - ./arch_config.yaml:/app/arch_config.yaml + - DEFAULT_MODEL=gpt-4o-mini + - OPENAI_API_KEY=your_secret_key jaeger: build: