apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-17 15:25:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

add support for openwebui

Browse source
This commit is contained in:
Adil Hafeez 2025-05-27 11:03:30 -07:00
parent 79cbcb5fe1
commit 2ccbcb3ff5
No known key found for this signature in database
GPG key ID: 9B18EF7691369645
6 changed files with 100 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

2
arch/envoy.template.yaml
View file

@ -728,7 +728,7 @@ static_resources:
- endpoint:
address:
socket_address:
address: 0.0.0.0
address: host.docker.internal
port_value: 9091
hostname: localhost

1
arch/tools/cli/docker_cli.py
View file

@ -52,6 +52,7 @@ def docker_start_archgw_detached(
port_mappings = [
f"{prompt_gateway_port}:{prompt_gateway_port}",
f"{llm_gateway_port}:{llm_gateway_port}",
f"{llm_gateway_port+1}:{llm_gateway_port+1}",
"19901:9901",
]
port_mappings_args = [item for port in port_mappings for item in ("-p", port)]

1
crates/brightstaff/src/handlers/mod.rs
View file

@ -1 +1,2 @@
pub mod chat_completions;
pub mod models;

34
crates/brightstaff/src/main.rs
View file

@ -1,4 +1,5 @@
use brightstaff::handlers::chat_completions::chat_completions;
use brightstaff::handlers::models::list_models;
use brightstaff::router::llm_router::RouterService;
use bytes::Bytes;
use common::configuration::Configuration;
@ -72,6 +73,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let arch_config = Arc::new(config);
let llm_providers = Arc::new(arch_config.llm_providers.clone());
debug!(
"arch_config: {:?}",
&serde_json::to_string(arch_config.as_ref()).unwrap()
@ -104,10 +107,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let router_service = Arc::clone(&router_service);
let llm_provider_endpoint = llm_provider_endpoint.clone();
let llm_providers = llm_providers.clone();
let service = service_fn(move |req| {
let router_service = Arc::clone(&router_service);
let parent_cx = extract_context_from_request(&req);
let llm_provider_endpoint = llm_provider_endpoint.clone();
let llm_providers = llm_providers.clone();
async move {
match (req.method(), req.uri().path()) {
@ -116,6 +121,35 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
.with_context(parent_cx)
.await
}
(&Method::GET, "/v1/models") => {
Ok(list_models(llm_providers).await)
}
(&Method::OPTIONS, "/v1/models") => {
let mut response = Response::new(empty());
*response.status_mut() = StatusCode::NO_CONTENT;
response.headers_mut().insert(
"Allow",
"GET, OPTIONS".parse().unwrap(),
);
response.headers_mut().insert(
"Access-Control-Allow-Origin",
"*".parse().unwrap(),
);
response.headers_mut().insert(
"Access-Control-Allow-Headers",
"Authorization, Content-Type".parse().unwrap(),
);
response.headers_mut().insert(
"Access-Control-Allow-Methods",
"GET, POST, OPTIONS".parse().unwrap(),
);
response.headers_mut().insert(
"Content-Type",
"application/json".parse().unwrap(),
);
Ok(response)
}
_ => {
let mut not_found = Response::new(empty());
*not_found.status_mut() = StatusCode::NOT_FOUND;

63
crates/common/src/api/open_ai.rs
View file

@ -1,7 +1,10 @@
use crate::consts::{ARCH_FC_MODEL_NAME, ASSISTANT_ROLE};
use crate::{
configuration::LlmProvider,
consts::{ARCH_FC_MODEL_NAME, ASSISTANT_ROLE},
};
use core::{panic, str};
use serde::{ser::SerializeMap, Deserialize, Serialize};
use serde_yaml::Value;
use core::panic;
use std::{
collections::{HashMap, VecDeque},
fmt::Display,
@ -420,6 +423,45 @@ pub fn to_server_events(chunks: Vec<ChatCompletionStreamResponse>) -> String {
response_str
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelDetail {
pub id: String,
pub object: String,
pub created: usize,
pub owned_by: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ModelObject {
#[serde(rename = "list")]
List,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Models {
pub object: ModelObject,
pub data: Vec<ModelDetail>,
}
impl From<Vec<LlmProvider>> for Models {
fn from(llm_providers: Vec<LlmProvider>) -> Self {
let data = llm_providers
.iter()
.map(|provider| ModelDetail {
id: provider.model.as_ref().unwrap().clone(),
object: "model".to_string(),
created: 1721172741,
owned_by: "system".to_string(),
})
.collect();
Models {
object: ModelObject::List,
data,
}
}
}
#[cfg(test)]
mod test {
use crate::api::open_ai::{ChatCompletionsRequest, ContentType, MultiPartContentType};
@ -775,7 +817,10 @@ data: [DONE]
if let Some(ContentType::MultiPart(multi_part_content)) =
chat_completions_request.messages[0].content.as_ref()
{
assert_eq!(multi_part_content[0].content_type, MultiPartContentType::Text);
assert_eq!(
multi_part_content[0].content_type,
MultiPartContentType::Text
);
assert_eq!(
multi_part_content[0].text,
Some("What city do you want to know the weather for?".to_string())
@ -815,22 +860,24 @@ data: [DONE]
chat_completions_request.messages[0].content.as_ref()
{
assert_eq!(multi_part_content.len(), 2);
assert_eq!(multi_part_content[0].content_type, MultiPartContentType::Text);
assert_eq!(
multi_part_content[0].content_type,
MultiPartContentType::Text
);
assert_eq!(
multi_part_content[0].text,
Some("What city do you want to know the weather for?".to_string())
);
assert_eq!(multi_part_content[1].content_type, MultiPartContentType::Text);
assert_eq!(
multi_part_content[1].text,
Some("hello world".to_string())
multi_part_content[1].content_type,
MultiPartContentType::Text
);
assert_eq!(multi_part_content[1].text, Some("hello world".to_string()));
} else {
panic!("Expected MultiPartContent");
}
}
#[test]
fn stream_chunk_parse_claude() {
const CHUNK_RESPONSE: &str = r#"data: {"id":"msg_01DZDMxYSgq8aPQxMQoBv6Kb","choices":[{"index":0,"delta":{"role":"assistant"}}],"created":1747685264,"model":"claude-3-7-sonnet-latest","object":"chat.completion.chunk"}

18
demos/use_cases/llm_routing/docker-compose.yaml
View file

@ -1,17 +1,15 @@
services:
chatbot_ui:
build:
context: ../../shared/chatbot_ui
dockerfile: Dockerfile
open-web-ui:
image: ghcr.io/open-webui/open-webui:main
container_name: open-webui
restart: always
ports:
- "18080:8080"
- "8080:8080"
environment:
- CHAT_COMPLETION_ENDPOINT=http://host.docker.internal:12000/v1
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- ./arch_config.yaml:/app/arch_config.yaml
- DEFAULT_MODEL=gpt-4o-mini
- OPENAI_API_KEY=your_secret_key
jaeger:
build: