apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-29 15:49:40 +02:00
Code Issues Projects Releases Packages Wiki Activity

fixed issue with groq LLMs that require the openai in the /v1/chat/completions path. My first change

Browse source
This commit is contained in:
Salman Paracha 2025-04-12 16:24:30 -07:00
parent 4d2d8bd7a1
commit 630081ebd3
5 changed files with 27 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/common/src/consts.rs
View file

@ -11,7 +11,7 @@ pub const MODEL_SERVER_NAME: &str = "model_server";
pub const ARCH_ROUTING_HEADER: &str = "x-arch-llm-provider";
pub const MESSAGES_KEY: &str = "messages";
pub const ARCH_PROVIDER_HINT_HEADER: &str = "x-arch-llm-provider-hint";
pub const CHAT_COMPLETIONS_PATH: &str = "/v1/chat/completions";
pub const CHAT_COMPLETIONS_PATH: [&str; 2] = ["/v1/chat/completions", "/openai/v1/chat/completions"];
pub const HEALTHZ_PATH: &str = "/healthz";
pub const ARCH_STATE_HEADER: &str = "x-arch-state";
pub const ARCH_FC_MODEL_NAME: &str = "Arch-Function-1.5B";

21
crates/llm_gateway/src/stream_context.rs
View file

@ -89,6 +89,23 @@ impl StreamContext {
provider_hint,
));
// Check if we need to modify the path based on the provider's base_url
let needs_openai_prefix = self
.llm_provider
.as_ref()
.and_then(|provider| provider.endpoint.as_ref())
.map(|url| url.contains("api.groq.com"))
.unwrap_or(false);
if needs_openai_prefix {
if let Some(path) = self.get_http_request_header(":path") {
if path.starts_with("/v1/") {
let new_path = format!("/openai{}", path);
self.set_http_request_header(":path", Some(new_path.as_str()));
}
}
}
debug!(
"request received: llm provider hint: {}, selected llm: {}, model: {}",
self.get_http_request_header(ARCH_PROVIDER_HINT_HEADER)
@ -237,8 +254,8 @@ impl HttpContext for StreamContext {
self.delete_content_length_header();
self.save_ratelimit_header();
self.is_chat_completions_request =
self.get_http_request_header(":path").unwrap_or_default() == CHAT_COMPLETIONS_PATH;
let request_path = self.get_http_request_header(":path").unwrap_or_default();
self.is_chat_completions_request = CHAT_COMPLETIONS_PATH.contains(&request_path.as_str());
self.request_id = self.get_http_request_header(REQUEST_ID_HEADER);
self.traceparent = self.get_http_request_header(TRACE_PARENT_HEADER);

2
crates/prompt_gateway/src/http_context.rs
View file

@ -61,7 +61,7 @@ impl HttpContext for StreamContext {
return Action::Continue;
}
self.is_chat_completions_request = request_path == CHAT_COMPLETIONS_PATH;
self.is_chat_completions_request = CHAT_COMPLETIONS_PATH.contains(&request_path.as_str());
debug!(
"on_http_request_headers S[{}] req_headers={:?}",