apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-04-30 03:16:28 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Introduce brightstaff a new terminal service for llm routing (#477)

Browse source
This commit is contained in:
Adil Hafeez 2025-05-19 09:59:22 -07:00 • committed by GitHub
parent 1f95fac4af
commit 27c0f2fdce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
36 changed files with 2817 additions and 150 deletions
Download patch file Download diff file Expand all files Collapse all files

35
crates/llm_gateway/src/stream_context.rs
View file

@ -228,6 +228,7 @@ impl HttpContext for StreamContext {
stream: None,
port: None,
rate_limits: None,
usage: None,
}));
} else {
self.select_llm_provider();
@ -316,10 +317,6 @@ impl HttpContext for StreamContext {
}
};
// remove metadata from the request body
//TODO: move this to prompt gateway
// deserialized_body.metadata = None;
// delete model key from message array
for message in deserialized_body.messages.iter_mut() {
message.model = None;
}
@ -342,24 +339,22 @@ impl HttpContext for StreamContext {
};
let model_requested = deserialized_body.model.clone();
if deserialized_body.model.is_empty() || deserialized_body.model.to_lowercase() == "none" {
deserialized_body.model = match model_name {
Some(model_name) => model_name.clone(),
None => {
if use_agent_orchestrator {
"agent_orchestrator".to_string()
} else {
self.send_server_error(
ServerError::BadRequest {
why: format!("No model specified in request and couldn't determine model name from arch_config. Model name in req: {}, arch_config, provider: {}, model: {:?}", deserialized_body.model, self.llm_provider().name, self.llm_provider().model).to_string(),
},
Some(StatusCode::BAD_REQUEST),
);
return Action::Continue;
}
deserialized_body.model = match model_name {
Some(model_name) => model_name.clone(),
None => {
if use_agent_orchestrator {
"agent_orchestrator".to_string()
} else {
self.send_server_error(
ServerError::BadRequest {
why: format!("No model specified in request and couldn't determine model name from arch_config. Model name in req: {}, arch_config, provider: {}, model: {:?}", deserialized_body.model, self.llm_provider().name, self.llm_provider().model).to_string(),
},
Some(StatusCode::BAD_REQUEST),
);
return Action::Continue;
}
}
}
};
info!(
"on_http_request_body: provider: {}, model requested: {}, model selected: {}",