apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-05-24 14:05:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

add health check endpoint for llm gateway (#420)

Browse source 
* add health check endpoint for llm gateway

* fix rust tests
This commit is contained in:
Adil Hafeez 2025-03-03 13:11:57 -08:00 • committed by GitHub
parent a402fee13b
commit 10cad4d0b7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 20 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
crates/llm_gateway/src/stream_context.rs
View file

@ -5,7 +5,7 @@ use common::api::open_ai::{
};
use common::configuration::LlmProvider;
use common::consts::{
ARCH_PROVIDER_HINT_HEADER, ARCH_ROUTING_HEADER, CHAT_COMPLETIONS_PATH,
ARCH_PROVIDER_HINT_HEADER, ARCH_ROUTING_HEADER, CHAT_COMPLETIONS_PATH, HEALTHZ_PATH,
RATELIMIT_SELECTOR_HEADER_KEY, REQUEST_ID_HEADER, TRACE_PARENT_HEADER,
};
use common::errors::ServerError;
@ -176,6 +176,12 @@ impl HttpContext for StreamContext {
// Envoy's HTTP model is event driven. The WASM ABI has given implementors events to hook onto
// the lifecycle of the http request and response.
fn on_http_request_headers(&mut self, _num_headers: usize, _end_of_stream: bool) -> Action {
let request_path = self.get_http_request_header(":path").unwrap_or_default();
if request_path == HEALTHZ_PATH {
self.send_http_response(200, vec![], None);
return Action::Continue;
}
self.select_llm_provider();
// if endpoint is not set then use provider name as routing header so envoy can resolve the cluster name