apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-17 15:25:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix ollama demo

Browse source
This commit is contained in:
Adil Hafeez 2025-03-25 13:23:07 -07:00
parent 9f59943041
commit b73b177b3b
No known key found for this signature in database
GPG key ID: 9B18EF7691369645
1 changed files with 8 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

13
crates/llm_gateway/src/stream_context.rs
View file

@ -89,7 +89,7 @@ impl StreamContext {
provider_hint,
));
trace!(
debug!(
"request received: llm provider hint: {}, selected llm: {}, model: {}",
self.get_http_request_header(ARCH_PROVIDER_HINT_HEADER)
.unwrap_or_default(),
@ -186,6 +186,12 @@ impl HttpContext for StreamContext {
// Envoy's HTTP model is event driven. The WASM ABI has given implementors events to hook onto
// the lifecycle of the http request and response.
fn on_http_request_headers(&mut self, _num_headers: usize, _end_of_stream: bool) -> Action {
debug!(
"on_http_request_headers [S={}] end_stream={}, headers: {:?}",
self.context_id,
_end_of_stream,
self.get_http_request_headers()
);
let request_path = self.get_http_request_header(":path").unwrap_or_default();
if request_path == HEALTHZ_PATH {
self.send_http_response(200, vec![], None);
@ -214,10 +220,7 @@ impl HttpContext for StreamContext {
}));
} else {
self.select_llm_provider();
self.add_http_request_header(
ARCH_ROUTING_HEADER,
&self.llm_provider().provider_interface.to_string(),
);
self.add_http_request_header(ARCH_ROUTING_HEADER, &self.llm_provider().name);
if let Err(error) = self.modify_auth_headers() {
// ensure that the provider has an endpoint if the access key is missing else return a bad request
if self.llm_provider.as_ref().unwrap().endpoint.is_none() && !use_agent_orchestrator