From b73b177b3be91c67b7c84c3dc8164d0e3f8584f5 Mon Sep 17 00:00:00 2001 From: Adil Hafeez Date: Tue, 25 Mar 2025 13:23:07 -0700 Subject: [PATCH] fix ollama demo --- crates/llm_gateway/src/stream_context.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/llm_gateway/src/stream_context.rs b/crates/llm_gateway/src/stream_context.rs index 20ca9d62..a59be141 100644 --- a/crates/llm_gateway/src/stream_context.rs +++ b/crates/llm_gateway/src/stream_context.rs @@ -89,7 +89,7 @@ impl StreamContext { provider_hint, )); - trace!( + debug!( "request received: llm provider hint: {}, selected llm: {}, model: {}", self.get_http_request_header(ARCH_PROVIDER_HINT_HEADER) .unwrap_or_default(), @@ -186,6 +186,12 @@ impl HttpContext for StreamContext { // Envoy's HTTP model is event driven. The WASM ABI has given implementors events to hook onto // the lifecycle of the http request and response. fn on_http_request_headers(&mut self, _num_headers: usize, _end_of_stream: bool) -> Action { + debug!( + "on_http_request_headers [S={}] end_stream={}, headers: {:?}", + self.context_id, + _end_of_stream, + self.get_http_request_headers() + ); let request_path = self.get_http_request_header(":path").unwrap_or_default(); if request_path == HEALTHZ_PATH { self.send_http_response(200, vec![], None); @@ -214,10 +220,7 @@ impl HttpContext for StreamContext { })); } else { self.select_llm_provider(); - self.add_http_request_header( - ARCH_ROUTING_HEADER, - &self.llm_provider().provider_interface.to_string(), - ); + self.add_http_request_header(ARCH_ROUTING_HEADER, &self.llm_provider().name); if let Err(error) = self.modify_auth_headers() { // ensure that the provider has an endpoint if the access key is missing else return a bad request if self.llm_provider.as_ref().unwrap().endpoint.is_none() && !use_agent_orchestrator