diff --git a/crates/hermesllm/src/providers/response.rs b/crates/hermesllm/src/providers/response.rs index 636d2a7c..a4f1c889 100644 --- a/crates/hermesllm/src/providers/response.rs +++ b/crates/hermesllm/src/providers/response.rs @@ -2,12 +2,12 @@ use crate::providers::id::ProviderId; use serde::Serialize; use std::error::Error; use std::fmt; +use std::convert::TryFrom; use crate::apis::openai::ChatCompletionsResponse; use crate::apis::OpenAISseIter; use crate::clients::endpoints::SupportedAPIs; -use std::convert::TryFrom; - +use crate::apis::anthropic::AnthropicSseIter; use crate::apis::anthropic::MessagesResponse; #[derive(Serialize)] @@ -17,7 +17,7 @@ pub enum ProviderResponseType { MessagesResponse(MessagesResponse), } -use crate::apis::anthropic::AnthropicSseIter; + pub enum ProviderStreamResponseIter { ChatCompletionsStream(OpenAISseIter>), diff --git a/crates/llm_gateway/src/stream_context.rs b/crates/llm_gateway/src/stream_context.rs index 9109fbeb..4e3aef4e 100644 --- a/crates/llm_gateway/src/stream_context.rs +++ b/crates/llm_gateway/src/stream_context.rs @@ -399,7 +399,10 @@ impl StreamContext { supported_api: SupportedAPIs, provider_id: ProviderId, ) -> Result, Action> { - debug!("non streaming response"); + debug!( + "no streaming response data (converted to utf8): {}", + String::from_utf8_lossy(body) + ); let response: ProviderResponseType = match (Some(&supported_api), self.resolved_api.as_ref()) { @@ -682,6 +685,10 @@ impl HttpContext for StreamContext { } }; + debug!( + "Setting HTTP request body {}", + String::from_utf8_lossy(&deserialized_body_bytes) + ); self.set_http_request_body(0, body_size, &deserialized_body_bytes); Action::Continue @@ -728,7 +735,7 @@ impl HttpContext for StreamContext { } let body = match self.read_response_body(body_size) { - Ok(b) => b, + Ok(bytes) => bytes, Err(action) => return action, }; @@ -741,8 +748,8 @@ impl HttpContext for StreamContext { if let Some(supported_api) = supported_api_opt { match self.handle_streaming_response(&body, supported_api, provider_id) { Ok(serialized_body) => { - // Write the normalized body back to the wire using the original body_size - self.set_http_response_body(0, body_size, &serialized_body); + // Pass-through: let the original streaming response continue unchanged + self.set_http_response_body(0, serialized_body.len(), &serialized_body); } Err(action) => return action, } @@ -753,8 +760,7 @@ impl HttpContext for StreamContext { if let Some(supported_api) = supported_api_opt { match self.handle_non_streaming_response(&body, supported_api, provider_id) { Ok(serialized_body) => { - // Write the normalized body back to the wire using the original body_size - self.set_http_response_body(0, body_size, &serialized_body); + self.set_http_response_body(0, serialized_body.len(), &serialized_body); } Err(action) => return action, }