diff --git a/crates/hermesllm/src/lib.rs b/crates/hermesllm/src/lib.rs index b26c4eb6..93146f8c 100644 --- a/crates/hermesllm/src/lib.rs +++ b/crates/hermesllm/src/lib.rs @@ -7,17 +7,19 @@ pub mod providers; pub enum Provider { Mistral, + Deepseek, Groq, Gemini, OpenAI, Claude, - Github + Github, } impl From<&str> for Provider { fn from(value: &str) -> Self { match value.to_lowercase().as_str() { "mistral" => Provider::Mistral, + "deepseek" => Provider::Deepseek, "groq" => Provider::Groq, "gemini" => Provider::Gemini, "openai" => Provider::OpenAI, @@ -32,6 +34,7 @@ impl Display for Provider { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Provider::Mistral => write!(f, "Mistral"), + Provider::Deepseek => write!(f, "Deepseek"), Provider::Groq => write!(f, "Groq"), Provider::Gemini => write!(f, "Gemini"), Provider::OpenAI => write!(f, "OpenAI"), diff --git a/crates/hermesllm/src/providers/openai/types.rs b/crates/hermesllm/src/providers/openai/types.rs index 265fd4fe..6d49d37d 100644 --- a/crates/hermesllm/src/providers/openai/types.rs +++ b/crates/hermesllm/src/providers/openai/types.rs @@ -1,7 +1,7 @@ use std::fmt::Display; use serde::{Deserialize, Serialize}; -use serde_json::{Value}; +use serde_json::Value; use serde_with::skip_serializing_none; use std::convert::TryFrom; use std::str; @@ -15,6 +15,11 @@ pub enum OpenAIError { JsonParseError(#[from] serde_json::Error), #[error("utf8 parsing error: {0}")] Utf8Error(#[from] std::str::Utf8Error), + #[error("invalid streaming data err {source}, data: {data}")] + InvalidStreamingData { + source: serde_json::Error, + data: String, + }, #[error("unsupported provider: {provider}")] UnsupportedProvider { provider: String }, } @@ -134,6 +139,7 @@ impl ChatCompletionsRequest { pub fn to_bytes(&self, provider: Provider) -> Result> { match provider { Provider::OpenAI + | Provider::Deepseek | Provider::Mistral | Provider::Groq | Provider::Gemini @@ -218,9 +224,18 @@ where if data == "[DONE]" { return None; } + + if data == r#"{"type": "ping"}"# { + continue; // Skip ping messages - that is usually from anthropic + } + return Some( - serde_json::from_str::(data) - .map_err(OpenAIError::from), + serde_json::from_str::(data).map_err(|e| { + OpenAIError::InvalidStreamingData { + source: e, + data: data.to_string(), + } + }), ); } } diff --git a/crates/llm_gateway/src/stream_context.rs b/crates/llm_gateway/src/stream_context.rs index 7ba0c9b3..88840b22 100644 --- a/crates/llm_gateway/src/stream_context.rs +++ b/crates/llm_gateway/src/stream_context.rs @@ -373,9 +373,11 @@ impl HttpContext for StreamContext { return Action::Continue; } + let llm_provider_str = self.llm_provider().provider_interface.to_string(); + let hermes_llm_provider = Provider::from(llm_provider_str.as_str()); + // convert chat completion request to llm provider specific request - let deserialized_body_bytes = match deserialized_body.to_bytes(hermesllm::Provider::OpenAI) - { + let deserialized_body_bytes = match deserialized_body.to_bytes(hermes_llm_provider) { Ok(bytes) => bytes, Err(e) => { warn!("Failed to serialize request body: {}", e);