apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-17 15:25:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

making first commit. still need to work on streaming respones

Browse source
This commit is contained in:
Salman Paracha 2025-11-24 15:23:59 -08:00
parent b01a81927d
commit 3e4c00dbc3
18 changed files with 11930 additions and 140 deletions
Download patch file Download diff file Expand all files Collapse all files

9
crates/brightstaff/src/handlers/router.rs
View file

@ -3,7 +3,7 @@ use common::configuration::{ModelAlias, ModelUsagePreference};
use common::consts::{ARCH_IS_STREAMING_HEADER, ARCH_PROVIDER_HINT_HEADER};
use hermesllm::apis::openai::ChatCompletionsRequest;
use hermesllm::clients::endpoints::SupportedUpstreamAPIs;
use hermesllm::clients::SupportedAPIs;
use hermesllm::clients::SupportedAPIsFromClients;
use hermesllm::{ProviderRequest, ProviderRequestType};
use http_body_util::combinators::BoxBody;
use http_body_util::{BodyExt, Full};
@ -39,7 +39,7 @@ pub async fn router_chat(
let mut client_request = match ProviderRequestType::try_from((
&chat_request_bytes[..],
&SupportedAPIs::from_endpoint(request_path.as_str()).unwrap(),
&SupportedAPIsFromClients::from_endpoint(request_path.as_str()).unwrap(),
)) {
Ok(request) => request,
Err(err) => {
@ -91,10 +91,11 @@ pub async fn router_chat(
Ok(
ProviderRequestType::MessagesRequest(_)
| ProviderRequestType::BedrockConverse(_)
| ProviderRequestType::BedrockConverseStream(_),
| ProviderRequestType::BedrockConverseStream(_)
| ProviderRequestType::ResponsesAPIRequest(_),
) => {
// This should not happen after conversion to OpenAI format
warn!("Unexpected: got MessagesRequest after converting to OpenAI format");
warn!("Unexpected: got non-ChatCompletions request after converting to OpenAI format");
let err_msg = "Request conversion failed".to_string();
let mut bad_request = Response::new(full(err_msg));
*bad_request.status_mut() = StatusCode::BAD_REQUEST;

4
crates/brightstaff/src/main.rs
View file

@ -6,7 +6,7 @@ use brightstaff::router::llm_router::RouterService;
use brightstaff::utils::tracing::init_tracer;
use bytes::Bytes;
use common::configuration::Configuration;
use common::consts::{CHAT_COMPLETIONS_PATH, MESSAGES_PATH};
use common::consts::{CHAT_COMPLETIONS_PATH, MESSAGES_PATH, OPENAI_RESPONSES_API_PATH};
use http_body_util::{combinators::BoxBody, BodyExt, Empty};
use hyper::body::Incoming;
use hyper::server::conn::http1;
@ -123,7 +123,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
async move {
match (req.method(), req.uri().path()) {
(&Method::POST, CHAT_COMPLETIONS_PATH | MESSAGES_PATH) => {
(&Method::POST, CHAT_COMPLETIONS_PATH | MESSAGES_PATH | OPENAI_RESPONSES_API_PATH) => {
let fully_qualified_url =
format!("{}{}", llm_provider_url, req.uri().path());
router_chat(req, router_service, fully_qualified_url, model_aliases)

1
crates/common/src/consts.rs
View file

@ -13,6 +13,7 @@ pub const MESSAGES_KEY: &str = "messages";
pub const ARCH_PROVIDER_HINT_HEADER: &str = "x-arch-llm-provider-hint";
pub const ARCH_IS_STREAMING_HEADER: &str = "x-arch-streaming-request";
pub const CHAT_COMPLETIONS_PATH: &str = "/v1/chat/completions";
pub const OPENAI_RESPONSES_API_PATH: &str = "/v1/responses";
pub const MESSAGES_PATH: &str = "/v1/messages";
pub const HEALTHZ_PATH: &str = "/healthz";
pub const X_ARCH_STATE_HEADER: &str = "x-arch-state";

4
crates/hermesllm/src/apis/mod.rs
View file

@ -2,6 +2,7 @@ pub mod amazon_bedrock;
pub mod amazon_bedrock_binary_frame;
pub mod anthropic;
pub mod openai;
pub mod openai_responses;
pub mod sse;
// Explicit exports to avoid naming conflicts
@ -88,8 +89,9 @@ mod tests {
fn test_all_variants_method() {
// Test that all_variants returns the expected variants
let openai_variants = OpenAIApi::all_variants();
assert_eq!(openai_variants.len(), 1);
assert_eq!(openai_variants.len(), 2);
assert!(openai_variants.contains(&OpenAIApi::ChatCompletions));
assert!(openai_variants.contains(&OpenAIApi::Responses));
let anthropic_variants = AnthropicApi::all_variants();
assert_eq!(anthropic_variants.len(), 1);

15
crates/hermesllm/src/apis/openai.rs
View file

@ -9,7 +9,7 @@ use super::ApiDefinition;
use crate::providers::request::{ProviderRequest, ProviderRequestError};
use crate::providers::response::{ProviderResponse, ProviderStreamResponse, TokenUsage};
use crate::transforms::lib::ExtractText;
use crate::CHAT_COMPLETIONS_PATH;
use crate::{CHAT_COMPLETIONS_PATH, OPENAI_RESPONSES_API_PATH};
// ============================================================================
// OPENAI API ENUMERATION
@ -19,6 +19,7 @@ use crate::CHAT_COMPLETIONS_PATH;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum OpenAIApi {
ChatCompletions,
Responses,
// Future APIs can be added here:
// Embeddings,
// FineTuning,
@ -29,12 +30,14 @@ impl ApiDefinition for OpenAIApi {
fn endpoint(&self) -> &'static str {
match self {
OpenAIApi::ChatCompletions => CHAT_COMPLETIONS_PATH,
OpenAIApi::Responses => OPENAI_RESPONSES_API_PATH,
}
}
fn from_endpoint(endpoint: &str) -> Option<Self> {
match endpoint {
CHAT_COMPLETIONS_PATH => Some(OpenAIApi::ChatCompletions),
OPENAI_RESPONSES_API_PATH => Some(OpenAIApi::Responses),
_ => None,
}
}
@ -42,23 +45,26 @@ impl ApiDefinition for OpenAIApi {
fn supports_streaming(&self) -> bool {
match self {
OpenAIApi::ChatCompletions => true,
OpenAIApi::Responses => true,
}
}
fn supports_tools(&self) -> bool {
match self {
OpenAIApi::ChatCompletions => true,
OpenAIApi::Responses => true,
}
}
fn supports_vision(&self) -> bool {
match self {
OpenAIApi::ChatCompletions => true,
OpenAIApi::Responses => true,
}
}
fn all_variants() -> Vec<Self> {
vec![OpenAIApi::ChatCompletions]
vec![OpenAIApi::ChatCompletions, OpenAIApi::Responses]
}
}
@ -1077,8 +1083,9 @@ mod tests {
// Test all_variants
let all_variants = OpenAIApi::all_variants();
assert_eq!(all_variants.len(), 1);
assert_eq!(all_variants[0], OpenAIApi::ChatCompletions);
assert_eq!(all_variants.len(), 2);
assert!(all_variants.contains(&OpenAIApi::ChatCompletions));
assert!(all_variants.contains(&OpenAIApi::Responses));
}
#[test]

1386
crates/hermesllm/src/apis/openai_responses.rs Normal file
View file

File diff suppressed because it is too large Load diff

56
crates/hermesllm/src/clients/endpoints.rs
View file

@ -4,9 +4,10 @@ use std::fmt;
/// Unified enum representing all supported API endpoints across providers
#[derive(Debug, Clone, PartialEq)]
pub enum SupportedAPIs {
pub enum SupportedAPIsFromClients {
OpenAIChatCompletions(OpenAIApi),
AnthropicMessagesAPI(AnthropicApi),
OpenAIResponsesAPI(OpenAIApi),
}
#[derive(Debug, Clone, PartialEq)]
@ -15,17 +16,21 @@ pub enum SupportedUpstreamAPIs {
AnthropicMessagesAPI(AnthropicApi),
AmazonBedrockConverse(AmazonBedrockApi),
AmazonBedrockConverseStream(AmazonBedrockApi),
OpenAIResponsesAPI(OpenAIApi),
}
impl fmt::Display for SupportedAPIs {
impl fmt::Display for SupportedAPIsFromClients {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
SupportedAPIs::OpenAIChatCompletions(api) => {
SupportedAPIsFromClients::OpenAIChatCompletions(api) => {
write!(f, "OpenAI ({})", api.endpoint())
}
SupportedAPIs::AnthropicMessagesAPI(api) => {
SupportedAPIsFromClients::AnthropicMessagesAPI(api) => {
write!(f, "Anthropic AI ({})", api.endpoint())
}
SupportedAPIsFromClients::OpenAIResponsesAPI(api) => {
write!(f, "OpenAI Responses ({})", api.endpoint())
}
}
}
}
@ -45,19 +50,22 @@ impl fmt::Display for SupportedUpstreamAPIs {
SupportedUpstreamAPIs::AmazonBedrockConverseStream(api) => {
write!(f, "Amazon Bedrock ({})", api.endpoint())
}
SupportedUpstreamAPIs::OpenAIResponsesAPI(api) => {
write!(f, "OpenAI Responses ({})", api.endpoint())
}
}
}
}
impl SupportedAPIs {
impl SupportedAPIsFromClients {
/// Create a SupportedApi from an endpoint path
pub fn from_endpoint(endpoint: &str) -> Option<Self> {
if let Some(openai_api) = OpenAIApi::from_endpoint(endpoint) {
return Some(SupportedAPIs::OpenAIChatCompletions(openai_api));
return Some(SupportedAPIsFromClients::OpenAIChatCompletions(openai_api));
}
if let Some(anthropic_api) = AnthropicApi::from_endpoint(endpoint) {
return Some(SupportedAPIs::AnthropicMessagesAPI(anthropic_api));
return Some(SupportedAPIsFromClients::AnthropicMessagesAPI(anthropic_api));
}
None
@ -66,8 +74,9 @@ impl SupportedAPIs {
/// Get the endpoint path for this API
pub fn endpoint(&self) -> &'static str {
match self {
SupportedAPIs::OpenAIChatCompletions(api) => api.endpoint(),
SupportedAPIs::AnthropicMessagesAPI(api) => api.endpoint(),
SupportedAPIsFromClients::OpenAIChatCompletions(api) => api.endpoint(),
SupportedAPIsFromClients::AnthropicMessagesAPI(api) => api.endpoint(),
SupportedAPIsFromClients::OpenAIResponsesAPI(api) => api.endpoint(),
}
}
@ -95,7 +104,7 @@ impl SupportedAPIs {
};
match self {
SupportedAPIs::AnthropicMessagesAPI(AnthropicApi::Messages) => match provider_id {
SupportedAPIsFromClients::AnthropicMessagesAPI(AnthropicApi::Messages) => match provider_id {
ProviderId::Anthropic => build_endpoint("/v1", "/messages"),
ProviderId::AmazonBedrock => {
if request_path.starts_with("/v1/") && !is_streaming {
@ -198,22 +207,23 @@ mod tests {
#[test]
fn test_is_supported_endpoint() {
// OpenAI endpoints
assert!(SupportedAPIs::from_endpoint("/v1/chat/completions").is_some());
assert!(SupportedAPIsFromClients::from_endpoint("/v1/chat/completions").is_some());
// Anthropic endpoints
assert!(SupportedAPIs::from_endpoint("/v1/messages").is_some());
assert!(SupportedAPIsFromClients::from_endpoint("/v1/messages").is_some());
// Unsupported endpoints
assert!(!SupportedAPIs::from_endpoint("/v1/unknown").is_some());
assert!(!SupportedAPIs::from_endpoint("/v2/chat").is_some());
assert!(!SupportedAPIs::from_endpoint("").is_some());
assert!(!SupportedAPIsFromClients::from_endpoint("/v1/unknown").is_some());
assert!(!SupportedAPIsFromClients::from_endpoint("/v2/chat").is_some());
assert!(!SupportedAPIsFromClients::from_endpoint("").is_some());
}
#[test]
fn test_supported_endpoints() {
let endpoints = supported_endpoints();
assert_eq!(endpoints.len(), 2); // We have 2 APIs defined
assert_eq!(endpoints.len(), 3); // We have 3 APIs defined
assert!(endpoints.contains(&"/v1/chat/completions"));
assert!(endpoints.contains(&"/v1/messages"));
assert!(endpoints.contains(&"/v1/responses"));
}
#[test]
@ -263,7 +273,7 @@ mod tests {
#[test]
fn test_target_endpoint_without_base_url_prefix() {
let api = SupportedAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
let api = SupportedAPIsFromClients::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
// Test default OpenAI provider
assert_eq!(
@ -340,7 +350,7 @@ mod tests {
#[test]
fn test_target_endpoint_with_base_url_prefix() {
let api = SupportedAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
let api = SupportedAPIsFromClients::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
// Test Zhipu with custom base_url_path_prefix
assert_eq!(
@ -405,7 +415,7 @@ mod tests {
#[test]
fn test_target_endpoint_with_empty_base_url_prefix() {
let api = SupportedAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
let api = SupportedAPIsFromClients::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
// Test with just slashes - trims to empty, uses provider default
assert_eq!(
@ -434,7 +444,7 @@ mod tests {
#[test]
fn test_amazon_bedrock_endpoints() {
let api = SupportedAPIs::AnthropicMessagesAPI(AnthropicApi::Messages);
let api = SupportedAPIsFromClients::AnthropicMessagesAPI(AnthropicApi::Messages);
// Test Bedrock non-streaming without prefix
assert_eq!(
@ -487,7 +497,7 @@ mod tests {
#[test]
fn test_anthropic_messages_endpoint() {
let api = SupportedAPIs::AnthropicMessagesAPI(AnthropicApi::Messages);
let api = SupportedAPIsFromClients::AnthropicMessagesAPI(AnthropicApi::Messages);
// Test Anthropic without prefix
assert_eq!(
@ -516,7 +526,7 @@ mod tests {
#[test]
fn test_non_v1_request_paths() {
let api = SupportedAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
let api = SupportedAPIsFromClients::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
// Test Groq with non-v1 path (should use default)
assert_eq!(
@ -557,7 +567,7 @@ mod tests {
#[test]
fn test_azure_openai_with_query_params() {
let api = SupportedAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
let api = SupportedAPIsFromClients::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
// Test Azure without prefix - should include query params
assert_eq!(

2
crates/hermesllm/src/clients/mod.rs
View file

@ -3,7 +3,7 @@ pub mod lib;
pub mod transformer;
// Re-export the main items for easier access
pub use endpoints::{identify_provider, SupportedAPIs};
pub use endpoints::{identify_provider, SupportedAPIsFromClients};
pub use lib::*;
// Note: transformer module contains TryFrom trait implementations that are automatically available

5
crates/hermesllm/src/lib.rs
View file

@ -18,6 +18,7 @@ pub use providers::response::{
//TODO: Refactor such that commons doesn't depend on Hermes. For now this will clean up strings
pub const CHAT_COMPLETIONS_PATH: &str = "/v1/chat/completions";
pub const OPENAI_RESPONSES_API_PATH: &str = "/v1/responses";
pub const MESSAGES_PATH: &str = "/v1/messages";
#[cfg(test)]
@ -42,9 +43,9 @@ mod tests {
data: [DONE]
"#;
use crate::clients::endpoints::SupportedAPIs;
use crate::clients::endpoints::SupportedAPIsFromClients;
let client_api =
SupportedAPIs::OpenAIChatCompletions(crate::apis::OpenAIApi::ChatCompletions);
SupportedAPIsFromClients::OpenAIChatCompletions(crate::apis::OpenAIApi::ChatCompletions);
let upstream_api =
SupportedUpstreamAPIs::OpenAIChatCompletions(crate::apis::OpenAIApi::ChatCompletions);

31
crates/hermesllm/src/providers/id.rs
View file

@ -1,5 +1,5 @@
use crate::apis::{AmazonBedrockApi, AnthropicApi, OpenAIApi};
use crate::clients::endpoints::{SupportedAPIs, SupportedUpstreamAPIs};
use crate::clients::endpoints::{SupportedAPIsFromClients, SupportedUpstreamAPIs};
use std::fmt::Display;
/// Provider identifier enum - simple enum for identifying providers
@ -51,19 +51,24 @@ impl ProviderId {
/// Given a client API, return the compatible upstream API for this provider
pub fn compatible_api_for_client(
&self,
client_api: &SupportedAPIs,
client_api: &SupportedAPIsFromClients,
is_streaming: bool,
) -> SupportedUpstreamAPIs {
match (self, client_api) {
// Claude/Anthropic providers natively support Anthropic APIs
(ProviderId::Anthropic, SupportedAPIs::AnthropicMessagesAPI(_)) => {
(ProviderId::Anthropic, SupportedAPIsFromClients::AnthropicMessagesAPI(_)) => {
SupportedUpstreamAPIs::AnthropicMessagesAPI(AnthropicApi::Messages)
}
(
ProviderId::Anthropic,
SupportedAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions),
SupportedAPIsFromClients::OpenAIChatCompletions(_),
) => SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions),
// Anthropic doesn't support Responses API, fall back to chat completions
(ProviderId::Anthropic, SupportedAPIsFromClients::OpenAIResponsesAPI(_)) => {
SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions)
}
// OpenAI-compatible providers only support OpenAI chat completions
(
ProviderId::OpenAI
@ -80,7 +85,7 @@ impl ProviderId {
| ProviderId::Moonshotai
| ProviderId::Zhipu
| ProviderId::Qwen,
SupportedAPIs::AnthropicMessagesAPI(_),
SupportedAPIsFromClients::AnthropicMessagesAPI(_),
) => SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions),
(
@ -98,11 +103,21 @@ impl ProviderId {
| ProviderId::Moonshotai
| ProviderId::Zhipu
| ProviderId::Qwen,
SupportedAPIs::OpenAIChatCompletions(_),
SupportedAPIsFromClients::OpenAIChatCompletions(_),
) => SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions),
// OpenAI Responses API - only OpenAI supports this
(ProviderId::OpenAI, SupportedAPIsFromClients::OpenAIResponsesAPI(_)) => {
SupportedUpstreamAPIs::OpenAIResponsesAPI(OpenAIApi::Responses)
}
// Non-OpenAI providers: if client requested the Responses API, fall back to Chat Completions
(_, SupportedAPIsFromClients::OpenAIResponsesAPI(_)) => {
SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions)
}
// Amazon Bedrock natively supports Bedrock APIs
(ProviderId::AmazonBedrock, SupportedAPIs::OpenAIChatCompletions(_)) => {
(ProviderId::AmazonBedrock, SupportedAPIsFromClients::OpenAIChatCompletions(_)) => {
if is_streaming {
SupportedUpstreamAPIs::AmazonBedrockConverseStream(
AmazonBedrockApi::ConverseStream,
@ -111,7 +126,7 @@ impl ProviderId {
SupportedUpstreamAPIs::AmazonBedrockConverse(AmazonBedrockApi::Converse)
}
}
(ProviderId::AmazonBedrock, SupportedAPIs::AnthropicMessagesAPI(_)) => {
(ProviderId::AmazonBedrock, SupportedAPIsFromClients::AnthropicMessagesAPI(_)) => {
if is_streaming {
SupportedUpstreamAPIs::AmazonBedrockConverseStream(
AmazonBedrockApi::ConverseStream,

535
crates/hermesllm/src/providers/request.rs
View file

@ -2,19 +2,21 @@ use crate::apis::anthropic::MessagesRequest;
use crate::apis::openai::ChatCompletionsRequest;
use crate::apis::amazon_bedrock::{ConverseRequest, ConverseStreamRequest};
use crate::clients::endpoints::SupportedAPIs;
use crate::apis::openai_responses::ResponsesAPIRequest;
use crate::clients::endpoints::SupportedAPIsFromClients;
use crate::clients::endpoints::SupportedUpstreamAPIs;
use serde_json::Value;
use std::collections::HashMap;
use std::error::Error;
use std::fmt;
#[derive(Clone)]
#[derive(Clone, Debug)]
pub enum ProviderRequestType {
ChatCompletionsRequest(ChatCompletionsRequest),
MessagesRequest(MessagesRequest),
BedrockConverse(ConverseRequest),
BedrockConverseStream(ConverseStreamRequest),
ResponsesAPIRequest(ResponsesAPIRequest),
//add more request types here
}
pub trait ProviderRequest: Send + Sync {
@ -49,6 +51,7 @@ impl ProviderRequest for ProviderRequestType {
Self::MessagesRequest(r) => r.model(),
Self::BedrockConverse(r) => r.model(),
Self::BedrockConverseStream(r) => r.model(),
Self::ResponsesAPIRequest(r) => r.model(),
}
}
@ -58,6 +61,7 @@ impl ProviderRequest for ProviderRequestType {
Self::MessagesRequest(r) => r.set_model(model),
Self::BedrockConverse(r) => r.set_model(model),
Self::BedrockConverseStream(r) => r.set_model(model),
Self::ResponsesAPIRequest(r) => r.set_model(model),
}
}
@ -67,6 +71,7 @@ impl ProviderRequest for ProviderRequestType {
Self::MessagesRequest(r) => r.is_streaming(),
Self::BedrockConverse(_) => false,
Self::BedrockConverseStream(_) => true,
Self::ResponsesAPIRequest(r) => r.is_streaming(),
}
}
@ -76,6 +81,7 @@ impl ProviderRequest for ProviderRequestType {
Self::MessagesRequest(r) => r.extract_messages_text(),
Self::BedrockConverse(r) => r.extract_messages_text(),
Self::BedrockConverseStream(r) => r.extract_messages_text(),
Self::ResponsesAPIRequest(r) => r.extract_messages_text(),
}
}
@ -85,6 +91,7 @@ impl ProviderRequest for ProviderRequestType {
Self::MessagesRequest(r) => r.get_recent_user_message(),
Self::BedrockConverse(r) => r.get_recent_user_message(),
Self::BedrockConverseStream(r) => r.get_recent_user_message(),
Self::ResponsesAPIRequest(r) => r.get_recent_user_message(),
}
}
@ -94,6 +101,7 @@ impl ProviderRequest for ProviderRequestType {
Self::MessagesRequest(r) => r.to_bytes(),
Self::BedrockConverse(r) => r.to_bytes(),
Self::BedrockConverseStream(r) => r.to_bytes(),
Self::ResponsesAPIRequest(r) => r.to_bytes(),
}
}
@ -103,6 +111,7 @@ impl ProviderRequest for ProviderRequestType {
Self::MessagesRequest(r) => r.metadata(),
Self::BedrockConverse(r) => r.metadata(),
Self::BedrockConverseStream(r) => r.metadata(),
Self::ResponsesAPIRequest(r) => r.metadata(),
}
}
@ -112,18 +121,19 @@ impl ProviderRequest for ProviderRequestType {
Self::MessagesRequest(r) => r.remove_metadata_key(key),
Self::BedrockConverse(r) => r.remove_metadata_key(key),
Self::BedrockConverseStream(r) => r.remove_metadata_key(key),
Self::ResponsesAPIRequest(r) => r.remove_metadata_key(key),
}
}
}
/// Parse the client API from a byte slice.
impl TryFrom<(&[u8], &SupportedAPIs)> for ProviderRequestType {
impl TryFrom<(&[u8], &SupportedAPIsFromClients)> for ProviderRequestType {
type Error = std::io::Error;
fn try_from((bytes, client_api): (&[u8], &SupportedAPIs)) -> Result<Self, Self::Error> {
fn try_from((bytes, client_api): (&[u8], &SupportedAPIsFromClients)) -> Result<Self, Self::Error> {
// Use SupportedApi to determine the appropriate request type
match client_api {
SupportedAPIs::OpenAIChatCompletions(_) => {
SupportedAPIsFromClients::OpenAIChatCompletions(_) => {
let chat_completion_request: ChatCompletionsRequest =
ChatCompletionsRequest::try_from(bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
@ -131,11 +141,20 @@ impl TryFrom<(&[u8], &SupportedAPIs)> for ProviderRequestType {
chat_completion_request,
))
}
SupportedAPIs::AnthropicMessagesAPI(_) => {
SupportedAPIsFromClients::AnthropicMessagesAPI(_) => {
let messages_request: MessagesRequest = MessagesRequest::try_from(bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
Ok(ProviderRequestType::MessagesRequest(messages_request))
}
SupportedAPIsFromClients::OpenAIResponsesAPI(_) => {
let responses_apirequest: ResponsesAPIRequest =
ResponsesAPIRequest::try_from(bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
Ok(ProviderRequestType::ResponsesAPIRequest(
responses_apirequest,
))
}
}
}
}
@ -148,17 +167,17 @@ impl TryFrom<(ProviderRequestType, &SupportedUpstreamAPIs)> for ProviderRequestT
(client_request, upstream_api): (ProviderRequestType, &SupportedUpstreamAPIs),
) -> Result<Self, Self::Error> {
match (client_request, upstream_api) {
// Same API - no conversion needed, just clone the reference
// ============================================================================
// ChatCompletionsRequest conversions
// ============================================================================
// ChatCompletions -> ChatCompletions (pass-through)
(
ProviderRequestType::ChatCompletionsRequest(chat_req),
SupportedUpstreamAPIs::OpenAIChatCompletions(_),
) => Ok(ProviderRequestType::ChatCompletionsRequest(chat_req)),
(
ProviderRequestType::MessagesRequest(messages_req),
SupportedUpstreamAPIs::AnthropicMessagesAPI(_),
) => Ok(ProviderRequestType::MessagesRequest(messages_req)),
// Cross-API conversion - cloning is necessary for transformation
// ChatCompletions -> Anthropic Messages
(
ProviderRequestType::ChatCompletionsRequest(chat_req),
SupportedUpstreamAPIs::AnthropicMessagesAPI(_),
@ -174,6 +193,54 @@ impl TryFrom<(ProviderRequestType, &SupportedUpstreamAPIs)> for ProviderRequestT
Ok(ProviderRequestType::MessagesRequest(messages_req))
}
// ChatCompletions -> Bedrock Converse
(
ProviderRequestType::ChatCompletionsRequest(chat_req),
SupportedUpstreamAPIs::AmazonBedrockConverse(_),
) => {
let bedrock_req = ConverseRequest::try_from(chat_req)
.map_err(|e| ProviderRequestError {
message: format!("Failed to convert ChatCompletionsRequest to Amazon Bedrock request: {}", e),
source: Some(Box::new(e))
})?;
Ok(ProviderRequestType::BedrockConverse(bedrock_req))
}
// ChatCompletions -> Bedrock Converse Stream
(
ProviderRequestType::ChatCompletionsRequest(chat_req),
SupportedUpstreamAPIs::AmazonBedrockConverseStream(_),
) => {
let bedrock_req = ConverseStreamRequest::try_from(chat_req)
.map_err(|e| ProviderRequestError {
message: format!("Failed to convert ChatCompletionsRequest to Amazon Bedrock Stream request: {}", e),
source: Some(Box::new(e))
})?;
Ok(ProviderRequestType::BedrockConverseStream(bedrock_req))
}
// ChatCompletions -> ResponsesAPI (not supported)
(
ProviderRequestType::ChatCompletionsRequest(_),
SupportedUpstreamAPIs::OpenAIResponsesAPI(_),
) => {
Err(ProviderRequestError {
message: "Conversion from ChatCompletionsRequest to ResponsesAPIRequest is not supported. ResponsesAPI can only be used as a client API, not as an upstream API.".to_string(),
source: None,
})
}
// ============================================================================
// MessagesRequest conversions
// ============================================================================
// MessagesRequest -> MessagesRequest (pass-through)
(
ProviderRequestType::MessagesRequest(messages_req),
SupportedUpstreamAPIs::AnthropicMessagesAPI(_),
) => Ok(ProviderRequestType::MessagesRequest(messages_req)),
// MessagesRequest -> ChatCompletions
(
ProviderRequestType::MessagesRequest(messages_req),
SupportedUpstreamAPIs::OpenAIChatCompletions(_),
@ -190,30 +257,7 @@ impl TryFrom<(ProviderRequestType, &SupportedUpstreamAPIs)> for ProviderRequestT
Ok(ProviderRequestType::ChatCompletionsRequest(chat_req))
}
// Cross-API conversions: OpenAI/Anthropic to Amazon Bedrock
(
ProviderRequestType::ChatCompletionsRequest(chat_req),
SupportedUpstreamAPIs::AmazonBedrockConverse(_),
) => {
let bedrock_req = ConverseRequest::try_from(chat_req)
.map_err(|e| ProviderRequestError {
message: format!("Failed to convert ChatCompletionsRequest to Amazon Bedrock request: {}", e),
source: Some(Box::new(e))
})?;
Ok(ProviderRequestType::BedrockConverse(bedrock_req))
}
(
ProviderRequestType::ChatCompletionsRequest(chat_req),
SupportedUpstreamAPIs::AmazonBedrockConverseStream(_),
) => {
let bedrock_req = ConverseStreamRequest::try_from(chat_req)
.map_err(|e| ProviderRequestError {
message: format!("Failed to convert ChatCompletionsRequest to Amazon Bedrock request: {}", e),
source: Some(Box::new(e))
})?;
Ok(ProviderRequestType::BedrockConverse(bedrock_req))
}
// MessagesRequest -> Bedrock Converse
(
ProviderRequestType::MessagesRequest(messages_req),
SupportedUpstreamAPIs::AmazonBedrockConverse(_),
@ -228,6 +272,8 @@ impl TryFrom<(ProviderRequestType, &SupportedUpstreamAPIs)> for ProviderRequestT
})?;
Ok(ProviderRequestType::BedrockConverse(bedrock_req))
}
// MessagesRequest -> Bedrock Converse Stream
(
ProviderRequestType::MessagesRequest(messages_req),
SupportedUpstreamAPIs::AmazonBedrockConverseStream(_),
@ -235,7 +281,101 @@ impl TryFrom<(ProviderRequestType, &SupportedUpstreamAPIs)> for ProviderRequestT
let bedrock_req = ConverseStreamRequest::try_from(messages_req).map_err(|e| {
ProviderRequestError {
message: format!(
"Failed to convert MessagesRequest to Amazon Bedrock request: {}",
"Failed to convert MessagesRequest to Amazon Bedrock Stream request: {}",
e
),
source: Some(Box::new(e)),
}
})?;
Ok(ProviderRequestType::BedrockConverseStream(bedrock_req))
}
// MessagesRequest -> ResponsesAPI (not supported)
(
ProviderRequestType::MessagesRequest(_),
SupportedUpstreamAPIs::OpenAIResponsesAPI(_),
) => {
Err(ProviderRequestError {
message: "Conversion from MessagesRequest to ResponsesAPIRequest is not supported. ResponsesAPI can only be used as a client API, not as an upstream API.".to_string(),
source: None,
})
}
// ============================================================================
// ResponsesAPIRequest conversions (only converts TO other formats)
// ============================================================================
// ResponsesAPI -> ResponsesAPI (pass-through, OpenAI only)
(
ProviderRequestType::ResponsesAPIRequest(responses_req),
SupportedUpstreamAPIs::OpenAIResponsesAPI(_),
) => Ok(ProviderRequestType::ResponsesAPIRequest(responses_req)),
// ResponsesAPI -> ChatCompletions (direct conversion)
(
ProviderRequestType::ResponsesAPIRequest(responses_req),
SupportedUpstreamAPIs::OpenAIChatCompletions(_),
) => {
let chat_req = ChatCompletionsRequest::try_from(responses_req).map_err(|e| {
ProviderRequestError {
message: format!(
"Failed to convert ResponsesAPIRequest to ChatCompletionsRequest: {}",
e
),
source: Some(Box::new(e)),
}
})?;
Ok(ProviderRequestType::ChatCompletionsRequest(chat_req))
}
// ResponsesAPI -> Anthropic Messages (via ChatCompletions)
(
ProviderRequestType::ResponsesAPIRequest(responses_req),
SupportedUpstreamAPIs::AnthropicMessagesAPI(_),
) => {
// Chain: ResponsesAPI -> ChatCompletions -> MessagesRequest
let chat_req = ChatCompletionsRequest::try_from(responses_req).map_err(|e| {
ProviderRequestError {
message: format!(
"Failed to convert ResponsesAPIRequest to ChatCompletionsRequest: {}",
e
),
source: Some(Box::new(e)),
}
})?;
let messages_req = MessagesRequest::try_from(chat_req).map_err(|e| {
ProviderRequestError {
message: format!(
"Failed to convert ChatCompletionsRequest to MessagesRequest: {}",
e
),
source: Some(Box::new(e)),
}
})?;
Ok(ProviderRequestType::MessagesRequest(messages_req))
}
// ResponsesAPI -> Bedrock Converse (via ChatCompletions)
(
ProviderRequestType::ResponsesAPIRequest(responses_req),
SupportedUpstreamAPIs::AmazonBedrockConverse(_),
) => {
// Chain: ResponsesAPI -> ChatCompletions -> ConverseRequest
let chat_req = ChatCompletionsRequest::try_from(responses_req).map_err(|e| {
ProviderRequestError {
message: format!(
"Failed to convert ResponsesAPIRequest to ChatCompletionsRequest: {}",
e
),
source: Some(Box::new(e)),
}
})?;
let bedrock_req = ConverseRequest::try_from(chat_req).map_err(|e| {
ProviderRequestError {
message: format!(
"Failed to convert ChatCompletionsRequest to Amazon Bedrock request: {}",
e
),
source: Some(Box::new(e)),
@ -244,13 +384,50 @@ impl TryFrom<(ProviderRequestType, &SupportedUpstreamAPIs)> for ProviderRequestT
Ok(ProviderRequestType::BedrockConverse(bedrock_req))
}
// Amazon Bedrock to other APIs conversions
// ResponsesAPI -> Bedrock Converse Stream (via ChatCompletions)
(
ProviderRequestType::ResponsesAPIRequest(responses_req),
SupportedUpstreamAPIs::AmazonBedrockConverseStream(_),
) => {
// Chain: ResponsesAPI -> ChatCompletions -> ConverseStreamRequest
let chat_req = ChatCompletionsRequest::try_from(responses_req).map_err(|e| {
ProviderRequestError {
message: format!(
"Failed to convert ResponsesAPIRequest to ChatCompletionsRequest: {}",
e
),
source: Some(Box::new(e)),
}
})?;
let bedrock_req = ConverseStreamRequest::try_from(chat_req).map_err(|e| {
ProviderRequestError {
message: format!(
"Failed to convert ChatCompletionsRequest to Amazon Bedrock Stream request: {}",
e
),
source: Some(Box::new(e)),
}
})?;
Ok(ProviderRequestType::BedrockConverseStream(bedrock_req))
}
// ============================================================================
// Amazon Bedrock conversions (not supported as client API)
// ============================================================================
(ProviderRequestType::BedrockConverse(_), _) => {
todo!("Amazon Bedrock to ChatCompletionsRequest conversion not implemented yet")
Err(ProviderRequestError {
message: "Amazon Bedrock Converse is not supported as a client API. Only OpenAI ChatCompletions, Anthropic Messages, and OpenAI Responses APIs are supported as client APIs.".to_string(),
source: None,
})
}
(ProviderRequestType::BedrockConverseStream(_), _) => {
todo!("Amazon Bedrock Stream to ChatCompletionsRequest conversion not implemented yet")
Err(ProviderRequestError {
message: "Amazon Bedrock Converse Stream is not supported as a client API. Only OpenAI ChatCompletions, Anthropic Messages, and OpenAI Responses APIs are supported as client APIs.".to_string(),
source: None,
})
}
}
}
@ -284,7 +461,7 @@ mod tests {
use crate::apis::anthropic::MessagesRequest as AnthropicMessagesRequest;
use crate::apis::openai::ChatCompletionsRequest;
use crate::apis::openai::OpenAIApi::ChatCompletions;
use crate::clients::endpoints::SupportedAPIs;
use crate::clients::endpoints::SupportedAPIsFromClients;
use crate::transforms::lib::ExtractText;
use serde_json::json;
@ -298,7 +475,7 @@ mod tests {
]
});
let bytes = serde_json::to_vec(&req).unwrap();
let api = SupportedAPIs::OpenAIChatCompletions(ChatCompletions);
let api = SupportedAPIsFromClients::OpenAIChatCompletions(ChatCompletions);
let result = ProviderRequestType::try_from((bytes.as_slice(), &api));
assert!(result.is_ok());
match result.unwrap() {
@ -321,7 +498,7 @@ mod tests {
]
});
let bytes = serde_json::to_vec(&req).unwrap();
let endpoint = SupportedAPIs::AnthropicMessagesAPI(Messages);
let endpoint = SupportedAPIsFromClients::AnthropicMessagesAPI(Messages);
let result = ProviderRequestType::try_from((bytes.as_slice(), &endpoint));
assert!(result.is_ok());
match result.unwrap() {
@ -343,7 +520,7 @@ mod tests {
]
});
let bytes = serde_json::to_vec(&req).unwrap();
let endpoint = SupportedAPIs::OpenAIChatCompletions(ChatCompletions);
let endpoint = SupportedAPIsFromClients::OpenAIChatCompletions(ChatCompletions);
let result = ProviderRequestType::try_from((bytes.as_slice(), &endpoint));
assert!(result.is_ok());
match result.unwrap() {
@ -366,7 +543,7 @@ mod tests {
});
let bytes = serde_json::to_vec(&req).unwrap();
// Intentionally use OpenAI endpoint for Anthropic payload
let endpoint = SupportedAPIs::OpenAIChatCompletions(ChatCompletions);
let endpoint = SupportedAPIsFromClients::OpenAIChatCompletions(ChatCompletions);
let result = ProviderRequestType::try_from((bytes.as_slice(), &endpoint));
// Should parse as ChatCompletionsRequest, not error
assert!(result.is_ok());
@ -486,4 +663,272 @@ mod tests {
let roundtrip_max_tokens = openai_req2.max_completion_tokens.or(openai_req2.max_tokens);
assert_eq!(original_max_tokens, roundtrip_max_tokens);
}
#[test]
fn test_responses_api_request_from_bytes() {
use crate::apis::openai::OpenAIApi::Responses;
let req = json!({
"model": "gpt-4o",
"input": "Hello, how are you?"
});
let bytes = serde_json::to_vec(&req).unwrap();
let api = SupportedAPIsFromClients::OpenAIResponsesAPI(Responses);
let result = ProviderRequestType::try_from((bytes.as_slice(), &api));
assert!(result.is_ok());
match result.unwrap() {
ProviderRequestType::ResponsesAPIRequest(r) => {
assert_eq!(r.model, "gpt-4o");
}
_ => panic!("Expected ResponsesAPIRequest variant"),
}
}
#[test]
fn test_responses_api_to_chat_completions_conversion() {
use crate::apis::openai::OpenAIApi::ChatCompletions;
use crate::apis::openai_responses::{InputParam, ResponsesAPIRequest};
let responses_req = ResponsesAPIRequest {
model: "gpt-4o".to_string(),
input: InputParam::Text("Hello, world!".to_string()),
temperature: Some(0.7),
top_p: Some(0.9),
max_output_tokens: Some(100),
stream: Some(false),
metadata: None,
tools: None,
tool_choice: None,
parallel_tool_calls: None,
instructions: None,
modalities: None,
user: None,
store: None,
reasoning_effort: None,
include: None,
audio: None,
text: None,
service_tier: None,
top_logprobs: None,
stream_options: None,
truncation: None,
conversation: None,
previous_response_id: None,
max_tool_calls: None,
background: None,
};
let upstream_api = SupportedUpstreamAPIs::OpenAIChatCompletions(ChatCompletions);
let result = ProviderRequestType::try_from((
ProviderRequestType::ResponsesAPIRequest(responses_req),
&upstream_api,
));
assert!(result.is_ok());
match result.unwrap() {
ProviderRequestType::ChatCompletionsRequest(chat_req) => {
assert_eq!(chat_req.model, "gpt-4o");
assert_eq!(chat_req.temperature, Some(0.7));
assert_eq!(chat_req.top_p, Some(0.9));
assert_eq!(chat_req.max_completion_tokens, Some(100));
assert_eq!(chat_req.messages.len(), 1);
}
_ => panic!("Expected ChatCompletionsRequest variant"),
}
}
#[test]
fn test_responses_api_to_anthropic_messages_conversion() {
use crate::apis::anthropic::AnthropicApi::Messages;
use crate::apis::openai_responses::{InputParam, ResponsesAPIRequest};
let responses_req = ResponsesAPIRequest {
model: "gpt-4o".to_string(),
input: InputParam::Text("Hello, Claude!".to_string()),
temperature: Some(0.8),
max_output_tokens: Some(150),
stream: Some(false),
metadata: None,
tools: None,
tool_choice: None,
parallel_tool_calls: None,
instructions: Some("You are a helpful assistant".to_string()),
modalities: None,
user: None,
store: None,
reasoning_effort: None,
include: None,
audio: None,
text: None,
service_tier: None,
top_p: None,
top_logprobs: None,
stream_options: None,
truncation: None,
conversation: None,
previous_response_id: None,
max_tool_calls: None,
background: None,
};
let upstream_api = SupportedUpstreamAPIs::AnthropicMessagesAPI(Messages);
let result = ProviderRequestType::try_from((
ProviderRequestType::ResponsesAPIRequest(responses_req),
&upstream_api,
));
assert!(result.is_ok());
match result.unwrap() {
ProviderRequestType::MessagesRequest(messages_req) => {
assert_eq!(messages_req.model, "gpt-4o");
assert_eq!(messages_req.temperature, Some(0.8));
assert_eq!(messages_req.max_tokens, 150);
// Instructions should be converted to system prompt via ChatCompletions conversion
// The conversion chain: ResponsesAPI -> ChatCompletions (system message) -> Anthropic (system prompt)
// But we need to check if the system prompt was actually set
assert_eq!(messages_req.messages.len(), 1);
}
_ => panic!("Expected MessagesRequest variant"),
}
}
#[test]
fn test_responses_api_to_bedrock_conversion() {
use crate::apis::amazon_bedrock::AmazonBedrockApi::Converse;
use crate::apis::openai_responses::{InputParam, ResponsesAPIRequest};
let responses_req = ResponsesAPIRequest {
model: "gpt-4o".to_string(),
input: InputParam::Text("Hello, Bedrock!".to_string()),
temperature: Some(0.5),
max_output_tokens: Some(200),
stream: Some(false),
metadata: None,
tools: None,
tool_choice: None,
parallel_tool_calls: None,
instructions: None,
modalities: None,
user: None,
store: None,
reasoning_effort: None,
include: None,
audio: None,
text: None,
service_tier: None,
top_p: None,
top_logprobs: None,
stream_options: None,
truncation: None,
conversation: None,
previous_response_id: None,
max_tool_calls: None,
background: None,
};
let upstream_api = SupportedUpstreamAPIs::AmazonBedrockConverse(Converse);
let result = ProviderRequestType::try_from((
ProviderRequestType::ResponsesAPIRequest(responses_req),
&upstream_api,
));
assert!(result.is_ok());
match result.unwrap() {
ProviderRequestType::BedrockConverse(bedrock_req) => {
assert_eq!(bedrock_req.model_id, "gpt-4o");
// Bedrock receives the converted request through ChatCompletions
assert!(!bedrock_req.messages.is_none());
}
_ => panic!("Expected BedrockConverse variant"),
}
}
#[test]
fn test_chat_completions_to_responses_api_not_supported() {
use crate::apis::openai::OpenAIApi::Responses;
use crate::apis::openai::{Message, MessageContent, Role};
let chat_req = ChatCompletionsRequest {
model: "gpt-4".to_string(),
messages: vec![Message {
role: Role::User,
content: MessageContent::Text("Hello!".to_string()),
name: None,
tool_calls: None,
tool_call_id: None,
}],
..Default::default()
};
let upstream_api = SupportedUpstreamAPIs::OpenAIResponsesAPI(Responses);
let result = ProviderRequestType::try_from((
ProviderRequestType::ChatCompletionsRequest(chat_req),
&upstream_api,
));
assert!(result.is_err());
let err = result.unwrap_err();
assert!(err.message.contains("ResponsesAPI can only be used as a client API"));
}
#[test]
fn test_anthropic_messages_to_responses_api_not_supported() {
use crate::apis::anthropic::MessagesRequest as AnthropicMessagesRequest;
use crate::apis::openai::OpenAIApi::Responses;
let messages_req = AnthropicMessagesRequest {
model: "claude-3-sonnet".to_string(),
messages: vec![crate::apis::anthropic::MessagesMessage {
role: crate::apis::anthropic::MessagesRole::User,
content: crate::apis::anthropic::MessagesMessageContent::Single(
"Hello!".to_string(),
),
}],
max_tokens: 100,
container: None,
mcp_servers: None,
service_tier: None,
thinking: None,
temperature: None,
top_p: None,
top_k: None,
stream: None,
stop_sequences: None,
system: None,
tools: None,
tool_choice: None,
metadata: None,
};
let upstream_api = SupportedUpstreamAPIs::OpenAIResponsesAPI(Responses);
let result = ProviderRequestType::try_from((
ProviderRequestType::MessagesRequest(messages_req),
&upstream_api,
));
assert!(result.is_err());
let err = result.unwrap_err();
assert!(err.message.contains("ResponsesAPI can only be used as a client API"));
}
#[test]
fn test_bedrock_as_client_api_not_supported() {
use crate::apis::openai::OpenAIApi::ChatCompletions;
// Create a simple Bedrock request (we'll use Default if available, or minimal construction)
let bedrock_req = ConverseRequest::default();
let upstream_api = SupportedUpstreamAPIs::OpenAIChatCompletions(ChatCompletions);
let result = ProviderRequestType::try_from((
ProviderRequestType::BedrockConverse(bedrock_req),
&upstream_api,
));
assert!(result.is_err());
let err = result.unwrap_err();
assert!(err.message.contains("not supported as a client API"));
assert!(err
.message
.contains("OpenAI ChatCompletions, Anthropic Messages, and OpenAI Responses"));
}
}

126
crates/hermesllm/src/providers/response.rs
View file

@ -9,8 +9,10 @@ use crate::apis::anthropic::MessagesResponse;
use crate::apis::anthropic::MessagesStreamEvent;
use crate::apis::openai::ChatCompletionsResponse;
use crate::apis::openai::ChatCompletionsStreamResponse;
use crate::apis::openai_responses::ResponsesAPIResponse;
use crate::apis::openai_responses::ResponseAPIStreamEvent;
use crate::apis::sse::SseEvent;
use crate::clients::endpoints::SupportedAPIs;
use crate::clients::endpoints::SupportedAPIsFromClients;
use crate::clients::endpoints::SupportedUpstreamAPIs;
use crate::providers::id::ProviderId;
@ -26,6 +28,7 @@ pub trait TokenUsage {
pub enum ProviderResponseType {
ChatCompletionsResponse(ChatCompletionsResponse),
MessagesResponse(MessagesResponse),
ResponsesAPIResponse(ResponsesAPIResponse),
}
#[derive(Serialize, Debug, Clone)]
@ -34,6 +37,7 @@ pub enum ProviderStreamResponseType {
ChatCompletionsStreamResponse(ChatCompletionsStreamResponse),
MessagesStreamEvent(MessagesStreamEvent),
ConverseStreamEvent(ConverseStreamEvent),
ResponseAPIStreamEvent(ResponseAPIStreamEvent)
}
pub trait ProviderResponse: Send + Sync {
@ -52,6 +56,7 @@ impl ProviderResponse for ProviderResponseType {
match self {
ProviderResponseType::ChatCompletionsResponse(resp) => resp.usage(),
ProviderResponseType::MessagesResponse(resp) => resp.usage(),
ProviderResponseType::ResponsesAPIResponse(resp) => resp.usage.as_ref().map(|u| u as &dyn TokenUsage),
}
}
@ -59,6 +64,11 @@ impl ProviderResponse for ProviderResponseType {
match self {
ProviderResponseType::ChatCompletionsResponse(resp) => resp.extract_usage_counts(),
ProviderResponseType::MessagesResponse(resp) => resp.extract_usage_counts(),
ProviderResponseType::ResponsesAPIResponse(resp) => {
resp.usage.as_ref().map(|u| {
(u.input_tokens as usize, u.output_tokens as usize, u.total_tokens as usize)
})
}
}
}
}
@ -82,6 +92,7 @@ impl ProviderStreamResponse for ProviderStreamResponseType {
ProviderStreamResponseType::ChatCompletionsStreamResponse(resp) => resp.content_delta(),
ProviderStreamResponseType::MessagesStreamEvent(resp) => resp.content_delta(),
ProviderStreamResponseType::ConverseStreamEvent(resp) => resp.content_delta(),
ProviderStreamResponseType::ResponseAPIStreamEvent(_resp) => None, // ResponsesAPI does not have content deltas
}
}
@ -90,6 +101,7 @@ impl ProviderStreamResponse for ProviderStreamResponseType {
ProviderStreamResponseType::ChatCompletionsStreamResponse(resp) => resp.is_final(),
ProviderStreamResponseType::MessagesStreamEvent(resp) => resp.is_final(),
ProviderStreamResponseType::ConverseStreamEvent(resp) => resp.is_final(),
ProviderStreamResponseType::ResponseAPIStreamEvent(resp) => resp.is_final(),
}
}
@ -98,6 +110,7 @@ impl ProviderStreamResponse for ProviderStreamResponseType {
ProviderStreamResponseType::ChatCompletionsStreamResponse(resp) => resp.role(),
ProviderStreamResponseType::MessagesStreamEvent(resp) => resp.role(),
ProviderStreamResponseType::ConverseStreamEvent(resp) => resp.role(),
ProviderStreamResponseType::ResponseAPIStreamEvent(resp) => resp.role(),
}
}
@ -106,6 +119,7 @@ impl ProviderStreamResponse for ProviderStreamResponseType {
ProviderStreamResponseType::ChatCompletionsStreamResponse(_resp) => None, // OpenAI doesn't use event types
ProviderStreamResponseType::MessagesStreamEvent(resp) => resp.event_type(),
ProviderStreamResponseType::ConverseStreamEvent(resp) => resp.event_type(), // Bedrock doesn't use event types
ProviderStreamResponseType::ResponseAPIStreamEvent(resp) => resp.event_type(),
}
}
}
@ -121,6 +135,10 @@ impl Into<String> for ProviderStreamResponseType {
// Use the Into<String> implementation for proper SSE formatting with event lines
event.into()
}
ProviderStreamResponseType::ResponseAPIStreamEvent(event) => {
// Use the Into<String> implementation for proper SSE formatting with event lines
event.into()
}
ProviderStreamResponseType::ChatCompletionsStreamResponse(_) => {
// For OpenAI, use simple data line format
let json = serde_json::to_string(&self).unwrap_or_default();
@ -131,17 +149,17 @@ impl Into<String> for ProviderStreamResponseType {
}
// --- Response transformation logic for client API compatibility ---
impl TryFrom<(&[u8], &SupportedAPIs, &ProviderId)> for ProviderResponseType {
impl TryFrom<(&[u8], &SupportedAPIsFromClients, &ProviderId)> for ProviderResponseType {
type Error = std::io::Error;
fn try_from(
(bytes, client_api, provider_id): (&[u8], &SupportedAPIs, &ProviderId),
(bytes, client_api, provider_id): (&[u8], &SupportedAPIsFromClients, &ProviderId),
) -> Result<Self, Self::Error> {
let upstream_api = provider_id.compatible_api_for_client(client_api, false);
match (&upstream_api, client_api) {
(
SupportedUpstreamAPIs::OpenAIChatCompletions(_),
SupportedAPIs::OpenAIChatCompletions(_),
SupportedAPIsFromClients::OpenAIChatCompletions(_),
) => {
let resp: ChatCompletionsResponse = ChatCompletionsResponse::try_from(bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
@ -149,7 +167,7 @@ impl TryFrom<(&[u8], &SupportedAPIs, &ProviderId)> for ProviderResponseType {
}
(
SupportedUpstreamAPIs::AnthropicMessagesAPI(_),
SupportedAPIs::AnthropicMessagesAPI(_),
SupportedAPIsFromClients::AnthropicMessagesAPI(_),
) => {
let resp: MessagesResponse = serde_json::from_slice(bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
@ -157,7 +175,7 @@ impl TryFrom<(&[u8], &SupportedAPIs, &ProviderId)> for ProviderResponseType {
}
(
SupportedUpstreamAPIs::AnthropicMessagesAPI(_),
SupportedAPIs::OpenAIChatCompletions(_),
SupportedAPIsFromClients::OpenAIChatCompletions(_),
) => {
let anthropic_resp: MessagesResponse = serde_json::from_slice(bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
@ -174,7 +192,7 @@ impl TryFrom<(&[u8], &SupportedAPIs, &ProviderId)> for ProviderResponseType {
}
(
SupportedUpstreamAPIs::OpenAIChatCompletions(_),
SupportedAPIs::AnthropicMessagesAPI(_),
SupportedAPIsFromClients::AnthropicMessagesAPI(_),
) => {
let openai_resp: ChatCompletionsResponse = ChatCompletionsResponse::try_from(bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
@ -191,7 +209,7 @@ impl TryFrom<(&[u8], &SupportedAPIs, &ProviderId)> for ProviderResponseType {
// Amazon Bedrock transformations
(
SupportedUpstreamAPIs::AmazonBedrockConverse(_),
SupportedAPIs::OpenAIChatCompletions(_),
SupportedAPIsFromClients::OpenAIChatCompletions(_),
) => {
let bedrock_resp: ConverseResponse = serde_json::from_slice(bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
@ -207,7 +225,7 @@ impl TryFrom<(&[u8], &SupportedAPIs, &ProviderId)> for ProviderResponseType {
}
(
SupportedUpstreamAPIs::AmazonBedrockConverse(_),
SupportedAPIs::AnthropicMessagesAPI(_),
SupportedAPIsFromClients::AnthropicMessagesAPI(_),
) => {
let bedrock_resp: ConverseResponse = serde_json::from_slice(bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
@ -221,6 +239,30 @@ impl TryFrom<(&[u8], &SupportedAPIs, &ProviderId)> for ProviderResponseType {
})?;
Ok(ProviderResponseType::MessagesResponse(messages_resp))
}
(
SupportedUpstreamAPIs::OpenAIResponsesAPI(_),
SupportedAPIsFromClients::OpenAIResponsesAPI(_),
) => {
let resp: ResponsesAPIResponse = ResponsesAPIResponse::try_from(bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
Ok(ProviderResponseType::ResponsesAPIResponse(resp))
}
(
SupportedUpstreamAPIs::OpenAIChatCompletions(_),
SupportedAPIsFromClients::OpenAIResponsesAPI(_),
) => {
let chat_completions_response: ChatCompletionsResponse = ChatCompletionsResponse::try_from(bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
// Transform to ResponsesAPI format using the transformer
let responses_resp: ResponsesAPIResponse = chat_completions_response.try_into().map_err(|e| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("Transformation error: {}", e),
)
})?;
Ok(ProviderResponseType::ResponsesAPIResponse(responses_resp))
}
_ => Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"Unsupported API combination for response transformation",
@ -230,14 +272,14 @@ impl TryFrom<(&[u8], &SupportedAPIs, &ProviderId)> for ProviderResponseType {
}
// Stream response transformation logic for client API compatibility
impl TryFrom<(&[u8], &SupportedAPIs, &SupportedUpstreamAPIs)> for ProviderStreamResponseType {
impl TryFrom<(&[u8], &SupportedAPIsFromClients, &SupportedUpstreamAPIs)> for ProviderStreamResponseType {
type Error = Box<dyn std::error::Error + Send + Sync>;
fn try_from(
(bytes, client_api, upstream_api): (&[u8], &SupportedAPIs, &SupportedUpstreamAPIs),
(bytes, client_api, upstream_api): (&[u8], &SupportedAPIsFromClients, &SupportedUpstreamAPIs),
) -> Result<Self, Self::Error> {
// Special case: Handle [DONE] marker for OpenAI -> Anthropic conversion
if bytes == b"[DONE]" && matches!(client_api, SupportedAPIs::AnthropicMessagesAPI(_)) {
if bytes == b"[DONE]" && matches!(client_api, SupportedAPIsFromClients::AnthropicMessagesAPI(_)) {
return Ok(ProviderStreamResponseType::MessagesStreamEvent(
crate::apis::anthropic::MessagesStreamEvent::MessageStop,
));
@ -246,7 +288,7 @@ impl TryFrom<(&[u8], &SupportedAPIs, &SupportedUpstreamAPIs)> for ProviderStream
// OpenAI upstream
(
SupportedUpstreamAPIs::OpenAIChatCompletions(_),
SupportedAPIs::OpenAIChatCompletions(_),
SupportedAPIsFromClients::OpenAIChatCompletions(_),
) => {
let resp = serde_json::from_slice(bytes)?;
Ok(ProviderStreamResponseType::ChatCompletionsStreamResponse(
@ -255,7 +297,7 @@ impl TryFrom<(&[u8], &SupportedAPIs, &SupportedUpstreamAPIs)> for ProviderStream
}
(
SupportedUpstreamAPIs::OpenAIChatCompletions(_),
SupportedAPIs::AnthropicMessagesAPI(_),
SupportedAPIsFromClients::AnthropicMessagesAPI(_),
) => {
let openai_resp: crate::apis::openai::ChatCompletionsStreamResponse =
serde_json::from_slice(bytes)?;
@ -268,14 +310,14 @@ impl TryFrom<(&[u8], &SupportedAPIs, &SupportedUpstreamAPIs)> for ProviderStream
// Anthropic upstream
(
SupportedUpstreamAPIs::AnthropicMessagesAPI(_),
SupportedAPIs::AnthropicMessagesAPI(_),
SupportedAPIsFromClients::AnthropicMessagesAPI(_),
) => {
let resp = serde_json::from_slice(bytes)?;
Ok(ProviderStreamResponseType::MessagesStreamEvent(resp))
}
(
SupportedUpstreamAPIs::AnthropicMessagesAPI(_),
SupportedAPIs::OpenAIChatCompletions(_),
SupportedAPIsFromClients::OpenAIChatCompletions(_),
) => {
let anthropic_resp: crate::apis::anthropic::MessagesStreamEvent =
serde_json::from_slice(bytes)?;
@ -288,7 +330,7 @@ impl TryFrom<(&[u8], &SupportedAPIs, &SupportedUpstreamAPIs)> for ProviderStream
// Amazon Bedrock ConverseStream upstream
(
SupportedUpstreamAPIs::AmazonBedrockConverseStream(_),
SupportedAPIs::AnthropicMessagesAPI(_),
SupportedAPIsFromClients::AnthropicMessagesAPI(_),
) => {
let bedrock_resp: crate::apis::amazon_bedrock::ConverseStreamEvent =
serde_json::from_slice(bytes)?;
@ -307,11 +349,11 @@ impl TryFrom<(&[u8], &SupportedAPIs, &SupportedUpstreamAPIs)> for ProviderStream
}
// TryFrom implementation to convert raw bytes to SseEvent with parsed provider response
impl TryFrom<(SseEvent, &SupportedAPIs, &SupportedUpstreamAPIs)> for SseEvent {
impl TryFrom<(SseEvent, &SupportedAPIsFromClients, &SupportedUpstreamAPIs)> for SseEvent {
type Error = Box<dyn std::error::Error + Send + Sync>;
fn try_from(
(sse_event, client_api, upstream_api): (SseEvent, &SupportedAPIs, &SupportedUpstreamAPIs),
(sse_event, client_api, upstream_api): (SseEvent, &SupportedAPIsFromClients, &SupportedUpstreamAPIs),
) -> Result<Self, Self::Error> {
// Create a new transformed event based on the original
let mut transformed_event = sse_event;
@ -320,7 +362,7 @@ impl TryFrom<(SseEvent, &SupportedAPIs, &SupportedUpstreamAPIs)> for SseEvent {
if transformed_event.is_done() {
// For OpenAI client API, keep [DONE] as-is
// For Anthropic client API, it will be transformed via ProviderStreamResponseType
if matches!(client_api, SupportedAPIs::OpenAIChatCompletions(_)) {
if matches!(client_api, SupportedAPIsFromClients::OpenAIChatCompletions(_)) {
// Keep the [DONE] marker as-is for OpenAI clients
transformed_event.sse_transform_buffer = "data: [DONE]".to_string();
return Ok(transformed_event);
@ -342,7 +384,7 @@ impl TryFrom<(SseEvent, &SupportedAPIs, &SupportedUpstreamAPIs)> for SseEvent {
match (client_api, upstream_api) {
(
SupportedAPIs::AnthropicMessagesAPI(_),
SupportedAPIsFromClients::AnthropicMessagesAPI(_),
SupportedUpstreamAPIs::OpenAIChatCompletions(_),
) => {
if let Some(provider_response) = &transformed_event.provider_stream_response {
@ -384,7 +426,7 @@ impl TryFrom<(SseEvent, &SupportedAPIs, &SupportedUpstreamAPIs)> for SseEvent {
}
}
(
SupportedAPIs::OpenAIChatCompletions(_),
SupportedAPIsFromClients::OpenAIChatCompletions(_),
SupportedUpstreamAPIs::AnthropicMessagesAPI(_),
) => {
if transformed_event.is_event_only() && transformed_event.event.is_some() {
@ -392,7 +434,7 @@ impl TryFrom<(SseEvent, &SupportedAPIs, &SupportedUpstreamAPIs)> for SseEvent {
}
}
(
SupportedAPIs::AnthropicMessagesAPI(_),
SupportedAPIsFromClients::AnthropicMessagesAPI(_),
SupportedUpstreamAPIs::AnthropicMessagesAPI(_),
) => {
// When both client and upstream are Anthropic, suppress event-only lines
@ -414,7 +456,7 @@ impl TryFrom<(SseEvent, &SupportedAPIs, &SupportedUpstreamAPIs)> for SseEvent {
impl
TryFrom<(
&aws_smithy_eventstream::frame::DecodedFrame,
&SupportedAPIs,
&SupportedAPIsFromClients,
&SupportedUpstreamAPIs,
)> for ProviderStreamResponseType
{
@ -423,7 +465,7 @@ impl
fn try_from(
(frame, client_api, upstream_api): (
&aws_smithy_eventstream::frame::DecodedFrame,
&SupportedAPIs,
&SupportedAPIsFromClients,
&SupportedUpstreamAPIs,
),
) -> Result<Self, Self::Error> {
@ -435,7 +477,7 @@ impl
match (upstream_api, client_api) {
(
SupportedUpstreamAPIs::AmazonBedrockConverseStream(_),
SupportedAPIs::AnthropicMessagesAPI(_),
SupportedAPIsFromClients::AnthropicMessagesAPI(_),
) => {
// Parse the DecodedFrame into ConverseStreamEvent
let bedrock_event =
@ -449,7 +491,7 @@ impl
}
(
SupportedUpstreamAPIs::AmazonBedrockConverseStream(_),
SupportedAPIs::OpenAIChatCompletions(_),
SupportedAPIsFromClients::OpenAIChatCompletions(_),
) => {
// Parse the DecodedFrame into ConverseStreamEvent
let bedrock_event =
@ -497,7 +539,7 @@ mod tests {
use crate::apis::anthropic::AnthropicApi;
use crate::apis::openai::OpenAIApi;
use crate::apis::sse::SseStreamIter;
use crate::clients::endpoints::SupportedAPIs;
use crate::clients::endpoints::SupportedAPIsFromClients;
use crate::providers::id::ProviderId;
use serde_json::json;
@ -521,7 +563,7 @@ mod tests {
let bytes = serde_json::to_vec(&resp).unwrap();
let result = ProviderResponseType::try_from((
bytes.as_slice(),
&SupportedAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions),
&SupportedAPIsFromClients::OpenAIChatCompletions(OpenAIApi::ChatCompletions),
&ProviderId::OpenAI,
));
assert!(result.is_ok());
@ -550,7 +592,7 @@ mod tests {
let bytes = serde_json::to_vec(&resp).unwrap();
let result = ProviderResponseType::try_from((
bytes.as_slice(),
&SupportedAPIs::AnthropicMessagesAPI(AnthropicApi::Messages),
&SupportedAPIsFromClients::AnthropicMessagesAPI(AnthropicApi::Messages),
&ProviderId::Anthropic,
));
assert!(result.is_ok());
@ -584,7 +626,7 @@ mod tests {
let bytes = serde_json::to_vec(&resp).unwrap();
let result = ProviderResponseType::try_from((
bytes.as_slice(),
&SupportedAPIs::AnthropicMessagesAPI(AnthropicApi::Messages),
&SupportedAPIsFromClients::AnthropicMessagesAPI(AnthropicApi::Messages),
&ProviderId::OpenAI,
));
assert!(result.is_ok());
@ -626,7 +668,7 @@ mod tests {
let bytes = serde_json::to_vec(&resp).unwrap();
let result = ProviderResponseType::try_from((
bytes.as_slice(),
&SupportedAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions),
&SupportedAPIsFromClients::OpenAIChatCompletions(OpenAIApi::ChatCompletions),
&ProviderId::Anthropic,
));
assert!(result.is_ok());
@ -860,7 +902,7 @@ mod tests {
// Test that [DONE] marker is properly converted to MessageStop in the transformation layer
let done_bytes = b"[DONE]";
let client_api = SupportedAPIs::AnthropicMessagesAPI(AnthropicApi::Messages);
let client_api = SupportedAPIsFromClients::AnthropicMessagesAPI(AnthropicApi::Messages);
let upstream_api = SupportedUpstreamAPIs::OpenAIChatCompletions(
crate::apis::openai::OpenAIApi::ChatCompletions,
);
@ -1069,7 +1111,7 @@ mod tests {
let mut decoder = BedrockBinaryFrameDecoder::new(&mut buffer);
let client_api =
SupportedAPIs::AnthropicMessagesAPI(crate::apis::anthropic::AnthropicApi::Messages);
SupportedAPIsFromClients::AnthropicMessagesAPI(crate::apis::anthropic::AnthropicApi::Messages);
let upstream_api = SupportedUpstreamAPIs::AmazonBedrockConverseStream(
crate::apis::amazon_bedrock::AmazonBedrockApi::ConverseStream,
);
@ -1156,7 +1198,7 @@ mod tests {
let mut decoder = BedrockBinaryFrameDecoder::new(&mut buffer);
let client_api =
SupportedAPIs::AnthropicMessagesAPI(crate::apis::anthropic::AnthropicApi::Messages);
SupportedAPIsFromClients::AnthropicMessagesAPI(crate::apis::anthropic::AnthropicApi::Messages);
let upstream_api = SupportedUpstreamAPIs::AmazonBedrockConverseStream(
crate::apis::amazon_bedrock::AmazonBedrockApi::ConverseStream,
);
@ -1267,7 +1309,7 @@ mod tests {
provider_stream_response: None,
};
let client_api = SupportedAPIs::AnthropicMessagesAPI(AnthropicApi::Messages);
let client_api = SupportedAPIsFromClients::AnthropicMessagesAPI(AnthropicApi::Messages);
let upstream_api = SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
// Transform the event
@ -1324,7 +1366,7 @@ mod tests {
provider_stream_response: None,
};
let client_api = SupportedAPIs::AnthropicMessagesAPI(AnthropicApi::Messages);
let client_api = SupportedAPIsFromClients::AnthropicMessagesAPI(AnthropicApi::Messages);
let upstream_api = SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
// Transform the event
@ -1380,7 +1422,7 @@ mod tests {
provider_stream_response: None,
};
let client_api = SupportedAPIs::AnthropicMessagesAPI(AnthropicApi::Messages);
let client_api = SupportedAPIsFromClients::AnthropicMessagesAPI(AnthropicApi::Messages);
let upstream_api = SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
// Transform the event
@ -1422,7 +1464,7 @@ mod tests {
provider_stream_response: None,
};
let client_api = SupportedAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
let client_api = SupportedAPIsFromClients::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
let upstream_api = SupportedUpstreamAPIs::AnthropicMessagesAPI(AnthropicApi::Messages);
// Transform the event
@ -1469,7 +1511,7 @@ mod tests {
provider_stream_response: None,
};
let client_api = SupportedAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
let client_api = SupportedAPIsFromClients::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
let upstream_api = SupportedUpstreamAPIs::AnthropicMessagesAPI(AnthropicApi::Messages);
// Transform the event
@ -1516,7 +1558,7 @@ mod tests {
provider_stream_response: None,
};
let client_api = SupportedAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
let client_api = SupportedAPIsFromClients::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
let upstream_api = SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
// Transform the event
@ -1560,7 +1602,7 @@ mod tests {
provider_stream_response: None,
};
let client_api = SupportedAPIs::AnthropicMessagesAPI(AnthropicApi::Messages);
let client_api = SupportedAPIsFromClients::AnthropicMessagesAPI(AnthropicApi::Messages);
let upstream_api = SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
// Transform the event

200
crates/hermesllm/src/transforms/request/from_openai.rs
View file

@ -12,6 +12,10 @@ use crate::apis::anthropic::{
use crate::apis::openai::{
ChatCompletionsRequest, Message, MessageContent, Role, Tool, ToolChoice, ToolChoiceType,
};
use crate::apis::openai_responses::{
ResponsesAPIRequest, InputContent, InputItem, InputParam, MessageRole, Modality, ReasoningEffort, Tool as ResponsesTool, ToolChoice as ResponsesToolChoice
};
use crate::clients::TransformError;
use crate::transforms::lib::ExtractText;
use crate::transforms::lib::*;
@ -244,6 +248,202 @@ impl TryFrom<Message> for BedrockMessage {
}
}
impl TryFrom<ResponsesAPIRequest> for ChatCompletionsRequest {
type Error = TransformError;
fn try_from(req: ResponsesAPIRequest) -> Result<Self, Self::Error> {
// Convert input to messages
let messages = match req.input {
InputParam::Text(text) => {
// Simple text input becomes a user message
vec![Message {
role: Role::User,
content: MessageContent::Text(text),
name: None,
tool_call_id: None,
tool_calls: None,
}]
}
InputParam::Items(items) => {
// Convert input items to messages
let mut converted_messages = Vec::new();
// Add instructions as system message if present
if let Some(instructions) = &req.instructions {
converted_messages.push(Message {
role: Role::System,
content: MessageContent::Text(instructions.clone()),
name: None,
tool_call_id: None,
tool_calls: None,
});
}
// Convert each input item
for item in items {
match item {
InputItem::Message(input_msg) => {
let role = match input_msg.role {
MessageRole::User => Role::User,
MessageRole::Assistant => Role::Assistant,
MessageRole::System => Role::System,
MessageRole::Developer => Role::System, // Map developer to system
};
// Convert content blocks
let content = if input_msg.content.len() == 1 {
// Single content item - check if it's simple text
match &input_msg.content[0] {
InputContent::InputText { text } => MessageContent::Text(text.clone()),
_ => {
// Convert to parts for non-text content
MessageContent::Parts(
input_msg.content.iter()
.filter_map(|c| match c {
InputContent::InputText { text } => {
Some(crate::apis::openai::ContentPart::Text { text: text.clone() })
}
InputContent::InputImage { image_url, .. } => {
Some(crate::apis::openai::ContentPart::ImageUrl {
image_url: crate::apis::openai::ImageUrl {
url: image_url.clone(),
detail: None,
}
})
}
InputContent::InputFile { .. } => None, // Skip files for now
InputContent::InputAudio { .. } => None, // Skip audio for now
})
.collect()
)
}
}
} else {
// Multiple content items - convert to parts
MessageContent::Parts(
input_msg.content.iter()
.filter_map(|c| match c {
InputContent::InputText { text } => {
Some(crate::apis::openai::ContentPart::Text { text: text.clone() })
}
InputContent::InputImage { image_url, .. } => {
Some(crate::apis::openai::ContentPart::ImageUrl {
image_url: crate::apis::openai::ImageUrl {
url: image_url.clone(),
detail: None,
}
})
}
InputContent::InputFile { .. } => None, // Skip files for now
InputContent::InputAudio { .. } => None, // Skip audio for now
})
.collect()
)
};
converted_messages.push(Message {
role,
content,
name: None,
tool_call_id: None,
tool_calls: None,
});
}
}
}
converted_messages
}
};
// Build the ChatCompletionsRequest
Ok(ChatCompletionsRequest {
model: req.model,
messages,
temperature: req.temperature,
top_p: req.top_p,
max_completion_tokens: req.max_output_tokens.map(|t| t as u32),
stream: req.stream,
metadata: req.metadata,
user: req.user,
store: req.store,
service_tier: req.service_tier,
top_logprobs: req.top_logprobs.map(|t| t as u32),
modalities: req.modalities.map(|mods| {
mods.into_iter().map(|m| {
match m {
Modality::Text => "text".to_string(),
Modality::Audio => "audio".to_string(),
}
}).collect()
}),
stream_options: req.stream_options.map(|opts| {
crate::apis::openai::StreamOptions {
include_usage: opts.include_usage,
}
}),
reasoning_effort: req.reasoning_effort.map(|effort| {
match effort {
ReasoningEffort::Low => "low".to_string(),
ReasoningEffort::Medium => "medium".to_string(),
ReasoningEffort::High => "high".to_string(),
}
}),
tools: req.tools.map(|tools| {
tools.into_iter().map(|tool| {
// Only convert Function tools - other types are not supported in ChatCompletions
match tool {
ResponsesTool::Function { function } => Ok(Tool {
tool_type: "function".to_string(),
function: crate::apis::openai::Function {
name: function.name,
description: function.description,
parameters: function.parameters.unwrap_or_else(|| serde_json::json!({
"type": "object",
"properties": {}
})),
strict: function.strict,
}
}),
ResponsesTool::FileSearch { .. } => Err(TransformError::UnsupportedConversion(
"FileSearch tool is not supported in ChatCompletions API. Only function tools are supported.".to_string()
)),
ResponsesTool::WebSearchPreview { .. } => Err(TransformError::UnsupportedConversion(
"WebSearchPreview tool is not supported in ChatCompletions API. Only function tools are supported.".to_string()
)),
ResponsesTool::CodeInterpreter => Err(TransformError::UnsupportedConversion(
"CodeInterpreter tool is not supported in ChatCompletions API. Only function tools are supported.".to_string()
)),
ResponsesTool::Computer { .. } => Err(TransformError::UnsupportedConversion(
"Computer tool is not supported in ChatCompletions API. Only function tools are supported.".to_string()
)),
}
}).collect::<Result<Vec<_>, _>>()
}).transpose()?,
tool_choice: req.tool_choice.map(|choice| {
match choice {
ResponsesToolChoice::String(s) => {
match s.as_str() {
"auto" => ToolChoice::Type(ToolChoiceType::Auto),
"required" => ToolChoice::Type(ToolChoiceType::Required),
"none" => ToolChoice::Type(ToolChoiceType::None),
_ => ToolChoice::Type(ToolChoiceType::Auto), // Default to auto for unknown strings
}
}
ResponsesToolChoice::Named { function, .. } => ToolChoice::Function {
choice_type: "function".to_string(),
function: crate::apis::openai::FunctionChoice { name: function.name }
}
}
}),
parallel_tool_calls: req.parallel_tool_calls,
..Default::default()
})
}
}
impl TryFrom<ChatCompletionsRequest> for AnthropicMessagesRequest {
type Error = TransformError;

366
crates/hermesllm/src/transforms/response/to_openai.rs
View file

@ -10,6 +10,7 @@ use crate::apis::openai::{
FunctionCallDelta, MessageContent, MessageDelta, ResponseMessage, Role, StreamChoice,
ToolCallDelta, Usage,
};
use crate::apis::openai_responses::ResponsesAPIResponse;
use crate::clients::TransformError;
use crate::transforms::lib::*;
@ -30,6 +31,163 @@ impl Into<Usage> for MessagesUsage {
}
}
impl TryFrom<ChatCompletionsResponse> for ResponsesAPIResponse {
type Error = TransformError;
fn try_from(resp: ChatCompletionsResponse) -> Result<Self, Self::Error> {
use crate::apis::openai_responses::{
IncompleteDetails, IncompleteReason, OutputContent, OutputItem, OutputItemStatus,
ResponseStatus, ResponseUsage, ResponsesAPIResponse,
};
// Convert the first choice's message to output items
let output = if let Some(choice) = resp.choices.first() {
let mut items = Vec::new();
// Create a message output item from the response message
let mut content = Vec::new();
// Add text content if present
if let Some(text) = &choice.message.content {
content.push(OutputContent::OutputText {
text: text.clone(),
annotations: vec![],
logprobs: None,
});
}
// Add audio content if present (audio is a Value, need to handle it carefully)
if let Some(audio) = &choice.message.audio {
// Audio is serde_json::Value, try to extract data and transcript
if let Some(audio_obj) = audio.as_object() {
let data = audio_obj
.get("data")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let transcript = audio_obj
.get("transcript")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
content.push(OutputContent::OutputAudio { data, transcript });
}
}
// Add refusal content if present
if let Some(refusal) = &choice.message.refusal {
content.push(OutputContent::Refusal {
refusal: refusal.clone(),
});
}
// Only add the message item if there's actual content (text, audio, or refusal)
// Don't add empty message items when there are only tool calls
if !content.is_empty() {
items.push(OutputItem::Message {
id: format!("msg_{}", resp.id),
status: OutputItemStatus::Completed,
role: match choice.message.role {
Role::User => "user".to_string(),
Role::Assistant => "assistant".to_string(),
Role::System => "system".to_string(),
Role::Tool => "tool".to_string(),
},
content,
});
}
// Add tool calls as function call items if present
if let Some(tool_calls) = &choice.message.tool_calls {
for tool_call in tool_calls {
items.push(OutputItem::FunctionCall {
id: format!("func_{}", tool_call.id),
status: OutputItemStatus::Completed,
call_id: tool_call.id.clone(),
name: Some(tool_call.function.name.clone()),
arguments: Some(tool_call.function.arguments.clone()),
});
}
}
items
} else {
vec![]
};
// Convert finish_reason to status
let status = if let Some(choice) = resp.choices.first() {
match choice.finish_reason {
Some(FinishReason::Stop) => ResponseStatus::Completed,
Some(FinishReason::ToolCalls) => ResponseStatus::Completed,
Some(FinishReason::Length) => ResponseStatus::Incomplete,
Some(FinishReason::ContentFilter) => ResponseStatus::Failed,
_ => ResponseStatus::Completed,
}
} else {
ResponseStatus::Completed
};
// Convert usage
let usage = ResponseUsage {
input_tokens: resp.usage.prompt_tokens as i32,
output_tokens: resp.usage.completion_tokens as i32,
total_tokens: resp.usage.total_tokens as i32,
input_tokens_details: resp.usage.prompt_tokens_details.map(|details| {
crate::apis::openai_responses::TokenDetails {
cached_tokens: details.cached_tokens.unwrap_or(0) as i32,
}
}),
output_tokens_details: resp.usage.completion_tokens_details.map(|details| {
crate::apis::openai_responses::OutputTokenDetails {
reasoning_tokens: details.reasoning_tokens.unwrap_or(0) as i32,
}
}),
};
// Set incomplete_details if status is incomplete
let incomplete_details = if matches!(status, ResponseStatus::Incomplete) {
Some(IncompleteDetails {
reason: IncompleteReason::MaxOutputTokens,
})
} else {
None
};
Ok(ResponsesAPIResponse {
id: resp.id,
object: "response".to_string(),
created_at: resp.created as i64,
status,
background: Some(false),
error: None,
incomplete_details,
instructions: None,
max_output_tokens: None,
max_tool_calls: None,
model: resp.model,
output,
usage: Some(usage),
parallel_tool_calls: true,
conversation: None,
previous_response_id: None,
tools: vec![],
tool_choice: "auto".to_string(),
temperature: 1.0,
top_p: 1.0,
metadata: resp.metadata.unwrap_or_default(),
truncation: None,
reasoning: None,
store: None,
text: None,
audio: None,
modalities: None,
service_tier: resp.service_tier,
top_logprobs: None,
})
}
}
impl TryFrom<MessagesResponse> for ChatCompletionsResponse {
type Error = TransformError;
@ -1166,4 +1324,212 @@ mod tests {
assert!(content.contains("Here's the analysis:"));
// Note: Image blocks are not converted to text in the current implementation
}
#[test]
fn test_chat_completions_to_responses_api_basic() {
use crate::apis::openai_responses::{OutputContent, OutputItem, ResponsesAPIResponse};
let chat_response = ChatCompletionsResponse {
id: "chatcmpl-123".to_string(),
object: Some("chat.completion".to_string()),
created: 1677652288,
model: "gpt-4".to_string(),
choices: vec![Choice {
index: 0,
message: crate::apis::openai::ResponseMessage {
role: Role::Assistant,
content: Some("Hello! How can I help you?".to_string()),
refusal: None,
annotations: None,
audio: None,
function_call: None,
tool_calls: None,
},
finish_reason: Some(FinishReason::Stop),
logprobs: None,
}],
usage: Usage {
prompt_tokens: 10,
completion_tokens: 20,
total_tokens: 30,
prompt_tokens_details: None,
completion_tokens_details: None,
},
system_fingerprint: None,
service_tier: Some("default".to_string()),
metadata: None,
};
let responses_api: ResponsesAPIResponse = chat_response.try_into().unwrap();
assert_eq!(responses_api.id, "chatcmpl-123");
assert_eq!(responses_api.object, "response");
assert_eq!(responses_api.model, "gpt-4");
// Check usage conversion
let usage = responses_api.usage.unwrap();
assert_eq!(usage.input_tokens, 10);
assert_eq!(usage.output_tokens, 20);
assert_eq!(usage.total_tokens, 30);
// Check output items
assert_eq!(responses_api.output.len(), 1);
match &responses_api.output[0] {
OutputItem::Message {
role,
content,
..
} => {
assert_eq!(role, "assistant");
assert_eq!(content.len(), 1);
match &content[0] {
OutputContent::OutputText { text, .. } => {
assert_eq!(text, "Hello! How can I help you?");
}
_ => panic!("Expected OutputText content"),
}
}
_ => panic!("Expected Message output item"),
}
}
#[test]
fn test_chat_completions_to_responses_api_with_tool_calls() {
use crate::apis::openai::{FunctionCall, ToolCall};
use crate::apis::openai_responses::{OutputItem, ResponsesAPIResponse};
let chat_response = ChatCompletionsResponse {
id: "chatcmpl-456".to_string(),
object: Some("chat.completion".to_string()),
created: 1677652300,
model: "gpt-4".to_string(),
choices: vec![Choice {
index: 0,
message: crate::apis::openai::ResponseMessage {
role: Role::Assistant,
content: Some("Let me check the weather.".to_string()),
refusal: None,
annotations: None,
audio: None,
function_call: None,
tool_calls: Some(vec![ToolCall {
id: "call_abc123".to_string(),
call_type: "function".to_string(),
function: FunctionCall {
name: "get_weather".to_string(),
arguments: r#"{"location":"San Francisco"}"#.to_string(),
},
}]),
},
finish_reason: Some(FinishReason::ToolCalls),
logprobs: None,
}],
usage: Usage {
prompt_tokens: 15,
completion_tokens: 25,
total_tokens: 40,
prompt_tokens_details: None,
completion_tokens_details: None,
},
system_fingerprint: None,
service_tier: None,
metadata: None,
};
let responses_api: ResponsesAPIResponse = chat_response.try_into().unwrap();
// Should have 2 output items: message + function call
assert_eq!(responses_api.output.len(), 2);
// Check message item
match &responses_api.output[0] {
OutputItem::Message { content, .. } => {
assert_eq!(content.len(), 1);
}
_ => panic!("Expected Message output item"),
}
// Check function call item
match &responses_api.output[1] {
OutputItem::FunctionCall {
call_id,
name,
arguments,
..
} => {
assert_eq!(call_id, "call_abc123");
assert_eq!(name.as_ref().unwrap(), "get_weather");
assert!(arguments.as_ref().unwrap().contains("San Francisco"));
}
_ => panic!("Expected FunctionCall output item"),
}
}
#[test]
fn test_chat_completions_to_responses_api_tool_calls_only() {
use crate::apis::openai::{FunctionCall, ToolCall};
use crate::apis::openai_responses::{OutputItem, ResponsesAPIResponse};
// Test the real-world case where content is null and there are only tool calls
let chat_response = ChatCompletionsResponse {
id: "chatcmpl-789".to_string(),
object: Some("chat.completion".to_string()),
created: 1764023939,
model: "gpt-4o-2024-08-06".to_string(),
choices: vec![Choice {
index: 0,
message: crate::apis::openai::ResponseMessage {
role: Role::Assistant,
content: None, // No text content, only tool calls
refusal: None,
annotations: None,
audio: None,
function_call: None,
tool_calls: Some(vec![ToolCall {
id: "call_oJBtqTJmRfBGlFS55QhMfUUV".to_string(),
call_type: "function".to_string(),
function: FunctionCall {
name: "get_weather".to_string(),
arguments: r#"{"location":"San Francisco, CA"}"#.to_string(),
},
}]),
},
finish_reason: Some(FinishReason::ToolCalls),
logprobs: None,
}],
usage: Usage {
prompt_tokens: 84,
completion_tokens: 17,
total_tokens: 101,
prompt_tokens_details: None,
completion_tokens_details: None,
},
system_fingerprint: Some("fp_7eeb46f068".to_string()),
service_tier: Some("default".to_string()),
metadata: None,
};
let responses_api: ResponsesAPIResponse = chat_response.try_into().unwrap();
// Should have only 1 output item: function call (no empty message item)
assert_eq!(responses_api.output.len(), 1);
// Check function call item
match &responses_api.output[0] {
OutputItem::FunctionCall {
call_id,
name,
arguments,
..
} => {
assert_eq!(call_id, "call_oJBtqTJmRfBGlFS55QhMfUUV");
assert_eq!(name.as_ref().unwrap(), "get_weather");
assert!(arguments.as_ref().unwrap().contains("San Francisco, CA"));
}
_ => panic!("Expected FunctionCall output item as first item"),
}
// Verify status is Completed for tool_calls finish reason
assert!(matches!(responses_api.status, crate::apis::openai_responses::ResponseStatus::Completed));
}
}

18
crates/llm_gateway/src/stream_context.rs
View file

@ -25,7 +25,7 @@ use common::{ratelimit, routing, tokenizer};
use hermesllm::apis::amazon_bedrock_binary_frame::BedrockBinaryFrameDecoder;
use hermesllm::apis::anthropic::{MessagesContentBlock, MessagesStreamEvent};
use hermesllm::apis::sse::{SseEvent, SseStreamIter};
use hermesllm::clients::endpoints::SupportedAPIs;
use hermesllm::clients::endpoints::SupportedAPIsFromClients;
use hermesllm::providers::response::ProviderResponse;
use hermesllm::{
DecodedFrame, ProviderId, ProviderRequest, ProviderRequestType, ProviderResponseType,
@ -38,7 +38,7 @@ pub struct StreamContext {
streaming_response: bool,
response_tokens: usize,
/// The API that is requested by the client (before compatibility mapping)
client_api: Option<SupportedAPIs>,
client_api: Option<SupportedAPIsFromClients>,
/// The API that should be used for the upstream provider (after compatibility mapping)
resolved_api: Option<SupportedUpstreamAPIs>,
llm_providers: Rc<LlmProviders>,
@ -172,7 +172,8 @@ impl StreamContext {
Some(
SupportedUpstreamAPIs::OpenAIChatCompletions(_)
| SupportedUpstreamAPIs::AmazonBedrockConverse(_)
| SupportedUpstreamAPIs::AmazonBedrockConverseStream(_),
| SupportedUpstreamAPIs::AmazonBedrockConverseStream(_)
| SupportedUpstreamAPIs::OpenAIResponsesAPI(_),
)
| None => {
// OpenAI and default: use Authorization Bearer token
@ -544,7 +545,7 @@ impl StreamContext {
fn handle_bedrock_binary_stream(
&mut self,
body: &[u8],
client_api: &SupportedAPIs,
client_api: &SupportedAPIsFromClients,
upstream_api: &SupportedUpstreamAPIs,
) -> Result<Vec<u8>, Action> {
// Initialize decoder if not present
@ -782,13 +783,14 @@ impl HttpContext for StreamContext {
self.select_llm_provider();
// Check if this is a supported API endpoint
if SupportedAPIs::from_endpoint(&request_path).is_none() {
if SupportedAPIsFromClients::from_endpoint(&request_path).is_none() {
self.send_http_response(404, vec![], Some(b"Unsupported endpoint"));
return Action::Continue;
}
// Get the SupportedApi for routing decisions
let supported_api: Option<SupportedAPIs> = SupportedAPIs::from_endpoint(&request_path);
let supported_api: Option<SupportedAPIsFromClients> =
SupportedAPIsFromClients::from_endpoint(&request_path);
self.client_api = supported_api;
// Debug: log provider, client API, resolved API, and request path
@ -1131,8 +1133,8 @@ impl HttpContext for StreamContext {
}
match self.client_api {
Some(SupportedAPIs::OpenAIChatCompletions(_)) => {}
Some(SupportedAPIs::AnthropicMessagesAPI(_)) => {}
Some(SupportedAPIsFromClients::OpenAIChatCompletions(_)) => {}
Some(SupportedAPIsFromClients::AnthropicMessagesAPI(_)) => {}
_ => {
let api_info = match &self.client_api {
Some(api) => format!("{}", api),

2756
demos/samples_python/weather_forecast/response.tx Normal file
View file

File diff suppressed because one or more lines are too long

1035
demos/samples_python/weather_forecast/response_chat.txt Normal file
View file

File diff suppressed because it is too large Load diff

5521
demos/samples_python/weather_forecast/response_chat_reasoning.txt Normal file
View file

File diff suppressed because it is too large Load diff