apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-04-28 18:36:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add support for v1/responses API (#622)

Browse source 
* making first commit. still need to work on streaming respones

* making first commit. still need to work on streaming respones

* stream buffer implementation with tests

* adding grok API keys to workflow

* fixed changes based on code review

* adding support for bedrock models

* fixed issues with translation to claude code

---------

Co-authored-by: Salman Paracha <salmanparacha@MacBook-Pro-342.local>
This commit is contained in:
Salman Paracha 2025-12-03 14:58:26 -08:00 • committed by GitHub
parent b01a81927d
commit a448c6e9cb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 7015 additions and 2955 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/hermesllm/src/transforms/mod.rs
View file

@ -11,11 +11,13 @@
pub mod lib;
pub mod request;
pub mod response;
pub mod response_streaming;
// Re-export commonly used items for convenience
pub use lib::*;
pub use request::*;
pub use response::*;
pub use response_streaming::*;
// ============================================================================
// CONSTANTS

200
crates/hermesllm/src/transforms/request/from_openai.rs
View file

@ -12,6 +12,10 @@ use crate::apis::anthropic::{
use crate::apis::openai::{
ChatCompletionsRequest, Message, MessageContent, Role, Tool, ToolChoice, ToolChoiceType,
};
use crate::apis::openai_responses::{
ResponsesAPIRequest, InputContent, InputItem, InputParam, MessageRole, Modality, ReasoningEffort, Tool as ResponsesTool, ToolChoice as ResponsesToolChoice
};
use crate::clients::TransformError;
use crate::transforms::lib::ExtractText;
use crate::transforms::lib::*;
@ -244,6 +248,202 @@ impl TryFrom<Message> for BedrockMessage {
}
}
impl TryFrom<ResponsesAPIRequest> for ChatCompletionsRequest {
type Error = TransformError;
fn try_from(req: ResponsesAPIRequest) -> Result<Self, Self::Error> {
// Convert input to messages
let messages = match req.input {
InputParam::Text(text) => {
// Simple text input becomes a user message
vec![Message {
role: Role::User,
content: MessageContent::Text(text),
name: None,
tool_call_id: None,
tool_calls: None,
}]
}
InputParam::Items(items) => {
// Convert input items to messages
let mut converted_messages = Vec::new();
// Add instructions as system message if present
if let Some(instructions) = &req.instructions {
converted_messages.push(Message {
role: Role::System,
content: MessageContent::Text(instructions.clone()),
name: None,
tool_call_id: None,
tool_calls: None,
});
}
// Convert each input item
for item in items {
match item {
InputItem::Message(input_msg) => {
let role = match input_msg.role {
MessageRole::User => Role::User,
MessageRole::Assistant => Role::Assistant,
MessageRole::System => Role::System,
MessageRole::Developer => Role::System, // Map developer to system
};
// Convert content blocks
let content = if input_msg.content.len() == 1 {
// Single content item - check if it's simple text
match &input_msg.content[0] {
InputContent::InputText { text } => MessageContent::Text(text.clone()),
_ => {
// Convert to parts for non-text content
MessageContent::Parts(
input_msg.content.iter()
.filter_map(|c| match c {
InputContent::InputText { text } => {
Some(crate::apis::openai::ContentPart::Text { text: text.clone() })
}
InputContent::InputImage { image_url, .. } => {
Some(crate::apis::openai::ContentPart::ImageUrl {
image_url: crate::apis::openai::ImageUrl {
url: image_url.clone(),
detail: None,
}
})
}
InputContent::InputFile { .. } => None, // Skip files for now
InputContent::InputAudio { .. } => None, // Skip audio for now
})
.collect()
)
}
}
} else {
// Multiple content items - convert to parts
MessageContent::Parts(
input_msg.content.iter()
.filter_map(|c| match c {
InputContent::InputText { text } => {
Some(crate::apis::openai::ContentPart::Text { text: text.clone() })
}
InputContent::InputImage { image_url, .. } => {
Some(crate::apis::openai::ContentPart::ImageUrl {
image_url: crate::apis::openai::ImageUrl {
url: image_url.clone(),
detail: None,
}
})
}
InputContent::InputFile { .. } => None, // Skip files for now
InputContent::InputAudio { .. } => None, // Skip audio for now
})
.collect()
)
};
converted_messages.push(Message {
role,
content,
name: None,
tool_call_id: None,
tool_calls: None,
});
}
}
}
converted_messages
}
};
// Build the ChatCompletionsRequest
Ok(ChatCompletionsRequest {
model: req.model,
messages,
temperature: req.temperature,
top_p: req.top_p,
max_completion_tokens: req.max_output_tokens.map(|t| t as u32),
stream: req.stream,
metadata: req.metadata,
user: req.user,
store: req.store,
service_tier: req.service_tier,
top_logprobs: req.top_logprobs.map(|t| t as u32),
modalities: req.modalities.map(|mods| {
mods.into_iter().map(|m| {
match m {
Modality::Text => "text".to_string(),
Modality::Audio => "audio".to_string(),
}
}).collect()
}),
stream_options: req.stream_options.map(|opts| {
crate::apis::openai::StreamOptions {
include_usage: opts.include_usage,
}
}),
reasoning_effort: req.reasoning_effort.map(|effort| {
match effort {
ReasoningEffort::Low => "low".to_string(),
ReasoningEffort::Medium => "medium".to_string(),
ReasoningEffort::High => "high".to_string(),
}
}),
tools: req.tools.map(|tools| {
tools.into_iter().map(|tool| {
// Only convert Function tools - other types are not supported in ChatCompletions
match tool {
ResponsesTool::Function { name, description, parameters, strict } => Ok(Tool {
tool_type: "function".to_string(),
function: crate::apis::openai::Function {
name,
description,
parameters: parameters.unwrap_or_else(|| serde_json::json!({
"type": "object",
"properties": {}
})),
strict,
}
}),
ResponsesTool::FileSearch { .. } => Err(TransformError::UnsupportedConversion(
"FileSearch tool is not supported in ChatCompletions API. Only function tools are supported.".to_string()
)),
ResponsesTool::WebSearchPreview { .. } => Err(TransformError::UnsupportedConversion(
"WebSearchPreview tool is not supported in ChatCompletions API. Only function tools are supported.".to_string()
)),
ResponsesTool::CodeInterpreter => Err(TransformError::UnsupportedConversion(
"CodeInterpreter tool is not supported in ChatCompletions API. Only function tools are supported.".to_string()
)),
ResponsesTool::Computer { .. } => Err(TransformError::UnsupportedConversion(
"Computer tool is not supported in ChatCompletions API. Only function tools are supported.".to_string()
)),
}
}).collect::<Result<Vec<_>, _>>()
}).transpose()?,
tool_choice: req.tool_choice.map(|choice| {
match choice {
ResponsesToolChoice::String(s) => {
match s.as_str() {
"auto" => ToolChoice::Type(ToolChoiceType::Auto),
"required" => ToolChoice::Type(ToolChoiceType::Required),
"none" => ToolChoice::Type(ToolChoiceType::None),
_ => ToolChoice::Type(ToolChoiceType::Auto), // Default to auto for unknown strings
}
}
ResponsesToolChoice::Named { function, .. } => ToolChoice::Function {
choice_type: "function".to_string(),
function: crate::apis::openai::FunctionChoice { name: function.name }
}
}
}),
parallel_tool_calls: req.parallel_tool_calls,
..Default::default()
})
}
}
impl TryFrom<ChatCompletionsRequest> for AnthropicMessagesRequest {
type Error = TransformError;

296
crates/hermesllm/src/transforms/response/to_anthropic.rs
View file

@ -1,16 +1,11 @@
use crate::apis::amazon_bedrock::{
ContentBlockDelta, ConverseOutput, ConverseResponse, ConverseStreamEvent, StopReason,
};
use crate::apis::amazon_bedrock::{ConverseOutput, ConverseResponse, StopReason};
use crate::apis::anthropic::{
MessagesContentBlock, MessagesContentDelta, MessagesMessageDelta, MessagesResponse,
MessagesRole, MessagesStopReason, MessagesStreamEvent, MessagesStreamMessage, MessagesUsage,
};
use crate::apis::openai::{
ChatCompletionsResponse, ChatCompletionsStreamResponse, Role, ToolCallDelta,
MessagesContentBlock, MessagesResponse,
MessagesRole, MessagesStopReason, MessagesUsage,
};
use crate::apis::openai::ChatCompletionsResponse;
use crate::clients::TransformError;
use crate::transforms::lib::*;
use serde_json::Value;
// ============================================================================
// STANDARD RUST TRAIT IMPLEMENTATIONS - Using Into/TryFrom for convenience
@ -120,289 +115,6 @@ impl TryFrom<ConverseResponse> for MessagesResponse {
}
}
impl TryFrom<ChatCompletionsStreamResponse> for MessagesStreamEvent {
type Error = TransformError;
fn try_from(resp: ChatCompletionsStreamResponse) -> Result<Self, Self::Error> {
if resp.choices.is_empty() {
return Ok(MessagesStreamEvent::Ping);
}
let choice = &resp.choices[0];
// Handle final chunk with usage
let has_usage = resp.usage.is_some();
if let Some(usage) = resp.usage {
if let Some(finish_reason) = &choice.finish_reason {
let anthropic_stop_reason: MessagesStopReason = finish_reason.clone().into();
return Ok(MessagesStreamEvent::MessageDelta {
delta: MessagesMessageDelta {
stop_reason: anthropic_stop_reason,
stop_sequence: None,
},
usage: usage.into(),
});
}
}
// Handle role start
if let Some(Role::Assistant) = choice.delta.role {
return Ok(MessagesStreamEvent::MessageStart {
message: MessagesStreamMessage {
id: resp.id,
obj_type: "message".to_string(),
role: MessagesRole::Assistant,
content: vec![],
model: resp.model,
stop_reason: None,
stop_sequence: None,
usage: MessagesUsage {
input_tokens: 0,
output_tokens: 0,
cache_creation_input_tokens: None,
cache_read_input_tokens: None,
},
},
});
}
// Handle content delta
if let Some(content) = &choice.delta.content {
if !content.is_empty() {
return Ok(MessagesStreamEvent::ContentBlockDelta {
index: 0,
delta: MessagesContentDelta::TextDelta {
text: content.clone(),
},
});
}
}
// Handle tool calls
if let Some(tool_calls) = &choice.delta.tool_calls {
return convert_tool_call_deltas(tool_calls.clone());
}
// Handle finish reason - generate MessageDelta only (MessageStop comes later)
if let Some(finish_reason) = &choice.finish_reason {
// If we have usage data, it was already handled above
// If not, we need to generate MessageDelta with default usage
if !has_usage {
let anthropic_stop_reason: MessagesStopReason = finish_reason.clone().into();
return Ok(MessagesStreamEvent::MessageDelta {
delta: MessagesMessageDelta {
stop_reason: anthropic_stop_reason,
stop_sequence: None,
},
usage: MessagesUsage {
input_tokens: 0,
output_tokens: 0,
cache_creation_input_tokens: None,
cache_read_input_tokens: None,
},
});
}
// If usage was already handled above, we don't need to do anything more here
// MessageStop will be handled when [DONE] is encountered
}
// Default to ping for unhandled cases
Ok(MessagesStreamEvent::Ping)
}
}
impl Into<String> for MessagesStreamEvent {
fn into(self) -> String {
let transformed_json = serde_json::to_string(&self).unwrap_or_default();
let event_type = match &self {
MessagesStreamEvent::MessageStart { .. } => "message_start",
MessagesStreamEvent::ContentBlockStart { .. } => "content_block_start",
MessagesStreamEvent::ContentBlockDelta { .. } => "content_block_delta",
MessagesStreamEvent::ContentBlockStop { .. } => "content_block_stop",
MessagesStreamEvent::MessageDelta { .. } => "message_delta",
MessagesStreamEvent::MessageStop => "message_stop",
MessagesStreamEvent::Ping => "ping",
};
let event = format!("event: {}\n", event_type);
let data = format!("data: {}\n\n", transformed_json);
event + &data
}
}
impl TryFrom<ConverseStreamEvent> for MessagesStreamEvent {
type Error = TransformError;
fn try_from(event: ConverseStreamEvent) -> Result<Self, Self::Error> {
match event {
// MessageStart - convert to Anthropic MessageStart
ConverseStreamEvent::MessageStart(start_event) => {
let role = match start_event.role {
crate::apis::amazon_bedrock::ConversationRole::User => MessagesRole::User,
crate::apis::amazon_bedrock::ConversationRole::Assistant => {
MessagesRole::Assistant
}
};
Ok(MessagesStreamEvent::MessageStart {
message: MessagesStreamMessage {
id: format!(
"bedrock-stream-{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos()
),
obj_type: "message".to_string(),
role,
content: vec![],
model: "bedrock-model".to_string(),
stop_reason: None,
stop_sequence: None,
usage: MessagesUsage {
input_tokens: 0,
output_tokens: 0,
cache_creation_input_tokens: None,
cache_read_input_tokens: None,
},
},
})
}
// ContentBlockStart - convert to Anthropic ContentBlockStart
ConverseStreamEvent::ContentBlockStart(start_event) => {
// Note: Bedrock sends tool_use_id and name at start, with input coming in subsequent deltas
// Anthropic expects the same pattern, so we initialize with an empty input object
match start_event.start {
crate::apis::amazon_bedrock::ContentBlockStart::ToolUse { tool_use } => {
Ok(MessagesStreamEvent::ContentBlockStart {
index: start_event.content_block_index as u32,
content_block: MessagesContentBlock::ToolUse {
id: tool_use.tool_use_id,
name: tool_use.name,
input: Value::Object(serde_json::Map::new()), // Empty - will be filled by deltas
cache_control: None,
},
})
}
}
}
// ContentBlockDelta - convert to Anthropic ContentBlockDelta
ConverseStreamEvent::ContentBlockDelta(delta_event) => {
let delta = match delta_event.delta {
ContentBlockDelta::Text { text } => MessagesContentDelta::TextDelta { text },
ContentBlockDelta::ToolUse { tool_use } => {
MessagesContentDelta::InputJsonDelta {
partial_json: tool_use.input,
}
}
};
Ok(MessagesStreamEvent::ContentBlockDelta {
index: delta_event.content_block_index as u32,
delta,
})
}
// ContentBlockStop - convert to Anthropic ContentBlockStop
ConverseStreamEvent::ContentBlockStop(stop_event) => {
Ok(MessagesStreamEvent::ContentBlockStop {
index: stop_event.content_block_index as u32,
})
}
// MessageStop - convert to Anthropic MessageDelta with stop reason + MessageStop
ConverseStreamEvent::MessageStop(stop_event) => {
let anthropic_stop_reason = match stop_event.stop_reason {
StopReason::EndTurn => MessagesStopReason::EndTurn,
StopReason::ToolUse => MessagesStopReason::ToolUse,
StopReason::MaxTokens => MessagesStopReason::MaxTokens,
StopReason::StopSequence => MessagesStopReason::EndTurn,
StopReason::GuardrailIntervened => MessagesStopReason::Refusal,
StopReason::ContentFiltered => MessagesStopReason::Refusal,
};
// Return MessageDelta (MessageStop will be sent separately by the streaming handler)
Ok(MessagesStreamEvent::MessageDelta {
delta: MessagesMessageDelta {
stop_reason: anthropic_stop_reason,
stop_sequence: None,
},
usage: MessagesUsage {
input_tokens: 0,
output_tokens: 0,
cache_creation_input_tokens: None,
cache_read_input_tokens: None,
},
})
}
// Metadata - convert usage information to MessageDelta
ConverseStreamEvent::Metadata(metadata_event) => {
Ok(MessagesStreamEvent::MessageDelta {
delta: MessagesMessageDelta {
stop_reason: MessagesStopReason::EndTurn,
stop_sequence: None,
},
usage: MessagesUsage {
input_tokens: metadata_event.usage.input_tokens,
output_tokens: metadata_event.usage.output_tokens,
cache_creation_input_tokens: metadata_event.usage.cache_write_input_tokens,
cache_read_input_tokens: metadata_event.usage.cache_read_input_tokens,
},
})
}
// Exception events - convert to Ping (could be enhanced to return error events)
ConverseStreamEvent::InternalServerException(_)
| ConverseStreamEvent::ModelStreamErrorException(_)
| ConverseStreamEvent::ServiceUnavailableException(_)
| ConverseStreamEvent::ThrottlingException(_)
| ConverseStreamEvent::ValidationException(_) => {
// TODO: Consider adding proper error handling/events
Ok(MessagesStreamEvent::Ping)
}
}
}
}
/// Convert tool call deltas to Anthropic stream events
fn convert_tool_call_deltas(
tool_calls: Vec<ToolCallDelta>,
) -> Result<MessagesStreamEvent, TransformError> {
for tool_call in tool_calls {
if let Some(id) = &tool_call.id {
// Tool call start
if let Some(function) = &tool_call.function {
if let Some(name) = &function.name {
return Ok(MessagesStreamEvent::ContentBlockStart {
index: tool_call.index,
content_block: MessagesContentBlock::ToolUse {
id: id.clone(),
name: name.clone(),
input: Value::Object(serde_json::Map::new()),
cache_control: None,
},
});
}
}
} else if let Some(function) = &tool_call.function {
if let Some(arguments) = &function.arguments {
// Tool arguments delta
return Ok(MessagesStreamEvent::ContentBlockDelta {
index: tool_call.index,
delta: MessagesContentDelta::InputJsonDelta {
partial_json: arguments.clone(),
},
});
}
}
}
// Fallback to ping if no valid tool call found
Ok(MessagesStreamEvent::Ping)
}
/// Convert Bedrock Message to Anthropic content blocks
///

810
crates/hermesllm/src/transforms/response/to_openai.rs
View file

@ -1,15 +1,13 @@
use crate::apis::amazon_bedrock::{
ConverseOutput, ConverseResponse, ConverseStreamEvent, StopReason,
ConverseOutput, ConverseResponse, StopReason,
};
use crate::apis::anthropic::{
MessagesContentBlock, MessagesContentDelta, MessagesResponse, MessagesStopReason,
MessagesStreamEvent, MessagesUsage,
MessagesContentBlock, MessagesResponse, MessagesUsage,
};
use crate::apis::openai::{
ChatCompletionsResponse, ChatCompletionsStreamResponse, Choice, FinishReason,
FunctionCallDelta, MessageContent, MessageDelta, ResponseMessage, Role, StreamChoice,
ToolCallDelta, Usage,
ChatCompletionsResponse, Choice, FinishReason, MessageContent, ResponseMessage, Role, Usage,
};
use crate::apis::openai_responses::ResponsesAPIResponse;
use crate::clients::TransformError;
use crate::transforms::lib::*;
@ -30,6 +28,163 @@ impl Into<Usage> for MessagesUsage {
}
}
impl TryFrom<ChatCompletionsResponse> for ResponsesAPIResponse {
type Error = TransformError;
fn try_from(resp: ChatCompletionsResponse) -> Result<Self, Self::Error> {
use crate::apis::openai_responses::{
IncompleteDetails, IncompleteReason, OutputContent, OutputItem, OutputItemStatus,
ResponseStatus, ResponseUsage, ResponsesAPIResponse,
};
// Convert the first choice's message to output items
let output = if let Some(choice) = resp.choices.first() {
let mut items = Vec::new();
// Create a message output item from the response message
let mut content = Vec::new();
// Add text content if present
if let Some(text) = &choice.message.content {
content.push(OutputContent::OutputText {
text: text.clone(),
annotations: vec![],
logprobs: None,
});
}
// Add audio content if present (audio is a Value, need to handle it carefully)
if let Some(audio) = &choice.message.audio {
// Audio is serde_json::Value, try to extract data and transcript
if let Some(audio_obj) = audio.as_object() {
let data = audio_obj
.get("data")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let transcript = audio_obj
.get("transcript")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
content.push(OutputContent::OutputAudio { data, transcript });
}
}
// Add refusal content if present
if let Some(refusal) = &choice.message.refusal {
content.push(OutputContent::Refusal {
refusal: refusal.clone(),
});
}
// Only add the message item if there's actual content (text, audio, or refusal)
// Don't add empty message items when there are only tool calls
if !content.is_empty() {
items.push(OutputItem::Message {
id: format!("msg_{}", resp.id),
status: OutputItemStatus::Completed,
role: match choice.message.role {
Role::User => "user".to_string(),
Role::Assistant => "assistant".to_string(),
Role::System => "system".to_string(),
Role::Tool => "tool".to_string(),
},
content,
});
}
// Add tool calls as function call items if present
if let Some(tool_calls) = &choice.message.tool_calls {
for tool_call in tool_calls {
items.push(OutputItem::FunctionCall {
id: format!("func_{}", tool_call.id),
status: OutputItemStatus::Completed,
call_id: tool_call.id.clone(),
name: Some(tool_call.function.name.clone()),
arguments: Some(tool_call.function.arguments.clone()),
});
}
}
items
} else {
vec![]
};
// Convert finish_reason to status
let status = if let Some(choice) = resp.choices.first() {
match choice.finish_reason {
Some(FinishReason::Stop) => ResponseStatus::Completed,
Some(FinishReason::ToolCalls) => ResponseStatus::Completed,
Some(FinishReason::Length) => ResponseStatus::Incomplete,
Some(FinishReason::ContentFilter) => ResponseStatus::Failed,
_ => ResponseStatus::Completed,
}
} else {
ResponseStatus::Completed
};
// Convert usage
let usage = ResponseUsage {
input_tokens: resp.usage.prompt_tokens as i32,
output_tokens: resp.usage.completion_tokens as i32,
total_tokens: resp.usage.total_tokens as i32,
input_tokens_details: resp.usage.prompt_tokens_details.map(|details| {
crate::apis::openai_responses::TokenDetails {
cached_tokens: details.cached_tokens.unwrap_or(0) as i32,
}
}),
output_tokens_details: resp.usage.completion_tokens_details.map(|details| {
crate::apis::openai_responses::OutputTokenDetails {
reasoning_tokens: details.reasoning_tokens.unwrap_or(0) as i32,
}
}),
};
// Set incomplete_details if status is incomplete
let incomplete_details = if matches!(status, ResponseStatus::Incomplete) {
Some(IncompleteDetails {
reason: IncompleteReason::MaxOutputTokens,
})
} else {
None
};
Ok(ResponsesAPIResponse {
id: resp.id,
object: "response".to_string(),
created_at: resp.created as i64,
status,
background: Some(false),
error: None,
incomplete_details,
instructions: None,
max_output_tokens: None,
max_tool_calls: None,
model: resp.model,
output,
usage: Some(usage),
parallel_tool_calls: true,
conversation: None,
previous_response_id: None,
tools: vec![],
tool_choice: "auto".to_string(),
temperature: 1.0,
top_p: 1.0,
metadata: resp.metadata.unwrap_or_default(),
truncation: None,
reasoning: None,
store: None,
text: None,
audio: None,
modalities: None,
service_tier: resp.service_tier,
top_logprobs: None,
})
}
}
impl TryFrom<MessagesResponse> for ChatCompletionsResponse {
type Error = TransformError;
@ -173,416 +328,6 @@ impl TryFrom<ConverseResponse> for ChatCompletionsResponse {
}
}
// ============================================================================
// STREAMING TRANSFORMATIONS
// ============================================================================
impl TryFrom<MessagesStreamEvent> for ChatCompletionsStreamResponse {
type Error = TransformError;
fn try_from(event: MessagesStreamEvent) -> Result<Self, Self::Error> {
match event {
MessagesStreamEvent::MessageStart { message } => Ok(create_openai_chunk(
&message.id,
&message.model,
MessageDelta {
role: Some(Role::Assistant),
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
None,
None,
)),
MessagesStreamEvent::ContentBlockStart { content_block, .. } => {
convert_content_block_start(content_block)
}
MessagesStreamEvent::ContentBlockDelta { delta, .. } => convert_content_delta(delta),
MessagesStreamEvent::ContentBlockStop { .. } => Ok(create_empty_openai_chunk()),
MessagesStreamEvent::MessageDelta { delta, usage } => {
let finish_reason: Option<FinishReason> = Some(delta.stop_reason.into());
let openai_usage: Option<Usage> = Some(usage.into());
Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
finish_reason,
openai_usage,
))
}
MessagesStreamEvent::MessageStop => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
Some(FinishReason::Stop),
None,
)),
MessagesStreamEvent::Ping => Ok(ChatCompletionsStreamResponse {
id: "stream".to_string(),
object: Some("chat.completion.chunk".to_string()),
created: current_timestamp(),
model: "unknown".to_string(),
choices: vec![],
usage: None,
system_fingerprint: None,
service_tier: None,
}),
}
}
}
impl TryFrom<ConverseStreamEvent> for ChatCompletionsStreamResponse {
type Error = TransformError;
fn try_from(event: ConverseStreamEvent) -> Result<Self, Self::Error> {
match event {
ConverseStreamEvent::MessageStart(start_event) => {
let role = match start_event.role {
crate::apis::amazon_bedrock::ConversationRole::User => Role::User,
crate::apis::amazon_bedrock::ConversationRole::Assistant => Role::Assistant,
};
Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: Some(role),
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
None,
None,
))
}
ConverseStreamEvent::ContentBlockStart(start_event) => {
use crate::apis::amazon_bedrock::ContentBlockStart;
match start_event.start {
ContentBlockStart::ToolUse { tool_use } => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: Some(vec![ToolCallDelta {
index: start_event.content_block_index as u32,
id: Some(tool_use.tool_use_id),
call_type: Some("function".to_string()),
function: Some(FunctionCallDelta {
name: Some(tool_use.name),
arguments: Some("".to_string()),
}),
}]),
},
None,
None,
)),
}
}
ConverseStreamEvent::ContentBlockDelta(delta_event) => {
use crate::apis::amazon_bedrock::ContentBlockDelta;
match delta_event.delta {
ContentBlockDelta::Text { text } => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: Some(text),
refusal: None,
function_call: None,
tool_calls: None,
},
None,
None,
)),
ContentBlockDelta::ToolUse { tool_use } => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: Some(vec![ToolCallDelta {
index: delta_event.content_block_index as u32,
id: None,
call_type: None,
function: Some(FunctionCallDelta {
name: None,
arguments: Some(tool_use.input),
}),
}]),
},
None,
None,
)),
}
}
ConverseStreamEvent::ContentBlockStop(_) => Ok(create_empty_openai_chunk()),
ConverseStreamEvent::MessageStop(stop_event) => {
let finish_reason = match stop_event.stop_reason {
StopReason::EndTurn => FinishReason::Stop,
StopReason::ToolUse => FinishReason::ToolCalls,
StopReason::MaxTokens => FinishReason::Length,
StopReason::StopSequence => FinishReason::Stop,
StopReason::GuardrailIntervened => FinishReason::ContentFilter,
StopReason::ContentFiltered => FinishReason::ContentFilter,
};
Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
Some(finish_reason),
None,
))
}
ConverseStreamEvent::Metadata(metadata_event) => {
let usage = Usage {
prompt_tokens: metadata_event.usage.input_tokens,
completion_tokens: metadata_event.usage.output_tokens,
total_tokens: metadata_event.usage.total_tokens,
prompt_tokens_details: None,
completion_tokens_details: None,
};
Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
None,
Some(usage),
))
}
// Error events - convert to empty chunks (errors should be handled elsewhere)
ConverseStreamEvent::InternalServerException(_)
| ConverseStreamEvent::ModelStreamErrorException(_)
| ConverseStreamEvent::ServiceUnavailableException(_)
| ConverseStreamEvent::ThrottlingException(_)
| ConverseStreamEvent::ValidationException(_) => Ok(create_empty_openai_chunk()),
}
}
}
/// Convert content block start to OpenAI chunk
fn convert_content_block_start(
content_block: MessagesContentBlock,
) -> Result<ChatCompletionsStreamResponse, TransformError> {
match content_block {
MessagesContentBlock::Text { .. } => {
// No immediate output for text block start
Ok(create_empty_openai_chunk())
}
MessagesContentBlock::ToolUse { id, name, .. }
| MessagesContentBlock::ServerToolUse { id, name, .. }
| MessagesContentBlock::McpToolUse { id, name, .. } => {
// Tool use start → OpenAI chunk with tool_calls
Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: Some(vec![ToolCallDelta {
index: 0,
id: Some(id),
call_type: Some("function".to_string()),
function: Some(FunctionCallDelta {
name: Some(name),
arguments: Some("".to_string()),
}),
}]),
},
None,
None,
))
}
_ => Err(TransformError::UnsupportedContent(
"Unsupported content block type in stream start".to_string(),
)),
}
}
/// Convert content delta to OpenAI chunk
fn convert_content_delta(
delta: MessagesContentDelta,
) -> Result<ChatCompletionsStreamResponse, TransformError> {
match delta {
MessagesContentDelta::TextDelta { text } => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: Some(text),
refusal: None,
function_call: None,
tool_calls: None,
},
None,
None,
)),
MessagesContentDelta::ThinkingDelta { thinking } => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: Some(format!("thinking: {}", thinking)),
refusal: None,
function_call: None,
tool_calls: None,
},
None,
None,
)),
MessagesContentDelta::InputJsonDelta { partial_json } => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: Some(vec![ToolCallDelta {
index: 0,
id: None,
call_type: None,
function: Some(FunctionCallDelta {
name: None,
arguments: Some(partial_json),
}),
}]),
},
None,
None,
)),
}
}
/// Helper to create OpenAI streaming chunk
fn create_openai_chunk(
id: &str,
model: &str,
delta: MessageDelta,
finish_reason: Option<FinishReason>,
usage: Option<Usage>,
) -> ChatCompletionsStreamResponse {
ChatCompletionsStreamResponse {
id: id.to_string(),
object: Some("chat.completion.chunk".to_string()),
created: current_timestamp(),
model: model.to_string(),
choices: vec![StreamChoice {
index: 0,
delta,
finish_reason,
logprobs: None,
}],
usage,
system_fingerprint: None,
service_tier: None,
}
}
/// Helper to create empty OpenAI streaming chunk
fn create_empty_openai_chunk() -> ChatCompletionsStreamResponse {
create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
None,
None,
)
}
/// Convert Anthropic content blocks to OpenAI message content
fn convert_anthropic_content_to_openai(
content: &[MessagesContentBlock],
) -> Result<MessageContent, TransformError> {
let mut text_parts = Vec::new();
for block in content {
match block {
MessagesContentBlock::Text { text, .. } => {
text_parts.push(text.clone());
}
MessagesContentBlock::Thinking { thinking, .. } => {
text_parts.push(format!("thinking: {}", thinking));
}
_ => {
// Skip other content types for basic text conversion
continue;
}
}
}
Ok(MessageContent::Text(text_parts.join("\n")))
}
// Stop Reason Conversions
impl Into<FinishReason> for MessagesStopReason {
fn into(self) -> FinishReason {
match self {
MessagesStopReason::EndTurn => FinishReason::Stop,
MessagesStopReason::MaxTokens => FinishReason::Length,
MessagesStopReason::StopSequence => FinishReason::Stop,
MessagesStopReason::ToolUse => FinishReason::ToolCalls,
MessagesStopReason::PauseTurn => FinishReason::Stop,
MessagesStopReason::Refusal => FinishReason::ContentFilter,
}
}
}
/// Convert Bedrock Message to OpenAI content and tool calls
/// This function extracts text content and tool calls from a Bedrock message
fn convert_bedrock_message_to_openai(
@ -627,6 +372,31 @@ fn convert_bedrock_message_to_openai(
Ok((content, tool_calls))
}
/// Convert Anthropic content blocks to OpenAI message content
fn convert_anthropic_content_to_openai(
content: &[MessagesContentBlock],
) -> Result<MessageContent, TransformError> {
let mut text_parts = Vec::new();
for block in content {
match block {
MessagesContentBlock::Text { text, .. } => {
text_parts.push(text.clone());
}
MessagesContentBlock::Thinking { thinking, .. } => {
text_parts.push(format!("thinking: {}", thinking));
}
_ => {
// Skip other content types for basic text conversion
continue;
}
}
}
Ok(MessageContent::Text(text_parts.join("\n")))
}
#[cfg(test)]
mod tests {
use super::*;
@ -1166,4 +936,212 @@ mod tests {
assert!(content.contains("Here's the analysis:"));
// Note: Image blocks are not converted to text in the current implementation
}
#[test]
fn test_chat_completions_to_responses_api_basic() {
use crate::apis::openai_responses::{OutputContent, OutputItem, ResponsesAPIResponse};
let chat_response = ChatCompletionsResponse {
id: "chatcmpl-123".to_string(),
object: Some("chat.completion".to_string()),
created: 1677652288,
model: "gpt-4".to_string(),
choices: vec![Choice {
index: 0,
message: crate::apis::openai::ResponseMessage {
role: Role::Assistant,
content: Some("Hello! How can I help you?".to_string()),
refusal: None,
annotations: None,
audio: None,
function_call: None,
tool_calls: None,
},
finish_reason: Some(FinishReason::Stop),
logprobs: None,
}],
usage: Usage {
prompt_tokens: 10,
completion_tokens: 20,
total_tokens: 30,
prompt_tokens_details: None,
completion_tokens_details: None,
},
system_fingerprint: None,
service_tier: Some("default".to_string()),
metadata: None,
};
let responses_api: ResponsesAPIResponse = chat_response.try_into().unwrap();
assert_eq!(responses_api.id, "chatcmpl-123");
assert_eq!(responses_api.object, "response");
assert_eq!(responses_api.model, "gpt-4");
// Check usage conversion
let usage = responses_api.usage.unwrap();
assert_eq!(usage.input_tokens, 10);
assert_eq!(usage.output_tokens, 20);
assert_eq!(usage.total_tokens, 30);
// Check output items
assert_eq!(responses_api.output.len(), 1);
match &responses_api.output[0] {
OutputItem::Message {
role,
content,
..
} => {
assert_eq!(role, "assistant");
assert_eq!(content.len(), 1);
match &content[0] {
OutputContent::OutputText { text, .. } => {
assert_eq!(text, "Hello! How can I help you?");
}
_ => panic!("Expected OutputText content"),
}
}
_ => panic!("Expected Message output item"),
}
}
#[test]
fn test_chat_completions_to_responses_api_with_tool_calls() {
use crate::apis::openai::{FunctionCall, ToolCall};
use crate::apis::openai_responses::{OutputItem, ResponsesAPIResponse};
let chat_response = ChatCompletionsResponse {
id: "chatcmpl-456".to_string(),
object: Some("chat.completion".to_string()),
created: 1677652300,
model: "gpt-4".to_string(),
choices: vec![Choice {
index: 0,
message: crate::apis::openai::ResponseMessage {
role: Role::Assistant,
content: Some("Let me check the weather.".to_string()),
refusal: None,
annotations: None,
audio: None,
function_call: None,
tool_calls: Some(vec![ToolCall {
id: "call_abc123".to_string(),
call_type: "function".to_string(),
function: FunctionCall {
name: "get_weather".to_string(),
arguments: r#"{"location":"San Francisco"}"#.to_string(),
},
}]),
},
finish_reason: Some(FinishReason::ToolCalls),
logprobs: None,
}],
usage: Usage {
prompt_tokens: 15,
completion_tokens: 25,
total_tokens: 40,
prompt_tokens_details: None,
completion_tokens_details: None,
},
system_fingerprint: None,
service_tier: None,
metadata: None,
};
let responses_api: ResponsesAPIResponse = chat_response.try_into().unwrap();
// Should have 2 output items: message + function call
assert_eq!(responses_api.output.len(), 2);
// Check message item
match &responses_api.output[0] {
OutputItem::Message { content, .. } => {
assert_eq!(content.len(), 1);
}
_ => panic!("Expected Message output item"),
}
// Check function call item
match &responses_api.output[1] {
OutputItem::FunctionCall {
call_id,
name,
arguments,
..
} => {
assert_eq!(call_id, "call_abc123");
assert_eq!(name.as_ref().unwrap(), "get_weather");
assert!(arguments.as_ref().unwrap().contains("San Francisco"));
}
_ => panic!("Expected FunctionCall output item"),
}
}
#[test]
fn test_chat_completions_to_responses_api_tool_calls_only() {
use crate::apis::openai::{FunctionCall, ToolCall};
use crate::apis::openai_responses::{OutputItem, ResponsesAPIResponse};
// Test the real-world case where content is null and there are only tool calls
let chat_response = ChatCompletionsResponse {
id: "chatcmpl-789".to_string(),
object: Some("chat.completion".to_string()),
created: 1764023939,
model: "gpt-4o-2024-08-06".to_string(),
choices: vec![Choice {
index: 0,
message: crate::apis::openai::ResponseMessage {
role: Role::Assistant,
content: None, // No text content, only tool calls
refusal: None,
annotations: None,
audio: None,
function_call: None,
tool_calls: Some(vec![ToolCall {
id: "call_oJBtqTJmRfBGlFS55QhMfUUV".to_string(),
call_type: "function".to_string(),
function: FunctionCall {
name: "get_weather".to_string(),
arguments: r#"{"location":"San Francisco, CA"}"#.to_string(),
},
}]),
},
finish_reason: Some(FinishReason::ToolCalls),
logprobs: None,
}],
usage: Usage {
prompt_tokens: 84,
completion_tokens: 17,
total_tokens: 101,
prompt_tokens_details: None,
completion_tokens_details: None,
},
system_fingerprint: Some("fp_7eeb46f068".to_string()),
service_tier: Some("default".to_string()),
metadata: None,
};
let responses_api: ResponsesAPIResponse = chat_response.try_into().unwrap();
// Should have only 1 output item: function call (no empty message item)
assert_eq!(responses_api.output.len(), 1);
// Check function call item
match &responses_api.output[0] {
OutputItem::FunctionCall {
call_id,
name,
arguments,
..
} => {
assert_eq!(call_id, "call_oJBtqTJmRfBGlFS55QhMfUUV");
assert_eq!(name.as_ref().unwrap(), "get_weather");
assert!(arguments.as_ref().unwrap().contains("San Francisco, CA"));
}
_ => panic!("Expected FunctionCall output item as first item"),
}
// Verify status is Completed for tool_calls finish reason
assert!(matches!(responses_api.status, crate::apis::openai_responses::ResponseStatus::Completed));
}
}

2
crates/hermesllm/src/transforms/response_streaming/mod.rs Normal file
View file

@ -0,0 +1,2 @@
pub mod to_anthropic_streaming;
pub mod to_openai_streaming;

281
crates/hermesllm/src/transforms/response_streaming/to_anthropic_streaming.rs Normal file
View file

@ -0,0 +1,281 @@
use crate::apis::amazon_bedrock::{
ContentBlockDelta, ConverseStreamEvent,
};
use crate::apis::anthropic::{
MessagesContentBlock, MessagesContentDelta, MessagesMessageDelta,
MessagesRole, MessagesStopReason, MessagesStreamEvent, MessagesStreamMessage, MessagesUsage,
};
use crate::apis::openai::{ ChatCompletionsStreamResponse, ToolCallDelta,
};
use crate::clients::TransformError;
use serde_json::Value;
impl TryFrom<ChatCompletionsStreamResponse> for MessagesStreamEvent {
type Error = TransformError;
fn try_from(resp: ChatCompletionsStreamResponse) -> Result<Self, Self::Error> {
if resp.choices.is_empty() {
return Ok(MessagesStreamEvent::Ping);
}
let choice = &resp.choices[0];
// Handle final chunk with usage
let has_usage = resp.usage.is_some();
if let Some(usage) = resp.usage {
if let Some(finish_reason) = &choice.finish_reason {
let anthropic_stop_reason: MessagesStopReason = finish_reason.clone().into();
return Ok(MessagesStreamEvent::MessageDelta {
delta: MessagesMessageDelta {
stop_reason: anthropic_stop_reason,
stop_sequence: None,
},
usage: usage.into(),
});
}
}
// NOTE: We do NOT emit MessageStart here anymore!
// The AnthropicMessagesStreamBuffer will inject message_start and content_block_start
// when it sees the first content_block_delta. This solves the problem where OpenAI
// sends both role and content in the same chunk - we can only return one event here,
// so we prioritize the content and let the buffer handle lifecycle events.
// Handle content delta (even if role is present in the same chunk)
if let Some(content) = &choice.delta.content {
if !content.is_empty() {
return Ok(MessagesStreamEvent::ContentBlockDelta {
index: 0,
delta: MessagesContentDelta::TextDelta {
text: content.clone(),
},
});
}
}
// Handle tool calls
if let Some(tool_calls) = &choice.delta.tool_calls {
return convert_tool_call_deltas(tool_calls.clone());
}
// Handle finish reason - generate MessageDelta only (MessageStop comes later)
if let Some(finish_reason) = &choice.finish_reason {
// If we have usage data, it was already handled above
// If not, we need to generate MessageDelta with default usage
if !has_usage {
let anthropic_stop_reason: MessagesStopReason = finish_reason.clone().into();
return Ok(MessagesStreamEvent::MessageDelta {
delta: MessagesMessageDelta {
stop_reason: anthropic_stop_reason,
stop_sequence: None,
},
usage: MessagesUsage {
input_tokens: 0,
output_tokens: 0,
cache_creation_input_tokens: None,
cache_read_input_tokens: None,
},
});
}
// If usage was already handled above, we don't need to do anything more here
// MessageStop will be handled when [DONE] is encountered
}
// Default to ping for unhandled cases
Ok(MessagesStreamEvent::Ping)
}
}
impl Into<String> for MessagesStreamEvent {
fn into(self) -> String {
let transformed_json = serde_json::to_string(&self).unwrap_or_default();
let event_type = match &self {
MessagesStreamEvent::MessageStart { .. } => "message_start",
MessagesStreamEvent::ContentBlockStart { .. } => "content_block_start",
MessagesStreamEvent::ContentBlockDelta { .. } => "content_block_delta",
MessagesStreamEvent::ContentBlockStop { .. } => "content_block_stop",
MessagesStreamEvent::MessageDelta { .. } => "message_delta",
MessagesStreamEvent::MessageStop => "message_stop",
MessagesStreamEvent::Ping => "ping",
};
let event = format!("event: {}\n", event_type);
let data = format!("data: {}\n\n", transformed_json);
event + &data
}
}
impl TryFrom<ConverseStreamEvent> for MessagesStreamEvent {
type Error = TransformError;
fn try_from(event: ConverseStreamEvent) -> Result<Self, Self::Error> {
match event {
// MessageStart - convert to Anthropic MessageStart
ConverseStreamEvent::MessageStart(start_event) => {
let role = match start_event.role {
crate::apis::amazon_bedrock::ConversationRole::User => MessagesRole::User,
crate::apis::amazon_bedrock::ConversationRole::Assistant => {
MessagesRole::Assistant
}
};
Ok(MessagesStreamEvent::MessageStart {
message: MessagesStreamMessage {
id: format!(
"bedrock-stream-{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos()
),
obj_type: "message".to_string(),
role,
content: vec![],
model: "bedrock-model".to_string(),
stop_reason: None,
stop_sequence: None,
usage: MessagesUsage {
input_tokens: 0,
output_tokens: 0,
cache_creation_input_tokens: None,
cache_read_input_tokens: None,
},
},
})
}
// ContentBlockStart - convert to Anthropic ContentBlockStart
ConverseStreamEvent::ContentBlockStart(start_event) => {
// Note: Bedrock sends tool_use_id and name at start, with input coming in subsequent deltas
// Anthropic expects the same pattern, so we initialize with an empty input object
match start_event.start {
crate::apis::amazon_bedrock::ContentBlockStart::ToolUse { tool_use } => {
Ok(MessagesStreamEvent::ContentBlockStart {
index: start_event.content_block_index as u32,
content_block: MessagesContentBlock::ToolUse {
id: tool_use.tool_use_id,
name: tool_use.name,
input: Value::Object(serde_json::Map::new()), // Empty - will be filled by deltas
cache_control: None,
},
})
}
}
}
// ContentBlockDelta - convert to Anthropic ContentBlockDelta
ConverseStreamEvent::ContentBlockDelta(delta_event) => {
let delta = match delta_event.delta {
ContentBlockDelta::Text { text } => MessagesContentDelta::TextDelta { text },
ContentBlockDelta::ToolUse { tool_use } => {
MessagesContentDelta::InputJsonDelta {
partial_json: tool_use.input,
}
}
};
Ok(MessagesStreamEvent::ContentBlockDelta {
index: delta_event.content_block_index as u32,
delta,
})
}
// ContentBlockStop - convert to Anthropic ContentBlockStop
ConverseStreamEvent::ContentBlockStop(stop_event) => {
Ok(MessagesStreamEvent::ContentBlockStop {
index: stop_event.content_block_index as u32,
})
}
// MessageStop - convert to Anthropic MessageDelta with stop reason
// Note: Bedrock sends Metadata separately with usage info, creating a second MessageDelta
// The client should merge these or use the final one with complete usage
ConverseStreamEvent::MessageStop(stop_event) => {
let anthropic_stop_reason = match stop_event.stop_reason {
crate::apis::amazon_bedrock::StopReason::EndTurn => MessagesStopReason::EndTurn,
crate::apis::amazon_bedrock::StopReason::ToolUse => MessagesStopReason::ToolUse,
crate::apis::amazon_bedrock::StopReason::MaxTokens => MessagesStopReason::MaxTokens,
crate::apis::amazon_bedrock::StopReason::StopSequence => MessagesStopReason::EndTurn,
crate::apis::amazon_bedrock::StopReason::GuardrailIntervened => MessagesStopReason::Refusal,
crate::apis::amazon_bedrock::StopReason::ContentFiltered => MessagesStopReason::Refusal,
};
Ok(MessagesStreamEvent::MessageDelta {
delta: MessagesMessageDelta {
stop_reason: anthropic_stop_reason,
stop_sequence: None,
},
usage: MessagesUsage {
input_tokens: 0,
output_tokens: 0,
cache_creation_input_tokens: None,
cache_read_input_tokens: None,
},
})
}
// Metadata - convert usage information to MessageDelta
ConverseStreamEvent::Metadata(metadata_event) => {
Ok(MessagesStreamEvent::MessageDelta {
delta: MessagesMessageDelta {
stop_reason: MessagesStopReason::EndTurn,
stop_sequence: None,
},
usage: MessagesUsage {
input_tokens: metadata_event.usage.input_tokens,
output_tokens: metadata_event.usage.output_tokens,
cache_creation_input_tokens: metadata_event.usage.cache_write_input_tokens,
cache_read_input_tokens: metadata_event.usage.cache_read_input_tokens,
},
})
}
// Exception events - convert to Ping (could be enhanced to return error events)
ConverseStreamEvent::InternalServerException(_)
| ConverseStreamEvent::ModelStreamErrorException(_)
| ConverseStreamEvent::ServiceUnavailableException(_)
| ConverseStreamEvent::ThrottlingException(_)
| ConverseStreamEvent::ValidationException(_) => {
// TODO: Consider adding proper error handling/events
Ok(MessagesStreamEvent::Ping)
}
}
}
}
/// Convert tool call deltas to Anthropic stream events
fn convert_tool_call_deltas(
tool_calls: Vec<ToolCallDelta>,
) -> Result<MessagesStreamEvent, TransformError> {
for tool_call in tool_calls {
if let Some(id) = &tool_call.id {
// Tool call start
if let Some(function) = &tool_call.function {
if let Some(name) = &function.name {
return Ok(MessagesStreamEvent::ContentBlockStart {
index: tool_call.index,
content_block: MessagesContentBlock::ToolUse {
id: id.clone(),
name: name.clone(),
input: Value::Object(serde_json::Map::new()),
cache_control: None,
},
});
}
}
} else if let Some(function) = &tool_call.function {
if let Some(arguments) = &function.arguments {
// Tool arguments delta
return Ok(MessagesStreamEvent::ContentBlockDelta {
index: tool_call.index,
delta: MessagesContentDelta::InputJsonDelta {
partial_json: arguments.clone(),
},
});
}
}
}
// Fallback to ping if no valid tool call found
Ok(MessagesStreamEvent::Ping)
}

527
crates/hermesllm/src/transforms/response_streaming/to_openai_streaming.rs Normal file
View file

@ -0,0 +1,527 @@
use crate::apis::amazon_bedrock::{ ConverseStreamEvent, StopReason};
use crate::apis::anthropic::{
MessagesContentBlock, MessagesContentDelta, MessagesStopReason, MessagesStreamEvent};
use crate::apis::openai::{ ChatCompletionsStreamResponse,FinishReason,
FunctionCallDelta, MessageDelta, Role, StreamChoice, ToolCallDelta, Usage,
};
use crate::apis::openai_responses::ResponsesAPIStreamEvent;
use crate::clients::TransformError;
use crate::transforms::lib::*;
// ============================================================================
// PROVIDER STREAMING TRANSFORMATIONS TO OPENAI FORMAT
// ============================================================================
//
// This module handles business logic for converting streaming events from
// various providers (Anthropic, Bedrock, etc.) into OpenAI's ChatCompletions format.
//
// # Architecture Separation
//
// **Provider Transformations** (this module):
// - Business logic for converting between provider formats
// - Uses Rust traits (TryFrom, Into) for type-safe conversions
// - Stateless event-by-event transformation
// - Example: MessagesStreamEvent → ChatCompletionsStreamResponse
//
// **Wire Format Buffering** (`apis/streaming_shapes/`):
// - SSE protocol handling (data:, event: lines)
// - State accumulation and lifecycle management
// - Buffering for stateful APIs (v1/responses)
// - Example: ChatCompletionsToResponsesTransformer
//
// # Flow
//
// ```text
// Anthropic Event → [Provider Transform] → OpenAI Event → [Wire Buffer] → SSE Wire Format
// (business) (this module) (protocol) (streaming_shapes) (network)
// ```
//
// ============================================================================
impl TryFrom<MessagesStreamEvent> for ChatCompletionsStreamResponse {
type Error = TransformError;
fn try_from(event: MessagesStreamEvent) -> Result<Self, Self::Error> {
match event {
MessagesStreamEvent::MessageStart { message } => Ok(create_openai_chunk(
&message.id,
&message.model,
MessageDelta {
role: Some(Role::Assistant),
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
None,
None,
)),
MessagesStreamEvent::ContentBlockStart { content_block, .. } => {
convert_content_block_start(content_block)
}
MessagesStreamEvent::ContentBlockDelta { delta, .. } => convert_content_delta(delta),
MessagesStreamEvent::ContentBlockStop { .. } => Ok(create_empty_openai_chunk()),
MessagesStreamEvent::MessageDelta { delta, usage } => {
let finish_reason: Option<FinishReason> = Some(delta.stop_reason.into());
let openai_usage: Option<Usage> = Some(usage.into());
Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
finish_reason,
openai_usage,
))
}
MessagesStreamEvent::MessageStop => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
Some(FinishReason::Stop),
None,
)),
MessagesStreamEvent::Ping => Ok(ChatCompletionsStreamResponse {
id: "stream".to_string(),
object: Some("chat.completion.chunk".to_string()),
created: current_timestamp(),
model: "unknown".to_string(),
choices: vec![],
usage: None,
system_fingerprint: None,
service_tier: None,
}),
}
}
}
impl TryFrom<ConverseStreamEvent> for ChatCompletionsStreamResponse {
type Error = TransformError;
fn try_from(event: ConverseStreamEvent) -> Result<Self, Self::Error> {
match event {
ConverseStreamEvent::MessageStart(start_event) => {
let role = match start_event.role {
crate::apis::amazon_bedrock::ConversationRole::User => Role::User,
crate::apis::amazon_bedrock::ConversationRole::Assistant => Role::Assistant,
};
Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: Some(role),
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
None,
None,
))
}
ConverseStreamEvent::ContentBlockStart(start_event) => {
use crate::apis::amazon_bedrock::ContentBlockStart;
match start_event.start {
ContentBlockStart::ToolUse { tool_use } => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: Some(vec![ToolCallDelta {
index: start_event.content_block_index as u32,
id: Some(tool_use.tool_use_id),
call_type: Some("function".to_string()),
function: Some(FunctionCallDelta {
name: Some(tool_use.name),
arguments: Some("".to_string()),
}),
}]),
},
None,
None,
)),
}
}
ConverseStreamEvent::ContentBlockDelta(delta_event) => {
use crate::apis::amazon_bedrock::ContentBlockDelta;
match delta_event.delta {
ContentBlockDelta::Text { text } => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: Some(text),
refusal: None,
function_call: None,
tool_calls: None,
},
None,
None,
)),
ContentBlockDelta::ToolUse { tool_use } => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: Some(vec![ToolCallDelta {
index: delta_event.content_block_index as u32,
id: None,
call_type: None,
function: Some(FunctionCallDelta {
name: None,
arguments: Some(tool_use.input),
}),
}]),
},
None,
None,
)),
}
}
ConverseStreamEvent::ContentBlockStop(_) => Ok(create_empty_openai_chunk()),
ConverseStreamEvent::MessageStop(stop_event) => {
let finish_reason = match stop_event.stop_reason {
StopReason::EndTurn => FinishReason::Stop,
StopReason::ToolUse => FinishReason::ToolCalls,
StopReason::MaxTokens => FinishReason::Length,
StopReason::StopSequence => FinishReason::Stop,
StopReason::GuardrailIntervened => FinishReason::ContentFilter,
StopReason::ContentFiltered => FinishReason::ContentFilter,
};
Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
Some(finish_reason),
None,
))
}
ConverseStreamEvent::Metadata(metadata_event) => {
let usage = Usage {
prompt_tokens: metadata_event.usage.input_tokens,
completion_tokens: metadata_event.usage.output_tokens,
total_tokens: metadata_event.usage.total_tokens,
prompt_tokens_details: None,
completion_tokens_details: None,
};
Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
None,
Some(usage),
))
}
// Error events - convert to empty chunks (errors should be handled elsewhere)
ConverseStreamEvent::InternalServerException(_)
| ConverseStreamEvent::ModelStreamErrorException(_)
| ConverseStreamEvent::ServiceUnavailableException(_)
| ConverseStreamEvent::ThrottlingException(_)
| ConverseStreamEvent::ValidationException(_) => Ok(create_empty_openai_chunk()),
}
}
}
/// Convert content block start to OpenAI chunk
fn convert_content_block_start(
content_block: MessagesContentBlock,
) -> Result<ChatCompletionsStreamResponse, TransformError> {
match content_block {
MessagesContentBlock::Text { .. } => {
// No immediate output for text block start
Ok(create_empty_openai_chunk())
}
MessagesContentBlock::ToolUse { id, name, .. }
| MessagesContentBlock::ServerToolUse { id, name, .. }
| MessagesContentBlock::McpToolUse { id, name, .. } => {
// Tool use start → OpenAI chunk with tool_calls
Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: Some(vec![ToolCallDelta {
index: 0,
id: Some(id),
call_type: Some("function".to_string()),
function: Some(FunctionCallDelta {
name: Some(name),
arguments: Some("".to_string()),
}),
}]),
},
None,
None,
))
}
_ => Err(TransformError::UnsupportedContent(
"Unsupported content block type in stream start".to_string(),
)),
}
}
/// Convert content delta to OpenAI chunk
fn convert_content_delta(
delta: MessagesContentDelta,
) -> Result<ChatCompletionsStreamResponse, TransformError> {
match delta {
MessagesContentDelta::TextDelta { text } => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: Some(text),
refusal: None,
function_call: None,
tool_calls: None,
},
None,
None,
)),
MessagesContentDelta::ThinkingDelta { thinking } => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: Some(format!("thinking: {}", thinking)),
refusal: None,
function_call: None,
tool_calls: None,
},
None,
None,
)),
MessagesContentDelta::InputJsonDelta { partial_json } => Ok(create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: Some(vec![ToolCallDelta {
index: 0,
id: None,
call_type: None,
function: Some(FunctionCallDelta {
name: None,
arguments: Some(partial_json),
}),
}]),
},
None,
None,
)),
}
}
/// Helper to create OpenAI streaming chunk
fn create_openai_chunk(
id: &str,
model: &str,
delta: MessageDelta,
finish_reason: Option<FinishReason>,
usage: Option<Usage>,
) -> ChatCompletionsStreamResponse {
ChatCompletionsStreamResponse {
id: id.to_string(),
object: Some("chat.completion.chunk".to_string()),
created: current_timestamp(),
model: model.to_string(),
choices: vec![StreamChoice {
index: 0,
delta,
finish_reason,
logprobs: None,
}],
usage,
system_fingerprint: None,
service_tier: None,
}
}
/// Helper to create empty OpenAI streaming chunk
fn create_empty_openai_chunk() -> ChatCompletionsStreamResponse {
create_openai_chunk(
"stream",
"unknown",
MessageDelta {
role: None,
content: None,
refusal: None,
function_call: None,
tool_calls: None,
},
None,
None,
)
}
// Stop Reason Conversions
impl Into<FinishReason> for MessagesStopReason {
fn into(self) -> FinishReason {
match self {
MessagesStopReason::EndTurn => FinishReason::Stop,
MessagesStopReason::MaxTokens => FinishReason::Length,
MessagesStopReason::StopSequence => FinishReason::Stop,
MessagesStopReason::ToolUse => FinishReason::ToolCalls,
MessagesStopReason::PauseTurn => FinishReason::Stop,
MessagesStopReason::Refusal => FinishReason::ContentFilter,
}
}
}
impl TryFrom<ChatCompletionsStreamResponse> for ResponsesAPIStreamEvent {
type Error = TransformError;
fn try_from(chunk: ChatCompletionsStreamResponse) -> Result<Self, TransformError> {
// Stateless conversion - just extract the delta information
// The buffer will manage state, item IDs, and sequence numbers
// Extract first choice if available
if let Some(choice) = chunk.choices.first() {
let delta = &choice.delta;
// Tool call with function name and/or arguments
if let Some(tool_calls) = &delta.tool_calls {
if let Some(tool_call) = tool_calls.first() {
// Extract call_id and name if available (metadata from initial event)
let call_id = tool_call.id.clone();
let function_name = tool_call.function.as_ref()
.and_then(|f| f.name.clone());
// Check if we have function metadata (name, id)
if let Some(function) = &tool_call.function {
// If we have arguments delta, return that
if let Some(args) = &function.arguments {
return Ok(ResponsesAPIStreamEvent::ResponseFunctionCallArgumentsDelta {
output_index: choice.index as i32,
item_id: "".to_string(), // Buffer will fill this
delta: args.clone(),
sequence_number: 0, // Buffer will fill this
call_id,
name: function_name,
});
}
// If we have function name but no arguments yet (initial tool call event)
// Return an empty arguments delta so the buffer knows to create the item
if function.name.is_some() {
return Ok(ResponsesAPIStreamEvent::ResponseFunctionCallArgumentsDelta {
output_index: choice.index as i32,
item_id: "".to_string(), // Buffer will fill this
delta: "".to_string(), // Empty delta signals this is the initial event
sequence_number: 0, // Buffer will fill this
call_id,
name: function_name,
});
}
}
}
}
// Text content delta
if let Some(content) = &delta.content {
if !content.is_empty() {
return Ok(ResponsesAPIStreamEvent::ResponseOutputTextDelta {
item_id: "".to_string(), // Buffer will fill this
output_index: choice.index as i32,
content_index: 0,
delta: content.clone(),
logprobs: vec![],
obfuscation: None,
sequence_number: 0, // Buffer will fill this
});
}
}
// Handle finish_reason - this is a completion signal
// Return an empty delta that the buffer can use to detect completion
if choice.finish_reason.is_some() {
// Return a minimal text delta to signal completion
// The buffer will handle the finish_reason and generate response.completed
return Ok(ResponsesAPIStreamEvent::ResponseOutputTextDelta {
item_id: "".to_string(), // Buffer will fill this
output_index: choice.index as i32,
content_index: 0,
delta: "".to_string(), // Empty delta signals completion
logprobs: vec![],
obfuscation: None,
sequence_number: 0, // Buffer will fill this
});
}
// Empty delta with role only (common at stream start)
if delta.role.is_some() {
// This is typically the first chunk establishing the assistant role
// Return an empty text delta that the buffer can use to initialize state
return Ok(ResponsesAPIStreamEvent::ResponseOutputTextDelta {
item_id: "".to_string(),
output_index: choice.index as i32,
content_index: 0,
delta: "".to_string(),
logprobs: vec![],
obfuscation: None,
sequence_number: 0,
});
}
}
// Empty chunk or no convertible content (e.g., keep-alive chunks with delta: {})
// These are valid in OpenAI streaming and should be silently ignored
// Return error so the caller can skip these chunks without warnings
Err(TransformError::UnsupportedConversion(
"Empty or keep-alive chunk with no convertible content".to_string(),
))
}
}