apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-29 15:49:40 +02:00
Code Issues Projects Releases Packages Wiki Activity

removed the dependency on try_streaming_from_bytes into a try_from trait implementation

Browse source
This commit is contained in:
Salman Paracha 2025-08-19 15:47:21 -07:00
parent 327b29ec6f
commit 6c1dc658cb
5 changed files with 94 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

6
crates/llm_gateway/src/stream_context.rs
View file

@ -10,9 +10,9 @@ use common::ratelimit::Header;
use common::stats::{IncrementingMetric, RecordingMetric};
use common::tracing::{Event, Span, TraceData, Traceparent};
use common::{ratelimit, routing, tokenizer};
use hermesllm::providers::response::ProviderStreamResponseIter;
use hermesllm::{
try_streaming_from_bytes, ProviderId, ProviderRequest, ProviderRequestType, ProviderResponse,
ProviderResponseType,
ProviderId, ProviderRequest, ProviderRequestType, ProviderResponse, ProviderResponseType,
};
use http::StatusCode;
use log::{debug, info, warn};
@ -572,7 +572,7 @@ impl HttpContext for StreamContext {
// Since all providers use OpenAI-compatible streaming format
let provider_id = self.get_provider_id();
match try_streaming_from_bytes(&body, &provider_id) {
match ProviderStreamResponseIter::try_from((&body[..], &provider_id)) {
Ok(mut streaming_response) => {
// Process each streaming chunk
while let Some(chunk_result) = streaming_response.next() {