apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-05 14:45:15 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix start time of llm filter (#278)

Browse source 
* fix start time of llm filter

* fix int tests
This commit is contained in:
Adil Hafeez 2024-11-17 17:01:19 -08:00 • committed by GitHub
parent df0cd50cbd
commit 097513ee60
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 16 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

9
crates/llm_gateway/src/stream_context.rs
View file

@ -40,6 +40,7 @@ pub struct StreamContext {
ttft_duration: Option<Duration>,
ttft_time: Option<SystemTime>,
pub traceparent: Option<String>,
request_body_sent_time: Option<SystemTime>,
user_message: Option<Message>,
}
@ -60,6 +61,7 @@ impl StreamContext {
traceparent: None,
ttft_time: None,
user_message: None,
request_body_sent_time: None,
}
}
fn llm_provider(&self) -> &LlmProvider {
@ -196,6 +198,11 @@ impl HttpContext for StreamContext {
fn on_http_request_body(&mut self, body_size: usize, end_of_stream: bool) -> Action {
// Let the client send the gateway all the data before sending to the LLM_provider.
// TODO: consider a streaming API.
if self.request_body_sent_time.is_none() {
self.request_body_sent_time = Some(get_current_time().unwrap());
}
if !end_of_stream {
return Action::Pause;
}
@ -351,7 +358,7 @@ impl HttpContext for StreamContext {
"upstream_llm_time".to_string(),
parent_trace_id.to_string(),
Some(parent_span_id.to_string()),
self.start_time
self.request_body_sent_time
.unwrap()
.duration_since(UNIX_EPOCH)
.unwrap()