apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-17 15:25:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix after merge

Browse source
This commit is contained in:
aayushwhiz 2024-11-08 11:58:54 -08:00
parent cb8e2a772b
commit 1f9d5860b5
1 changed files with 38 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

62
crates/llm_gateway/src/stream_context.rs
View file

@ -194,6 +194,20 @@ impl HttpContext for StreamContext {
);
self.request_id = self.get_http_request_header(REQUEST_ID_HEADER);
//start the timing for the request using get_current_time()
match get_current_time() {
Ok(current_time) => {
self.start_time = Some(current_time);
self.ttft_recorded = false;
self.ttft_duration = None;
}
Err(e) => {
warn!("Failed to get current time: {:?}", e);
self.start_time = None;
}
}
Action::Continue
}
@ -494,30 +508,30 @@ impl HttpContext for StreamContext {
.completion_tokens;
}
// Compute TFT if not already recorded
if !self.ttft_recorded {
if let Some(start_time) = self.start_time {
match get_current_time() {
Ok(current_time) => match current_time.duration_since(start_time) {
Ok(duration) => {
let duration_ms = duration.as_millis();
debug!("Time to First Token (TFT): {} milliseconds", duration_ms);
self.ttft_duration = Some(duration);
self.metrics.time_to_first_token.record(duration_ms as u64);
}
Err(e) => {
warn!("SystemTime error: {:?}", e);
}
},
Err(e) => {
warn!("Failed to get current time: {:?}", e);
}
}
self.ttft_recorded = true;
} else {
warn!("Start time was not recorded");
}
}
// // Compute TFT if not already recorded
// if !self.ttft_recorded {
// if let Some(start_time) = self.start_time {
// match get_current_time() {
// Ok(current_time) => match current_time.duration_since(start_time) {
// Ok(duration) => {
// let duration_ms = duration.as_millis();
// debug!("Time to First Token (TFT): {} milliseconds", duration_ms);
// self.ttft_duration = Some(duration);
// self.metrics.time_to_first_token.record(duration_ms as u64);
// }
// Err(e) => {
// warn!("SystemTime error: {:?}", e);
// }
// },
// Err(e) => {
// warn!("Failed to get current time: {:?}", e);
// }
// }
// self.ttft_recorded = true;
// } else {
// warn!("Start time was not recorded");
// }
// }
}
debug!(