apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-17 15:25:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

add in debug logs for new stats and update integration tests

Browse source
This commit is contained in:
aayushwhiz 2024-11-14 13:56:56 -08:00
parent 1e6d25a90e
commit 312b21ef7d
2 changed files with 4 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/llm_gateway/src/stream_context.rs
View file

@ -302,9 +302,11 @@ impl HttpContext for StreamContext {
- self.ttft_duration.unwrap().as_millis() as u64)
/ self.response_tokens as u64;
debug!("Time per output token: {} milliseconds", tpot);
// Record the time per output token
self.metrics.time_per_output_token.record(tpot);
debug!("Tokens per second: {}", 1000 / tpot);
// Record the tokens per second
self.metrics.tokens_per_second.record(1000 / tpot);
}

2
crates/llm_gateway/tests/integration.rs
View file

@ -75,6 +75,8 @@ fn setup_filter(module: &mut Tester, config: &str) -> i32 {
.expect_metric_creation(MetricType::Gauge, "active_http_calls")
.expect_metric_creation(MetricType::Counter, "ratelimited_rq")
.expect_metric_creation(MetricType::Histogram, "time_to_first_token")
.expect_metric_creation(MetricType::Histogram, "time_per_output_token")
.expect_metric_creation(MetricType::Histogram, "tokens_per_second")
.expect_metric_creation(MetricType::Histogram, "request_latency")
.expect_metric_creation(MetricType::Histogram, "output_sequence_length")
.expect_metric_creation(MetricType::Histogram, "input_sequence_length")