apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-04-30 03:16:28 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add in tpot (#269)

Browse source 
* add in tpot and tokens per second

* add in debug logs for new stats and update integration tests

* update shared dashboard to include new stats
This commit is contained in:
Aayush 2024-11-14 15:03:08 -08:00 • committed by GitHub
parent 9eeb790c7f
commit 1d229cba8f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 252 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/llm_gateway/tests/integration.rs
View file

@ -75,6 +75,8 @@ fn setup_filter(module: &mut Tester, config: &str) -> i32 {
.expect_metric_creation(MetricType::Gauge, "active_http_calls")
.expect_metric_creation(MetricType::Counter, "ratelimited_rq")
.expect_metric_creation(MetricType::Histogram, "time_to_first_token")
.expect_metric_creation(MetricType::Histogram, "time_per_output_token")
.expect_metric_creation(MetricType::Histogram, "tokens_per_second")
.expect_metric_creation(MetricType::Histogram, "request_latency")
.expect_metric_creation(MetricType::Histogram, "output_sequence_length")
.expect_metric_creation(MetricType::Histogram, "input_sequence_length")