apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-07-23 16:51:04 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'main' into adil/add_acm_demo

Browse source
This commit is contained in:
Adil Hafeez 2025-01-22 14:10:18 -08:00
commit 613192f71c
13 changed files with 165 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

11
crates/llm_gateway/src/stream_context.rs
View file

@ -486,11 +486,14 @@ impl HttpContext for StreamContext {
let tokens_str = chat_completions_chunk_response_events.to_string();
//HACK: add support for tokenizing mistral and other models
//filed issue https://github.com/katanemo/arch/issues/222
if model.as_ref().unwrap().starts_with("mistral")
|| model.as_ref().unwrap().starts_with("ministral")
{
model = Some("gpt-4".to_string());
if !model.as_ref().unwrap().starts_with("gpt") {
warn!(
"tiktoken_rs: unsupported model: {}, using gpt-4 to compute token count",
model.as_ref().unwrap()
);
}
model = Some("gpt-4".to_string());
let token_count =
match tokenizer::token_count(model.as_ref().unwrap().as_str(), tokens_str.as_str())
{