apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-04-28 18:36:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

release 0.3.2 (#507)

Browse source
This commit is contained in:
Adil Hafeez 2025-06-13 17:02:20 -07:00 • committed by GitHub
parent 0d6e06dfec
commit 7baec20772
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 153 additions and 132 deletions
Download patch file Download diff file Expand all files Collapse all files

12
crates/common/src/tokenizer.rs
View file

@ -6,15 +6,21 @@ pub fn token_count(model_name: &str, text: &str) -> Result<usize, String> {
//HACK: add support for tokenizing mistral and other models
//filed issue https://github.com/katanemo/arch/issues/222
let updated_model = match model_name.starts_with("gpt") {
let updated_model = match model_name.starts_with("gpt-4") {
false => {
debug!(
"tiktoken_rs: unsupported model: {}, using gpt-4 to compute token count",
model_name
);
"gpt-4"
"gpt-4o"
}
true => {
if model_name.starts_with("gpt-4.1") {
"gpt-4o"
} else {
model_name
}
}
true => model_name,
};
// Consideration: is it more expensive to instantiate the BPE object every time, or to contend the singleton?