apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-07-02 15:51:02 +02:00
Code Issues Projects Releases Packages Wiki Activity

fixed integration tests and cleaned up logs

Browse source
This commit is contained in:
Salman Paracha 2025-09-04 21:49:54 -07:00
parent ee52c608f7
commit 00c543667d
4 changed files with 172 additions and 119 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/common/src/tokenizer.rs
View file

@ -2,7 +2,7 @@ use log::debug;
#[allow(dead_code)]
pub fn token_count(model_name: &str, text: &str) -> Result<usize, String> {
debug!("getting token count model={}", model_name);
debug!("TOKENIZER: computing token count for model={}", model_name);
//HACK: add support for tokenizing mistral and other models
//filed issue https://github.com/katanemo/arch/issues/222