apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-04-27 17:56:28 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use better logs (#452)

Browse source
This commit is contained in:
Adil Hafeez 2025-03-27 10:40:20 -07:00 • committed by GitHub
parent 76ec5cda68
commit de221525de
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 200 additions and 156 deletions
Download patch file Download diff file Expand all files Collapse all files

6
crates/common/src/tokenizer.rs
View file

@ -1,14 +1,14 @@
use log::trace;
use log::{debug};
#[allow(dead_code)]
pub fn token_count(model_name: &str, text: &str) -> Result<usize, String> {
trace!("getting token count model={}", model_name);
debug!("getting token count model={}", model_name);
//HACK: add support for tokenizing mistral and other models
//filed issue https://github.com/katanemo/arch/issues/222
let updated_model = match model_name.starts_with("gpt") {
false => {
trace!(
debug!(
"tiktoken_rs: unsupported model: {}, using gpt-4 to compute token count",
model_name
);