apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-29 15:49:40 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix tracing

Browse source
This commit is contained in:
Adil Hafeez 2025-01-16 14:37:09 -08:00
parent 2413f56980
commit 5017e7931e
2 changed files with 8 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/common/src/tokenizer.rs
View file

@ -1,4 +1,4 @@
use log::trace;
use log::debug;
#[derive(thiserror::Error, Debug, PartialEq, Eq)]
#[allow(dead_code)]
@ -9,7 +9,7 @@ pub enum Error {
#[allow(dead_code)]
pub fn token_count(model_name: &str, text: &str) -> Result<usize, Error> {
trace!("getting token count model={}", model_name);
debug!("getting token count model={}", model_name);
// Consideration: is it more expensive to instantiate the BPE object every time, or to contend the singleton?
let bpe = tiktoken_rs::get_bpe_from_model(model_name).map_err(|_| Error::UnknownModel {
model_name: model_name.to_string(),