apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-17 15:25:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix tracing

Browse source
This commit is contained in:
Adil Hafeez 2025-01-14 11:56:10 -08:00
parent 35065e2e41
commit 9570b167db
2 changed files with 13 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/common/src/tokenizer.rs
View file

@ -1,4 +1,4 @@
use log::debug;
use log::trace;
#[derive(thiserror::Error, Debug, PartialEq, Eq)]
#[allow(dead_code)]
@ -9,7 +9,7 @@ pub enum Error {
#[allow(dead_code)]
pub fn token_count(model_name: &str, text: &str) -> Result<usize, Error> {
debug!("getting token count model={}", model_name);
trace!("getting token count model={}", model_name);
// Consideration: is it more expensive to instantiate the BPE object every time, or to contend the singleton?
let bpe = tiktoken_rs::get_bpe_from_model(model_name).map_err(|_| Error::UnknownModel {
model_name: model_name.to_string(),