apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-29 15:49:40 +02:00
Code Issues Projects Releases Packages Wiki Activity

Use heuristic based tokenizer

Browse source
This commit is contained in:
Adil Hafeez 2025-05-27 17:35:43 -07:00
parent d1542b988a
commit a74118238c
No known key found for this signature in database
GPG key ID: 9B18EF7691369645
3 changed files with 41 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

13
crates/common/src/tokenizer.rs
View file

@ -1,18 +1,5 @@
use log::debug;
pub trait Tokenizer {
/// Returns the number of tokens in the given text.
fn token_count(&self, text: &str, model_name: &str) -> Result<usize, String>;
}
pub struct TiktokenTokenizer {}
impl Tokenizer for TiktokenTokenizer {
fn token_count(&self, text: &str, model_name: &str) -> Result<usize, String> {
token_count(model_name, text)
}
}
#[allow(dead_code)]
pub fn token_count(model_name: &str, text: &str) -> Result<usize, String> {
debug!("getting token count model={}", model_name);