mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-24 21:41:04 +02:00
9 lines
237 B
Python
9 lines
237 B
Python
# pageindex/tokens.py
|
|
# Shared by parser and index layers (avoids a reverse dependency).
|
|
|
|
|
|
def count_tokens(text, model=None):
|
|
if not text:
|
|
return 0
|
|
import litellm
|
|
return litellm.token_counter(model=model, text=text)
|