apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-05-01 11:26:22 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Port metering to new API, not tested. (#354)

Browse source 
- Port metering to new API
- Moved price list to configuration
- Added tg-set-token-costs and tg-show-token-costs utils.
This commit is contained in:
cybermaggedon 2025-04-28 21:26:38 +01:00 • committed by GitHub
parent 5af7909122
commit 9508ac6c69
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 249 additions and 144 deletions
Download patch file Download diff file Expand all files Collapse all files

78
trustgraph-flow/trustgraph/metering/counter.py
View file

@ -3,22 +3,19 @@ Simple token counter for each LLM response.
"""
from prometheus_client import Counter
from . pricelist import price_list
import json
from .. schema import TextCompletionResponse, Error
from .. schema import text_completion_response_queue
from .. log_level import LogLevel
from .. base import Consumer
from .. base import FlowProcessor, ConsumerSpec
module = "metering"
default_ident = "metering"
default_input_queue = text_completion_response_queue
default_subscriber = module
class Processor(Consumer):
class Processor(FlowProcessor):
def __init__(self, **params):
id = params.get("id", default_ident)
if not hasattr(__class__, "input_token_metric"):
__class__.input_token_metric = Counter(
'input_tokens', 'Input token count'
@ -39,40 +36,63 @@ class Processor(Consumer):
'output_cost', 'Output cost'
)
input_queue = params.get("input_queue", default_input_queue)
subscriber = params.get("subscriber", default_subscriber)
super(Processor, self).__init__(
**params | {
"input_queue": input_queue,
"subscriber": subscriber,
"input_schema": TextCompletionResponse,
"id": id,
}
)
self.register_config_handler(self.on_cost_config)
self.register_specification(
ConsumerSpec(
name = "input",
schema = TextCompletionResponse,
handler = self.on_message,
)
)
self.prices = {}
self.config_key = "token-costs"
# Load token costs from the config service
async def on_cost_config(self, config, version):
print("Loading configuration version", version)
if self.config_key not in config:
print(f"No key {self.config_key} in config", flush=True)
return
config = config[self.config_key]
self.prices = {
k: json.loads(v)
for k, v in config.items()
}
def get_prices(self, prices, modelname):
for model in prices["price_list"]:
if model["model_name"] == modelname:
return model["input_price"], model["output_price"]
if modelname in self.prices:
model = self.prices[modelname]
return model["input_price"], model["output_price"]
return None, None # Return None if model is not found
async def handle(self, msg):
async def on_message(self, msg, consumer, flow):
v = msg.value()
modelname = v.model
# Sender-produced ID
id = msg.properties()["id"]
print(f"Handling response {id}...", flush=True)
num_in = v.in_token
num_out = v.out_token
__class__.input_token_metric.inc(num_in)
__class__.output_token_metric.inc(num_out)
model_input_price, model_output_price = self.get_prices(price_list, modelname)
model_input_price, model_output_price = self.get_prices(
price_list, modelname
)
if model_input_price == None:
cost_per_call = f"Model Not Found in Price list"
@ -91,10 +111,8 @@ class Processor(Consumer):
@staticmethod
def add_args(parser):
Consumer.add_args(
parser, default_input_queue, default_subscriber,
)
FlowProcessor.add_args(parser)
def run():
Processor.launch(module, __doc__)
Processor.launch(default_ident, __doc__)

104
trustgraph-flow/trustgraph/metering/pricelist.py
View file

@ -1,104 +0,0 @@
price_list = {
"price_list": [
{
"model_name": "mistral.mistral-large-2407-v1:0",
"input_price": 0.000004,
"output_price": 0.000012
},
{
"model_name": "meta.llama3-1-405b-instruct-v1:0",
"input_price": 0.00000532,
"output_price": 0.000016
},
{
"model_name": "mistral.mixtral-8x7b-instruct-v0:1",
"input_price": 0.00000045,
"output_price": 0.0000007
},
{
"model_name": "meta.llama3-1-70b-instruct-v1:0",
"input_price": 0.00000099,
"output_price": 0.00000099
},
{
"model_name": "meta.llama3-1-8b-instruct-v1:0",
"input_price": 0.00000022,
"output_price": 0.00000022
},
{
"model_name": "anthropic.claude-3-haiku-20240307-v1:0",
"input_price": 0.00000025,
"output_price": 0.00000125
},
{
"model_name": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"input_price": 0.000003,
"output_price": 0.000015
},
{
"model_name": "cohere.command-r-plus-v1:0",
"input_price": 0.0000030,
"output_price": 0.0000150
},
{
"model_name": "ollama",
"input_price": 0,
"output_price": 0
},
{
"model_name": "claude-3-haiku-20240307",
"input_price": 0.00000025,
"output_price": 0.00000125
},
{
"model_name": "claude-3-5-sonnet-20240620",
"input_price": 0.000003,
"output_price": 0.000015
},
{
"model_name": "claude-3-opus-20240229",
"input_price": 0.000015,
"output_price": 0.000075
},
{
"model_name": "claude-3-sonnet-20240229",
"input_price": 0.000003,
"output_price": 0.000015
},
{
"model_name": "command-r-08-202",
"input_price": 0.0000025,
"output_price": 0.000010
},
{
"model_name": "c4ai-aya-23-8b",
"input_price": 0,
"output_price": 0
},
{
"model_name": "llama.cpp",
"input_price": 0,
"output_price": 0
},
{
"model_name": "gpt-4o",
"input_price": 0.000005,
"output_price": 0.000015
},
{
"model_name": "gpt-4o-2024-08-06",
"input_price": 0.0000025,
"output_price": 0.000010
},
{
"model_name": "gpt-4o-2024-05-13",
"input_price": 0.000005,
"output_price": 0.000015
},
{
"model_name": "gpt-4o-mini",
"input_price": 0.00000015,
"output_price": 0.0000006
},
]
}