mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-06-09 06:45:13 +02:00
Port metering to new API, not tested. (#354)
- Port metering to new API - Moved price list to configuration - Added tg-set-token-costs and tg-show-token-costs utils.
This commit is contained in:
cybermaggedon
GitHub
parent
5af7909122
commit
9508ac6c69
6 changed files with 249 additions and 144 deletions
111
trustgraph-cli/scripts/tg-set-token-costs
Executable file
111
trustgraph-cli/scripts/tg-set-token-costs
Executable file
|
|
@ -0,0 +1,111 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Dumps out the current prompts
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from trustgraph.api import Api, ConfigKey, ConfigValue
|
||||
import json
|
||||
import tabulate
|
||||
import textwrap
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
|
||||
def set_costs(api_url, model, input_costs, output_costs):
|
||||
|
||||
api = Api(api_url)
|
||||
|
||||
api.config_put([
|
||||
ConfigValue(
|
||||
type="token-costs", key=model,
|
||||
value=json.dumps({
|
||||
"input_price": input_costs / 1000000,
|
||||
"output_price": output_costs / 1000000,
|
||||
})
|
||||
),
|
||||
])
|
||||
|
||||
def set_prompt(url, id, prompt, response, schema):
|
||||
|
||||
api = Api(url)
|
||||
|
||||
values = api.config_get([
|
||||
ConfigKey(type="prompt", key="template-index")
|
||||
])
|
||||
|
||||
ix = json.loads(values[0].value)
|
||||
|
||||
object = {
|
||||
"id": id,
|
||||
"prompt": prompt,
|
||||
}
|
||||
|
||||
if response:
|
||||
object["response-type"] = response
|
||||
else:
|
||||
object["response-type"] = "text"
|
||||
|
||||
if schema:
|
||||
object["schema"] = schema
|
||||
|
||||
if id not in ix:
|
||||
ix.append(id)
|
||||
|
||||
values = api.config_put([
|
||||
ConfigValue(
|
||||
type="prompt", key="template-index", value=json.dumps(ix)
|
||||
),
|
||||
ConfigValue(
|
||||
type="prompt", key=f"template.{id}", value=json.dumps(object)
|
||||
)
|
||||
])
|
||||
|
||||
print("Prompt set.")
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-show-prompts',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--api-url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--model',
|
||||
required=True,
|
||||
help=f'Model ID',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-i', '--input-costs',
|
||||
required=True,
|
||||
type=float,
|
||||
help=f'Input costs in $ per 1M tokens',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-o', '--output-costs',
|
||||
required=True,
|
||||
type=float,
|
||||
help=f'Input costs in $ per 1M tokens',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
set_costs(**vars(args))
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
79
trustgraph-cli/scripts/tg-show-token-costs
Executable file
79
trustgraph-cli/scripts/tg-show-token-costs
Executable file
|
|
@ -0,0 +1,79 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Dumps out the current prompts
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from trustgraph.api import Api, ConfigKey
|
||||
import json
|
||||
import tabulate
|
||||
import textwrap
|
||||
|
||||
tabulate.PRESERVE_WHITESPACE = True
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
|
||||
def show_config(url):
|
||||
|
||||
api = Api(url)
|
||||
|
||||
models = api.config_list("token-costs")
|
||||
|
||||
costs = []
|
||||
|
||||
def fmt(x):
|
||||
return "{price:.3f}".format(price = 1000000 * x)
|
||||
|
||||
for model in models:
|
||||
|
||||
try:
|
||||
values = json.loads(api.config_get([
|
||||
ConfigKey(type="token-costs", key=model),
|
||||
])[0].value)
|
||||
costs.append((
|
||||
model,
|
||||
fmt(values.get("input_price")),
|
||||
fmt(values.get("output_price")),
|
||||
))
|
||||
except:
|
||||
costs.append((
|
||||
model, "-", "-"
|
||||
))
|
||||
|
||||
print(tabulate.tabulate(
|
||||
costs,
|
||||
tablefmt = "pretty",
|
||||
headers = ["model", "input, $/Mt", "output, $/Mt"],
|
||||
colalign = ["left", "right", "right"],
|
||||
# stralign = ["left", "decimal", "decimal"]
|
||||
))
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-show-prompts',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--api-url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
show_config(
|
||||
url=args.api_url,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
|
|
@ -45,6 +45,8 @@ setuptools.setup(
|
|||
],
|
||||
scripts=[
|
||||
"scripts/tg-dump-msgpack",
|
||||
"scripts/tg-delete-flow-class",
|
||||
"scripts/tg-get-flow-class",
|
||||
"scripts/tg-graph-show",
|
||||
"scripts/tg-graph-to-turtle",
|
||||
"scripts/tg-init-pulsar",
|
||||
|
|
@ -54,24 +56,24 @@ setuptools.setup(
|
|||
"scripts/tg-invoke-graph-rag",
|
||||
"scripts/tg-invoke-llm",
|
||||
"scripts/tg-invoke-prompt",
|
||||
"scripts/tg-load-kg-core",
|
||||
"scripts/tg-load-doc-embeds",
|
||||
"scripts/tg-load-kg-core",
|
||||
"scripts/tg-load-pdf",
|
||||
"scripts/tg-load-text",
|
||||
"scripts/tg-load-turtle",
|
||||
"scripts/tg-processor-state",
|
||||
"scripts/tg-save-kg-core",
|
||||
"scripts/tg-put-flow-class",
|
||||
"scripts/tg-save-doc-embeds",
|
||||
"scripts/tg-save-kg-core",
|
||||
"scripts/tg-set-prompt",
|
||||
"scripts/tg-set-token-costs",
|
||||
"scripts/tg-show-config",
|
||||
"scripts/tg-show-flows",
|
||||
"scripts/tg-show-flow-classes",
|
||||
"scripts/tg-get-flow-class",
|
||||
"scripts/tg-show-flows",
|
||||
"scripts/tg-show-prompts",
|
||||
"scripts/tg-show-token-costs",
|
||||
"scripts/tg-show-tools",
|
||||
"scripts/tg-start-flow",
|
||||
"scripts/tg-stop-flow",
|
||||
"scripts/tg-delete-flow-class",
|
||||
"scripts/tg-put-flow-class",
|
||||
"scripts/tg-set-prompt",
|
||||
"scripts/tg-show-tools",
|
||||
"scripts/tg-show-prompts",
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import base64
|
|||
from langchain_community.document_loaders import PyPDFLoader
|
||||
|
||||
from ... schema import Document, TextDocument, Metadata
|
||||
from ... log_level import LogLevel
|
||||
from ... base import FlowProcessor, ConsumerSpec, ProducerSpec
|
||||
|
||||
default_ident = "pdf-decoder"
|
||||
|
|
|
|||
|
|
@ -3,22 +3,19 @@ Simple token counter for each LLM response.
|
|||
"""
|
||||
|
||||
from prometheus_client import Counter
|
||||
from . pricelist import price_list
|
||||
import json
|
||||
|
||||
from .. schema import TextCompletionResponse, Error
|
||||
from .. schema import text_completion_response_queue
|
||||
from .. log_level import LogLevel
|
||||
from .. base import Consumer
|
||||
from .. base import FlowProcessor, ConsumerSpec
|
||||
|
||||
module = "metering"
|
||||
default_ident = "metering"
|
||||
|
||||
default_input_queue = text_completion_response_queue
|
||||
default_subscriber = module
|
||||
|
||||
class Processor(Consumer):
|
||||
class Processor(FlowProcessor):
|
||||
|
||||
def __init__(self, **params):
|
||||
|
||||
id = params.get("id", default_ident)
|
||||
|
||||
if not hasattr(__class__, "input_token_metric"):
|
||||
__class__.input_token_metric = Counter(
|
||||
'input_tokens', 'Input token count'
|
||||
|
|
@ -39,40 +36,63 @@ class Processor(Consumer):
|
|||
'output_cost', 'Output cost'
|
||||
)
|
||||
|
||||
input_queue = params.get("input_queue", default_input_queue)
|
||||
subscriber = params.get("subscriber", default_subscriber)
|
||||
|
||||
super(Processor, self).__init__(
|
||||
**params | {
|
||||
"input_queue": input_queue,
|
||||
"subscriber": subscriber,
|
||||
"input_schema": TextCompletionResponse,
|
||||
"id": id,
|
||||
}
|
||||
)
|
||||
|
||||
self.register_config_handler(self.on_cost_config)
|
||||
|
||||
self.register_specification(
|
||||
ConsumerSpec(
|
||||
name = "input",
|
||||
schema = TextCompletionResponse,
|
||||
handler = self.on_message,
|
||||
)
|
||||
)
|
||||
|
||||
self.prices = {}
|
||||
|
||||
self.config_key = "token-costs"
|
||||
|
||||
# Load token costs from the config service
|
||||
async def on_cost_config(self, config, version):
|
||||
|
||||
print("Loading configuration version", version)
|
||||
|
||||
if self.config_key not in config:
|
||||
print(f"No key {self.config_key} in config", flush=True)
|
||||
return
|
||||
|
||||
config = config[self.config_key]
|
||||
|
||||
self.prices = {
|
||||
k: json.loads(v)
|
||||
for k, v in config.items()
|
||||
}
|
||||
|
||||
def get_prices(self, prices, modelname):
|
||||
for model in prices["price_list"]:
|
||||
if model["model_name"] == modelname:
|
||||
return model["input_price"], model["output_price"]
|
||||
|
||||
if modelname in self.prices:
|
||||
model = self.prices[modelname]
|
||||
return model["input_price"], model["output_price"]
|
||||
return None, None # Return None if model is not found
|
||||
|
||||
async def handle(self, msg):
|
||||
async def on_message(self, msg, consumer, flow):
|
||||
|
||||
v = msg.value()
|
||||
|
||||
modelname = v.model
|
||||
|
||||
# Sender-produced ID
|
||||
id = msg.properties()["id"]
|
||||
|
||||
print(f"Handling response {id}...", flush=True)
|
||||
|
||||
num_in = v.in_token
|
||||
num_out = v.out_token
|
||||
|
||||
__class__.input_token_metric.inc(num_in)
|
||||
__class__.output_token_metric.inc(num_out)
|
||||
|
||||
model_input_price, model_output_price = self.get_prices(price_list, modelname)
|
||||
model_input_price, model_output_price = self.get_prices(
|
||||
price_list, modelname
|
||||
)
|
||||
|
||||
if model_input_price == None:
|
||||
cost_per_call = f"Model Not Found in Price list"
|
||||
|
|
@ -91,10 +111,8 @@ class Processor(Consumer):
|
|||
@staticmethod
|
||||
def add_args(parser):
|
||||
|
||||
Consumer.add_args(
|
||||
parser, default_input_queue, default_subscriber,
|
||||
)
|
||||
FlowProcessor.add_args(parser)
|
||||
|
||||
def run():
|
||||
|
||||
Processor.launch(module, __doc__)
|
||||
Processor.launch(default_ident, __doc__)
|
||||
|
|
|
|||
|
|
@ -1,104 +0,0 @@
|
|||
price_list = {
|
||||
"price_list": [
|
||||
{
|
||||
"model_name": "mistral.mistral-large-2407-v1:0",
|
||||
"input_price": 0.000004,
|
||||
"output_price": 0.000012
|
||||
},
|
||||
{
|
||||
"model_name": "meta.llama3-1-405b-instruct-v1:0",
|
||||
"input_price": 0.00000532,
|
||||
"output_price": 0.000016
|
||||
},
|
||||
{
|
||||
"model_name": "mistral.mixtral-8x7b-instruct-v0:1",
|
||||
"input_price": 0.00000045,
|
||||
"output_price": 0.0000007
|
||||
},
|
||||
{
|
||||
"model_name": "meta.llama3-1-70b-instruct-v1:0",
|
||||
"input_price": 0.00000099,
|
||||
"output_price": 0.00000099
|
||||
},
|
||||
{
|
||||
"model_name": "meta.llama3-1-8b-instruct-v1:0",
|
||||
"input_price": 0.00000022,
|
||||
"output_price": 0.00000022
|
||||
},
|
||||
{
|
||||
"model_name": "anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"input_price": 0.00000025,
|
||||
"output_price": 0.00000125
|
||||
},
|
||||
{
|
||||
"model_name": "anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"input_price": 0.000003,
|
||||
"output_price": 0.000015
|
||||
},
|
||||
{
|
||||
"model_name": "cohere.command-r-plus-v1:0",
|
||||
"input_price": 0.0000030,
|
||||
"output_price": 0.0000150
|
||||
},
|
||||
{
|
||||
"model_name": "ollama",
|
||||
"input_price": 0,
|
||||
"output_price": 0
|
||||
},
|
||||
{
|
||||
"model_name": "claude-3-haiku-20240307",
|
||||
"input_price": 0.00000025,
|
||||
"output_price": 0.00000125
|
||||
},
|
||||
{
|
||||
"model_name": "claude-3-5-sonnet-20240620",
|
||||
"input_price": 0.000003,
|
||||
"output_price": 0.000015
|
||||
},
|
||||
{
|
||||
"model_name": "claude-3-opus-20240229",
|
||||
"input_price": 0.000015,
|
||||
"output_price": 0.000075
|
||||
},
|
||||
{
|
||||
"model_name": "claude-3-sonnet-20240229",
|
||||
"input_price": 0.000003,
|
||||
"output_price": 0.000015
|
||||
},
|
||||
{
|
||||
"model_name": "command-r-08-202",
|
||||
"input_price": 0.0000025,
|
||||
"output_price": 0.000010
|
||||
},
|
||||
{
|
||||
"model_name": "c4ai-aya-23-8b",
|
||||
"input_price": 0,
|
||||
"output_price": 0
|
||||
},
|
||||
{
|
||||
"model_name": "llama.cpp",
|
||||
"input_price": 0,
|
||||
"output_price": 0
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4o",
|
||||
"input_price": 0.000005,
|
||||
"output_price": 0.000015
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4o-2024-08-06",
|
||||
"input_price": 0.0000025,
|
||||
"output_price": 0.000010
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4o-2024-05-13",
|
||||
"input_price": 0.000005,
|
||||
"output_price": 0.000015
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4o-mini",
|
||||
"input_price": 0.00000015,
|
||||
"output_price": 0.0000006
|
||||
},
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue