diff --git a/trustgraph-cli/scripts/tg-set-token-costs b/trustgraph-cli/scripts/tg-set-token-costs new file mode 100755 index 00000000..ebd98a5c --- /dev/null +++ b/trustgraph-cli/scripts/tg-set-token-costs @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +""" +Dumps out the current prompts +""" + +import argparse +import os +from trustgraph.api import Api, ConfigKey, ConfigValue +import json +import tabulate +import textwrap + +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') + +def set_costs(api_url, model, input_costs, output_costs): + + api = Api(api_url) + + api.config_put([ + ConfigValue( + type="token-costs", key=model, + value=json.dumps({ + "input_price": input_costs / 1000000, + "output_price": output_costs / 1000000, + }) + ), + ]) + +def set_prompt(url, id, prompt, response, schema): + + api = Api(url) + + values = api.config_get([ + ConfigKey(type="prompt", key="template-index") + ]) + + ix = json.loads(values[0].value) + + object = { + "id": id, + "prompt": prompt, + } + + if response: + object["response-type"] = response + else: + object["response-type"] = "text" + + if schema: + object["schema"] = schema + + if id not in ix: + ix.append(id) + + values = api.config_put([ + ConfigValue( + type="prompt", key="template-index", value=json.dumps(ix) + ), + ConfigValue( + type="prompt", key=f"template.{id}", value=json.dumps(object) + ) + ]) + + print("Prompt set.") + +def main(): + + parser = argparse.ArgumentParser( + prog='tg-show-prompts', + description=__doc__, + ) + + parser.add_argument( + '-u', '--api-url', + default=default_url, + help=f'API URL (default: {default_url})', + ) + + parser.add_argument( + '--model', + required=True, + help=f'Model ID', + ) + + parser.add_argument( + '-i', '--input-costs', + required=True, + type=float, + help=f'Input costs in $ per 1M tokens', + ) + + parser.add_argument( + '-o', '--output-costs', + required=True, + type=float, + help=f'Input costs in $ per 1M tokens', + ) + + args = parser.parse_args() + + try: + + set_costs(**vars(args)) + + except Exception as e: + + print("Exception:", e, flush=True) + +main() + diff --git a/trustgraph-cli/scripts/tg-show-token-costs b/trustgraph-cli/scripts/tg-show-token-costs new file mode 100755 index 00000000..d9868721 --- /dev/null +++ b/trustgraph-cli/scripts/tg-show-token-costs @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +""" +Dumps out the current prompts +""" + +import argparse +import os +from trustgraph.api import Api, ConfigKey +import json +import tabulate +import textwrap + +tabulate.PRESERVE_WHITESPACE = True + +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') + +def show_config(url): + + api = Api(url) + + models = api.config_list("token-costs") + + costs = [] + + def fmt(x): + return "{price:.3f}".format(price = 1000000 * x) + + for model in models: + + try: + values = json.loads(api.config_get([ + ConfigKey(type="token-costs", key=model), + ])[0].value) + costs.append(( + model, + fmt(values.get("input_price")), + fmt(values.get("output_price")), + )) + except: + costs.append(( + model, "-", "-" + )) + + print(tabulate.tabulate( + costs, + tablefmt = "pretty", + headers = ["model", "input, $/Mt", "output, $/Mt"], + colalign = ["left", "right", "right"], +# stralign = ["left", "decimal", "decimal"] + )) + +def main(): + + parser = argparse.ArgumentParser( + prog='tg-show-prompts', + description=__doc__, + ) + + parser.add_argument( + '-u', '--api-url', + default=default_url, + help=f'API URL (default: {default_url})', + ) + + args = parser.parse_args() + + try: + + show_config( + url=args.api_url, + ) + + except Exception as e: + + print("Exception:", e, flush=True) + +main() + diff --git a/trustgraph-cli/setup.py b/trustgraph-cli/setup.py index d927c3ac..47258648 100644 --- a/trustgraph-cli/setup.py +++ b/trustgraph-cli/setup.py @@ -45,6 +45,8 @@ setuptools.setup( ], scripts=[ "scripts/tg-dump-msgpack", + "scripts/tg-delete-flow-class", + "scripts/tg-get-flow-class", "scripts/tg-graph-show", "scripts/tg-graph-to-turtle", "scripts/tg-init-pulsar", @@ -54,24 +56,24 @@ setuptools.setup( "scripts/tg-invoke-graph-rag", "scripts/tg-invoke-llm", "scripts/tg-invoke-prompt", - "scripts/tg-load-kg-core", "scripts/tg-load-doc-embeds", + "scripts/tg-load-kg-core", "scripts/tg-load-pdf", "scripts/tg-load-text", "scripts/tg-load-turtle", "scripts/tg-processor-state", - "scripts/tg-save-kg-core", + "scripts/tg-put-flow-class", "scripts/tg-save-doc-embeds", + "scripts/tg-save-kg-core", + "scripts/tg-set-prompt", + "scripts/tg-set-token-costs", "scripts/tg-show-config", - "scripts/tg-show-flows", "scripts/tg-show-flow-classes", - "scripts/tg-get-flow-class", + "scripts/tg-show-flows", + "scripts/tg-show-prompts", + "scripts/tg-show-token-costs", + "scripts/tg-show-tools", "scripts/tg-start-flow", "scripts/tg-stop-flow", - "scripts/tg-delete-flow-class", - "scripts/tg-put-flow-class", - "scripts/tg-set-prompt", - "scripts/tg-show-tools", - "scripts/tg-show-prompts", ] ) diff --git a/trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py b/trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py index d0669a59..3f836832 100755 --- a/trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py +++ b/trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py @@ -9,7 +9,6 @@ import base64 from langchain_community.document_loaders import PyPDFLoader from ... schema import Document, TextDocument, Metadata -from ... log_level import LogLevel from ... base import FlowProcessor, ConsumerSpec, ProducerSpec default_ident = "pdf-decoder" diff --git a/trustgraph-flow/trustgraph/metering/counter.py b/trustgraph-flow/trustgraph/metering/counter.py index c721c065..12d5683e 100644 --- a/trustgraph-flow/trustgraph/metering/counter.py +++ b/trustgraph-flow/trustgraph/metering/counter.py @@ -3,22 +3,19 @@ Simple token counter for each LLM response. """ from prometheus_client import Counter -from . pricelist import price_list +import json from .. schema import TextCompletionResponse, Error -from .. schema import text_completion_response_queue -from .. log_level import LogLevel -from .. base import Consumer +from .. base import FlowProcessor, ConsumerSpec -module = "metering" +default_ident = "metering" -default_input_queue = text_completion_response_queue -default_subscriber = module - -class Processor(Consumer): +class Processor(FlowProcessor): def __init__(self, **params): + id = params.get("id", default_ident) + if not hasattr(__class__, "input_token_metric"): __class__.input_token_metric = Counter( 'input_tokens', 'Input token count' @@ -39,40 +36,63 @@ class Processor(Consumer): 'output_cost', 'Output cost' ) - input_queue = params.get("input_queue", default_input_queue) - subscriber = params.get("subscriber", default_subscriber) - super(Processor, self).__init__( **params | { - "input_queue": input_queue, - "subscriber": subscriber, - "input_schema": TextCompletionResponse, + "id": id, } ) + self.register_config_handler(self.on_cost_config) + + self.register_specification( + ConsumerSpec( + name = "input", + schema = TextCompletionResponse, + handler = self.on_message, + ) + ) + + self.prices = {} + + self.config_key = "token-costs" + + # Load token costs from the config service + async def on_cost_config(self, config, version): + + print("Loading configuration version", version) + + if self.config_key not in config: + print(f"No key {self.config_key} in config", flush=True) + return + + config = config[self.config_key] + + self.prices = { + k: json.loads(v) + for k, v in config.items() + } + def get_prices(self, prices, modelname): - for model in prices["price_list"]: - if model["model_name"] == modelname: - return model["input_price"], model["output_price"] + + if modelname in self.prices: + model = self.prices[modelname] + return model["input_price"], model["output_price"] return None, None # Return None if model is not found - async def handle(self, msg): + async def on_message(self, msg, consumer, flow): v = msg.value() + modelname = v.model - - # Sender-produced ID - id = msg.properties()["id"] - - print(f"Handling response {id}...", flush=True) - num_in = v.in_token num_out = v.out_token __class__.input_token_metric.inc(num_in) __class__.output_token_metric.inc(num_out) - model_input_price, model_output_price = self.get_prices(price_list, modelname) + model_input_price, model_output_price = self.get_prices( + price_list, modelname + ) if model_input_price == None: cost_per_call = f"Model Not Found in Price list" @@ -91,10 +111,8 @@ class Processor(Consumer): @staticmethod def add_args(parser): - Consumer.add_args( - parser, default_input_queue, default_subscriber, - ) + FlowProcessor.add_args(parser) def run(): - Processor.launch(module, __doc__) + Processor.launch(default_ident, __doc__) diff --git a/trustgraph-flow/trustgraph/metering/pricelist.py b/trustgraph-flow/trustgraph/metering/pricelist.py deleted file mode 100644 index e890d0e1..00000000 --- a/trustgraph-flow/trustgraph/metering/pricelist.py +++ /dev/null @@ -1,104 +0,0 @@ -price_list = { - "price_list": [ - { - "model_name": "mistral.mistral-large-2407-v1:0", - "input_price": 0.000004, - "output_price": 0.000012 - }, - { - "model_name": "meta.llama3-1-405b-instruct-v1:0", - "input_price": 0.00000532, - "output_price": 0.000016 - }, - { - "model_name": "mistral.mixtral-8x7b-instruct-v0:1", - "input_price": 0.00000045, - "output_price": 0.0000007 - }, - { - "model_name": "meta.llama3-1-70b-instruct-v1:0", - "input_price": 0.00000099, - "output_price": 0.00000099 - }, - { - "model_name": "meta.llama3-1-8b-instruct-v1:0", - "input_price": 0.00000022, - "output_price": 0.00000022 - }, - { - "model_name": "anthropic.claude-3-haiku-20240307-v1:0", - "input_price": 0.00000025, - "output_price": 0.00000125 - }, - { - "model_name": "anthropic.claude-3-5-sonnet-20240620-v1:0", - "input_price": 0.000003, - "output_price": 0.000015 - }, - { - "model_name": "cohere.command-r-plus-v1:0", - "input_price": 0.0000030, - "output_price": 0.0000150 - }, - { - "model_name": "ollama", - "input_price": 0, - "output_price": 0 - }, - { - "model_name": "claude-3-haiku-20240307", - "input_price": 0.00000025, - "output_price": 0.00000125 - }, - { - "model_name": "claude-3-5-sonnet-20240620", - "input_price": 0.000003, - "output_price": 0.000015 - }, - { - "model_name": "claude-3-opus-20240229", - "input_price": 0.000015, - "output_price": 0.000075 - }, - { - "model_name": "claude-3-sonnet-20240229", - "input_price": 0.000003, - "output_price": 0.000015 - }, - { - "model_name": "command-r-08-202", - "input_price": 0.0000025, - "output_price": 0.000010 - }, - { - "model_name": "c4ai-aya-23-8b", - "input_price": 0, - "output_price": 0 - }, - { - "model_name": "llama.cpp", - "input_price": 0, - "output_price": 0 - }, - { - "model_name": "gpt-4o", - "input_price": 0.000005, - "output_price": 0.000015 - }, - { - "model_name": "gpt-4o-2024-08-06", - "input_price": 0.0000025, - "output_price": 0.000010 - }, - { - "model_name": "gpt-4o-2024-05-13", - "input_price": 0.000005, - "output_price": 0.000015 - }, - { - "model_name": "gpt-4o-mini", - "input_price": 0.00000015, - "output_price": 0.0000006 - }, - ] -} \ No newline at end of file