apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-24 12:41:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Metrics working

Browse source
This commit is contained in:
Cyber MacGeddon 2025-04-14 22:39:09 +01:00
parent 726baa9932
commit 9fdc408a95
6 changed files with 152 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

1
trustgraph-base/trustgraph/base/__init__.py
View file

@ -5,4 +5,5 @@ from . consumer import Consumer
from . producer import Producer from . producer import Producer
from . publisher import Publisher from . publisher import Publisher
from . subscriber import Subscriber from . subscriber import Subscriber
from . metrics import ProcessorMetrics, ConsumerMetrics, ProducerMetrics

19
trustgraph-base/trustgraph/base/async_processor.py
View file

@ -73,16 +73,25 @@ class AsyncProcessor:
while True: while True:
await asyncio.sleep(2) await asyncio.sleep(2)
def subscribe(self, queue, subscriber, schema, handler): def subscribe(self, queue, subscriber, schema, handler, metrics=None):
return Consumer( return Consumer(
self.taskgroup, self.client, queue, subscriber, schema, handler taskgroup = self.taskgroup,
client = self.client,
queue = queue,
subscriber = subscriber,
schema = schema,
handler = handler,
metrics = metrics,
) )
def publish(self, queue, schema): def publish(self, queue, schema, metrics=None):
return Producer( return Producer(
self.client, queue, schema client = self.client,
queue = queue,
schema = schema,
metrics = metrics,
) )
def set_processor_state(self, flow, state): def set_processor_state(self, flow, state):
@ -166,7 +175,7 @@ class AsyncProcessor:
print("Exception group:") print("Exception group:")
for se in e.exceptions: for se in e.exceptions:
print(" Type:", type(se)) print(" Type:", type(se))
print(f" Exception: {se}") print(f" Exception: {se}")
except Exception as e: except Exception as e:

48
trustgraph-base/trustgraph/base/consumer.py
View file

@ -8,7 +8,8 @@ class Consumer:
def __init__( def __init__(
self, taskgroup, client, queue, subscriber, schema, self, taskgroup, client, queue, subscriber, schema,
handler handler, rate_limit_retry_time = 10,
rate_limit_timeout = 7200, metrics = None
): ):
self.taskgroup = taskgroup self.taskgroup = taskgroup
@ -17,10 +18,14 @@ class Consumer:
self.subscriber = subscriber self.subscriber = subscriber
self.schema = schema self.schema = schema
self.handler = handler self.handler = handler
self.rate_limit_retry_time = rate_limit_retry_time
self.rate_limit_timeout = rate_limit_timeout
self.running = True self.running = True
self.task = None self.task = None
self.metrics = metrics
async def start(self): async def start(self):
self.running = True self.running = True
@ -29,17 +34,23 @@ class Consumer:
self.queue, self.subscriber, self.schema self.queue, self.subscriber, self.schema
) )
# Puts it in the stopped state, the run thread should set running
if self.metrics:
self.metrics.state("stopped")
self.task = self.taskgroup.create_task(self.run()) self.task = self.taskgroup.create_task(self.run())
async def run(self): async def run(self):
if self.metrics:
print("RUNNING")
self.metrics.state("running")
while self.running: while self.running:
msg = await asyncio.to_thread(self.consumer.receive) msg = await asyncio.to_thread(self.consumer.receive)
# expiry = time.time() + self.rate_limit_timeout expiry = time.time() + self.rate_limit_timeout
expiry = time.time() + 10
# This loop is for retry on rate-limit / resource limits # This loop is for retry on rate-limit / resource limits
while True: while True:
@ -52,8 +63,8 @@ class Consumer:
# be retried # be retried
self.consumer.negative_acknowledge(msg) self.consumer.negative_acknowledge(msg)
# FIXME if self.metrics:
# __class__.processing_metric.labels(status="error").inc() self.metrics.process("error")
# Break out of retry loop, processes next message # Break out of retry loop, processes next message
break break
@ -61,15 +72,22 @@ class Consumer:
try: try:
print("Handle...") print("Handle...")
# FIXME
# with __class__.request_metric.time(): if self.metrics:
await self.handler(msg, self.consumer)
with self.metrics.record_time():
await self.handler(msg, self.consumer)
else:
await self.handler(msg, self.consumer)
print("Handled.") print("Handled.")
# Acknowledge successful processing of the message # Acknowledge successful processing of the message
self.consumer.acknowledge(msg) self.consumer.acknowledge(msg)
# __class__.processing_metric.labels(status="success").inc() if self.metrics:
self.metrics.process("success")
# Break out of retry loop # Break out of retry loop
break break
@ -79,10 +97,11 @@ class Consumer:
print("TooManyRequests: will retry...", flush=True) print("TooManyRequests: will retry...", flush=True)
# FIXME # FIXME
# __class__.rate_limit_metric.inc() if self.metrics:
self.metrics.rate_limit()
# Sleep # Sleep
time.sleep(self.rate_limit_retry) await asyncio.sleep(self.rate_limit_retry)
# Contine from retry loop, just causes a reprocessing # Contine from retry loop, just causes a reprocessing
continue continue
@ -95,8 +114,11 @@ class Consumer:
# be retried # be retried
self.consumer.negative_acknowledge(msg) self.consumer.negative_acknowledge(msg)
# __class__.processing_metric.labels(status="error").inc() if self.metrics:
self.metrics.process("error")
# Break out of retry loop, processes next message # Break out of retry loop, processes next message
break break
if self.metrics:
self.metrics.state("stopped")

82
trustgraph-base/trustgraph/base/metrics.py Normal file
View file

@ -0,0 +1,82 @@
from prometheus_client import start_http_server, Info, Enum, Histogram
from prometheus_client import Counter
class ConsumerMetrics:
def __init__(self, id, flow=None):
self.id = id
self.flow = flow
if not hasattr(__class__, "state_metric"):
__class__.state_metric = Enum(
'consumer_state', 'Consumer state',
["id", "flow"],
states=['stopped', 'running']
)
if not hasattr(__class__, "request_metric"):
__class__.request_metric = Histogram(
'request_latency', 'Request latency (seconds)',
["id", "flow"],
)
if not hasattr(__class__, "processing_metric"):
__class__.processing_metric = Counter(
'processing_count', 'Processing count',
["id", "flow", "status"]
)
if not hasattr(__class__, "rate_limit_metric"):
__class__.rate_limit_metric = Counter(
'rate_limit_count', 'Rate limit event count',
["id", "flow"]
)
def process(self, status):
__class__.processing_metric.labels(
id=self.id, flow=self.flow, status=status
).inc()
def rate_limit(self):
__class__.rate_limit_metric.labels(
id=self.id, flow=self.flow
).inc()
def state(self, state):
__class__.state_metric.labels(
id=self.id, flow=self.flow
).state(state)
def record_time(self):
return __class__.request_metric.labels(
id=self.id, flow=self.flow
).time()
class ProducerMetrics:
def __init__(self, id, flow=None):
self.id = id
self.flow = flow
if not hasattr(__class__, "output_metric"):
__class__.output_metric = Counter(
'output_count', 'Output items created',
["id", "flow"]
)
def inc(self):
__class__.output_metric.labels(id=self.id, flow=self.flow).inc()
class ProcessorMetrics:
def __init__(self, id):
self.id = id
if not hasattr(__class__, "pubsub_metric"):
__class__.pubsub_metric = Info(
'pubsub', 'Pub/sub configuration',
["id"]
)
def info(self, info):
__class__.pubsub_metric.labels(id=self.id).info(info)

11
trustgraph-base/trustgraph/base/producer.py
View file

@ -1,18 +1,19 @@
class Producer: class Producer:
def __init__(self, client, queue, schema): def __init__(self, client, queue, schema, metrics=None):
self.client = client self.client = client
self.queue = queue self.queue = queue
self.schema = schema self.schema = schema
self.producer = None self.producer = self.client.publish(self.queue, self.schema)
self.metrics = metrics
async def send(self, msg, properties={}): async def send(self, msg, properties={}):
if self.producer == None: if self.metrics:
self.producer = self.client.publish(self.queue, self.schema) self.metrics.inc()
self.producer.send(msg, properties) self.producer.send(msg, properties)

37
trustgraph-flow/trustgraph/config/service/service.py
View file

@ -15,6 +15,7 @@ from trustgraph.log_level import LogLevel
from trustgraph.base import AsyncProcessor, Consumer, Producer from trustgraph.base import AsyncProcessor, Consumer, Producer
from . config import Configuration from . config import Configuration
from ... base import ProcessorMetrics, ConsumerMetrics, ProducerMetrics
module = "config-svc" module = "config-svc"
@ -40,19 +41,13 @@ class Processor(AsyncProcessor):
response_schema = ConfigResponse response_schema = ConfigResponse
push_schema = ConfigResponse push_schema = ConfigResponse
self.set_processor_state(id, "starting") ProcessorMetrics(id=id).info(
self.set_pubsub_info(
id
{ {
"id": id,
"subscriber": subscriber, "subscriber": subscriber,
"request_queue": request_queue, "request_queue": request_queue,
"request_schema": request_schema.__name__, "request_schema": request_schema.__name__,
"response_queue": response_queue, "response_queue": response_queue,
"response_schema": request_schema.__name__, "response_schema": request_schema.__name__,
# "rate_limit_retry": str(self.rate_limit_retry),
# "rate_limit_timeout": str(self.rate_limit_timeout),
} }
) )
@ -64,25 +59,20 @@ class Processor(AsyncProcessor):
} }
) )
if not hasattr(__class__, "request_metric"): self.request_metrics = ConsumerMetrics(id + "-request")
__class__.request_metric = Histogram( self.response_metrics = ProducerMetrics(id + "-response")
'request_latency', 'Request latency (seconds)' self.push_metrics = ProducerMetrics(id + "-push")
)
if not hasattr(__class__, "processing_metric"):
__class__.processing_metric = Counter(
'processing_count', 'Processing count',
["status"]
)
self.push_pub = self.publish( self.push_pub = self.publish(
queue = push_queue, queue = push_queue,
schema = ConfigPush schema = ConfigPush,
metrics = self.push_metrics,
) )
self.out_pub = self.publish( self.response_pub = self.publish(
queue = response_queue, queue = response_queue,
schema = ConfigResponse schema = ConfigResponse,
metrics = self.response_metrics,
) )
self.subs = self.subscribe( self.subs = self.subscribe(
@ -90,6 +80,7 @@ class Processor(AsyncProcessor):
subscriber = subscriber, subscriber = subscriber,
schema = request_schema, schema = request_schema,
handler = self.on_message, handler = self.on_message,
metrics = self.request_metrics,
) )
self.config = Configuration() self.config = Configuration()
@ -130,7 +121,7 @@ class Processor(AsyncProcessor):
resp = await self.config.handle(v) resp = await self.config.handle(v)
await self.out_pub.send(resp, properties={"id": id}) await self.response_pub.send(resp, properties={"id": id})
consumer.acknowledge(msg) consumer.acknowledge(msg)
@ -144,7 +135,7 @@ class Processor(AsyncProcessor):
text=None, text=None,
) )
await self.out_pub.send(resp, properties={"id": id}) await self.response_pub.send(resp, properties={"id": id})
consumer.acknowledge(msg) consumer.acknowledge(msg)
@ -166,7 +157,7 @@ class Processor(AsyncProcessor):
) )
parser.add_argument( parser.add_argument(
'-P', '--push-queue', '--push-queue',
default=default_push_queue, default=default_push_queue,
help=f'Config push queue (default: {default_push_queue})' help=f'Config push queue (default: {default_push_queue})'
) )