From 3c5f6edc5454156654c24a41f55f9cce800abc0d Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Wed, 23 Apr 2025 10:46:38 +0100 Subject: [PATCH] Make metrics uniform --- .../trustgraph/base/async_processor.py | 2 +- .../trustgraph/base/consumer_spec.py | 2 +- trustgraph-base/trustgraph/base/metrics.py | 46 +++++++++++-------- .../trustgraph/base/producer_spec.py | 2 +- .../trustgraph/base/request_response_spec.py | 2 +- .../trustgraph/base/subscriber_spec.py | 2 +- .../trustgraph/config/service/service.py | 20 ++++++-- 7 files changed, 48 insertions(+), 28 deletions(-) diff --git a/trustgraph-base/trustgraph/base/async_processor.py b/trustgraph-base/trustgraph/base/async_processor.py index 15a8a4a0..696c167f 100644 --- a/trustgraph-base/trustgraph/base/async_processor.py +++ b/trustgraph-base/trustgraph/base/async_processor.py @@ -33,7 +33,7 @@ class AsyncProcessor: self.pulsar_client = PulsarClient(**params) # Initialise metrics, records the parameters - ProcessorMetrics(id=self.id).info({ + ProcessorMetrics(processor = self.id).info({ k: str(params[k]) for k in params if k != "id" diff --git a/trustgraph-base/trustgraph/base/consumer_spec.py b/trustgraph-base/trustgraph/base/consumer_spec.py index aaeca677..c4a37ef6 100644 --- a/trustgraph-base/trustgraph/base/consumer_spec.py +++ b/trustgraph-base/trustgraph/base/consumer_spec.py @@ -12,7 +12,7 @@ class ConsumerSpec(Spec): def add(self, flow, processor, definition): consumer_metrics = ConsumerMetrics( - flow.id, f"{flow.name}-{self.name}" + processor = flow.id, flow = flow.name, name = self.name, ) consumer = Consumer( diff --git a/trustgraph-base/trustgraph/base/metrics.py b/trustgraph-base/trustgraph/base/metrics.py index 5d87849f..6e46e896 100644 --- a/trustgraph-base/trustgraph/base/metrics.py +++ b/trustgraph-base/trustgraph/base/metrics.py @@ -4,79 +4,89 @@ from prometheus_client import Counter class ConsumerMetrics: - def __init__(self, id, flow=None): + def __init__(self, processor, flow, name): - self.id = id + self.processor = processor self.flow = flow + self.name = name if not hasattr(__class__, "state_metric"): __class__.state_metric = Enum( 'consumer_state', 'Consumer state', - ["id", "flow"], + ["processor", "flow", "name"], states=['stopped', 'running'] ) + if not hasattr(__class__, "request_metric"): __class__.request_metric = Histogram( 'request_latency', 'Request latency (seconds)', - ["id", "flow"], + ["processor", "flow", "name"], ) + if not hasattr(__class__, "processing_metric"): __class__.processing_metric = Counter( 'processing_count', 'Processing count', - ["id", "flow", "status"] + ["processor", "flow", "name"], ) + if not hasattr(__class__, "rate_limit_metric"): __class__.rate_limit_metric = Counter( 'rate_limit_count', 'Rate limit event count', - ["id", "flow"] + ["processor", "flow", "name"], ) def process(self, status): __class__.processing_metric.labels( - id=self.id, flow=self.flow, status=status + processor = self.processor, flow=self.flow, status=status ).inc() def rate_limit(self): __class__.rate_limit_metric.labels( - id=self.id, flow=self.flow + processor = self.processor, flow=self.flow ).inc() def state(self, state): __class__.state_metric.labels( - id=self.id, flow=self.flow + processor = self.processor, flow=self.flow ).state(state) def record_time(self): return __class__.request_metric.labels( - id=self.id, flow=self.flow + processor = self.processor, flow=self.flow ).time() class ProducerMetrics: - def __init__(self, id, flow=None): - self.id = id + def __init__(self, processor, flow, name): + + self.processor = processor self.flow = flow + self.name = name if not hasattr(__class__, "output_metric"): __class__.output_metric = Counter( 'output_count', 'Output items created', - ["id", "flow"] + ["processor", "flow", "name"], ) def inc(self): - __class__.output_metric.labels(id=self.id, flow=self.flow).inc() + __class__.output_metric.labels( + processor = self.processor, flow = self.flow, name = self.name + ).inc() class ProcessorMetrics: - def __init__(self, id): + def __init__(self, processor): - self.id = id + self.processor = processor if not hasattr(__class__, "processor_metric"): __class__.processor_metric = Info( 'processor', 'Processor configuration', - ["id"] + ["processor"] ) def info(self, info): - __class__.processor_metric.labels(id=self.id).info(info) + __class__.processor_metric.labels( + processor = self.processor + ).info(info) diff --git a/trustgraph-base/trustgraph/base/producer_spec.py b/trustgraph-base/trustgraph/base/producer_spec.py index 9007f48b..0549f949 100644 --- a/trustgraph-base/trustgraph/base/producer_spec.py +++ b/trustgraph-base/trustgraph/base/producer_spec.py @@ -11,7 +11,7 @@ class ProducerSpec(Spec): def add(self, flow, processor, definition): producer_metrics = ProducerMetrics( - flow.id, f"{flow.name}-{self.name}" + processor = flow.id, flow = flow.name, name = self.name ) producer = Producer( diff --git a/trustgraph-base/trustgraph/base/request_response_spec.py b/trustgraph-base/trustgraph/base/request_response_spec.py index dcfcbf9b..7f82e163 100644 --- a/trustgraph-base/trustgraph/base/request_response_spec.py +++ b/trustgraph-base/trustgraph/base/request_response_spec.py @@ -117,7 +117,7 @@ class RequestResponseSpec(Spec): def add(self, flow, processor, definition): producer_metrics = ProducerMetrics( - flow.id, f"{flow.name}-{self.response_name}" + process = flow.id, name = flow.name, name = self.response_name ) rr = self.impl( diff --git a/trustgraph-base/trustgraph/base/subscriber_spec.py b/trustgraph-base/trustgraph/base/subscriber_spec.py index 2f89290b..6c3d7c80 100644 --- a/trustgraph-base/trustgraph/base/subscriber_spec.py +++ b/trustgraph-base/trustgraph/base/subscriber_spec.py @@ -13,7 +13,7 @@ class SubscriberSpec(Spec): # FIXME: Metrics not used subscriber_metrics = ConsumerMetrics( - flow.id, f"{flow.name}-{self.name}" + processor = flow.id, flow = flow.name, name = self.name ) subscriber = Subscriber( diff --git a/trustgraph-flow/trustgraph/config/service/service.py b/trustgraph-flow/trustgraph/config/service/service.py index 5be27e90..972de45e 100644 --- a/trustgraph-flow/trustgraph/config/service/service.py +++ b/trustgraph-flow/trustgraph/config/service/service.py @@ -68,12 +68,22 @@ class Processor(AsyncProcessor): } ) - config_request_metrics = ConsumerMetrics(id + "-config-request") - config_response_metrics = ProducerMetrics(id + "-config-response") - config_push_metrics = ProducerMetrics(id + "-config-push") + config_request_metrics = ConsumerMetrics( + processor = self.id, flow = None, name = "config-request" + ) + config_response_metrics = ProducerMetrics( + processor = self.id, flow = None, name = "config-response" + ) + config_push_metrics = ProducerMetrics( + processor = self.id, flow = None, name = "config-push" + ) - flow_request_metrics = ConsumerMetrics(id + "-flow-request") - flow_response_metrics = ProducerMetrics(id + "-flow-response") + flow_request_metrics = ConsumerMetrics( + processor = self.id, flow = None, name = "flow-request" + ) + flow_response_metrics = ProducerMetrics( + processor = self.id, flow = None, name = "flow-response" + ) self.config_request_consumer = Consumer( taskgroup = self.taskgroup,