diff --git a/trustgraph-base/trustgraph/base/base_processor.py b/trustgraph-base/trustgraph/base/base_processor.py index faf761bf..d5627fba 100644 --- a/trustgraph-base/trustgraph/base/base_processor.py +++ b/trustgraph-base/trustgraph/base/base_processor.py @@ -7,14 +7,110 @@ from pulsar.schema import JsonSchema import _pulsar import time import uuid -from prometheus_client import start_http_server, Info +from prometheus_client import start_http_server, Info, Enum from .. schema import ConfigPush, config_push_queue from .. log_level import LogLevel +from .. exceptions import TooManyRequests default_config_queue = config_push_queue config_subscriber_id = str(uuid.uuid4()) +class Subscription: + def __init__(self, consumer, handler, taskgroup): + self.running = True + self.task = None + self.consumer = consumer + self.handle = handler + self.taskgroup=taskgroup + + async def start(self): + self.running = True + self.task = self.taskgroup.create_task(self.run()) + + async def run(self): + + while self.running: + + print("Waiting...") + msg = await asyncio.to_thread(self.consumer.receive) + print("Got", msg) + +# expiry = time.time() + self.rate_limit_timeout + expiry = time.time() + 10 + + # This loop is for retry on rate-limit / resource limits + while True: + + if time.time() > expiry: + + print("Gave up waiting for rate-limit retry", flush=True) + + # Message failed to be processed, this causes it to + # be retried + self.consumer.negative_acknowledge(msg) + + # FIXME +# __class__.processing_metric.labels(status="error").inc() + + # Break out of retry loop, processes next message + break + + try: + + print("Handle...") + # FIXME +# with __class__.request_metric.time(): + await self.handle(msg, self.consumer) + print("Handled.") + + # Acknowledge successful processing of the message + self.consumer.acknowledge(msg) + +# __class__.processing_metric.labels(status="success").inc() + + # Break out of retry loop + break + + except TooManyRequests: + + print("TooManyRequests: will retry...", flush=True) + + # FIXME +# __class__.rate_limit_metric.inc() + + # Sleep + time.sleep(self.rate_limit_retry) + + # Contine from retry loop, just causes a reprocessing + continue + + except Exception as e: + + print("Exception:", e, flush=True) + + # Message failed to be processed, this causes it to + # be retried + self.consumer.negative_acknowledge(msg) + +# __class__.processing_metric.labels(status="error").inc() + + # Break out of retry loop, processes next message + break + +class Publisher: + + def __init__(self, producer): + self.producer = producer +# self.running = True + + async def send(self, msg, properties={}): + self.producer.send(msg, properties) + + # FIXME +# __class__.output_metric.inc() + + class BaseProcessor: default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650') @@ -22,6 +118,10 @@ class BaseProcessor: def __init__(self, **params): + self.taskgroup = params.get("taskgroup") + if self.taskgroup is None: + raise RuntimeError("Essential taskgroup missing") + self.client = None if not hasattr(__class__, "params_metric"): @@ -68,7 +168,7 @@ class BaseProcessor: self.config_push_queue, config_subscriber_id, consumer_type=pulsar.ConsumerType.Shared, initial_position=pulsar.InitialPosition.Earliest, - schema=JsonSchema(ConfigPush), + schema=JsonSchema(ConfigPush), ) self.config_handlers = [] @@ -154,22 +254,71 @@ class BaseProcessor: await h(v.version, v.config) async def run(self): - raise RuntimeError("Something should have implemented the run method") + while True: + await asyncio.sleep(2) + +# raise RuntimeError("Something should have implemented the run method") + + def subscribe(self, input_queue, subscriber, schema, handler): + + consumer = self.client.subscribe( + input_queue, subscriber, + consumer_type=pulsar.ConsumerType.Shared, + schema=JsonSchema(schema), + ) + + s = Subscription(consumer, handler, self.taskgroup) + + return s + + def publish(self, output_queue, schema): + + producer = self.client.create_producer( + topic=output_queue, + schema=JsonSchema(schema), + chunking_enabled=True, + ) + + p = Publisher(producer) + + return p + + def set_processor_state(self, flow, state): + + if not hasattr(__class__, "state_metric"): + __class__.state_metric = Enum( + 'processor_state', 'Processor state', + ["flow"], + states=['starting', 'running', 'stopped'] + ) + + __class__.state_metric.labels("flow").state(state) + + def set_pubsub_info(self, flow, info) : + + if not hasattr(__class__, "pubsub_metric"): + __class__.pubsub_metric = Info( + 'pubsub', 'Pub/sub configuration', + ["flow"] + ) + + __class__.pubsub_metric.labels(flow=flow).info(info) @classmethod async def launch_async(cls, args, ident): - p = cls(**args) - # FIXME: Two sort of 'ident' things going on here? - p.module = ident - p.config_ident = args.ident + async with asyncio.TaskGroup() as tg: - await p.start() + p = cls(taskgroup=tg, **args) - task1 = asyncio.create_task(p.run_config_queue()) - task2 = asyncio.create_task(p.run()) + # FIXME: Two sort of 'ident' things going on here? + p.module = ident + p.config_ident = args.get("ident", "FIXME") - await asyncio.gather(task1, task2) + await p.start() + + task1 = tg.create_task(p.run_config_queue()) + task2 = tg.create_task(p.run()) @classmethod def launch(cls, ident, doc): @@ -215,6 +364,10 @@ class BaseProcessor: print(type(e)) + print(e.message) + print(e.exceptions) + print(e) + print("Exception:", e, flush=True) print("Will retry...", flush=True) diff --git a/trustgraph-base/trustgraph/base/consumer.py b/trustgraph-base/trustgraph/base/consumer.py index fdbe5531..6ba2650e 100644 --- a/trustgraph-base/trustgraph/base/consumer.py +++ b/trustgraph-base/trustgraph/base/consumer.py @@ -26,7 +26,6 @@ class Consumer(BaseProcessor): super(Consumer, self).__init__(**params) - self.input_queue = params.get("input_queue") self.subscriber = params.get("subscriber") self.input_schema = params.get("input_schema") @@ -42,46 +41,67 @@ class Consumer(BaseProcessor): if not hasattr(__class__, "request_metric"): __class__.request_metric = Histogram( - 'request_latency', 'Request latency (seconds)' + 'request_latency', 'Request latency (seconds)', + ["flow"] ) if not hasattr(__class__, "pubsub_metric"): __class__.pubsub_metric = Info( - 'pubsub', 'Pub/sub configuration' + 'pubsub', 'Pub/sub configuration', + ["flow"] ) if not hasattr(__class__, "processing_metric"): __class__.processing_metric = Counter( - 'processing_count', 'Processing count', ["status"] + 'processing_count', 'Processing count', + ["flow", "status"] ) if not hasattr(__class__, "rate_limit_metric"): __class__.rate_limit_metric = Counter( 'rate_limit_count', 'Rate limit event count', + ["flow"] ) - __class__.pubsub_metric.info({ - "input_queue": self.input_queue, - "subscriber": self.subscriber, - "input_schema": self.input_schema.__name__, - "rate_limit_retry": str(self.rate_limit_retry), - "rate_limit_timeout": str(self.rate_limit_timeout), - }) + # __class__.pubsub_metric.info({ + # "input_queue": self.input_queue, + # "subscriber": self.subscriber, + # "input_schema": self.input_schema.__name__, + # "rate_limit_retry": str(self.rate_limit_retry), + # "rate_limit_timeout": str(self.rate_limit_timeout), + # }) - self.consumer = self.client.subscribe( - self.input_queue, self.subscriber, - consumer_type=pulsar.ConsumerType.Shared, - schema=JsonSchema(self.input_schema), - ) + # self.consumer = self.client.subscribe( + # self.input_queue, self.subscriber, + # consumer_type=pulsar.ConsumerType.Shared, + # schema=JsonSchema(self.input_schema), + # ) + + self.config_queues = {} + self.config_handlers = [self.on_config_pubsub] print("Initialised consumer.", flush=True) + async def on_config_pubsub(self, version, config): + + print("Configuring using config", version) + + flows = config.get("flows", {}).get(self.ident, {}) + + print(flows) + + print("Configuration completed OK") + async def run(self): __class__.state_metric.state('running') while True: + await asyncio.sleep(1) + + continue + msg = await asyncio.to_thread(self.consumer.receive) expiry = time.time() + self.rate_limit_timeout @@ -145,12 +165,6 @@ class Consumer(BaseProcessor): BaseProcessor.add_args(parser) - parser.add_argument( - '-i', '--input-queue', - default=default_input_queue, - help=f'Input queue (default: {default_input_queue})' - ) - parser.add_argument( '-s', '--subscriber', default=default_subscriber, diff --git a/trustgraph-base/trustgraph/base/producer.py b/trustgraph-base/trustgraph/base/producer.py index bc2d7791..619670db 100644 --- a/trustgraph-base/trustgraph/base/producer.py +++ b/trustgraph-base/trustgraph/base/producer.py @@ -54,3 +54,4 @@ class Producer(BaseProcessor): default=default_output_queue, help=f'Output queue (default: {default_output_queue})' ) + diff --git a/trustgraph-flow/trustgraph/config/service/service.py b/trustgraph-flow/trustgraph/config/service/service.py index c17ad2ec..2c2f143b 100644 --- a/trustgraph-flow/trustgraph/config/service/service.py +++ b/trustgraph-flow/trustgraph/config/service/service.py @@ -5,13 +5,14 @@ using the API. """ from pulsar.schema import JsonSchema +from prometheus_client import Histogram, Counter from trustgraph.schema import ConfigRequest, ConfigResponse, ConfigPush from trustgraph.schema import ConfigValue, Error from trustgraph.schema import config_request_queue, config_response_queue from trustgraph.schema import config_push_queue from trustgraph.log_level import LogLevel -from trustgraph.base import ConsumerProducer +from trustgraph.base import ConsumerProducer, BaseProcessor module = "config-svc" @@ -33,30 +34,67 @@ class Configuration(dict): self[key] = ConfigurationItems() return dict.__getitem__(self, key) -class Processor(ConsumerProducer): +class Processor(BaseProcessor): def __init__(self, **params): - + input_queue = params.get("input_queue", default_input_queue) output_queue = params.get("output_queue", default_output_queue) push_queue = params.get("push_queue", default_push_queue) subscriber = params.get("subscriber", default_subscriber) - super(Processor, self).__init__( - **params | { + + input_schema = ConfigRequest + output_schema = ConfigResponse + push_schema = ConfigResponse + + self.set_processor_state("FIXME", "starting") + + self.set_pubsub_info( + "FIXME", + { "input_queue": input_queue, - "output_queue": output_queue, - "push_queue": output_queue, "subscriber": subscriber, - "input_schema": ConfigRequest, - "output_schema": ConfigResponse, - "push_schema": ConfigPush, + "input_schema": input_schema.__name__, +# "rate_limit_retry": str(self.rate_limit_retry), +# "rate_limit_timeout": str(self.rate_limit_timeout), } ) - self.push_prod = self.client.create_producer( - topic=push_queue, - schema=JsonSchema(ConfigPush), + super(Processor, self).__init__( + **params | { + "input_schema": input_schema.__name__, + "output_schema": output_schema.__name__, + "push_schema": push_schema.__name__, + } + ) + + self.subs = self.subscribe( + input_queue=input_queue, + subscriber=subscriber, + schema=input_schema, + handler=self.on_message, + ) + + if not hasattr(__class__, "request_metric"): + __class__.request_metric = Histogram( + 'request_latency', 'Request latency (seconds)' + ) + + if not hasattr(__class__, "processing_metric"): + __class__.processing_metric = Counter( + 'processing_count', 'Processing count', + ["status"] + ) + + self.push_pub = self.publish( + output_queue = push_queue, + schema = ConfigPush + ) + + self.out_pub = self.publish( + output_queue = output_queue, + schema = ConfigResponse ) # FIXME: The state is held internally. This only works if there's @@ -67,8 +105,11 @@ class Processor(ConsumerProducer): # Version counter self.version = 0 + print("Service initialised.") + async def start(self): await self.push() + await self.subs.start() async def handle_get(self, v, id): @@ -226,10 +267,12 @@ class Processor(ConsumerProducer): config = self.config, error = None, ) - self.push_prod.send(resp) + + await self.push_pub.send(resp) + print("Pushed.") - async def handle(self, msg): + async def on_message(self, msg, consumer): v = msg.value() @@ -276,9 +319,9 @@ class Processor(ConsumerProducer): ) ) - await self.send(resp, properties={"id": id}) + await self.out_pub.send(resp, properties={"id": id}) - self.consumer.acknowledge(msg) + consumer.acknowledge(msg) except Exception as e: @@ -289,8 +332,8 @@ class Processor(ConsumerProducer): ), text=None, ) - await self.send(resp, properties={"id": id}) - self.consumer.acknowledge(msg) + await self.out_pub.send(resp, properties={"id": id}) + consumer.acknowledge(msg) @staticmethod def add_args(parser): @@ -300,6 +343,12 @@ class Processor(ConsumerProducer): default_output_queue, ) + parser.add_argument( + '-i', '--input-queue', + default=default_input_queue, + help=f'Input queue (default: {default_input_queue})' + ) + parser.add_argument( '-q', '--push-queue', default=default_push_queue,