New API emerging

trustgraph-base/trustgraph/base/base_processor.py

@@ -7,14 +7,110 @@ from pulsar.schema import JsonSchema
 import _pulsar
 import time
 import uuid
-from prometheus_client import start_http_server, Info
+from prometheus_client import start_http_server, Info, Enum
 
 from .. schema import ConfigPush, config_push_queue
 from .. log_level import LogLevel
+from .. exceptions import TooManyRequests
 
 default_config_queue = config_push_queue
 config_subscriber_id = str(uuid.uuid4())
 
+class Subscription:
+    def __init__(self, consumer, handler, taskgroup):
+        self.running = True
+        self.task = None
+        self.consumer = consumer
+        self.handle = handler
+        self.taskgroup=taskgroup
+
+    async def start(self):
+        self.running = True
+        self.task = self.taskgroup.create_task(self.run())
+
+    async def run(self):
+
+        while self.running:
+
+            print("Waiting...")
+            msg = await asyncio.to_thread(self.consumer.receive)
+            print("Got", msg)
+
+#             expiry = time.time() + self.rate_limit_timeout
+            expiry = time.time() + 10
+
+            # This loop is for retry on rate-limit / resource limits
+            while True:
+
+                if time.time() > expiry:
+
+                    print("Gave up waiting for rate-limit retry", flush=True)
+
+                    # Message failed to be processed, this causes it to
+                    # be retried
+                    self.consumer.negative_acknowledge(msg)
+
+                    # FIXME
+#                    __class__.processing_metric.labels(status="error").inc()
+
+                    # Break out of retry loop, processes next message
+                    break
+
+                try:
+
+                    print("Handle...")
+                    # FIXME
+#                    with __class__.request_metric.time():
+                    await self.handle(msg, self.consumer)
+                    print("Handled.")
+
+                    # Acknowledge successful processing of the message
+                    self.consumer.acknowledge(msg)
+
+#                    __class__.processing_metric.labels(status="success").inc()
+
+                    # Break out of retry loop
+                    break
+
+                except TooManyRequests:
+
+                    print("TooManyRequests: will retry...", flush=True)
+
+                    # FIXME
+#                     __class__.rate_limit_metric.inc()
+
+                    # Sleep
+                    time.sleep(self.rate_limit_retry)
+
+                    # Contine from retry loop, just causes a reprocessing
+                    continue
+
+                except Exception as e:
+
+                    print("Exception:", e, flush=True)
+
+                    # Message failed to be processed, this causes it to
+                    # be retried
+                    self.consumer.negative_acknowledge(msg)
+
+#                    __class__.processing_metric.labels(status="error").inc()
+
+                    # Break out of retry loop, processes next message
+                    break
+
+class Publisher:
+
+    def __init__(self, producer):
+        self.producer = producer
+#         self.running = True
+
+    async def send(self, msg, properties={}):
+        self.producer.send(msg, properties)
+
+        # FIXME
+#         __class__.output_metric.inc()
+
+                
 class BaseProcessor:
 
     default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
@@ -22,6 +118,10 @@ class BaseProcessor:
 
     def __init__(self, **params):
 
+        self.taskgroup = params.get("taskgroup")
+        if self.taskgroup is None:
+            raise RuntimeError("Essential taskgroup missing")
+
         self.client = None
 
         if not hasattr(__class__, "params_metric"):
@@ -68,7 +168,7 @@ class BaseProcessor:
             self.config_push_queue, config_subscriber_id,
             consumer_type=pulsar.ConsumerType.Shared,
             initial_position=pulsar.InitialPosition.Earliest,
-            schema=JsonSchema(ConfigPush),         
+            schema=JsonSchema(ConfigPush),
         )
 
         self.config_handlers = []
@@ -154,22 +254,71 @@ class BaseProcessor:
                 await h(v.version, v.config)
 
     async def run(self):
-        raise RuntimeError("Something should have implemented the run method")
+        while True:
+            await asyncio.sleep(2)
+
+#       raise RuntimeError("Something should have implemented the run method")
+
+    def subscribe(self, input_queue, subscriber, schema, handler):
+
+        consumer = self.client.subscribe(
+            input_queue, subscriber,
+            consumer_type=pulsar.ConsumerType.Shared,
+            schema=JsonSchema(schema),
+        )
+
+        s = Subscription(consumer, handler, self.taskgroup)
+
+        return s
+
+    def publish(self, output_queue, schema):
+
+        producer = self.client.create_producer(
+            topic=output_queue,
+            schema=JsonSchema(schema),
+            chunking_enabled=True,
+        )
+
+        p = Publisher(producer)
+
+        return p
+
+    def set_processor_state(self, flow, state):
+
+        if not hasattr(__class__, "state_metric"):
+            __class__.state_metric = Enum(
+                'processor_state', 'Processor state',
+                ["flow"],
+                states=['starting', 'running', 'stopped']
+            )
+
+        __class__.state_metric.labels("flow").state(state)
+
+    def set_pubsub_info(self, flow, info) :
+
+        if not hasattr(__class__, "pubsub_metric"):
+            __class__.pubsub_metric = Info(
+                'pubsub', 'Pub/sub configuration',
+                ["flow"]
+            )
+
+        __class__.pubsub_metric.labels(flow=flow).info(info)
 
     @classmethod
     async def launch_async(cls, args, ident):
-        p = cls(**args)
 
-        # FIXME: Two sort of 'ident' things going on here?
+        async with asyncio.TaskGroup() as tg:
-        p.module = ident
-        p.config_ident = args.ident
 
-        await p.start()
+            p = cls(taskgroup=tg, **args)
 
-        task1 = asyncio.create_task(p.run_config_queue())
+            # FIXME: Two sort of 'ident' things going on here?
-        task2 = asyncio.create_task(p.run())
+            p.module = ident
+            p.config_ident = args.get("ident", "FIXME")
 
-        await asyncio.gather(task1, task2)
+            await p.start()
+
+            task1 = tg.create_task(p.run_config_queue())
+            task2 = tg.create_task(p.run())
 
     @classmethod
     def launch(cls, ident, doc):
@@ -215,6 +364,10 @@ class BaseProcessor:
 
                 print(type(e))
 
+                print(e.message)
+                print(e.exceptions)
+                print(e)
+
                 print("Exception:", e, flush=True)
                 print("Will retry...", flush=True)
 

trustgraph-base/trustgraph/base/consumer.py

@@ -26,7 +26,6 @@ class Consumer(BaseProcessor):
 
         super(Consumer, self).__init__(**params)
 
-        self.input_queue = params.get("input_queue")
         self.subscriber = params.get("subscriber")
         self.input_schema = params.get("input_schema")
 
@@ -42,46 +41,67 @@ class Consumer(BaseProcessor):
 
         if not hasattr(__class__, "request_metric"):
             __class__.request_metric = Histogram(
-                'request_latency', 'Request latency (seconds)'
+                'request_latency', 'Request latency (seconds)',
+                ["flow"]
             )
 
         if not hasattr(__class__, "pubsub_metric"):
             __class__.pubsub_metric = Info(
-                'pubsub', 'Pub/sub configuration'
+                'pubsub', 'Pub/sub configuration',
+                ["flow"]
             )
 
         if not hasattr(__class__, "processing_metric"):
             __class__.processing_metric = Counter(
-                'processing_count', 'Processing count', ["status"]
+                'processing_count', 'Processing count',
+                ["flow", "status"]
             )
 
         if not hasattr(__class__, "rate_limit_metric"):
             __class__.rate_limit_metric = Counter(
                 'rate_limit_count', 'Rate limit event count',
+                ["flow"]
             )
 
-        __class__.pubsub_metric.info({
+        # __class__.pubsub_metric.info({
-            "input_queue": self.input_queue,
+        #     "input_queue": self.input_queue,
-            "subscriber": self.subscriber,
+        #     "subscriber": self.subscriber,
-            "input_schema": self.input_schema.__name__,
+        #     "input_schema": self.input_schema.__name__,
-            "rate_limit_retry": str(self.rate_limit_retry),
+        #     "rate_limit_retry": str(self.rate_limit_retry),
-            "rate_limit_timeout": str(self.rate_limit_timeout),
+        #     "rate_limit_timeout": str(self.rate_limit_timeout),
-        })
+        # })
 
-        self.consumer = self.client.subscribe(
+        # self.consumer = self.client.subscribe(
-            self.input_queue, self.subscriber,
+        #     self.input_queue, self.subscriber,
-            consumer_type=pulsar.ConsumerType.Shared,
+        #     consumer_type=pulsar.ConsumerType.Shared,
-            schema=JsonSchema(self.input_schema),
+        #     schema=JsonSchema(self.input_schema),
-        )
+        # )
+
+        self.config_queues = {}
+        self.config_handlers = [self.on_config_pubsub]
 
         print("Initialised consumer.", flush=True)
 
+    async def on_config_pubsub(self, version, config):
+
+        print("Configuring using config", version)
+
+        flows = config.get("flows", {}).get(self.ident, {})
+
+        print(flows)
+
+        print("Configuration completed OK")
+
     async def run(self):
 
         __class__.state_metric.state('running')
 
         while True:
 
+            await asyncio.sleep(1)
+
+            continue
+
             msg = await asyncio.to_thread(self.consumer.receive)
 
             expiry = time.time() + self.rate_limit_timeout
@@ -145,12 +165,6 @@ class Consumer(BaseProcessor):
 
         BaseProcessor.add_args(parser)
 
-        parser.add_argument(
-            '-i', '--input-queue',
-            default=default_input_queue,
-            help=f'Input queue (default: {default_input_queue})'
-        )
-
         parser.add_argument(
             '-s', '--subscriber',
             default=default_subscriber,

trustgraph-base/trustgraph/base/producer.py

@@ -54,3 +54,4 @@ class Producer(BaseProcessor):
             default=default_output_queue,
             help=f'Output queue (default: {default_output_queue})'
         )
+

trustgraph-flow/trustgraph/config/service/service.py

@@ -5,13 +5,14 @@ using the API.
 """
 
 from pulsar.schema import JsonSchema
+from prometheus_client import Histogram, Counter
 
 from trustgraph.schema import ConfigRequest, ConfigResponse, ConfigPush
 from trustgraph.schema import ConfigValue, Error
 from trustgraph.schema import config_request_queue, config_response_queue
 from trustgraph.schema import config_push_queue
 from trustgraph.log_level import LogLevel
-from trustgraph.base import ConsumerProducer
+from trustgraph.base import ConsumerProducer, BaseProcessor
 
 module = "config-svc"
 
@@ -33,30 +34,67 @@ class Configuration(dict):
             self[key] = ConfigurationItems()
         return dict.__getitem__(self, key)
         
-class Processor(ConsumerProducer):
+class Processor(BaseProcessor):
 
     def __init__(self, **params):
-
+        
         input_queue = params.get("input_queue", default_input_queue)
         output_queue = params.get("output_queue", default_output_queue)
         push_queue = params.get("push_queue", default_push_queue)
         subscriber = params.get("subscriber", default_subscriber)
 
-        super(Processor, self).__init__(
+
-            **params | {
+        input_schema = ConfigRequest
+        output_schema = ConfigResponse
+        push_schema = ConfigResponse
+
+        self.set_processor_state("FIXME", "starting")
+
+        self.set_pubsub_info(
+            "FIXME",
+            {
                 "input_queue": input_queue,
-                "output_queue": output_queue,
-                "push_queue": output_queue,
                 "subscriber": subscriber,
-                "input_schema": ConfigRequest,
+                "input_schema": input_schema.__name__,
-                "output_schema": ConfigResponse,
+#                 "rate_limit_retry": str(self.rate_limit_retry),
-                "push_schema": ConfigPush,
+#                 "rate_limit_timeout": str(self.rate_limit_timeout),
             }
         )
 
-        self.push_prod = self.client.create_producer(
+        super(Processor, self).__init__(
-            topic=push_queue,
+            **params | {
-            schema=JsonSchema(ConfigPush),
+                "input_schema": input_schema.__name__,
+                "output_schema": output_schema.__name__,
+                "push_schema": push_schema.__name__,
+            }
+        )
+
+        self.subs = self.subscribe(
+            input_queue=input_queue,
+            subscriber=subscriber,
+            schema=input_schema,
+            handler=self.on_message,
+        )
+
+        if not hasattr(__class__, "request_metric"):
+            __class__.request_metric = Histogram(
+                'request_latency', 'Request latency (seconds)'
+            )
+
+        if not hasattr(__class__, "processing_metric"):
+            __class__.processing_metric = Counter(
+                'processing_count', 'Processing count',
+                ["status"]
+            )
+
+        self.push_pub = self.publish(
+            output_queue = push_queue,
+            schema = ConfigPush
+        )
+
+        self.out_pub = self.publish(
+            output_queue = output_queue,
+            schema = ConfigResponse
         )
 
         # FIXME: The state is held internally. This only works if there's
@@ -67,8 +105,11 @@ class Processor(ConsumerProducer):
         # Version counter
         self.version = 0
 
+        print("Service initialised.")
+
     async def start(self):
         await self.push()
+        await self.subs.start()
         
     async def handle_get(self, v, id):
 
@@ -226,10 +267,12 @@ class Processor(ConsumerProducer):
             config = self.config,
             error = None,
         )
-        self.push_prod.send(resp)
+
+        await self.push_pub.send(resp)
+
         print("Pushed.")
         
-    async def handle(self, msg):
+    async def on_message(self, msg, consumer):
 
         v = msg.value()
 
@@ -276,9 +319,9 @@ class Processor(ConsumerProducer):
                     )
                 )
 
-            await self.send(resp, properties={"id": id})
+            await self.out_pub.send(resp, properties={"id": id})
 
-            self.consumer.acknowledge(msg)
+            consumer.acknowledge(msg)
 
         except Exception as e:
                 
@@ -289,8 +332,8 @@ class Processor(ConsumerProducer):
                 ),
                 text=None,
             )
-            await self.send(resp, properties={"id": id})
+            await self.out_pub.send(resp, properties={"id": id})
-            self.consumer.acknowledge(msg)
+            consumer.acknowledge(msg)
 
     @staticmethod
     def add_args(parser):
@@ -300,6 +343,12 @@ class Processor(ConsumerProducer):
             default_output_queue,
         )
 
+        parser.add_argument(
+            '-i', '--input-queue',
+            default=default_input_queue,
+            help=f'Input queue (default: {default_input_queue})'
+        )
+
         parser.add_argument(
             '-q', '--push-queue',
             default=default_push_queue,