Request/response spec working, implemented in extract-defs and vertexai

trustgraph-base/trustgraph/base/__init__.py

@@ -7,7 +7,9 @@ from . publisher import Publisher
 from . subscriber import Subscriber
 from . metrics import ProcessorMetrics, ConsumerMetrics, ProducerMetrics
 from . flow_processor import FlowProcessor
-from . flow_processor import ConsumerSpec, SettingSpec
-from . flow_processor import SubscriberSpec, ProducerSpec
-from . request_response import RequestResponseService
+from . consumer_spec import ConsumerSpec
+from . setting_spec import SettingSpec
+from . producer_spec import ProducerSpec
+from . subscriber_spec import SubscriberSpec
+from . request_response_spec import RequestResponseSpec
 

trustgraph-base/trustgraph/base/async_processor.py

@@ -30,7 +30,7 @@ class AsyncProcessor:
         self.id = params.get("id")
 
         # Register a pulsar client
-        self.client = PulsarClient(**params)
+        self.pulsar_client = PulsarClient(**params)
 
         # Initialise metrics, records the parameters
         ProcessorMetrics(id=self.id).info({
@@ -58,10 +58,17 @@ class AsyncProcessor:
         config_subscriber_id = str(uuid.uuid4())
 
         # Subscribe to config queue
-        self.config_sub_task = self.subscribe(
+        self.config_sub_task = Consumer(
+
+            taskgroup = self.taskgroup,
+            client = self.client,
+            subscriber = config_subscriber_id,
             flow = None,
-            queue = self.config_push_queue, subscriber = config_subscriber_id,
-            schema = ConfigPush, handler = self.on_config_change,
+
+            topic = self.config_push_queue,
+            schema = ConfigPush,
+
+            handler = self.on_config_change,
 
             # This causes new subscriptions to view the entire history of
             # configuration
@@ -85,6 +92,10 @@ class AsyncProcessor:
     @property
     def pulsar_host(self): return self.client.pulsar_host
 
+    # Returns the pulsar client
+    @property
+    def client(self): return self.pulsar_client.client
+
     # Register a new event handler for configuration change
     def register_config_handler(self, handler):
         self.config_handlers.append(handler)
@@ -112,33 +123,6 @@ class AsyncProcessor:
         while self.running:
             await asyncio.sleep(2)
 
-    # Subscribe to a topic, returns a Consumer
-    def subscribe(
-            self, flow, queue, subscriber, schema, handler, metrics=None,
-            start_of_messages=False
-    ):
-
-        return Consumer(
-            flow = flow,
-            taskgroup = self.taskgroup,
-            client = self.client,
-            queue = queue,
-            subscriber = subscriber,
-            schema = schema,
-            handler = handler,
-            metrics = metrics,
-            start_of_messages=start_of_messages,
-        )
-
-    # Open a mechanism to publish messages to a topic.  Returns a subscriber
-    def publish(self, queue, schema, metrics=None):
-        return Producer(
-            client = self.client,
-            queue = queue,
-            schema = schema,
-            metrics = metrics,
-        )
-
     # Startup fabric.  This runs in 'async' mode, creates a taskgroup and
     # runs the producer.
     @classmethod

trustgraph-base/trustgraph/base/consumer.py

@@ -1,4 +1,6 @@
 
+from pulsar.schema import JsonSchema
+import pulsar
 import _pulsar
 import asyncio
 import time
@@ -8,19 +10,18 @@ from .. exceptions import TooManyRequests
 class Consumer:
 
     def __init__(
-            self, taskgroup, flow, client, queue, subscriber, schema,
+            self, taskgroup, flow, client, topic, subscriber, schema,
             handler, 
             metrics = None,
             start_of_messages=False,
             rate_limit_retry_time = 10, rate_limit_timeout = 7200,
             reconnect_time = 5,
-            
     ):
 
         self.taskgroup = taskgroup
         self.flow = flow
         self.client = client
-        self.queue = queue
+        self.topic = topic
         self.subscriber = subscriber
         self.schema = schema
         self.handler = handler
@@ -43,7 +44,8 @@ class Consumer:
         self.running = False
 
         if hasattr(self, "consumer"):
-            self.consumer.close()
+            if self.consumer:
+                self.consumer.close()
 
     async def stop(self):
 
@@ -69,13 +71,19 @@ class Consumer:
 
             try:
 
-                print(self.queue, "subscribing...", flush=True)
+                print(self.topic, "subscribing...", flush=True)
+
+                if self.start_of_messages:
+                    pos = pulsar.InitialPosition.Earliest
+                else:
+                    pos = pulsar.InitialPosition.Latest
 
                 self.consumer = await asyncio.to_thread(
                     self.client.subscribe,
-                    queue = self.queue, subscriber = self.subscriber,
-                    schema = self.schema,
-                    start_of_messages = self.start_of_messages
+                    topic = self.topic,
+                    subscription_name = self.subscriber,
+                    schema = JsonSchema(self.schema),
+                    initial_position = pos
                 )
 
             except Exception as e:
@@ -84,7 +92,7 @@ class Consumer:
                 await asyncio.sleep(self.reconnect_time)
                 continue
 
-            print(self.queue, "subscribed", flush=True)
+            print(self.topic, "subscribed", flush=True)
 
             if self.metrics:
                 self.metrics.state("running")

trustgraph-base/trustgraph/base/consumer_spec.py

@@ -0,0 +1,36 @@
+
+from . metrics import ConsumerMetrics
+from . consumer import Consumer
+from . spec import Spec
+
+class ConsumerSpec(Spec):
+    def __init__(self, name, schema, handler):
+        self.name = name
+        self.schema = schema
+        self.handler = handler
+
+    def add(self, flow, processor, definition):
+
+        consumer_metrics = ConsumerMetrics(
+            flow.id, f"{flow.name}-{self.name}"
+        )
+
+        consumer = Consumer(
+            taskgroup = processor.taskgroup,
+            flow = flow,
+            client = processor.client,
+            topic = definition[self.name],
+            subscriber = flow.id,
+            schema = self.schema,
+            handler = self.handler,
+            metrics = consumer_metrics,
+        )
+
+        # Consumer handle gets access to producers and other
+        # metadata
+        consumer.id = flow.id
+        consumer.name = self.name
+        consumer.flow = flow
+
+        flow.consumer[self.name] = consumer
+

trustgraph-base/trustgraph/base/flow.py

@@ -0,0 +1,32 @@
+
+import asyncio
+
+class Flow:
+    def __init__(self, id, flow, processor, defn):
+
+        self.id = id
+        self.name = flow
+
+        self.producer = {}
+
+        # Consumers and publishers.  Is this a bit untidy?
+        self.consumer = {}
+
+        self.setting = {}
+
+        for spec in processor.specifications:
+            spec.add(self, processor, defn)
+
+    async def start(self):
+        for c in self.consumer.values():
+            await c.start()
+
+    async def stop(self):
+        for c in self.consumer.values():
+            await c.stop()
+
+    def __call__(self, key):
+        if key in self.producer: return self.producer[key]
+        if key in self.consumer: return self.consumer[key]
+        if key in self.setting: return self.setting[key].value
+        return None

trustgraph-base/trustgraph/base/flow_processor.py

@@ -11,124 +11,10 @@ from .. schema import Error
 from .. schema import config_request_queue, config_response_queue
 from .. schema import config_push_queue
 from .. log_level import LogLevel
-from .. base import AsyncProcessor, Consumer, Producer, Subscriber
-from . metrics import ConsumerMetrics, ProducerMetrics
+from . async_processor import AsyncProcessor
+from . subscriber import Subscriber
+from . flow import Flow
 
-class Spec:
-    pass
-
-class SubscriberSpec(Spec):
-    def __init__(self, name, schema):
-        self.name = name
-        self.schema = schema
-
-    def add(self, flow, processor, definition):
-
-        subscriber_metrics = ConsumerMetrics(
-            flow.id, f"{flow.name}-{self.name}"
-        )
-
-        subscriber = Subscriber(
-            pulsar_client = processor.client.client,
-            topic = definition[self.name],
-            subscription = flow.id,
-            consumer_name = flow.id,
-            schema = self.schema,
-        )
-
-        # Put it in the consumer map, does that work?
-        # It means it gets start/stop call.
-        flow.consumer[self.name] = subscriber
-
-        if not hasattr(flow, self.name):
-            setattr(flow, self.name, subscriber)
-
-class ConsumerSpec(Spec):
-    def __init__(self, name, schema, handler):
-        self.name = name
-        self.schema = schema
-        self.handler = handler
-
-    def add(self, flow, processor, definition):
-
-        consumer_metrics = ConsumerMetrics(
-            flow.id, f"{flow.name}-{self.name}"
-        )
-
-        consumer = processor.subscribe(
-            flow = flow,
-            queue = definition[self.name],
-            subscriber = flow.id,
-            schema = self.schema,
-            handler = self.handler,
-            metrics = consumer_metrics,
-        )
-
-        # Consumer handle gets access to producers and other
-        # metadata
-        consumer.id = flow.id
-        consumer.name = self.name
-        consumer.flow = flow
-
-        flow.consumer[self.name] = consumer
-
-        if not hasattr(flow, self.name):
-            setattr(flow, self.name, consumer)
-
-class ProducerSpec(Spec):
-    def __init__(self, name, schema):
-        self.name = name
-        self.schema = schema
-
-    def add(self, flow, processor, definition):
-
-        producer_metrics = ProducerMetrics(
-            flow.id, f"{flow.name}-{self.name}"
-        )
-
-        producer = processor.publish(
-            queue = definition[self.name],
-            schema = self.schema,
-            metrics = producer_metrics,
-        )
-
-        flow.producer[self.name] = producer
-
-        if not hasattr(self, self.name):
-            setattr(flow, self.name, producer)
-
-class SettingSpec(Spec):
-    def __init__(self, name):
-        self.name = name
-
-    def add(self, flow, processor, definition):
-
-        flow.config[self.name] = definition[self.name]
-
-        if not hasattr(flow, self.name):
-            setattr(flow, self.name, definition[self.name])
-
-class Flow:
-    def __init__(self, id, flow, processor, defn):
-
-        self.id = id
-        self.name = flow
-
-        self.producer = {}
-        self.consumer = {}
-        self.setting = {}
-
-        for spec in processor.specifications:
-            spec.add(self, processor, defn)
-
-    async def start(self):
-        for c in self.consumer.values():
-            await c.start()
-
-    async def stop(self):
-        for c in self.consumer.values():
-            await c.stop()
-                
 # Parent class for configurable processors, configured with flows by
 # the config service
 class FlowProcessor(AsyncProcessor):
@@ -151,18 +37,6 @@ class FlowProcessor(AsyncProcessor):
 
         print("Service initialised.")
 
-    # Register a new consumer name
-    def register_consumer(self, name, schema, handler):
-        self.specifications.append(ConsumerSpec(name, schema, handler))
-
-    # Register a producer name
-    def register_producer(self, name, schema):
-        self.specifications.append(ProducerSpec(name, schema))
-
-    # Register a configuration variable
-    def register_config(self, name):
-        self.specifications.append(SettingSpec(name))
-
     # Register a configuration variable
     def register_specification(self, spec):
         self.specifications.append(spec)

trustgraph-base/trustgraph/base/producer.py

@@ -1,11 +1,12 @@
 
+from pulsar.schema import JsonSchema
 import asyncio
 
 class Producer:
 
-    def __init__(self, client, queue, schema, metrics=None):
+    def __init__(self, client, topic, schema, metrics=None):
         self.client = client
-        self.queue = queue
+        self.topic = topic
         self.schema = schema
 
         self.metrics = metrics
@@ -34,9 +35,12 @@ class Producer:
         while self.running and self.producer is None:
 
             try:
-                print("Connect publisher to", self.queue, "...", flush=True)
-                self.producer = self.client.publish(self.queue, self.schema)
-                print("Connected to", self.queue, flush=True)
+                print("Connect publisher to", self.topic, "...", flush=True)
+                self.producer = self.client.create_producer(
+                    topic = self.topic,
+                    schema = JsonSchema(self.schema)
+                )
+                print("Connected to", self.topic, flush=True)
             except Exception as e:
                 print("Exception:", e, flush=True)
                 await asyncio.sleep(2)

trustgraph-base/trustgraph/base/producer_spec.py

@@ -0,0 +1,25 @@
+
+from . producer import Producer
+from . metrics import ProducerMetrics
+from . spec import Spec
+
+class ProducerSpec(Spec):
+    def __init__(self, name, schema):
+        self.name = name
+        self.schema = schema
+
+    def add(self, flow, processor, definition):
+
+        producer_metrics = ProducerMetrics(
+            flow.id, f"{flow.name}-{self.name}"
+        )
+
+        producer = Producer(
+            client = processor.client,
+            topic = definition[self.name],
+            schema = self.schema,
+            metrics = producer_metrics,
+        )
+
+        flow.producer[self.name] = producer
+

trustgraph-base/trustgraph/base/publisher.py

@@ -7,9 +7,9 @@ import pulsar
 
 class Publisher:
 
-    def __init__(self, pulsar_client, topic, schema=None, max_size=10,
+    def __init__(self, client, topic, schema=None, max_size=10,
                  chunking_enabled=True):
-        self.client = pulsar_client
+        self.client = client
         self.topic = topic
         self.schema = schema
         self.q = asyncio.Queue(maxsize=max_size)

trustgraph-base/trustgraph/base/pubsub.py

@@ -51,30 +51,6 @@ class PulsarClient:
             if self.client:
                 self.client.close()
 
-    def subscribe(self, queue, subscriber, schema,
-                  start_of_messages=False):
-
-        if start_of_messages:
-            pos = pulsar.InitialPosition.Earliest
-        else:
-            pos =  pulsar.InitialPosition.Latest
-
-        return self.client.subscribe(
-            topic = queue,
-            subscription_name = subscriber,
-            consumer_type = pulsar.ConsumerType.Shared,
-            schema = JsonSchema(schema),
-            initial_position = pos,
-        )
-
-    def publish(self, queue, schema):
-
-        return self.client.create_producer(
-            topic=queue,
-            schema=JsonSchema(schema),
-            chunking_enabled=True,
-        )
-
     @staticmethod
     def add_args(parser):
 

trustgraph-base/trustgraph/base/request_response.py

@@ -1,73 +0,0 @@
-
-import json
-from pulsar.schema import JsonSchema
-
-from .. schema import Error
-from .. schema import config_request_queue, config_response_queue
-from .. schema import config_push_queue
-from .. log_level import LogLevel
-from .. base import AsyncProcessor, Consumer, Producer
-
-from .. base import ProcessorMetrics, ConsumerMetrics, ProducerMetrics
-from . flow_processor import FlowProcessor
-
-class RequestResponseService(FlowProcessor):
-
-    def __init__(self, **params):
-
-        super(RequestResponseService, self).__init__(**params)
-
-        self.response_schema = params.get("responsedrequest_schema")
-
-        # These can be overriden by a derived class
-        self.consumer_spec = [
-            ("request", params.get("request_schema"), self.on_message)
-        ]
-        self.producer_spec = [
-            ("response", params.get("response_schema"))
-        ]
-
-        print("Service initialised.")
-
-    async def on_message(self, message, consumer, flow):
-        
-        v = message.value()
-
-        # Sender-produced ID
-        id = message.properties()["id"]
-
-        print(f"Handling input {id}...", flush=True)
-
-        try:
-            resp = await self.on_request(v, consumer, flow)
-
-            print("Send response...", flush=True)
-
-            await flow.producer["response"].send(resp, properties={"id": id})
-
-            return
-
-        except Exception as e:
-
-            print("Exception:", e, flush=True)
-            print("Send error response...", flush=True)
-            r = self.response_schema(
-                error=Error(
-                    type="internal-error",
-                    message = str(e)
-                )
-            )
-
-            await flow.producer["response"].send(r, properties={"id": id})
-
-        print("Done.", flush=True)
-
-    @staticmethod
-    def add_args(parser):
-
-        FlowProcessor.add_args(parser)
-
-def run():
-
-    Processor.launch(module, __doc__)
-

trustgraph-base/trustgraph/base/request_response_spec.py

@@ -0,0 +1,104 @@
+
+import uuid
+import asyncio
+
+from . subscriber import Subscriber
+from . producer import Producer
+from . spec import Spec
+from . metrics import ConsumerMetrics, ProducerMetrics
+
+class RequestResponse(Subscriber):
+
+    def __init__(
+            self, client, subscription, consumer_name,
+            request_topic, request_schema,
+            request_metrics,
+            response_topic, response_schema,
+            response_metrics,
+    ):
+
+        super(RequestResponse, self).__init__(
+            client = client,
+            subscription = subscription,
+            consumer_name = consumer_name,
+            topic = response_topic,
+            schema = response_schema,
+        )
+
+        self.producer = Producer(
+            client = client,
+            topic = request_topic,
+            schema = request_schema,
+            metrics = request_metrics,
+        )
+
+    async def start(self):
+        await self.producer.start()
+        await super(RequestResponse, self).start()
+
+    async def stop(self):
+        await self.producer.stop()
+        await super(RequestResponse, self).stop()
+
+    async def request(self, req, timeout=300):
+
+        id = str(uuid.uuid4())
+
+        q = await self.subscribe(id)
+
+        try:
+
+            await self.producer.send(
+                req,
+                properties={"id": id}
+            )
+
+            resp = await asyncio.wait_for(
+                q.get(),
+                timeout=timeout
+            )
+
+            return resp
+
+        except Exception as e:
+
+            print("Exception:", e)
+            raise e
+
+        finally:
+
+            await self.unsubscribe(id)
+
+# This deals with the request/response case.  The caller needs to
+# use another service in request/response mode.  Uses two topics:
+# - we send on the request topic as a producer
+# - we receive on the response topic as a subscriber
+class RequestResponseSpec(Spec):
+    def __init__(
+            self, request_name, request_schema, response_name, response_schema
+    ):
+        self.request_name = request_name
+        self.request_schema = request_schema
+        self.response_name = response_name
+        self.response_schema = response_schema
+
+    def add(self, flow, processor, definition):
+
+        producer_metrics = ProducerMetrics(
+            flow.id, f"{flow.name}-{self.response_name}"
+        )
+
+        rr = RequestResponse(
+            client = processor.client,
+            subscription = flow.id,
+            consumer_name = flow.id,
+            request_topic = definition[self.request_name],
+            request_schema = self.request_schema,
+            request_metrics = producer_metrics,
+            response_topic = definition[self.response_name],
+            response_schema = self.response_schema,
+            response_metrics = None,
+        )
+
+        flow.consumer[self.request_name] = rr
+

trustgraph-base/trustgraph/base/setting_spec.py

@@ -0,0 +1,19 @@
+
+from . spec import Spec
+
+class Setting:
+    def __init__(self, value):
+        self.value = value
+    async def start():
+        pass
+    async def stop():
+        pass
+        
+class SettingSpec(Spec):
+    def __init__(self, name):
+        self.name = name
+
+    def add(self, flow, processor, definition):
+
+        flow.config[self.name] = Setting(definition[self.name])
+

trustgraph-base/trustgraph/base/spec.py

@@ -0,0 +1,4 @@
+
+class Spec:
+    pass
+

trustgraph-base/trustgraph/base/subscriber.py

@@ -6,9 +6,9 @@ import time
 
 class Subscriber:
 
-    def __init__(self, pulsar_client, topic, subscription, consumer_name,
+    def __init__(self, client, topic, subscription, consumer_name,
                  schema=None, max_size=100):
-        self.client = pulsar_client
+        self.client = client
         self.topic = topic
         self.subscription = subscription
         self.consumer_name = consumer_name

trustgraph-base/trustgraph/base/subscriber_spec.py

@@ -0,0 +1,30 @@
+
+from . metrics import ConsumerMetrics
+from . subscriber import Subscriber
+from . spec import Spec
+
+class SubscriberSpec(Spec):
+
+    def __init__(self, name, schema):
+        self.name = name
+        self.schema = schema
+
+    def add(self, flow, processor, definition):
+
+        # FIXME: Metrics not used
+        subscriber_metrics = ConsumerMetrics(
+            flow.id, f"{flow.name}-{self.name}"
+        )
+
+        subscriber = Subscriber(
+            client = processor.client,
+            topic = definition[self.name],
+            subscription = flow.id,
+            consumer_name = flow.id,
+            schema = self.schema,
+        )
+
+        # Put it in the consumer map, does that work?
+        # It means it gets start/stop call.
+        flow.consumer[self.name] = subscriber
+