Reduc pulsar connections (#189)

trustgraph-flow/trustgraph/api/gateway/endpoint.py

@@ -41,10 +41,15 @@ class ServiceEndpoint:
 
         self.operation = "service"
 
-    async def start(self):
+    async def start(self, client):
 
-        self.pub_task = asyncio.create_task(self.pub.run())
+        self.pub_task = asyncio.create_task(self.pub.run(client))
-        self.sub_task = asyncio.create_task(self.sub.run())
+        self.sub_task = asyncio.create_task(self.sub.run(client))
+
+    async def join(self):
+        
+        await self.pub_task
+        await self.sub_task
 
     def add_routes(self, app):
 

trustgraph-flow/trustgraph/api/gateway/graph_embeddings_load.py

@@ -29,10 +29,10 @@ class GraphEmbeddingsLoadEndpoint(SocketEndpoint):
             schema=JsonSchema(GraphEmbeddings)
         )
 
-    async def start(self):
+    async def start(self, client):
 
         self.task = asyncio.create_task(
-            self.publisher.run()
+            self.publisher.run(client)
         )
 
     async def listener(self, ws, running):

trustgraph-flow/trustgraph/api/gateway/graph_embeddings_stream.py

@@ -28,10 +28,10 @@ class GraphEmbeddingsStreamEndpoint(SocketEndpoint):
             schema=JsonSchema(GraphEmbeddings)
         )
 
-    async def start(self):
+    async def start(self, client):
 
         self.task = asyncio.create_task(
-            self.subscriber.run()
+            self.subscriber.run(client)
         )
 
     async def async_thread(self, ws, running):

trustgraph-flow/trustgraph/api/gateway/publisher.py

@@ -1,6 +1,5 @@
 
 import asyncio
-import aiopulsar
 
 class Publisher:
 
@@ -12,12 +11,11 @@ class Publisher:
         self.q = asyncio.Queue(maxsize=max_size)
         self.chunking_enabled = chunking_enabled
 
-    async def run(self):
+    async def run(self, client):
 
         while True:
 
             try:
-                async with aiopulsar.connect(self.pulsar_host) as client:
                 async with client.create_producer(
                         topic=self.topic,
                         schema=self.schema,

trustgraph-flow/trustgraph/api/gateway/service.py

@@ -17,6 +17,7 @@ from aiohttp import web
 import logging
 import os
 import base64
+import aiopulsar
 
 import pulsar
 from pulsar.schema import JsonSchema
@@ -237,13 +238,35 @@ class Api:
                 { "error": str(e) }
             )
 
-    async def app_factory(self):
+    async def run_endpoints(self):
+
+        async with aiopulsar.connect(self.pulsar_host) as client:
 
             for ep in self.endpoints:
-            await ep.start()
+                await ep.start(client)
 
-        self.doc_ingest_pub_task = asyncio.create_task(self.document_out.run())
+                self.doc_ingest_pub_task = asyncio.create_task(
-        self.text_ingest_pub_task = asyncio.create_task(self.text_out.run())
+                    self.document_out.run(client)
+                )
+
+                self.text_ingest_pub_task = asyncio.create_task(
+                    self.text_out.run(client)
+                )
+
+            print("Endpoints are running...")
+
+            # They never exit
+            for ep in self.endpoints:
+                await ep.join()
+
+            await self.doc_ingest_pub_task
+            await self.text_ingest_pub_task
+
+            print("Endpoints are stopped.")
+
+    async def app_factory(self):
+
+        self.endpoint_task = asyncio.create_task(self.run_endpoints())
 
         return self.app
 

trustgraph-flow/trustgraph/api/gateway/socket.py

@@ -76,6 +76,12 @@ class SocketEndpoint:
     async def start(self):
         pass
 
+    async def join(self):
+
+        # Nothing to wait for
+        while True:
+            await asyncio.sleep(100)
+
     def add_routes(self, app):
 
         app.add_routes([

trustgraph-flow/trustgraph/api/gateway/subscriber.py

@@ -1,6 +1,5 @@
 
 import asyncio
-import aiopulsar
 
 class Subscriber:
 
@@ -14,10 +13,9 @@ class Subscriber:
         self.q = {}
         self.full = {}
 
-    async def run(self):
+    async def run(self, client):
         while True:
             try:
-                async with aiopulsar.connect(self.pulsar_host) as client:
                 async with client.subscribe(
                     topic=self.topic,
                     subscription_name=self.subscription,

trustgraph-flow/trustgraph/api/gateway/triples_load.py

@@ -27,10 +27,10 @@ class TriplesLoadEndpoint(SocketEndpoint):
             schema=JsonSchema(Triples)
         )
 
-    async def start(self):
+    async def start(self, client):
 
         self.task = asyncio.create_task(
-            self.publisher.run()
+            self.publisher.run(client)
         )
 
     async def listener(self, ws, running):

trustgraph-flow/trustgraph/api/gateway/triples_stream.py

@@ -26,10 +26,10 @@ class TriplesStreamEndpoint(SocketEndpoint):
             schema=JsonSchema(Triples)
         )
 
-    async def start(self):
+    async def start(self, client):
 
         self.task = asyncio.create_task(
-            self.subscriber.run()
+            self.subscriber.run(client)
         )
 
     async def async_thread(self, ws, running):