mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-17 17:21:02 +02:00
input/output is working
This commit is contained in:
Cyber MacGeddon
parent
98faec7798
commit
60ca5aa3a6
8 changed files with 122 additions and 35 deletions
|
|
@ -47,6 +47,18 @@ class AsyncProcessor:
|
|||
|
||||
self.config_handlers = []
|
||||
|
||||
self.config_sub_task = self.subscribe(
|
||||
self.config_push_queue, config_subscriber_id,
|
||||
schema=ConfigPush, handler=self.on_config_change,
|
||||
start_of_messages=True
|
||||
)
|
||||
|
||||
self.running = True
|
||||
|
||||
def stop(self):
|
||||
self.client.close()
|
||||
self.running = False
|
||||
|
||||
@property
|
||||
def pulsar_host(self): return self.client.pulsar_host
|
||||
|
||||
|
|
@ -55,14 +67,21 @@ class AsyncProcessor:
|
|||
|
||||
async def start(self):
|
||||
|
||||
self.config_sub_task = self.subscribe(
|
||||
self.config_push_queue, config_subscriber_id,
|
||||
schema=ConfigPush, handler=self.on_config_change
|
||||
)
|
||||
print("STARTING CONFIG SUB", flush=True)
|
||||
await self.config_sub_task.start()
|
||||
|
||||
async def on_config_change(self, config, version):
|
||||
async def on_config_change(self, message, consumer):
|
||||
|
||||
config = message.value().config
|
||||
version = message.value().version
|
||||
|
||||
consumer.acknowledge(message)
|
||||
|
||||
print("Config change event", config, version, flush=True)
|
||||
for ch in self.config_handlers:
|
||||
ch(config, version)
|
||||
print("Invoke... handler...", flush=True)
|
||||
await ch(config, version)
|
||||
|
||||
|
||||
async def run_config_queue(self):
|
||||
|
||||
|
|
@ -73,11 +92,15 @@ class AsyncProcessor:
|
|||
print("Config thread running", flush=True)
|
||||
|
||||
async def run(self):
|
||||
while True:
|
||||
while self.running:
|
||||
await asyncio.sleep(2)
|
||||
|
||||
def subscribe(self, queue, subscriber, schema, handler, metrics=None):
|
||||
def subscribe(
|
||||
self, queue, subscriber, schema, handler, metrics=None,
|
||||
start_of_messages=False
|
||||
):
|
||||
|
||||
print("Processing subscription!!!!")
|
||||
return Consumer(
|
||||
taskgroup = self.taskgroup,
|
||||
client = self.client,
|
||||
|
|
@ -86,6 +109,7 @@ class AsyncProcessor:
|
|||
schema = schema,
|
||||
handler = handler,
|
||||
metrics = metrics,
|
||||
start_of_messages=start_of_messages,
|
||||
)
|
||||
|
||||
def publish(self, queue, schema, metrics=None):
|
||||
|
|
@ -123,15 +147,23 @@ class AsyncProcessor:
|
|||
|
||||
async with asyncio.TaskGroup() as tg:
|
||||
|
||||
p = cls(**args | { "taskgroup": tg })
|
||||
try:
|
||||
print("CREATING...", flush=True)
|
||||
|
||||
# FIXME: Two sort of 'ident' things going on here?
|
||||
p.module = ident
|
||||
p.config_ident = args.get("ident", "FIXME")
|
||||
p = cls(**args | { "taskgroup": tg })
|
||||
|
||||
await p.start()
|
||||
# FIXME: Two sort of 'ident' things going on here?
|
||||
p.module = ident
|
||||
p.config_ident = args.get("ident", "FIXME")
|
||||
|
||||
task2 = tg.create_task(p.run())
|
||||
print("STARTING...", flush=True)
|
||||
await p.start()
|
||||
|
||||
task2 = tg.create_task(p.run())
|
||||
|
||||
except Exception as e:
|
||||
print("Exception, dropping out", flush=True)
|
||||
raise e
|
||||
|
||||
@classmethod
|
||||
def launch(cls, ident, doc):
|
||||
|
|
@ -152,13 +184,15 @@ class AsyncProcessor:
|
|||
args = parser.parse_args()
|
||||
args = vars(args)
|
||||
|
||||
print(args)
|
||||
print(args, flush=True)
|
||||
|
||||
if args["metrics"]:
|
||||
start_http_server(args["metrics_port"])
|
||||
|
||||
while True:
|
||||
|
||||
print("Starting...", flush=True)
|
||||
|
||||
try:
|
||||
|
||||
asyncio.run(cls.launch_async(
|
||||
|
|
@ -166,23 +200,23 @@ class AsyncProcessor:
|
|||
))
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("Keyboard interrupt.")
|
||||
print("Keyboard interrupt.", flush=True)
|
||||
return
|
||||
|
||||
except _pulsar.Interrupted:
|
||||
print("Pulsar Interrupted.")
|
||||
print("Pulsar Interrupted.", flush=True)
|
||||
return
|
||||
|
||||
except ExceptionGroup as e:
|
||||
|
||||
print("Exception group:")
|
||||
print("Exception group:", flush=True)
|
||||
|
||||
for se in e.exceptions:
|
||||
print(" Type:", type(se))
|
||||
print(f" Exception: {se}")
|
||||
print(" Type:", type(se), flush=True)
|
||||
print(f" Exception: {se}", flush=True)
|
||||
|
||||
except Exception as e:
|
||||
print("Type:", type(e))
|
||||
print("Type:", type(e), flush=True)
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
print("Will retry...", flush=True)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ class Consumer:
|
|||
def __init__(
|
||||
self, taskgroup, client, queue, subscriber, schema,
|
||||
handler, rate_limit_retry_time = 10,
|
||||
rate_limit_timeout = 7200, metrics = None
|
||||
rate_limit_timeout = 7200, metrics = None,
|
||||
start_of_messages=False,
|
||||
):
|
||||
|
||||
self.taskgroup = taskgroup
|
||||
|
|
@ -20,19 +21,30 @@ class Consumer:
|
|||
self.handler = handler
|
||||
self.rate_limit_retry_time = rate_limit_retry_time
|
||||
self.rate_limit_timeout = rate_limit_timeout
|
||||
self.start_of_messages = start_of_messages
|
||||
|
||||
self.running = True
|
||||
self.task = None
|
||||
|
||||
self.metrics = metrics
|
||||
|
||||
print("Consumer: __init__d")
|
||||
|
||||
async def stop(self):
|
||||
|
||||
self.running = False
|
||||
|
||||
async def start(self):
|
||||
|
||||
self.running = True
|
||||
|
||||
print("Subscribing...", flush=True)
|
||||
|
||||
self.consumer = self.client.subscribe(
|
||||
self.queue, self.subscriber, self.schema
|
||||
self.queue, self.subscriber, self.schema,
|
||||
start_of_messages = self.start_of_messages,
|
||||
)
|
||||
print("Subscribed.", flush=True)
|
||||
|
||||
# Puts it in the stopped state, the run thread should set running
|
||||
if self.metrics:
|
||||
|
|
@ -40,6 +52,8 @@ class Consumer:
|
|||
|
||||
self.task = self.taskgroup.create_task(self.run())
|
||||
|
||||
print("Subscriber started", flush=True)
|
||||
|
||||
async def run(self):
|
||||
|
||||
if self.metrics:
|
||||
|
|
@ -48,8 +62,12 @@ class Consumer:
|
|||
|
||||
while self.running:
|
||||
|
||||
print("Consumer: WAIT FOR MESSAGE", flush=True)
|
||||
|
||||
msg = await asyncio.to_thread(self.consumer.receive)
|
||||
|
||||
print("Consumer: MESSAGE RECEIVED", flush=True)
|
||||
|
||||
expiry = time.time() + self.rate_limit_timeout
|
||||
|
||||
# This loop is for retry on rate-limit / resource limits
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
|
||||
import json
|
||||
from pulsar.schema import JsonSchema
|
||||
|
||||
from .. schema import Error
|
||||
|
|
@ -18,6 +19,8 @@ class InputOutputProcessor(AsyncProcessor):
|
|||
self.input_schema = params.get("input_schema")
|
||||
self.output_schema = params.get("output_schema")
|
||||
|
||||
print("alsdkjasdlasdjkL")
|
||||
|
||||
ProcessorMetrics(id=self.id).info(
|
||||
{
|
||||
"subscriber": self.subscriber,
|
||||
|
|
@ -26,6 +29,7 @@ class InputOutputProcessor(AsyncProcessor):
|
|||
}
|
||||
)
|
||||
|
||||
print("ASD")
|
||||
super(InputOutputProcessor, self).__init__(
|
||||
**params | {
|
||||
"id": self.id,
|
||||
|
|
@ -34,6 +38,7 @@ class InputOutputProcessor(AsyncProcessor):
|
|||
}
|
||||
)
|
||||
|
||||
print("alsdkj")
|
||||
self.on_config(self.on_configuration)
|
||||
|
||||
self.subs = {}
|
||||
|
|
@ -48,7 +53,7 @@ class InputOutputProcessor(AsyncProcessor):
|
|||
|
||||
self.subs[flow] = self.subscribe(
|
||||
queue = defn["input"],
|
||||
subscriber = subscriber,
|
||||
subscriber = self.subscriber,
|
||||
schema = self.input_schema,
|
||||
handler = self.on_message,
|
||||
metrics = input_metrics,
|
||||
|
|
@ -60,31 +65,44 @@ class InputOutputProcessor(AsyncProcessor):
|
|||
metrics = output_metrics,
|
||||
)
|
||||
|
||||
self.subs[flow].start()
|
||||
await self.subs[flow].start()
|
||||
|
||||
print("Started flow for", flow)
|
||||
|
||||
async def stop_handler(self, flow):
|
||||
print("Stopping ", flow, flush=True)
|
||||
await self.subs[flow].stop()
|
||||
del self.subs[flow]
|
||||
del self.pubs[flow]
|
||||
|
||||
|
||||
async def on_configuration(self, config, version):
|
||||
|
||||
print("Got config version", version)
|
||||
|
||||
if "flows" not in config: return
|
||||
|
||||
if self.id in config["flows"]:
|
||||
|
||||
wanted_keys = config["flows"][self.id].keys()
|
||||
flow_config = json.loads(config["flows"][self.id])
|
||||
|
||||
wanted_keys = flow_config.keys()
|
||||
current_keys = self.subs.keys()
|
||||
|
||||
for key in wanted_keys:
|
||||
if key not in current_keys:
|
||||
self.start_handler(key, config["flows"][key])
|
||||
await self.start_handler(key, flow_config[key])
|
||||
|
||||
for key in current_keys:
|
||||
if key not in wanted_keys:
|
||||
self.stop_handler(key, config["flows"][key])
|
||||
await self.stop_handler(key)
|
||||
|
||||
print("Handled config update")
|
||||
|
||||
async def start(self):
|
||||
pass
|
||||
|
||||
print("INPUT OUTPU START")
|
||||
await super(InputOutputProcessor, self).start()
|
||||
|
||||
@staticmethod
|
||||
def add_args(parser, default_subscriber):
|
||||
|
|
|
|||
|
|
@ -51,12 +51,18 @@ class PulsarClient:
|
|||
if self.client:
|
||||
self.client.close()
|
||||
|
||||
def subscribe(self, queue, subscriber, schema):
|
||||
def subscribe(self, queue, subscriber, schema, start_of_messages=False):
|
||||
|
||||
if start_of_messages:
|
||||
pos = pulsar.InitialPosition.Earliest
|
||||
else:
|
||||
pos = pulsar.InitialPosition.Latest
|
||||
|
||||
return self.client.subscribe(
|
||||
queue, subscriber,
|
||||
consumer_type=pulsar.ConsumerType.Shared,
|
||||
schema=JsonSchema(schema),
|
||||
initial_position=pos,
|
||||
)
|
||||
|
||||
def publish(self, queue, schema):
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@ class Subscriber:
|
|||
|
||||
def run(self):
|
||||
|
||||
print("SUBSCRIBER LOOP KICKS IN", flush=True)
|
||||
|
||||
while self.running:
|
||||
|
||||
try:
|
||||
|
|
@ -42,10 +44,14 @@ class Subscriber:
|
|||
schema=self.schema,
|
||||
)
|
||||
|
||||
print("SUBSCRIBER RUNNING...", flush=True)
|
||||
|
||||
while self.running:
|
||||
|
||||
msg = consumer.receive()
|
||||
|
||||
print("GOT MESSAGE...", flush=True)
|
||||
|
||||
# Acknowledge successful reception of the message
|
||||
consumer.acknowledge(msg)
|
||||
|
||||
|
|
@ -73,7 +79,7 @@ class Subscriber:
|
|||
pass
|
||||
|
||||
except Exception as e:
|
||||
print("Exception:", e, flush=True)
|
||||
print("Subscriber exception:", e, flush=True)
|
||||
|
||||
# If handler drops out, sleep a retry
|
||||
time.sleep(2)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue