apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-18 17:51:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

New API emerging

Browse source
This commit is contained in:
Cyber MacGeddon 2025-04-14 09:05:42 +01:00
parent df6a11bb86
commit e66c06f99a
4 changed files with 269 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

58
trustgraph-base/trustgraph/base/consumer.py
View file

@ -26,7 +26,6 @@ class Consumer(BaseProcessor):
super(Consumer, self).__init__(**params)
self.input_queue = params.get("input_queue")
self.subscriber = params.get("subscriber")
self.input_schema = params.get("input_schema")
@ -42,46 +41,67 @@ class Consumer(BaseProcessor):
if not hasattr(__class__, "request_metric"):
__class__.request_metric = Histogram(
'request_latency', 'Request latency (seconds)'
'request_latency', 'Request latency (seconds)',
["flow"]
)
if not hasattr(__class__, "pubsub_metric"):
__class__.pubsub_metric = Info(
'pubsub', 'Pub/sub configuration'
'pubsub', 'Pub/sub configuration',
["flow"]
)
if not hasattr(__class__, "processing_metric"):
__class__.processing_metric = Counter(
'processing_count', 'Processing count', ["status"]
'processing_count', 'Processing count',
["flow", "status"]
)
if not hasattr(__class__, "rate_limit_metric"):
__class__.rate_limit_metric = Counter(
'rate_limit_count', 'Rate limit event count',
["flow"]
)
__class__.pubsub_metric.info({
"input_queue": self.input_queue,
"subscriber": self.subscriber,
"input_schema": self.input_schema.__name__,
"rate_limit_retry": str(self.rate_limit_retry),
"rate_limit_timeout": str(self.rate_limit_timeout),
})
# __class__.pubsub_metric.info({
# "input_queue": self.input_queue,
# "subscriber": self.subscriber,
# "input_schema": self.input_schema.__name__,
# "rate_limit_retry": str(self.rate_limit_retry),
# "rate_limit_timeout": str(self.rate_limit_timeout),
# })
self.consumer = self.client.subscribe(
self.input_queue, self.subscriber,
consumer_type=pulsar.ConsumerType.Shared,
schema=JsonSchema(self.input_schema),
)
# self.consumer = self.client.subscribe(
# self.input_queue, self.subscriber,
# consumer_type=pulsar.ConsumerType.Shared,
# schema=JsonSchema(self.input_schema),
# )
self.config_queues = {}
self.config_handlers = [self.on_config_pubsub]
print("Initialised consumer.", flush=True)
async def on_config_pubsub(self, version, config):
print("Configuring using config", version)
flows = config.get("flows", {}).get(self.ident, {})
print(flows)
print("Configuration completed OK")
async def run(self):
__class__.state_metric.state('running')
while True:
await asyncio.sleep(1)
continue
msg = await asyncio.to_thread(self.consumer.receive)
expiry = time.time() + self.rate_limit_timeout
@ -145,12 +165,6 @@ class Consumer(BaseProcessor):
BaseProcessor.add_args(parser)
parser.add_argument(
'-i', '--input-queue',
default=default_input_queue,
help=f'Input queue (default: {default_input_queue})'
)
parser.add_argument(
'-s', '--subscriber',
default=default_subscriber,