mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-05-29 17:25:15 +02:00
31328317fd
- Keeps processing in different flows separate so that data can go to different stores / collections etc. - Potentially supports different processing flows - Tidies the processing API with common base-classes for e.g. LLMs, and automatic configuration of 'clients' to use the right queue names in a flow
30 lines
788 B
Python
30 lines
788 B
Python
|
|
from . metrics import ConsumerMetrics
|
|
from . subscriber import Subscriber
|
|
from . spec import Spec
|
|
|
|
class SubscriberSpec(Spec):
|
|
|
|
def __init__(self, name, schema):
|
|
self.name = name
|
|
self.schema = schema
|
|
|
|
def add(self, flow, processor, definition):
|
|
|
|
# FIXME: Metrics not used
|
|
subscriber_metrics = ConsumerMetrics(
|
|
flow.id, f"{flow.name}-{self.name}"
|
|
)
|
|
|
|
subscriber = Subscriber(
|
|
client = processor.client,
|
|
topic = definition[self.name],
|
|
subscription = flow.id,
|
|
consumer_name = flow.id,
|
|
schema = self.schema,
|
|
)
|
|
|
|
# Put it in the consumer map, does that work?
|
|
# It means it gets start/stop call.
|
|
flow.consumer[self.name] = subscriber
|
|
|