apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-28 01:46:22 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Revert "Feature/configure flows (#345)"

Browse source 
This reverts commit a9197d11ee.
This commit is contained in:
Cyber MacGeddon 2025-04-25 19:02:08 +01:00
parent 3adb3cf59c
commit 1822ca395f
125 changed files with 2628 additions and 3751 deletions
Download patch file Download diff file Expand all files Collapse all files

42
trustgraph-base/trustgraph/base/publisher.py
View file

@ -1,52 +1,47 @@
from pulsar.schema import JsonSchema
import asyncio
import queue
import time
import pulsar
import threading
class Publisher:
def __init__(self, client, topic, schema=None, max_size=10,
def __init__(self, pulsar_client, topic, schema=None, max_size=10,
chunking_enabled=True):
self.client = client
self.client = pulsar_client
self.topic = topic
self.schema = schema
self.q = asyncio.Queue(maxsize=max_size)
self.q = queue.Queue(maxsize=max_size)
self.chunking_enabled = chunking_enabled
self.running = True
async def start(self):
self.task = asyncio.create_task(self.run())
def start(self):
self.task = threading.Thread(target=self.run)
self.task.start()
async def stop(self):
def stop(self):
self.running = False
async def join(self):
await self.stop()
await self.task
def join(self):
self.stop()
self.task.join()
async def run(self):
def run(self):
while self.running:
try:
producer = self.client.create_producer(
topic=self.topic,
schema=JsonSchema(self.schema),
schema=self.schema,
chunking_enabled=self.chunking_enabled,
)
while self.running:
try:
id, item = await asyncio.wait_for(
self.q.get(),
timeout=0.5
)
except asyncio.TimeoutError:
continue
except asyncio.QueueEmpty:
id, item = self.q.get(timeout=0.5)
except queue.Empty:
continue
if id:
@ -60,6 +55,7 @@ class Publisher:
# If handler drops out, sleep a retry
time.sleep(2)
async def send(self, id, item):
await self.q.put((id, item))
def send(self, id, msg):
self.q.put((id, msg))