apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-26 17:06:22 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Feature/librarian (#307)

Browse source 
* Bring QDrant up-to-date

* Tables for data from queue outputs

- Pass single Pulsar client to everything in gateway & librarian
- Pulsar listener-name support in gateway
- PDF and text load working in librarian

* Complete Cassandra schema

* Add librarian support to templates
This commit is contained in:
cybermaggedon 2025-02-12 23:39:24 +00:00 • committed by GitHub
parent f350abb415
commit f7df2df266
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
35 changed files with 500 additions and 145 deletions
Download patch file Download diff file Expand all files Collapse all files

14
trustgraph-base/trustgraph/base/publisher.py
View file

@ -6,14 +6,13 @@ import threading
class Publisher:
def __init__(self, pulsar_host, topic, schema=None, max_size=10,
chunking_enabled=True, listener=None):
self.pulsar_host = pulsar_host
def __init__(self, pulsar_client, topic, schema=None, max_size=10,
chunking_enabled=True):
self.client = pulsar_client
self.topic = topic
self.schema = schema
self.q = queue.Queue(maxsize=max_size)
self.chunking_enabled = chunking_enabled
self.listener_name = listener
self.running = True
def start(self):
@ -33,11 +32,8 @@ class Publisher:
try:
client = pulsar.Client(
self.pulsar_host, listener_name=self.listener_name
)
producer = client.create_producer(
print(self.chunking_enabled)
producer = self.client.create_producer(
topic=self.topic,
schema=self.schema,
chunking_enabled=self.chunking_enabled,

14
trustgraph-base/trustgraph/base/subscriber.py
View file

@ -6,9 +6,9 @@ import time
class Subscriber:
def __init__(self, pulsar_host, topic, subscription, consumer_name,
schema=None, max_size=100, listener=None):
self.pulsar_host = pulsar_host
def __init__(self, pulsar_client, topic, subscription, consumer_name,
schema=None, max_size=100):
self.client = pulsar_client
self.topic = topic
self.subscription = subscription
self.consumer_name = consumer_name
@ -17,7 +17,6 @@ class Subscriber:
self.full = {}
self.max_size = max_size
self.lock = threading.Lock()
self.listener_name = listener
self.running = True
def start(self):
@ -36,12 +35,7 @@ class Subscriber:
try:
client = pulsar.Client(
self.pulsar_host,
listener_name=self.listener_name,
)
consumer = client.subscribe(
consumer = self.client.subscribe(
topic=self.topic,
subscription_name=self.subscription,
consumer_name=self.consumer_name,