From 9df348489bcacb5fc0180d8163b3b8b7d7a80dac Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Wed, 4 Mar 2026 16:36:00 +0000 Subject: [PATCH] Move chunker/librarian comms to base class --- .../trustgraph/base/chunking_service.py | 14 +++++++++++++- .../trustgraph/chunking/recursive/chunker.py | 9 --------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/trustgraph-base/trustgraph/base/chunking_service.py b/trustgraph-base/trustgraph/base/chunking_service.py index 16a8678c..890ed3f5 100644 --- a/trustgraph-base/trustgraph/base/chunking_service.py +++ b/trustgraph-base/trustgraph/base/chunking_service.py @@ -190,4 +190,16 @@ class ChunkingService(FlowProcessor): @staticmethod def add_args(parser): """Add chunking service arguments to parser""" - FlowProcessor.add_args(parser) \ No newline at end of file + FlowProcessor.add_args(parser) + + parser.add_argument( + '--librarian-request-queue', + default=default_librarian_request_queue, + help=f'Librarian request queue (default: {default_librarian_request_queue})', + ) + + parser.add_argument( + '--librarian-response-queue', + default=default_librarian_response_queue, + help=f'Librarian response queue (default: {default_librarian_response_queue})', + ) \ No newline at end of file diff --git a/trustgraph-flow/trustgraph/chunking/recursive/chunker.py b/trustgraph-flow/trustgraph/chunking/recursive/chunker.py index c338c3d8..4915d0a4 100755 --- a/trustgraph-flow/trustgraph/chunking/recursive/chunker.py +++ b/trustgraph-flow/trustgraph/chunking/recursive/chunker.py @@ -4,27 +4,18 @@ Simple decoder, accepts text documents on input, outputs chunks from the as text as separate output objects. """ -import asyncio -import base64 import logging -import uuid from langchain_text_splitters import RecursiveCharacterTextSplitter from prometheus_client import Histogram from ... schema import TextDocument, Chunk -from ... schema import LibrarianRequest, LibrarianResponse -from ... schema import librarian_request_queue, librarian_response_queue from ... base import ChunkingService, ConsumerSpec, ProducerSpec -from ... base import Consumer, Producer, ConsumerMetrics, ProducerMetrics # Module logger logger = logging.getLogger(__name__) default_ident = "chunker" -default_librarian_request_queue = librarian_request_queue -default_librarian_response_queue = librarian_response_queue - class Processor(ChunkingService):