diff --git a/trustgraph-base/trustgraph/base/chunking_service.py b/trustgraph-base/trustgraph/base/chunking_service.py index 16a8678c..890ed3f5 100644 --- a/trustgraph-base/trustgraph/base/chunking_service.py +++ b/trustgraph-base/trustgraph/base/chunking_service.py @@ -190,4 +190,16 @@ class ChunkingService(FlowProcessor): @staticmethod def add_args(parser): """Add chunking service arguments to parser""" - FlowProcessor.add_args(parser) \ No newline at end of file + FlowProcessor.add_args(parser) + + parser.add_argument( + '--librarian-request-queue', + default=default_librarian_request_queue, + help=f'Librarian request queue (default: {default_librarian_request_queue})', + ) + + parser.add_argument( + '--librarian-response-queue', + default=default_librarian_response_queue, + help=f'Librarian response queue (default: {default_librarian_response_queue})', + ) \ No newline at end of file diff --git a/trustgraph-flow/trustgraph/chunking/recursive/chunker.py b/trustgraph-flow/trustgraph/chunking/recursive/chunker.py index c338c3d8..4915d0a4 100755 --- a/trustgraph-flow/trustgraph/chunking/recursive/chunker.py +++ b/trustgraph-flow/trustgraph/chunking/recursive/chunker.py @@ -4,27 +4,18 @@ Simple decoder, accepts text documents on input, outputs chunks from the as text as separate output objects. """ -import asyncio -import base64 import logging -import uuid from langchain_text_splitters import RecursiveCharacterTextSplitter from prometheus_client import Histogram from ... schema import TextDocument, Chunk -from ... schema import LibrarianRequest, LibrarianResponse -from ... schema import librarian_request_queue, librarian_response_queue from ... base import ChunkingService, ConsumerSpec, ProducerSpec -from ... base import Consumer, Producer, ConsumerMetrics, ProducerMetrics # Module logger logger = logging.getLogger(__name__) default_ident = "chunker" -default_librarian_request_queue = librarian_request_queue -default_librarian_response_queue = librarian_response_queue - class Processor(ChunkingService):