Move chunker/librarian comms to base class

trustgraph-base/trustgraph/base/chunking_service.py

@@ -190,4 +190,16 @@ class ChunkingService(FlowProcessor):
     @staticmethod
     def add_args(parser):
         """Add chunking service arguments to parser"""
-        FlowProcessor.add_args(parser)
+        FlowProcessor.add_args(parser)
+
+        parser.add_argument(
+            '--librarian-request-queue',
+            default=default_librarian_request_queue,
+            help=f'Librarian request queue (default: {default_librarian_request_queue})',
+        )
+
+        parser.add_argument(
+            '--librarian-response-queue',
+            default=default_librarian_response_queue,
+            help=f'Librarian response queue (default: {default_librarian_response_queue})',
+        )

trustgraph-flow/trustgraph/chunking/recursive/chunker.py

@@ -4,27 +4,18 @@ Simple decoder, accepts text documents on input, outputs chunks from the
 as text as separate output objects.
 """
 
-import asyncio
-import base64
 import logging
-import uuid
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from prometheus_client import Histogram
 
 from ... schema import TextDocument, Chunk
-from ... schema import LibrarianRequest, LibrarianResponse
-from ... schema import librarian_request_queue, librarian_response_queue
 from ... base import ChunkingService, ConsumerSpec, ProducerSpec
-from ... base import Consumer, Producer, ConsumerMetrics, ProducerMetrics
 
 # Module logger
 logger = logging.getLogger(__name__)
 
 default_ident = "chunker"
 
-default_librarian_request_queue = librarian_request_queue
-default_librarian_response_queue = librarian_response_queue
-
 
 class Processor(ChunkingService):