apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-22 03:31:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Move chunker/librarian comms to base class

Browse source
This commit is contained in:
Cyber MacGeddon 2026-03-04 16:36:00 +00:00
parent 64447f6d60
commit 9df348489b
2 changed files with 13 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

12
trustgraph-base/trustgraph/base/chunking_service.py
View file

@ -191,3 +191,15 @@ class ChunkingService(FlowProcessor):
def add_args(parser): def add_args(parser):
"""Add chunking service arguments to parser""" """Add chunking service arguments to parser"""
FlowProcessor.add_args(parser) FlowProcessor.add_args(parser)
parser.add_argument(
'--librarian-request-queue',
default=default_librarian_request_queue,
help=f'Librarian request queue (default: {default_librarian_request_queue})',
)
parser.add_argument(
'--librarian-response-queue',
default=default_librarian_response_queue,
help=f'Librarian response queue (default: {default_librarian_response_queue})',
)

9
trustgraph-flow/trustgraph/chunking/recursive/chunker.py
View file

@ -4,27 +4,18 @@ Simple decoder, accepts text documents on input, outputs chunks from the
as text as separate output objects. as text as separate output objects.
""" """
import asyncio
import base64
import logging import logging
import uuid
from langchain_text_splitters import RecursiveCharacterTextSplitter from langchain_text_splitters import RecursiveCharacterTextSplitter
from prometheus_client import Histogram from prometheus_client import Histogram
from ... schema import TextDocument, Chunk from ... schema import TextDocument, Chunk
from ... schema import LibrarianRequest, LibrarianResponse
from ... schema import librarian_request_queue, librarian_response_queue
from ... base import ChunkingService, ConsumerSpec, ProducerSpec from ... base import ChunkingService, ConsumerSpec, ProducerSpec
from ... base import Consumer, Producer, ConsumerMetrics, ProducerMetrics
# Module logger # Module logger
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
default_ident = "chunker" default_ident = "chunker"
default_librarian_request_queue = librarian_request_queue
default_librarian_response_queue = librarian_response_queue
class Processor(ChunkingService): class Processor(ChunkingService):