apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-17 09:11:03 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

More refactoring

Browse source
This commit is contained in:
Cyber MacGeddon 2025-04-16 20:50:41 +01:00
parent 4c22f23f3a
commit 51000e39ff
5 changed files with 11 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

2
trustgraph-base/trustgraph/base/flow_processor.py
View file

@ -105,7 +105,7 @@ class FlowProcessor(AsyncProcessor):
consumer = self.subscribe(
flow = flow_obj,
queue = defn[name],
subscriber = self.subscriber,
subscriber = self.id,
schema = schema,
handler = handler,
metrics = consumer_metrics,

4
trustgraph-base/trustgraph/base/request_response.py
View file

@ -63,9 +63,9 @@ class RequestResponseService(FlowProcessor):
print("Done.", flush=True)
@staticmethod
def add_args(parser, default_subscriber):
def add_args(parser):
FlowProcessor.add_args(parser, default_subscriber)
FlowProcessor.add_args(parser)
def run():

13
trustgraph-flow/trustgraph/chunking/recursive/chunker.py
View file

@ -12,23 +12,18 @@ from ... schema import text_ingest_queue, chunk_ingest_queue
from ... log_level import LogLevel
from ... base import FlowProcessor
module = "chunker"
default_subscriber = module
default_ident = "chunker"
class Processor(FlowProcessor):
def __init__(self, **params):
id = params.get("id")
subscriber = params.get("subscriber", default_subscriber)
id = params.get("id", default_ident)
chunk_size = params.get("chunk_size", 2000)
chunk_overlap = params.get("chunk_overlap", 100)
super(Processor, self).__init__(
**params | {
"subscriber": subscriber,
}
**params | { "id": id }
)
if not hasattr(__class__, "chunk_metric"):
@ -88,7 +83,7 @@ class Processor(FlowProcessor):
@staticmethod
def add_args(parser):
FlowProcessor.add_args(parser, default_subscriber)
FlowProcessor.add_args(parser)
parser.add_argument(
'-z', '--chunk-size',

11
trustgraph-flow/trustgraph/chunking/token/chunker.py
View file

@ -12,23 +12,18 @@ from ... schema import text_ingest_queue, chunk_ingest_queue
from ... log_level import LogLevel
from ... base import FlowProcessor
module = "chunker"
default_subscriber = module
default_ident = "chunker"
class Processor(FlowProcessor):
def __init__(self, **params):
id = params.get("id")
subscriber = params.get("subscriber", default_subscriber)
chunk_size = params.get("chunk_size", 250)
chunk_overlap = params.get("chunk_overlap", 15)
super(Processor, self).__init__(
**params | {
"subscriber": subscriber,
}
**params | { "id": id }
)
if not hasattr(__class__, "chunk_metric"):
@ -87,7 +82,7 @@ class Processor(FlowProcessor):
@staticmethod
def add_args(parser):
FlowProcessor.add_args(parser, default_subscriber)
FlowProcessor.add_args(parser)
parser.add_argument(
'-z', '--chunk-size',

2
trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py
View file

@ -73,7 +73,7 @@ class Processor(FlowProcessor):
@staticmethod
def add_args(parser):
FlowProcessor.add_args(parser, default_ident)
FlowProcessor.add_args(parser)
def run():