apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-09 05:12:12 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Concurrency implemented in more services (#409)

Browse source
This commit is contained in:
cybermaggedon 2025-06-04 11:45:21 +01:00 • committed by GitHub
parent e10e9d2295
commit 5364b1fad5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 63 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

13
trustgraph-flow/trustgraph/extract/kg/definitions/extract.py
View file

@ -21,16 +21,19 @@ RDF_LABEL_VALUE = Value(value=RDF_LABEL, is_uri=True)
SUBJECT_OF_VALUE = Value(value=SUBJECT_OF, is_uri=True)
default_ident = "kg-extract-definitions"
default_concurrency = 1
class Processor(FlowProcessor):
def __init__(self, **params):
id = params.get("id")
concurrency = params.get("concurrency", 1)
super(Processor, self).__init__(
**params | {
"id": id,
"concurrency": concurrency,
}
)
@ -38,7 +41,8 @@ class Processor(FlowProcessor):
ConsumerSpec(
name = "input",
schema = Chunk,
handler = self.on_message
handler = self.on_message,
concurrency = concurrency,
)
)
@ -190,6 +194,13 @@ class Processor(FlowProcessor):
@staticmethod
def add_args(parser):
parser.add_argument(
'-c', '--concurrency',
type=int,
default=default_concurrency,
help=f'Concurrent processing threads (default: {default_concurrency})'
)
FlowProcessor.add_args(parser)
def run():

13
trustgraph-flow/trustgraph/extract/kg/relationships/extract.py
View file

@ -20,16 +20,19 @@ RDF_LABEL_VALUE = Value(value=RDF_LABEL, is_uri=True)
SUBJECT_OF_VALUE = Value(value=SUBJECT_OF, is_uri=True)
default_ident = "kg-extract-relationships"
default_concurrency = 1
class Processor(FlowProcessor):
def __init__(self, **params):
id = params.get("id")
concurrency = params.get("concurrency", 1)
super(Processor, self).__init__(
**params | {
"id": id,
"concurrency": concurrency,
}
)
@ -37,7 +40,8 @@ class Processor(FlowProcessor):
ConsumerSpec(
name = "input",
schema = Chunk,
handler = self.on_message
handler = self.on_message,
concurrency = concurrency,
)
)
@ -192,6 +196,13 @@ class Processor(FlowProcessor):
@staticmethod
def add_args(parser):
parser.add_argument(
'-c', '--concurrency',
type=int,
default=default_concurrency,
help=f'Concurrent processing threads (default: {default_concurrency})'
)
FlowProcessor.add_args(parser)
def run():