- Changed module names to idents in all processors

- Add --id argument to base
- Tweaked config into handler
This commit is contained in:
Cyber MacGeddon 2025-04-10 19:14:31 +01:00
parent a06a814a41
commit df6a11bb86
58 changed files with 82 additions and 67 deletions

View file

@ -71,6 +71,8 @@ class BaseProcessor:
schema=JsonSchema(ConfigPush),
)
self.config_handlers = []
def __del__(self):
if hasattr(self, "client"):
@ -148,18 +150,20 @@ class BaseProcessor:
v = msg.value()
print("Got config version", v.version, flush=True)
await self.on_config(v.version, v.config)
async def on_config(self, version, config):
pass
for h in self.config_handlers:
await h(v.version, v.config)
async def run(self):
raise RuntimeError("Something should have implemented the run method")
@classmethod
async def launch_async(cls, args, prog):
async def launch_async(cls, args, ident):
p = cls(**args)
p.module = prog
# FIXME: Two sort of 'ident' things going on here?
p.module = ident
p.config_ident = args.ident
await p.start()
task1 = asyncio.create_task(p.run_config_queue())
@ -168,12 +172,18 @@ class BaseProcessor:
await asyncio.gather(task1, task2)
@classmethod
def launch(cls, prog, doc):
def launch(cls, ident, doc):
parser = argparse.ArgumentParser(
prog=prog,
prog=ident,
description=doc
)
parser.add_argument(
'--id',
default=ident,
help=f'Configuration identity (default: {ident})',
)
cls.add_args(parser)
@ -189,7 +199,9 @@ class BaseProcessor:
try:
asyncio.run(cls.launch_async(args, prog))
asyncio.run(cls.launch_async(
args, ident
))
except KeyboardInterrupt:
print("Keyboard interrupt.")

View file

@ -17,7 +17,7 @@ from .... log_level import LogLevel
from .... base import ConsumerProducer
from .... exceptions import TooManyRequests
module = ".".join(__name__.split(".")[1:-1])
module = "text-completion"
default_input_queue = text_completion_request_queue
default_output_queue = text_completion_response_queue

View file

@ -12,7 +12,7 @@ from trustgraph.schema import embeddings_response_queue
from trustgraph.log_level import LogLevel
from trustgraph.base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "embeddings"
default_input_queue = embeddings_request_queue
default_output_queue = embeddings_response_queue

View file

@ -25,7 +25,7 @@ from . agent_manager import AgentManager
from . types import Final, Action, Tool, Argument
module = ".".join(__name__.split(".")[1:-1])
module = "agent"
default_input_queue = agent_request_queue
default_output_queue = agent_response_queue
@ -102,6 +102,8 @@ class Processor(ConsumerProducer):
additional_context="",
)
self.config_handlers.append(self.on_config)
async def on_config(self, version, config):
print("Loading configuration version", version)

View file

@ -12,7 +12,7 @@ from ... schema import text_ingest_queue, chunk_ingest_queue
from ... log_level import LogLevel
from ... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "chunker"
default_input_queue = text_ingest_queue
default_output_queue = chunk_ingest_queue

View file

@ -12,7 +12,7 @@ from ... schema import text_ingest_queue, chunk_ingest_queue
from ... log_level import LogLevel
from ... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "chunker"
default_input_queue = text_ingest_queue
default_output_queue = chunk_ingest_queue

View file

@ -13,7 +13,7 @@ from trustgraph.schema import config_push_queue
from trustgraph.log_level import LogLevel
from trustgraph.base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "config-svc"
default_input_queue = config_request_queue
default_output_queue = config_response_queue

View file

@ -19,7 +19,7 @@ from ... schema import document_ingest_queue, text_ingest_queue
from ... log_level import LogLevel
from ... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "ocr"
default_input_queue = document_ingest_queue
default_output_queue = text_ingest_queue

View file

@ -13,7 +13,7 @@ from ... schema import document_ingest_queue, text_ingest_queue
from ... log_level import LogLevel
from ... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "pdf-decoder"
default_input_queue = document_ingest_queue
default_output_queue = text_ingest_queue

View file

@ -13,7 +13,7 @@ from ... clients.embeddings_client import EmbeddingsClient
from ... log_level import LogLevel
from ... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "document-embeddings"
default_input_queue = chunk_ingest_queue
default_output_queue = document_embeddings_store_queue

View file

@ -11,7 +11,7 @@ from ... base import ConsumerProducer
from fastembed import TextEmbedding
import os
module = ".".join(__name__.split(".")[1:-1])
module = "embeddings"
default_input_queue = embeddings_request_queue
default_output_queue = embeddings_response_queue

View file

@ -13,7 +13,7 @@ from ... clients.embeddings_client import EmbeddingsClient
from ... log_level import LogLevel
from ... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "graph-embeddings"
default_input_queue = entity_contexts_ingest_queue
default_output_queue = graph_embeddings_store_queue

View file

@ -11,7 +11,7 @@ from ... base import ConsumerProducer
from ollama import Client
import os
module = ".".join(__name__.split(".")[1:-1])
module = "embeddings"
default_input_queue = embeddings_request_queue
default_output_queue = embeddings_response_queue

View file

@ -11,7 +11,7 @@ from trustgraph.log_level import LogLevel
from trustgraph.base import ConsumerProducer
import requests
module = ".".join(__name__.split(".")[1:-1])
module = "wikipedia"
default_input_queue = encyclopedia_lookup_request_queue
default_output_queue = encyclopedia_lookup_response_queue

View file

@ -23,7 +23,7 @@ DEFINITION_VALUE = Value(value=DEFINITION, is_uri=True)
RDF_LABEL_VALUE = Value(value=RDF_LABEL, is_uri=True)
SUBJECT_OF_VALUE = Value(value=SUBJECT_OF, is_uri=True)
module = ".".join(__name__.split(".")[1:-1])
module = "kg-extract-definitions"
default_input_queue = chunk_ingest_queue
default_output_queue = triples_store_queue

View file

@ -20,7 +20,7 @@ from .... base import ConsumerProducer
RDF_LABEL_VALUE = Value(value=RDF_LABEL, is_uri=True)
SUBJECT_OF_VALUE = Value(value=SUBJECT_OF, is_uri=True)
module = ".".join(__name__.split(".")[1:-1])
module = "kg-extract-relationships"
default_input_queue = chunk_ingest_queue
default_output_queue = triples_store_queue

View file

@ -18,7 +18,7 @@ from .... base import ConsumerProducer
DEFINITION_VALUE = Value(value=DEFINITION, is_uri=True)
module = ".".join(__name__.split(".")[1:-1])
module = "kg-extract-topics"
default_input_queue = chunk_ingest_queue
default_output_queue = triples_store_queue

View file

@ -3,7 +3,7 @@ API gateway. Offers HTTP services which are translated to interaction on the
Pulsar bus.
"""
module = ".".join(__name__.split(".")[1:-1])
module = "api-gateway"
# FIXME: Subscribes to Pulsar unnecessarily, should only do it when there
# are active listeners

View file

@ -35,7 +35,7 @@ from .. exceptions import RequestError
from . librarian import Librarian
module = ".".join(__name__.split(".")[1:-1])
module = "librarian"
default_input_queue = librarian_request_queue
default_output_queue = librarian_response_queue

View file

@ -10,12 +10,11 @@ from .. schema import text_completion_response_queue
from .. log_level import LogLevel
from .. base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "metering"
default_input_queue = text_completion_response_queue
default_subscriber = module
class Processor(Consumer):
def __init__(self, **params):

View file

@ -27,7 +27,7 @@ from .... clients.llm_client import LlmClient
from . prompts import to_definitions, to_relationships, to_topics
from . prompts import to_kg_query, to_document_query, to_rows
module = ".".join(__name__.split(".")[1:-1])
module = "prompt"
default_input_queue = prompt_request_queue
default_output_queue = prompt_response_queue

View file

@ -18,7 +18,7 @@ from .... clients.llm_client import LlmClient
from . prompt_manager import PromptConfiguration, Prompt, PromptManager
module = ".".join(__name__.split(".")[1:-1])
module = "prompt"
default_input_queue = prompt_request_queue
default_output_queue = prompt_response_queue
@ -71,6 +71,8 @@ class Processor(ConsumerProducer):
self.llm = Llm(self.llm)
self.config_handlers.append(self.on_config)
# Null configuration, should reload quickly
self.manager = PromptManager(
llm = self.llm,

View file

@ -16,7 +16,7 @@ from .... log_level import LogLevel
from .... base import ConsumerProducer
from .... exceptions import TooManyRequests
module = ".".join(__name__.split(".")[1:-1])
module = "text-completion"
default_input_queue = text_completion_request_queue
default_output_queue = text_completion_response_queue

View file

@ -16,7 +16,7 @@ from .... log_level import LogLevel
from .... base import ConsumerProducer
from .... exceptions import TooManyRequests
module = ".".join(__name__.split(".")[1:-1])
module = "text-completion"
default_input_queue = text_completion_request_queue
default_output_queue = text_completion_response_queue

View file

@ -15,7 +15,7 @@ from .... log_level import LogLevel
from .... base import ConsumerProducer
from .... exceptions import TooManyRequests
module = ".".join(__name__.split(".")[1:-1])
module = "text-completion"
default_input_queue = text_completion_request_queue
default_output_queue = text_completion_response_queue

View file

@ -15,7 +15,7 @@ from .... log_level import LogLevel
from .... base import ConsumerProducer
from .... exceptions import TooManyRequests
module = ".".join(__name__.split(".")[1:-1])
module = "text-completion"
default_input_queue = text_completion_request_queue
default_output_queue = text_completion_response_queue

View file

@ -17,7 +17,7 @@ from .... log_level import LogLevel
from .... base import ConsumerProducer
from .... exceptions import TooManyRequests
module = ".".join(__name__.split(".")[1:-1])
module = "text-completion"
default_input_queue = text_completion_request_queue
default_output_queue = text_completion_response_queue

View file

@ -14,7 +14,7 @@ from .... log_level import LogLevel
from .... base import ConsumerProducer
from .... exceptions import TooManyRequests
module = ".".join(__name__.split(".")[1:-1])
module = "text-completion"
default_input_queue = text_completion_request_queue
default_output_queue = text_completion_response_queue

View file

@ -15,7 +15,7 @@ from .... log_level import LogLevel
from .... base import ConsumerProducer
from .... exceptions import TooManyRequests
module = ".".join(__name__.split(".")[1:-1])
module = "text-completion"
default_input_queue = text_completion_request_queue
default_output_queue = text_completion_response_queue

View file

@ -15,7 +15,7 @@ from .... log_level import LogLevel
from .... base import ConsumerProducer
from .... exceptions import TooManyRequests
module = ".".join(__name__.split(".")[1:-1])
module = "text-completion"
default_input_queue = text_completion_request_queue
default_output_queue = text_completion_response_queue

View file

@ -15,7 +15,7 @@ from .... log_level import LogLevel
from .... base import ConsumerProducer
from .... exceptions import TooManyRequests
module = ".".join(__name__.split(".")[1:-1])
module = "text-completion"
default_input_queue = text_completion_request_queue
default_output_queue = text_completion_response_queue

View file

@ -15,7 +15,7 @@ from .... log_level import LogLevel
from .... base import ConsumerProducer
from .... exceptions import TooManyRequests
module = ".".join(__name__.split(".")[1:-1])
module = "text-completion"
default_input_queue = text_completion_request_queue
default_output_queue = text_completion_response_queue

View file

@ -11,7 +11,7 @@ from .... schema import document_embeddings_request_queue
from .... schema import document_embeddings_response_queue
from .... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "de-query"
default_input_queue = document_embeddings_request_queue
default_output_queue = document_embeddings_response_queue

View file

@ -16,7 +16,7 @@ from .... schema import document_embeddings_request_queue
from .... schema import document_embeddings_response_queue
from .... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "de-query"
default_input_queue = document_embeddings_request_queue
default_output_queue = document_embeddings_response_queue

View file

@ -15,7 +15,7 @@ from .... schema import document_embeddings_request_queue
from .... schema import document_embeddings_response_queue
from .... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "de-query"
default_input_queue = document_embeddings_request_queue
default_output_queue = document_embeddings_response_queue

View file

@ -11,7 +11,7 @@ from .... schema import graph_embeddings_request_queue
from .... schema import graph_embeddings_response_queue
from .... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "ge-query"
default_input_queue = graph_embeddings_request_queue
default_output_queue = graph_embeddings_response_queue

View file

@ -16,7 +16,7 @@ from .... schema import graph_embeddings_request_queue
from .... schema import graph_embeddings_response_queue
from .... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "ge-query"
default_input_queue = graph_embeddings_request_queue
default_output_queue = graph_embeddings_response_queue

View file

@ -15,7 +15,7 @@ from .... schema import graph_embeddings_request_queue
from .... schema import graph_embeddings_response_queue
from .... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "ge-query"
default_input_queue = graph_embeddings_request_queue
default_output_queue = graph_embeddings_response_queue

View file

@ -11,7 +11,7 @@ from .... schema import triples_request_queue
from .... schema import triples_response_queue
from .... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "triples-query"
default_input_queue = triples_request_queue
default_output_queue = triples_response_queue

View file

@ -13,7 +13,7 @@ from .... schema import triples_request_queue
from .... schema import triples_response_queue
from .... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "triples-query"
default_input_queue = triples_request_queue
default_output_queue = triples_response_queue

View file

@ -13,7 +13,7 @@ from .... schema import triples_request_queue
from .... schema import triples_response_queue
from .... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "triples-query"
default_input_queue = triples_request_queue
default_output_queue = triples_response_queue

View file

@ -13,7 +13,7 @@ from .... schema import triples_request_queue
from .... schema import triples_response_queue
from .... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "triples-query"
default_input_queue = triples_request_queue
default_output_queue = triples_response_queue

View file

@ -16,7 +16,7 @@ from ... log_level import LogLevel
from ... document_rag import DocumentRag
from ... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "document-rag"
default_input_queue = document_rag_request_queue
default_output_queue = document_rag_response_queue

View file

@ -18,7 +18,7 @@ from ... log_level import LogLevel
from ... graph_rag import GraphRag
from ... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "graph-rag"
default_input_queue = graph_rag_request_queue
default_output_queue = graph_rag_response_queue

View file

@ -10,7 +10,7 @@ from .... schema import document_embeddings_store_queue
from .... log_level import LogLevel
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "de-write"
default_input_queue = document_embeddings_store_queue
default_subscriber = module

View file

@ -16,7 +16,7 @@ from .... schema import document_embeddings_store_queue
from .... log_level import LogLevel
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "de-write"
default_input_queue = document_embeddings_store_queue
default_subscriber = module

View file

@ -13,7 +13,7 @@ from .... schema import document_embeddings_store_queue
from .... log_level import LogLevel
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "de-write"
default_input_queue = document_embeddings_store_queue
default_subscriber = module

View file

@ -9,7 +9,7 @@ from .... log_level import LogLevel
from .... direct.milvus_graph_embeddings import EntityVectors
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "ge-write"
default_input_queue = graph_embeddings_store_queue
default_subscriber = module

View file

@ -15,7 +15,7 @@ from .... schema import graph_embeddings_store_queue
from .... log_level import LogLevel
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "ge-write"
default_input_queue = graph_embeddings_store_queue
default_subscriber = module

View file

@ -13,7 +13,7 @@ from .... schema import graph_embeddings_store_queue
from .... log_level import LogLevel
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "ge-write"
default_input_queue = graph_embeddings_store_queue
default_subscriber = module

View file

@ -9,7 +9,7 @@ from .... log_level import LogLevel
from .... direct.milvus_object_embeddings import ObjectVectors
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "oe-write"
default_input_queue = object_embeddings_store_queue
default_subscriber = module

View file

@ -17,7 +17,7 @@ from .... schema import rows_store_queue
from .... log_level import LogLevel
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "rows-write"
ssl_context = SSLContext(PROTOCOL_TLSv1_2)
default_input_queue = rows_store_queue

View file

@ -15,7 +15,7 @@ from .... schema import triples_store_queue
from .... log_level import LogLevel
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "triples-write"
default_input_queue = triples_store_queue
default_subscriber = module

View file

@ -16,7 +16,7 @@ from .... schema import triples_store_queue
from .... log_level import LogLevel
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "triples-write"
default_input_queue = triples_store_queue
default_subscriber = module

View file

@ -16,7 +16,7 @@ from .... schema import triples_store_queue
from .... log_level import LogLevel
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "triples-write"
default_input_queue = triples_store_queue
default_subscriber = module

View file

@ -16,7 +16,7 @@ from .... schema import triples_store_queue
from .... log_level import LogLevel
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
module = "triples-write"
default_input_queue = triples_store_queue
default_subscriber = module

View file

@ -14,7 +14,7 @@ from ... schema import document_ingest_queue, text_ingest_queue
from ... log_level import LogLevel
from ... base import ConsumerProducer
module = ".".join(__name__.split(".")[1:-1])
module = "ocr"
default_input_queue = document_ingest_queue
default_output_queue = text_ingest_queue

View file

@ -30,7 +30,7 @@ from .... log_level import LogLevel
from .... base import ConsumerProducer
from .... exceptions import TooManyRequests
module = ".".join(__name__.split(".")[1:-1])
module = "text-completion"
default_input_queue = text_completion_request_queue
default_output_queue = text_completion_response_queue