Refactor subscriber/module names, queue definitions and schema

scripts/chunker-recursive

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-from trustgraph.chunker.recursive import run
+from trustgraph.chunking.recursive import run
 
 run()
 

scripts/ge-write-milvus

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.graph_embeddings.milvus import run
+
+run()
+

scripts/graph-rag

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-from trustgraph.rag.graph import run
+from trustgraph.retrieval.graph_rag import run
 
 run()
 

scripts/graph-write-cassandra

@@ -1,6 +0,0 @@
-#!/usr/bin/env python3
-
-from trustgraph.graph.cassandra_write import run
-
-run()
-

scripts/llm-azure-text

@@ -1,6 +0,0 @@
-#!/usr/bin/env python3
-
-from trustgraph.llm.azure_text import run
-
-run()
-

scripts/llm-claude-text

@@ -1,6 +0,0 @@
-#!/usr/bin/env python3
-
-from trustgraph.llm.claude_text import run
-
-run()
-

scripts/llm-ollama-text

@@ -1,6 +0,0 @@
-#!/usr/bin/env python3
-
-from trustgraph.llm.ollama_text import run
-
-run()
-

scripts/llm-vertexai-text

@@ -1,6 +0,0 @@
-#!/usr/bin/env python3
-
-from trustgraph.llm.vertexai_text import run
-
-run()
-

scripts/pdf-decoder

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-from trustgraph.decoder.pdf import run
+from trustgraph.decoding.pdf import run
 
 run()
 

scripts/text-completion-azure

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.azure import run
+
+run()
+

scripts/text-completion-claude

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.claude import run
+
+run()
+

scripts/text-completion-ollama

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.ollama import run
+
+run()
+

scripts/text-completion-vertexai

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.vertexai import run
+
+run()
+

scripts/triples-write-cassandra

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.triples.cassandra import run
+
+run()
+

scripts/vector-write-milvus

@@ -1,6 +0,0 @@
-#!/usr/bin/env python3
-
-from trustgraph.vector.milvus_write import run
-
-run()
-

trustgraph/base/__init__.py

@@ -1,3 +1,6 @@
 
-from . processor import *
+from . base_processor import BaseProcessor
+from . consumer import Consumer
+from . producer import Producer
+from . consumer_producer import ConsumerProducer
 

trustgraph/base/base_processor.py

@@ -0,0 +1,117 @@
+
+import os
+import argparse
+import pulsar
+import _pulsar
+import time
+from prometheus_client import start_http_server, Info
+
+from .. log_level import LogLevel
+
+class BaseProcessor:
+
+    default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
+
+    def __init__(self, **params):
+
+        self.client = None
+
+        if not hasattr(__class__, "params_metric"):
+            __class__.params_metric = Info(
+                'params', 'Parameters configuration'
+            )
+
+        # FIXME: Maybe outputs information it should not
+        __class__.params_metric.info({
+            k: str(params[k])
+            for k in params
+        })
+
+        pulsar_host = params.get("pulsar_host", self.default_pulsar_host)
+        log_level = params.get("log_level", LogLevel.INFO)
+
+        self.pulsar_host = pulsar_host
+
+        self.client = pulsar.Client(
+            pulsar_host,
+            logger=pulsar.ConsoleLogger(log_level.to_pulsar())
+        )
+
+    def __del__(self):
+
+        if self.client:
+            self.client.close()
+
+    @staticmethod
+    def add_args(parser):
+
+        parser.add_argument(
+            '-p', '--pulsar-host',
+            default=__class__.default_pulsar_host,
+            help=f'Pulsar host (default: {__class__.default_pulsar_host})',
+        )
+
+        parser.add_argument(
+            '-l', '--log-level',
+            type=LogLevel,
+            default=LogLevel.INFO,
+            choices=list(LogLevel),
+            help=f'Output queue (default: info)'
+        )
+
+        parser.add_argument(
+            '-M', '--metrics-enabled',
+            type=bool,
+            default=True,
+            help=f'Pulsar host (default: true)',
+        )
+
+        parser.add_argument(
+            '-P', '--metrics-port',
+            type=int,
+            default=8000,
+            help=f'Pulsar host (default: 8000)',
+        )
+
+    def run(self):
+        raise RuntimeError("Something should have implemented the run method")
+
+    @classmethod
+    def start(cls, prog, doc):
+
+        while True:
+
+            parser = argparse.ArgumentParser(
+                prog=prog,
+                description=doc
+            )
+
+            cls.add_args(parser)
+
+            args = parser.parse_args()
+            args = vars(args)
+
+            if args["metrics_enabled"]:
+                start_http_server(args["metrics_port"])
+
+            try:
+
+                p = cls(**args)
+                p.run()
+
+            except KeyboardInterrupt:
+                print("Keyboard interrupt.")
+                return
+
+            except _pulsar.Interrupted:
+                print("Pulsar Interrupted.")
+                return
+
+            except Exception as e:
+
+                print(type(e))
+
+                print("Exception:", e, flush=True)
+                print("Will retry...", flush=True)
+
+                time.sleep(10)

trustgraph/base/consumer.py

@@ -0,0 +1,87 @@
+
+from pulsar.schema import JsonSchema
+from prometheus_client import start_http_server, Histogram, Info, Counter
+
+from . base_processor import BaseProcessor
+
+class Consumer(BaseProcessor):
+
+    def __init__(self, **params):
+
+        super(Consumer, self).__init__(**params)
+
+        input_queue = params.get("input_queue")
+        subscriber = params.get("subscriber")
+        input_schema = params.get("input_schema")
+
+        if input_schema == None:
+            raise RuntimeError("input_schema must be specified")
+
+        if not hasattr(__class__, "request_metric"):
+            __class__.request_metric = Histogram(
+                'request_latency', 'Request latency (seconds)'
+            )
+
+        if not hasattr(__class__, "pubsub_metric"):
+            __class__.pubsub_metric = Info(
+                'pubsub', 'Pub/sub configuration'
+            )
+
+        if not hasattr(__class__, "processing_metric"):
+            __class__.processing_metric = Counter(
+                'processing_count', 'Processing count', ["status"]
+            )
+
+        __class__.pubsub_metric.info({
+            "input_queue": input_queue,
+            "subscriber": subscriber,
+            "input_schema": input_schema.__name__,
+        })
+
+        self.consumer = self.client.subscribe(
+            input_queue, subscriber,
+            schema=JsonSchema(input_schema),
+        )
+
+    def run(self):
+
+        while True:
+
+            msg = self.consumer.receive()
+
+            try:
+
+                with __class__.request_metric.time():
+                    self.handle(msg)
+
+                # Acknowledge successful processing of the message
+                self.consumer.acknowledge(msg)
+
+                __class__.processing_metric.labels(status="success").inc()
+
+            except Exception as e:
+
+                print("Exception:", e, flush=True)
+
+                # Message failed to be processed
+                self.consumer.negative_acknowledge(msg)
+
+                __class__.processing_metric.labels(status="error").inc()
+
+    @staticmethod
+    def add_args(parser, default_input_queue, default_subscriber):
+
+        BaseProcessor.add_args(parser)
+
+        parser.add_argument(
+            '-i', '--input-queue',
+            default=default_input_queue,
+            help=f'Input queue (default: {default_input_queue})'
+        )
+
+        parser.add_argument(
+            '-s', '--subscriber',
+            default=default_subscriber,
+            help=f'Queue subscriber name (default: {default_subscriber})'
+        )
+

trustgraph/base/consumer_producer.py

@@ -0,0 +1,168 @@
+
+from pulsar.schema import JsonSchema
+from prometheus_client import Histogram, Info, Counter
+
+from . base_processor import BaseProcessor
+
+# FIXME: Derive from consumer?  And producer?
+
+class ConsumerProducer(BaseProcessor):
+
+    def __init__(self, **params):
+
+        input_queue = params.get("input_queue")
+        output_queue = params.get("output_queue")
+        subscriber = params.get("subscriber")
+        input_schema = params.get("input_schema")
+        output_schema = params.get("output_schema")
+
+        if not hasattr(__class__, "request_metric"):
+            __class__.request_metric = Histogram(
+                'request_latency', 'Request latency (seconds)'
+            )
+
+        if not hasattr(__class__, "output_metric"):
+            __class__.output_metric = Counter(
+                'output_count', 'Output items created'
+            )
+
+        if not hasattr(__class__, "pubsub_metric"):
+            __class__.pubsub_metric = Info(
+                'pubsub', 'Pub/sub configuration'
+            )
+
+        if not hasattr(__class__, "processing_metric"):
+            __class__.processing_metric = Counter(
+                'processing_count', 'Processing count', ["status"]
+            )
+
+        __class__.pubsub_metric.info({
+            "input_queue": input_queue,
+            "output_queue": output_queue,
+            "subscriber": subscriber,
+            "input_schema": input_schema.__name__,
+            "output_schema": output_schema.__name__,
+        })
+
+        super(ConsumerProducer, self).__init__(**params)
+
+        if input_schema == None:
+            raise RuntimeError("input_schema must be specified")
+
+        if output_schema == None:
+            raise RuntimeError("output_schema must be specified")
+
+        self.consumer = self.client.subscribe(
+            input_queue, subscriber,
+            schema=JsonSchema(input_schema),
+        )
+
+        self.producer = self.client.create_producer(
+            topic=output_queue,
+            schema=JsonSchema(output_schema),
+        )
+
+    def run(self):
+
+        while True:
+
+            msg = self.consumer.receive()
+
+            try:
+
+                with __class__.request_metric.time():
+                    resp = self.handle(msg)
+
+                # Acknowledge successful processing of the message
+                self.consumer.acknowledge(msg)
+
+                __class__.processing_metric.labels(status="success").inc()
+
+            except Exception as e:
+
+                print("Exception:", e, flush=True)
+
+                # Message failed to be processed
+                self.consumer.negative_acknowledge(msg)
+
+                __class__.processing_metric.labels(status="error").inc()
+
+    def send(self, msg, properties={}):
+        self.producer.send(msg, properties)
+        __class__.output_metric.inc()
+
+    @staticmethod
+    def add_args(
+            parser, default_input_queue, default_subscriber,
+            default_output_queue,
+    ):
+
+        BaseProcessor.add_args(parser)
+
+        parser.add_argument(
+            '-i', '--input-queue',
+            default=default_input_queue,
+            help=f'Input queue (default: {default_input_queue})'
+        )
+
+        parser.add_argument(
+            '-s', '--subscriber',
+            default=default_subscriber,
+            help=f'Queue subscriber name (default: {default_subscriber})'
+        )
+
+        parser.add_argument(
+            '-o', '--output-queue',
+            default=default_output_queue,
+            help=f'Output queue (default: {default_output_queue})'
+        )
+
+class Producer(BaseProcessor):
+
+    def __init__(self, **params):
+
+        output_queue = params.get("output_queue")
+        output_schema = params.get("output_schema")
+
+        if not hasattr(__class__, "output_metric"):
+            __class__.output_metric = Counter(
+                'output_count', 'Output items created'
+            )
+
+        if not hasattr(__class__, "pubsub_metric"):
+            __class__.pubsub_metric = Info(
+                'pubsub', 'Pub/sub configuration'
+            )
+
+        __class__.pubsub_metric.info({
+            "output_queue": output_queue,
+            "output_schema": output_schema.__name__,
+        })
+
+        super(Producer, self).__init__(**params)
+
+        if output_schema == None:
+            raise RuntimeError("output_schema must be specified")
+
+        self.producer = self.client.create_producer(
+            topic=output_queue,
+            schema=JsonSchema(output_schema),
+        )
+
+    def send(self, msg, properties={}):
+        self.producer.send(msg, properties)
+        __class__.output_metric.inc()
+
+    @staticmethod
+    def add_args(
+            parser, default_input_queue, default_subscriber,
+            default_output_queue,
+    ):
+
+        BaseProcessor.add_args(parser)
+
+        parser.add_argument(
+            '-o', '--output-queue',
+            default=default_output_queue,
+            help=f'Output queue (default: {default_output_queue})'
+        )

trustgraph/base/processor.py

@@ -1,360 +0,0 @@
-
-import os
-import argparse
-import pulsar
-import _pulsar
-import time
-from pulsar.schema import JsonSchema
-from prometheus_client import start_http_server, Histogram, Info, Counter
-
-from .. log_level import LogLevel
-
-class BaseProcessor:
-
-    default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
-
-    def __init__(self, **params):
-
-        self.client = None
-
-        if not hasattr(__class__, "params_metric"):
-            __class__.params_metric = Info(
-                'params', 'Parameters configuration'
-            )
-
-        # FIXME: Maybe outputs information it should not
-        __class__.params_metric.info({
-            k: str(params[k])
-            for k in params
-        })
-
-        pulsar_host = params.get("pulsar_host", self.default_pulsar_host)
-        log_level = params.get("log_level", LogLevel.INFO)
-
-        self.pulsar_host = pulsar_host
-
-        self.client = pulsar.Client(
-            pulsar_host,
-            logger=pulsar.ConsoleLogger(log_level.to_pulsar())
-        )
-
-    def __del__(self):
-
-        if self.client:
-            self.client.close()
-
-    @staticmethod
-    def add_args(parser):
-
-        parser.add_argument(
-            '-p', '--pulsar-host',
-            default=__class__.default_pulsar_host,
-            help=f'Pulsar host (default: {__class__.default_pulsar_host})',
-        )
-
-        parser.add_argument(
-            '-l', '--log-level',
-            type=LogLevel,
-            default=LogLevel.INFO,
-            choices=list(LogLevel),
-            help=f'Output queue (default: info)'
-        )
-
-        parser.add_argument(
-            '-M', '--metrics-enabled',
-            type=bool,
-            default=True,
-            help=f'Pulsar host (default: true)',
-        )
-
-        parser.add_argument(
-            '-P', '--metrics-port',
-            type=int,
-            default=8000,
-            help=f'Pulsar host (default: 8000)',
-        )
-
-    def run(self):
-        raise RuntimeError("Something should have implemented the run method")
-
-    @classmethod
-    def start(cls, prog, doc):
-
-        while True:
-
-            parser = argparse.ArgumentParser(
-                prog=prog,
-                description=doc
-            )
-
-            cls.add_args(parser)
-
-            args = parser.parse_args()
-            args = vars(args)
-
-            if args["metrics_enabled"]:
-                start_http_server(args["metrics_port"])
-
-            try:
-
-                p = cls(**args)
-                p.run()
-
-            except KeyboardInterrupt:
-                print("Keyboard interrupt.")
-                return
-
-            except _pulsar.Interrupted:
-                print("Pulsar Interrupted.")
-                return
-
-            except Exception as e:
-
-                print(type(e))
-
-                print("Exception:", e, flush=True)
-                print("Will retry...", flush=True)
-
-                time.sleep(10)
-
-class Consumer(BaseProcessor):
-
-    def __init__(self, **params):
-
-        super(Consumer, self).__init__(**params)
-
-        input_queue = params.get("input_queue")
-        subscriber = params.get("subscriber")
-        input_schema = params.get("input_schema")
-
-        if input_schema == None:
-            raise RuntimeError("input_schema must be specified")
-
-        if not hasattr(__class__, "request_metric"):
-            __class__.request_metric = Histogram(
-                'request_latency', 'Request latency (seconds)'
-            )
-
-        if not hasattr(__class__, "pubsub_metric"):
-            __class__.pubsub_metric = Info(
-                'pubsub', 'Pub/sub configuration'
-            )
-
-        if not hasattr(__class__, "processing_metric"):
-            __class__.processing_metric = Counter(
-                'processing_count', 'Processing count', ["status"]
-            )
-
-        __class__.pubsub_metric.info({
-            "input_queue": input_queue,
-            "subscriber": subscriber,
-            "input_schema": input_schema.__name__,
-        })
-
-        self.consumer = self.client.subscribe(
-            input_queue, subscriber,
-            schema=JsonSchema(input_schema),
-        )
-
-    def run(self):
-
-        while True:
-
-            msg = self.consumer.receive()
-
-            try:
-
-                with __class__.request_metric.time():
-                    self.handle(msg)
-
-                # Acknowledge successful processing of the message
-                self.consumer.acknowledge(msg)
-
-                __class__.processing_metric.labels(status="success").inc()
-
-            except Exception as e:
-
-                print("Exception:", e, flush=True)
-
-                # Message failed to be processed
-                self.consumer.negative_acknowledge(msg)
-
-                __class__.processing_metric.labels(status="error").inc()
-
-    @staticmethod
-    def add_args(parser, default_input_queue, default_subscriber):
-
-        BaseProcessor.add_args(parser)
-
-        parser.add_argument(
-            '-i', '--input-queue',
-            default=default_input_queue,
-            help=f'Input queue (default: {default_input_queue})'
-        )
-
-        parser.add_argument(
-            '-s', '--subscriber',
-            default=default_subscriber,
-            help=f'Queue subscriber name (default: {default_subscriber})'
-        )
-
-class ConsumerProducer(BaseProcessor):
-
-    def __init__(self, **params):
-
-        input_queue = params.get("input_queue")
-        output_queue = params.get("output_queue")
-        subscriber = params.get("subscriber")
-        input_schema = params.get("input_schema")
-        output_schema = params.get("output_schema")
-
-        if not hasattr(__class__, "request_metric"):
-            __class__.request_metric = Histogram(
-                'request_latency', 'Request latency (seconds)'
-            )
-
-        if not hasattr(__class__, "output_metric"):
-            __class__.output_metric = Counter(
-                'output_count', 'Output items created'
-            )
-
-        if not hasattr(__class__, "pubsub_metric"):
-            __class__.pubsub_metric = Info(
-                'pubsub', 'Pub/sub configuration'
-            )
-
-        if not hasattr(__class__, "processing_metric"):
-            __class__.processing_metric = Counter(
-                'processing_count', 'Processing count', ["status"]
-            )
-
-        __class__.pubsub_metric.info({
-            "input_queue": input_queue,
-            "output_queue": output_queue,
-            "subscriber": subscriber,
-            "input_schema": input_schema.__name__,
-            "output_schema": output_schema.__name__,
-        })
-
-        super(ConsumerProducer, self).__init__(**params)
-
-        if input_schema == None:
-            raise RuntimeError("input_schema must be specified")
-
-        if output_schema == None:
-            raise RuntimeError("output_schema must be specified")
-
-        self.consumer = self.client.subscribe(
-            input_queue, subscriber,
-            schema=JsonSchema(input_schema),
-        )
-
-        self.producer = self.client.create_producer(
-            topic=output_queue,
-            schema=JsonSchema(output_schema),
-        )
-
-    def run(self):
-
-        while True:
-
-            msg = self.consumer.receive()
-
-            try:
-
-                with __class__.request_metric.time():
-                    resp = self.handle(msg)
-
-                # Acknowledge successful processing of the message
-                self.consumer.acknowledge(msg)
-
-                __class__.processing_metric.labels(status="success").inc()
-
-            except Exception as e:
-
-                print("Exception:", e, flush=True)
-
-                # Message failed to be processed
-                self.consumer.negative_acknowledge(msg)
-
-                __class__.processing_metric.labels(status="error").inc()
-
-    def send(self, msg, properties={}):
-        self.producer.send(msg, properties)
-        __class__.output_metric.inc()
-
-    @staticmethod
-    def add_args(
-            parser, default_input_queue, default_subscriber,
-            default_output_queue,
-    ):
-
-        BaseProcessor.add_args(parser)
-
-        parser.add_argument(
-            '-i', '--input-queue',
-            default=default_input_queue,
-            help=f'Input queue (default: {default_input_queue})'
-        )
-
-        parser.add_argument(
-            '-s', '--subscriber',
-            default=default_subscriber,
-            help=f'Queue subscriber name (default: {default_subscriber})'
-        )
-
-        parser.add_argument(
-            '-o', '--output-queue',
-            default=default_output_queue,
-            help=f'Output queue (default: {default_output_queue})'
-        )
-
-class Producer(BaseProcessor):
-
-    def __init__(self, **params):
-
-        output_queue = params.get("output_queue")
-        output_schema = params.get("output_schema")
-
-        if not hasattr(__class__, "output_metric"):
-            __class__.output_metric = Counter(
-                'output_count', 'Output items created'
-            )
-
-        if not hasattr(__class__, "pubsub_metric"):
-            __class__.pubsub_metric = Info(
-                'pubsub', 'Pub/sub configuration'
-            )
-
-        __class__.pubsub_metric.info({
-            "output_queue": output_queue,
-            "output_schema": output_schema.__name__,
-        })
-
-        super(Producer, self).__init__(**params)
-
-        if output_schema == None:
-            raise RuntimeError("output_schema must be specified")
-
-        self.producer = self.client.create_producer(
-            topic=output_queue,
-            schema=JsonSchema(output_schema),
-        )
-
-    def send(self, msg, properties={}):
-        self.producer.send(msg, properties)
-        __class__.output_metric.inc()
-
-    @staticmethod
-    def add_args(
-            parser, default_input_queue, default_subscriber,
-            default_output_queue,
-    ):
-
-        BaseProcessor.add_args(parser)
-
-        parser.add_argument(
-            '-o', '--output-queue',
-            default=default_output_queue,
-            help=f'Output queue (default: {default_output_queue})'
-        )

trustgraph/base/producer.py

@@ -0,0 +1,55 @@
+
+from pulsar.schema import JsonSchema
+from prometheus_client import Info, Counter
+
+from . base_processor import BaseProcessor
+
+class Producer(BaseProcessor):
+
+    def __init__(self, **params):
+
+        output_queue = params.get("output_queue")
+        output_schema = params.get("output_schema")
+
+        if not hasattr(__class__, "output_metric"):
+            __class__.output_metric = Counter(
+                'output_count', 'Output items created'
+            )
+
+        if not hasattr(__class__, "pubsub_metric"):
+            __class__.pubsub_metric = Info(
+                'pubsub', 'Pub/sub configuration'
+            )
+
+        __class__.pubsub_metric.info({
+            "output_queue": output_queue,
+            "output_schema": output_schema.__name__,
+        })
+
+        super(Producer, self).__init__(**params)
+
+        if output_schema == None:
+            raise RuntimeError("output_schema must be specified")
+
+        self.producer = self.client.create_producer(
+            topic=output_queue,
+            schema=JsonSchema(output_schema),
+        )
+
+    def send(self, msg, properties={}):
+        self.producer.send(msg, properties)
+        __class__.output_metric.inc()
+
+    @staticmethod
+    def add_args(
+            parser, default_input_queue, default_subscriber,
+            default_output_queue,
+    ):
+
+        BaseProcessor.add_args(parser)
+
+        parser.add_argument(
+            '-o', '--output-queue',
+            default=default_output_queue,
+            help=f'Output queue (default: {default_output_queue})'
+        )

trustgraph/chunking/recursive/chunker.py

@@ -8,12 +8,15 @@ from langchain_text_splitters import RecursiveCharacterTextSplitter
 
 
 from ... schema import TextDocument, Chunk, Source
+from ... schema import text_ingest_queue, chunk_ingest_queue
 from ... log_level import LogLevel
 from ... base import ConsumerProducer
 
-default_input_queue = 'text-doc-load'
+module = ".".join(__name__.split(".")[1:-1])
-default_output_queue = 'chunk-load'
+
-default_subscriber = 'chunker-recursive'
+default_input_queue = text_ingest_queue
+default_output_queue = chunk_ingest_queue
+default_subscriber = module
 
 class Processor(ConsumerProducer):
 
@@ -92,5 +95,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start('chunker', __doc__)
+    Processor.start(module, __doc__)
 

trustgraph/decoding/pdf/pdf_decoder.py

@@ -9,12 +9,15 @@ import base64
 from langchain_community.document_loaders import PyPDFLoader
 
 from ... schema import Document, TextDocument, Source
+from ... schema import document_ingest_queue, text_ingest_queue
 from ... log_level import LogLevel
 from ... base import ConsumerProducer
 
-default_input_queue = 'document-load'
+module = ".".join(__name__.split(".")[1:-1])
-default_output_queue = 'text-doc-load'
+
-default_subscriber = 'pdf-decoder'
+default_input_queue = document_ingest_queue
+default_output_queue = text_ingest_queue
+default_subscriber = module
 
 class Processor(ConsumerProducer):
 
@@ -80,5 +83,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start("pdf-decoder", __doc__)
+    Processor.start(module, __doc__)
 

trustgraph/embeddings/hf/hf.py

@@ -10,9 +10,11 @@ from ... schema import EmbeddingsRequest, EmbeddingsResponse
 from ... log_level import LogLevel
 from ... base import ConsumerProducer
 
+module = ".".join(__name__.split(".")[1:-1])
+
 default_input_queue = 'embeddings'
 default_output_queue = 'embeddings-response'
-default_subscriber = 'embeddings-hf'
+default_subscriber = module
 default_model="all-MiniLM-L6-v2"
 
 class Processor(ConsumerProducer):
@@ -70,5 +72,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start("embeddings-hf", __doc__)
+    Processor.start(module, __doc__)
 

trustgraph/embeddings/ollama/processor.py

@@ -9,9 +9,11 @@ from ... schema import EmbeddingsRequest, EmbeddingsResponse
 from ... log_level import LogLevel
 from ... base import ConsumerProducer
 
+module = ".".join(__name__.split(".")[1:-1])
+
 default_input_queue = 'embeddings'
 default_output_queue = 'embeddings-response'
-default_subscriber = 'embeddings-ollama'
+default_subscriber = module
 default_model="mxbai-embed-large"
 default_ollama = 'http://localhost:11434'
 
@@ -77,5 +79,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start('embeddings-ollama', __doc__)
+    Processor.start(module, __doc__)
 

trustgraph/embeddings/vectorize/vectorize.py

@@ -4,14 +4,17 @@ Vectorizer, calls the embeddings service to get embeddings for a chunk.
 Input is text chunk, output is chunk and vectors.
 """
 
-from ... schema import Chunk, VectorsChunk
+from ... schema import Chunk, ChunkEmbeddings
+from ... schema import chunk_ingest_queue, chunk_embeddings_ingest_queue
 from ... embeddings_client import EmbeddingsClient
 from ... log_level import LogLevel
 from ... base import ConsumerProducer
 
-default_input_queue = 'chunk-load'
+module = ".".join(__name__.split(".")[1:-1])
-default_output_queue = 'vectors-chunk-load'
+
-default_subscriber = 'embeddings-vectorizer'
+default_input_queue = chunk_ingest_queue
+default_output_queue = chunk_embeddings_ingest_queue
+default_subscriber = module
 
 class Processor(ConsumerProducer):
 
@@ -27,7 +30,7 @@ class Processor(ConsumerProducer):
                 "output_queue": output_queue,
                 "subscriber": subscriber,
                 "input_schema": Chunk,
-                "output_schema": VectorsChunk,
+                "output_schema": ChunkEmbeddings,
             }
         )
 
@@ -70,5 +73,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start("embeddings-vectorize", __doc__)
+    Processor.start(module, __doc__)
 

trustgraph/kg/extract_definitions/extract.py

@@ -1,13 +1,14 @@
 
 """
-Simple decoder, accepts vector+text chunks input, applies entity analysis to
+Simple decoder, accepts embeddings+text chunks input, applies entity analysis to
 get entity definitions which are output as graph edges.
 """
 
 import urllib.parse
 import json
 
-from ... schema import VectorsChunk, Triple, Source, Value
+from ... schema import ChunkEmbeddings, Triple, Source, Value
+from ... schema import chunk_embeddings_ingest_queue, triple_ingest_queue
 from ... log_level import LogLevel
 from ... llm_client import LlmClient
 from ... prompts import to_definitions
@@ -16,9 +17,11 @@ from ... base import ConsumerProducer
 
 DEFINITION_VALUE = Value(value=DEFINITION, is_uri=True)
 
-default_input_queue = 'vectors-chunk-load'
+module = ".".join(__name__.split(".")[1:-1])
-default_output_queue = 'graph-load'
+
-default_subscriber = 'kg-extract-definitions'
+default_input_queue = chunk_embeddings_ingest_queue
+default_output_queue = triple_ingest_queue
+default_subscriber = module
 
 class Processor(ConsumerProducer):
 
@@ -33,7 +36,7 @@ class Processor(ConsumerProducer):
                 "input_queue": input_queue,
                 "output_queue": output_queue,
                 "subscriber": subscriber,
-                "input_schema": VectorsChunk,
+                "input_schema": ChunkEmbeddings,
                 "output_schema": Triple,
             }
         )
@@ -101,5 +104,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start("kg-extract-definitions", __doc__)
+    Processor.start(module, __doc__)
 

trustgraph/kg/extract_relationships/extract.py

@@ -10,7 +10,8 @@ import json
 import os
 from pulsar.schema import JsonSchema
 
-from ... schema import VectorsChunk, Triple, VectorsAssociation, Source, Value
+from ... schema import ChunkEmbeddings, Triple, GraphEmbeddings, Source, Value
+from ... schema import chunk_embeddings_ingest_queue, triples_store_queue, graph_embeddings_store_queue
 from ... log_level import LogLevel
 from ... llm_client import LlmClient
 from ... prompts import to_relationships
@@ -19,10 +20,12 @@ from ... base import ConsumerProducer
 
 RDF_LABEL_VALUE = Value(value=RDF_LABEL, is_uri=True)
 
-default_input_queue = 'vectors-chunk-load'
+module = ".".join(__name__.split(".")[1:-1])
-default_output_queue = 'graph-load'
+
-default_subscriber = 'kg-extract-relationships'
+default_input_queue = chunk_embeddings_ingest_queue
-default_vector_queue='vectors-load'
+default_output_queue = triples_store_queue
+default_vector_queue = graph_embeddings_store_queue
+default_subscriber = module
 
 class Processor(ConsumerProducer):
 
@@ -171,5 +174,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start("kg-extract-relationships", __doc__)
+    Processor.start(module, __doc__)
 

trustgraph/model/text_completion/azure/llm.py

@@ -7,13 +7,17 @@ serverless endpoint service.  Input is prompt, output is response.
 import requests
 import json
 
-from ... schema import TextCompletionRequest, TextCompletionResponse
+from .... schema import TextCompletionRequest, TextCompletionResponse
-from ... log_level import LogLevel
+from .... schema import text_completion_request_queue
-from ... base import ConsumerProducer
+from .... schema import text_completion_response_queue
+from .... log_level import LogLevel
+from .... base import ConsumerProducer
 
-default_input_queue = 'llm-complete-text'
+module = ".".join(__name__.split(".")[1:-1])
-default_output_queue = 'llm-complete-text-response'
+
-default_subscriber = 'llm-azure-text'
+default_input_queue = text_completion_request_queue
+default_output_queue = text_completion_response_queue
+default_subscriber = module
 
 class Processor(ConsumerProducer):
 
@@ -121,4 +125,4 @@ class Processor(ConsumerProducer):
 
 def run():
     
-    Processor.start("llm-azure-text", __doc__)
+    Processor.start(module, __doc__)

trustgraph/model/text_completion/claude/llm.py

@@ -6,13 +6,17 @@ Input is prompt, output is response.
 
 import anthropic
 
-from ... schema import TextCompletionRequest, TextCompletionResponse
+from .... schema import TextCompletionRequest, TextCompletionResponse
-from ... log_level import LogLevel
+from .... schema import text_completion_request_queue
-from ... base import ConsumerProducer
+from .... schema import text_completion_response_queue
+from .... log_level import LogLevel
+from .... base import ConsumerProducer
 
-default_input_queue = 'llm-complete-text'
+module = ".".join(__name__.split(".")[1:-1])
-default_output_queue = 'llm-complete-text-response'
+
-default_subscriber = 'llm-claude-text'
+default_input_queue = text_completion_request_queue
+default_output_queue = text_completion_response_queue
+default_subscriber = module
 default_model = 'claude-3-5-sonnet-20240620'
 
 class Processor(ConsumerProducer):
@@ -101,6 +105,6 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start("llm-claude-text", __doc__)
+    Processor.start(module, __doc__)
 
     

trustgraph/model/text_completion/ollama/llm.py

@@ -7,13 +7,17 @@ Input is prompt, output is response.
 from langchain_community.llms import Ollama
 from prometheus_client import Histogram, Info, Counter
 
-from ... schema import TextCompletionRequest, TextCompletionResponse
+from .... schema import TextCompletionRequest, TextCompletionResponse
-from ... log_level import LogLevel
+from .... schema import text_completion_request_queue
-from ... base import ConsumerProducer
+from .... schema import text_completion_response_queue
+from .... log_level import LogLevel
+from .... base import ConsumerProducer
 
-default_input_queue = 'llm-complete-text'
+module = ".".join(__name__.split(".")[1:-1])
-default_output_queue = 'llm-complete-text-response'
+
-default_subscriber = 'llm-ollama-text'
+default_input_queue = text_completion_request_queue
+default_output_queue = text_completion_response_queue
+default_subscriber = module
 default_model = 'gemma2'
 default_ollama = 'http://localhost:11434'
 
@@ -93,6 +97,6 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start("llm-ollama-text", __doc__)
+    Processor.start(module, __doc__)
 
     

trustgraph/model/text_completion/vertexai/llm.py

@@ -21,13 +21,17 @@ from vertexai.preview.generative_models import (
     Tool,
 )
 
-from ... schema import TextCompletionRequest, TextCompletionResponse
+from .... schema import TextCompletionRequest, TextCompletionResponse
-from ... log_level import LogLevel
+from .... schema import text_completion_request_queue
-from ... base import ConsumerProducer
+from .... schema import text_completion_response_queue
+from .... log_level import LogLevel
+from .... base import ConsumerProducer
 
-default_input_queue = 'llm-complete-text'
+module = ".".join(__name__.split(".")[1:-1])
-default_output_queue = 'llm-complete-text-response'
+
-default_subscriber = 'llm-vertexai-text'
+default_input_queue = text_completion_request_queue
+default_output_queue = text_completion_response_queue
+default_subscriber = module
 
 class Processor(ConsumerProducer):
 
@@ -169,5 +173,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start("llm-vertexai-text", __doc__)
+    Processor.start(module, __doc__)
 

trustgraph/retrieval/graph_rag/rag.py

@@ -5,13 +5,16 @@ Input is query, output is response.
 """
 
 from ... schema import GraphRagQuery, GraphRagResponse
+from ... schema import graph_rag_request_queue, graph_rag_response_queue
 from ... log_level import LogLevel
 from ... graph_rag import GraphRag
 from ... base import ConsumerProducer
 
-default_input_queue = 'graph-rag-query'
+module = ".".join(__name__.split(".")[1:-1])
-default_output_queue = 'graph-rag-response'
+
-default_subscriber = 'graph-rag'
+default_input_queue = graph_rag_request_queue
+default_output_queue = graph_rag_response_queue
+default_subscriber = module
 default_graph_hosts = 'localhost'
 default_vector_store = 'http://localhost:19530'
 
@@ -112,5 +115,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start('graph-rag', __doc__)
+    Processor.start(module, __doc__)
 

trustgraph/schema.py

@@ -3,11 +3,7 @@ from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double
 
 from enum import Enum
 
-#class Command(Enum):
+############################################################################
-#    reindex = 1
-
-#class IndexCommand(Record):
-#    command = Command
 
 class Value(Record):
     value = String()
@@ -23,20 +19,27 @@ class Document(Record):
     source = Source()
     data = Bytes()
 
+document_ingest_queue = 'document-load'
+text_ingest_queue = 'text-document-load'
+
 class TextDocument(Record):
     source = Source()
     text = Bytes()
 
+chunk_ingest_queue = 'chunk-load'
+
 class Chunk(Record):
     source = Source()
     chunk = Bytes()
 
-class VectorsChunk(Record):
+class ChunkEmbeddings(Record):
     source = Source()
     vectors = Array(Array(Double()))
     chunk = Bytes()
 
-class VectorsAssociation(Record):
+chunk_embeddings_ingest_queue = 'chunk-embeddings-load'
+
+class GraphEmbeddings(Record):
     source = Source()
     vectors = Array(Array(Double()))
     entity = Value()
@@ -47,6 +50,14 @@ class Triple(Record):
     p = Value()
     o = Value()
 
+triples_store_queue = 'triples-store'
+
+# chunk_embeddings_store_queue = 'chunk-embeddings-store'
+graph_embeddings_store_queue = 'graph-embeddings-store'
+
+text_completion_request_queue = 'text-completion'
+text_completion_response_queue = 'text-completion-response'
+
 class TextCompletionRequest(Record):
     prompt = String()
 
@@ -65,3 +76,6 @@ class GraphRagQuery(Record):
 class GraphRagResponse(Record):
     response = String()
 
+graph_rag_request_queue = 'graph-rag'
+graph_rag_response_queue = 'graph-rag-response'
+

trustgraph/storage/graph_embeddings/milvus/write.py

@@ -3,13 +3,16 @@
 Accepts entity/vector pairs and writes them to a Milvus store.
 """
 
-from ... schema import VectorsAssociation
+from .... schema import VectorsAssociation
-from ... log_level import LogLevel
+from .... schema import graph_embeddings_store_queue
-from ... triple_vectors import TripleVectors
+from .... log_level import LogLevel
-from ... base import Consumer
+from .... triple_vectors import TripleVectors
+from .... base import Consumer
 
-default_input_queue = 'vectors-load'
+module = ".".join(__name__.split(".")[1:-1])
-default_subscriber = 'vector-write-milvus'
+
+default_input_queue = graph_embeddings_store_queue
+default_subscriber = module
 default_store_uri = 'http://localhost:19530'
 
 class Processor(Consumer):
@@ -54,6 +57,5 @@ class Processor(Consumer):
 
 def run():
 
-    Processor.start("vector-write-milvus", __doc__)
+    Processor.start(module, __doc__)
 
-    

trustgraph/storage/triples/cassandra/write.py

@@ -9,13 +9,16 @@ import os
 import argparse
 import time
 
-from ... trustgraph import TrustGraph
+from .... trustgraph import TrustGraph
-from ... schema import Triple
+from .... schema import Triple
-from ... log_level import LogLevel
+from .... schema import triples_store_queue
-from ... base import Consumer
+from .... log_level import LogLevel
+from .... base import Consumer
 
-default_input_queue = 'graph-load'
+module = ".".join(__name__.split(".")[1:-1])
-default_subscriber = 'graph-write-cassandra'
+
+default_input_queue = triples_store_queue
+default_subscriber = module
 default_graph_host='localhost'
 
 class Processor(Consumer):
@@ -61,5 +64,5 @@ class Processor(Consumer):
 
 def run():
 
-    Processor.start("graph-write-cassandra", __doc__)
+    Processor.start(module, __doc__)