diff --git a/docker-compose-azure.yaml b/docker-compose-azure.yaml index f48df808..bb54e965 100644 --- a/docker-compose-azure.yaml +++ b/docker-compose-azure.yaml @@ -119,7 +119,7 @@ services: restart: on-failure:100 pdf-decoder: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "pdf-decoder" - "-p" @@ -127,7 +127,7 @@ services: restart: on-failure:100 chunker: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "chunker-recursive" - "-p" @@ -135,7 +135,7 @@ services: restart: on-failure:100 vectorize: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "embeddings-vectorize" - "-p" @@ -143,7 +143,7 @@ services: restart: on-failure:100 embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "embeddings-hf" - "-p" @@ -153,7 +153,7 @@ services: restart: on-failure:100 kg-extract-definitions: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "kg-extract-definitions" - "-p" @@ -161,7 +161,7 @@ services: restart: on-failure:100 kg-extract-relationships: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "kg-extract-relationships" - "-p" @@ -169,7 +169,7 @@ services: restart: on-failure:100 vector-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "vector-write-milvus" - "-p" @@ -179,7 +179,7 @@ services: restart: on-failure:100 graph-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "graph-write-cassandra" - "-p" @@ -189,7 +189,7 @@ services: restart: on-failure:100 llm: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "llm-azure-text" - "-p" @@ -201,7 +201,7 @@ services: restart: on-failure:100 graph-rag: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "graph-rag" - "-p" diff --git a/docker-compose-claude.yaml b/docker-compose-claude.yaml index fc63ef14..ef3c124b 100644 --- a/docker-compose-claude.yaml +++ b/docker-compose-claude.yaml @@ -119,7 +119,7 @@ services: restart: on-failure:100 pdf-decoder: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "pdf-decoder" - "-p" @@ -127,7 +127,7 @@ services: restart: on-failure:100 chunker: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "chunker-recursive" - "-p" @@ -135,7 +135,7 @@ services: restart: on-failure:100 vectorize: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "embeddings-vectorize" - "-p" @@ -143,7 +143,7 @@ services: restart: on-failure:100 embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "embeddings-hf" - "-p" @@ -153,7 +153,7 @@ services: restart: on-failure:100 kg-extract-definitions: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "kg-extract-definitions" - "-p" @@ -161,7 +161,7 @@ services: restart: on-failure:100 kg-extract-relationships: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "kg-extract-relationships" - "-p" @@ -169,7 +169,7 @@ services: restart: on-failure:100 vector-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "vector-write-milvus" - "-p" @@ -179,7 +179,7 @@ services: restart: on-failure:100 graph-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "graph-write-cassandra" - "-p" @@ -189,7 +189,7 @@ services: restart: on-failure:100 llm: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "llm-claude-text" - "-p" @@ -199,7 +199,7 @@ services: restart: on-failure:100 graph-rag: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "graph-rag" - "-p" diff --git a/docker-compose-ollama.yaml b/docker-compose-ollama.yaml index 2e66727b..43be203a 100644 --- a/docker-compose-ollama.yaml +++ b/docker-compose-ollama.yaml @@ -119,7 +119,7 @@ services: restart: on-failure:100 pdf-decoder: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "pdf-decoder" - "-p" @@ -127,7 +127,7 @@ services: restart: on-failure:100 chunker: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "chunker-recursive" - "-p" @@ -135,7 +135,7 @@ services: restart: on-failure:100 vectorize: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "embeddings-vectorize" - "-p" @@ -143,7 +143,7 @@ services: restart: on-failure:100 embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "embeddings-hf" - "-p" @@ -153,7 +153,7 @@ services: restart: on-failure:100 kg-extract-definitions: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "kg-extract-definitions" - "-p" @@ -161,7 +161,7 @@ services: restart: on-failure:100 kg-extract-relationships: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "kg-extract-relationships" - "-p" @@ -169,9 +169,9 @@ services: restart: on-failure:100 vector-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - - "vector-write-milvus" + - "ge-write-milvus" - "-p" - "pulsar://pulsar:6650" - "-t" @@ -179,9 +179,9 @@ services: restart: on-failure:100 graph-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - - "graph-write-cassandra" + - "triples-write-cassandra" - "-p" - "pulsar://pulsar:6650" - "-g" @@ -189,9 +189,9 @@ services: restart: on-failure:100 llm: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - - "llm-ollama-text" + - "text-completion-ollama" - "-p" - "pulsar://pulsar:6650" - "-r" @@ -199,7 +199,7 @@ services: restart: on-failure:100 graph-rag: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "graph-rag" - "-p" diff --git a/docker-compose-vertexai.yaml b/docker-compose-vertexai.yaml index 2a69ee69..9e46909d 100644 --- a/docker-compose-vertexai.yaml +++ b/docker-compose-vertexai.yaml @@ -119,7 +119,7 @@ services: restart: on-failure:100 pdf-decoder: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "pdf-decoder" - "-p" @@ -127,7 +127,7 @@ services: restart: on-failure:100 chunker: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "chunker-recursive" - "-p" @@ -135,7 +135,7 @@ services: restart: on-failure:100 vectorize: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "embeddings-vectorize" - "-p" @@ -143,7 +143,7 @@ services: restart: on-failure:100 embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "embeddings-hf" - "-p" @@ -153,7 +153,7 @@ services: restart: on-failure:100 kg-extract-definitions: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "kg-extract-definitions" - "-p" @@ -161,7 +161,7 @@ services: restart: on-failure:100 kg-extract-relationships: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "kg-extract-relationships" - "-p" @@ -169,7 +169,7 @@ services: restart: on-failure:100 vector-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "vector-write-milvus" - "-p" @@ -179,7 +179,7 @@ services: restart: on-failure:100 graph-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "graph-write-cassandra" - "-p" @@ -189,7 +189,7 @@ services: restart: on-failure:100 llm: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "llm-vertexai-text" - "-p" @@ -203,7 +203,7 @@ services: restart: on-failure:100 graph-rag: - image: docker.io/trustgraph/trustgraph-flow:0.4.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.0 command: - "graph-rag" - "-p" diff --git a/setup.py b/setup.py index 586c4b2b..61cce124 100644 --- a/setup.py +++ b/setup.py @@ -50,21 +50,21 @@ setuptools.setup( "scripts/embeddings-hf", "scripts/embeddings-ollama", "scripts/embeddings-vectorize", + "scripts/ge-write-milvus", "scripts/graph-rag", "scripts/graph-show", "scripts/graph-to-turtle", - "scripts/graph-write-cassandra", "scripts/init-pulsar-manager", "scripts/kg-extract-definitions", "scripts/kg-extract-relationships", - "scripts/llm-azure-text", - "scripts/llm-claude-text", - "scripts/llm-ollama-text", - "scripts/llm-vertexai-text", "scripts/loader", "scripts/pdf-decoder", "scripts/query", "scripts/run-processing", - "scripts/vector-write-milvus", + "scripts/text-completion-azure", + "scripts/text-completion-claude", + "scripts/text-completion-ollama", + "scripts/text-completion-vertexai", + "scripts/triples-write-cassandra", ] ) diff --git a/trustgraph/embeddings/vectorize/vectorize.py b/trustgraph/embeddings/vectorize/vectorize.py index aebc4159..eaf9b05e 100755 --- a/trustgraph/embeddings/vectorize/vectorize.py +++ b/trustgraph/embeddings/vectorize/vectorize.py @@ -38,7 +38,7 @@ class Processor(ConsumerProducer): def emit(self, source, chunk, vectors): - r = VectorsChunk(source=source, chunk=chunk, vectors=vectors) + r = ChunkEmbeddings(source=source, chunk=chunk, vectors=vectors) self.producer.send(r) def handle(self, msg): diff --git a/trustgraph/embeddings_client.py b/trustgraph/embeddings_client.py index bac41074..e09216d0 100644 --- a/trustgraph/embeddings_client.py +++ b/trustgraph/embeddings_client.py @@ -3,7 +3,8 @@ import pulsar import _pulsar from pulsar.schema import JsonSchema -from trustgraph.schema import EmbeddingsRequest, EmbeddingsResponse +from . schema import EmbeddingsRequest, EmbeddingsResponse +from . schema import embeddings_request_queue, embeddings_response_queue import hashlib import uuid @@ -31,13 +32,13 @@ class EmbeddingsClient: ) self.producer = self.client.create_producer( - topic='embeddings', + topic=embeddings_request_queue, schema=JsonSchema(EmbeddingsRequest), chunking_enabled=True, ) self.consumer = self.client.subscribe( - 'embeddings-response', client_id, + embeddings_response_queue, client_id, schema=JsonSchema(EmbeddingsResponse), ) diff --git a/trustgraph/graph_rag_client.py b/trustgraph/graph_rag_client.py index 6f48e772..c6634623 100644 --- a/trustgraph/graph_rag_client.py +++ b/trustgraph/graph_rag_client.py @@ -3,7 +3,9 @@ import pulsar import _pulsar from pulsar.schema import JsonSchema -from trustgraph.schema import GraphRagQuery, GraphRagResponse +from . schema import GraphRagQuery, GraphRagResponse +from . schema import graph_rag_request_queue, graph_rag_response_queue + import hashlib import uuid @@ -29,13 +31,13 @@ class GraphRagClient: ) self.producer = self.client.create_producer( - topic='graph-rag-query', + topic=graph_rag_request_queue, schema=JsonSchema(GraphRagQuery), chunking_enabled=True, ) self.consumer = self.client.subscribe( - 'graph-rag-response', client_id, + graph_rag_response_queue, client_id, schema=JsonSchema(GraphRagResponse), ) diff --git a/trustgraph/kg/extract_definitions/extract.py b/trustgraph/kg/extract_definitions/extract.py index d0b06502..22b428b9 100755 --- a/trustgraph/kg/extract_definitions/extract.py +++ b/trustgraph/kg/extract_definitions/extract.py @@ -8,7 +8,7 @@ import urllib.parse import json from ... schema import ChunkEmbeddings, Triple, Source, Value -from ... schema import chunk_embeddings_ingest_queue, triple_ingest_queue +from ... schema import chunk_embeddings_ingest_queue, triples_store_queue from ... log_level import LogLevel from ... llm_client import LlmClient from ... prompts import to_definitions @@ -20,7 +20,7 @@ DEFINITION_VALUE = Value(value=DEFINITION, is_uri=True) module = ".".join(__name__.split(".")[1:-1]) default_input_queue = chunk_embeddings_ingest_queue -default_output_queue = triple_ingest_queue +default_output_queue = triples_store_queue default_subscriber = module class Processor(ConsumerProducer): diff --git a/trustgraph/kg/extract_relationships/extract.py b/trustgraph/kg/extract_relationships/extract.py index 2d945718..3ef180d5 100755 --- a/trustgraph/kg/extract_relationships/extract.py +++ b/trustgraph/kg/extract_relationships/extract.py @@ -41,14 +41,14 @@ class Processor(ConsumerProducer): "input_queue": input_queue, "output_queue": output_queue, "subscriber": subscriber, - "input_schema": VectorsChunk, + "input_schema": ChunkEmbeddings, "output_schema": Triple, } ) self.vec_prod = self.client.create_producer( topic=vector_queue, - schema=JsonSchema(VectorsAssociation), + schema=JsonSchema(GraphEmbeddings), ) __class__.pubsub_metric.info({ @@ -56,9 +56,9 @@ class Processor(ConsumerProducer): "output_queue": output_queue, "vector_queue": vector_queue, "subscriber": subscriber, - "input_schema": VectorsChunk.__name__, + "input_schema": ChunkEmbeddings.__name__, "output_schema": Triple.__name__, - "vector_schema": VectorsAssociation.__name__, + "vector_schema": GraphEmbeddings.__name__, }) self.llm = LlmClient(pulsar_host=self.pulsar_host) diff --git a/trustgraph/llm_client.py b/trustgraph/llm_client.py index 4d392f0e..aa034b77 100644 --- a/trustgraph/llm_client.py +++ b/trustgraph/llm_client.py @@ -3,10 +3,13 @@ import pulsar import _pulsar from pulsar.schema import JsonSchema -from trustgraph.schema import TextCompletionRequest, TextCompletionResponse import hashlib import uuid +from . schema import TextCompletionRequest, TextCompletionResponse +from . schema import text_completion_request_queue +from . schema import text_completion_response_queue + # Ugly ERROR=_pulsar.LoggerLevel.Error WARN=_pulsar.LoggerLevel.Warn @@ -29,13 +32,13 @@ class LlmClient: ) self.producer = self.client.create_producer( - topic='llm-complete-text', + topic=text_completion_request_queue, schema=JsonSchema(TextCompletionRequest), chunking_enabled=True, ) self.consumer = self.client.subscribe( - 'llm-complete-text-response', client_id, + text_completion_response_queue, client_id, schema=JsonSchema(TextCompletionResponse), )