apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-06 20:02:11 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Fix more naming problems

Browse source
This commit is contained in:
Cyber MacGeddon 2024-07-23 21:02:21 +01:00
parent 02fb1e3e77
commit 9a62413d49
11 changed files with 71 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

20
docker-compose-azure.yaml
View file

@ -119,7 +119,7 @@ services:
restart: on-failure:100 restart: on-failure:100
pdf-decoder: pdf-decoder:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
@ -127,7 +127,7 @@ services:
restart: on-failure:100 restart: on-failure:100
chunker: chunker:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
@ -135,7 +135,7 @@ services:
restart: on-failure:100 restart: on-failure:100
vectorize: vectorize:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
@ -143,7 +143,7 @@ services:
restart: on-failure:100 restart: on-failure:100
embeddings: embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
@ -153,7 +153,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-definitions: kg-extract-definitions:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
@ -161,7 +161,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-relationships: kg-extract-relationships:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
@ -169,7 +169,7 @@ services:
restart: on-failure:100 restart: on-failure:100
vector-write: vector-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "vector-write-milvus" - "vector-write-milvus"
- "-p" - "-p"
@ -179,7 +179,7 @@ services:
restart: on-failure:100 restart: on-failure:100
graph-write: graph-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "graph-write-cassandra" - "graph-write-cassandra"
- "-p" - "-p"
@ -189,7 +189,7 @@ services:
restart: on-failure:100 restart: on-failure:100
llm: llm:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "llm-azure-text" - "llm-azure-text"
- "-p" - "-p"
@ -201,7 +201,7 @@ services:
restart: on-failure:100 restart: on-failure:100
graph-rag: graph-rag:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "graph-rag" - "graph-rag"
- "-p" - "-p"

20
docker-compose-claude.yaml
View file

@ -119,7 +119,7 @@ services:
restart: on-failure:100 restart: on-failure:100
pdf-decoder: pdf-decoder:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
@ -127,7 +127,7 @@ services:
restart: on-failure:100 restart: on-failure:100
chunker: chunker:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
@ -135,7 +135,7 @@ services:
restart: on-failure:100 restart: on-failure:100
vectorize: vectorize:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
@ -143,7 +143,7 @@ services:
restart: on-failure:100 restart: on-failure:100
embeddings: embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
@ -153,7 +153,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-definitions: kg-extract-definitions:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
@ -161,7 +161,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-relationships: kg-extract-relationships:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
@ -169,7 +169,7 @@ services:
restart: on-failure:100 restart: on-failure:100
vector-write: vector-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "vector-write-milvus" - "vector-write-milvus"
- "-p" - "-p"
@ -179,7 +179,7 @@ services:
restart: on-failure:100 restart: on-failure:100
graph-write: graph-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "graph-write-cassandra" - "graph-write-cassandra"
- "-p" - "-p"
@ -189,7 +189,7 @@ services:
restart: on-failure:100 restart: on-failure:100
llm: llm:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "llm-claude-text" - "llm-claude-text"
- "-p" - "-p"
@ -199,7 +199,7 @@ services:
restart: on-failure:100 restart: on-failure:100
graph-rag: graph-rag:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "graph-rag" - "graph-rag"
- "-p" - "-p"

26
docker-compose-ollama.yaml
View file

@ -119,7 +119,7 @@ services:
restart: on-failure:100 restart: on-failure:100
pdf-decoder: pdf-decoder:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
@ -127,7 +127,7 @@ services:
restart: on-failure:100 restart: on-failure:100
chunker: chunker:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
@ -135,7 +135,7 @@ services:
restart: on-failure:100 restart: on-failure:100
vectorize: vectorize:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
@ -143,7 +143,7 @@ services:
restart: on-failure:100 restart: on-failure:100
embeddings: embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
@ -153,7 +153,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-definitions: kg-extract-definitions:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
@ -161,7 +161,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-relationships: kg-extract-relationships:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
@ -169,9 +169,9 @@ services:
restart: on-failure:100 restart: on-failure:100
vector-write: vector-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "vector-write-milvus" - "ge-write-milvus"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-t" - "-t"
@ -179,9 +179,9 @@ services:
restart: on-failure:100 restart: on-failure:100
graph-write: graph-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "graph-write-cassandra" - "triples-write-cassandra"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-g" - "-g"
@ -189,9 +189,9 @@ services:
restart: on-failure:100 restart: on-failure:100
llm: llm:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "llm-ollama-text" - "text-completion-ollama"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-r" - "-r"
@ -199,7 +199,7 @@ services:
restart: on-failure:100 restart: on-failure:100
graph-rag: graph-rag:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "graph-rag" - "graph-rag"
- "-p" - "-p"

20
docker-compose-vertexai.yaml
View file

@ -119,7 +119,7 @@ services:
restart: on-failure:100 restart: on-failure:100
pdf-decoder: pdf-decoder:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
@ -127,7 +127,7 @@ services:
restart: on-failure:100 restart: on-failure:100
chunker: chunker:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
@ -135,7 +135,7 @@ services:
restart: on-failure:100 restart: on-failure:100
vectorize: vectorize:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
@ -143,7 +143,7 @@ services:
restart: on-failure:100 restart: on-failure:100
embeddings: embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
@ -153,7 +153,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-definitions: kg-extract-definitions:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
@ -161,7 +161,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-relationships: kg-extract-relationships:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
@ -169,7 +169,7 @@ services:
restart: on-failure:100 restart: on-failure:100
vector-write: vector-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "vector-write-milvus" - "vector-write-milvus"
- "-p" - "-p"
@ -179,7 +179,7 @@ services:
restart: on-failure:100 restart: on-failure:100
graph-write: graph-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "graph-write-cassandra" - "graph-write-cassandra"
- "-p" - "-p"
@ -189,7 +189,7 @@ services:
restart: on-failure:100 restart: on-failure:100
llm: llm:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "llm-vertexai-text" - "llm-vertexai-text"
- "-p" - "-p"
@ -203,7 +203,7 @@ services:
restart: on-failure:100 restart: on-failure:100
graph-rag: graph-rag:
image: docker.io/trustgraph/trustgraph-flow:0.4.2 image: docker.io/trustgraph/trustgraph-flow:0.5.0
command: command:
- "graph-rag" - "graph-rag"
- "-p" - "-p"

12
setup.py
View file

@ -50,21 +50,21 @@ setuptools.setup(
"scripts/embeddings-hf", "scripts/embeddings-hf",
"scripts/embeddings-ollama", "scripts/embeddings-ollama",
"scripts/embeddings-vectorize", "scripts/embeddings-vectorize",
"scripts/ge-write-milvus",
"scripts/graph-rag", "scripts/graph-rag",
"scripts/graph-show", "scripts/graph-show",
"scripts/graph-to-turtle", "scripts/graph-to-turtle",
"scripts/graph-write-cassandra",
"scripts/init-pulsar-manager", "scripts/init-pulsar-manager",
"scripts/kg-extract-definitions", "scripts/kg-extract-definitions",
"scripts/kg-extract-relationships", "scripts/kg-extract-relationships",
"scripts/llm-azure-text",
"scripts/llm-claude-text",
"scripts/llm-ollama-text",
"scripts/llm-vertexai-text",
"scripts/loader", "scripts/loader",
"scripts/pdf-decoder", "scripts/pdf-decoder",
"scripts/query", "scripts/query",
"scripts/run-processing", "scripts/run-processing",
"scripts/vector-write-milvus", "scripts/text-completion-azure",
"scripts/text-completion-claude",
"scripts/text-completion-ollama",
"scripts/text-completion-vertexai",
"scripts/triples-write-cassandra",
] ]
) )

2
trustgraph/embeddings/vectorize/vectorize.py
View file

@ -38,7 +38,7 @@ class Processor(ConsumerProducer):
def emit(self, source, chunk, vectors): def emit(self, source, chunk, vectors):
r = VectorsChunk(source=source, chunk=chunk, vectors=vectors) r = ChunkEmbeddings(source=source, chunk=chunk, vectors=vectors)
self.producer.send(r) self.producer.send(r)
def handle(self, msg): def handle(self, msg):

7
trustgraph/embeddings_client.py
View file

@ -3,7 +3,8 @@
import pulsar import pulsar
import _pulsar import _pulsar
from pulsar.schema import JsonSchema from pulsar.schema import JsonSchema
from trustgraph.schema import EmbeddingsRequest, EmbeddingsResponse from . schema import EmbeddingsRequest, EmbeddingsResponse
from . schema import embeddings_request_queue, embeddings_response_queue
import hashlib import hashlib
import uuid import uuid
@ -31,13 +32,13 @@ class EmbeddingsClient:
) )
self.producer = self.client.create_producer( self.producer = self.client.create_producer(
topic='embeddings', topic=embeddings_request_queue,
schema=JsonSchema(EmbeddingsRequest), schema=JsonSchema(EmbeddingsRequest),
chunking_enabled=True, chunking_enabled=True,
) )
self.consumer = self.client.subscribe( self.consumer = self.client.subscribe(
'embeddings-response', client_id, embeddings_response_queue, client_id,
schema=JsonSchema(EmbeddingsResponse), schema=JsonSchema(EmbeddingsResponse),
) )

8
trustgraph/graph_rag_client.py
View file

@ -3,7 +3,9 @@
import pulsar import pulsar
import _pulsar import _pulsar
from pulsar.schema import JsonSchema from pulsar.schema import JsonSchema
from trustgraph.schema import GraphRagQuery, GraphRagResponse from . schema import GraphRagQuery, GraphRagResponse
from . schema import graph_rag_request_queue, graph_rag_response_queue
import hashlib import hashlib
import uuid import uuid
@ -29,13 +31,13 @@ class GraphRagClient:
) )
self.producer = self.client.create_producer( self.producer = self.client.create_producer(
topic='graph-rag-query', topic=graph_rag_request_queue,
schema=JsonSchema(GraphRagQuery), schema=JsonSchema(GraphRagQuery),
chunking_enabled=True, chunking_enabled=True,
) )
self.consumer = self.client.subscribe( self.consumer = self.client.subscribe(
'graph-rag-response', client_id, graph_rag_response_queue, client_id,
schema=JsonSchema(GraphRagResponse), schema=JsonSchema(GraphRagResponse),
) )

4
trustgraph/kg/extract_definitions/extract.py
View file

@ -8,7 +8,7 @@ import urllib.parse
import json import json
from ... schema import ChunkEmbeddings, Triple, Source, Value from ... schema import ChunkEmbeddings, Triple, Source, Value
from ... schema import chunk_embeddings_ingest_queue, triple_ingest_queue from ... schema import chunk_embeddings_ingest_queue, triples_store_queue
from ... log_level import LogLevel from ... log_level import LogLevel
from ... llm_client import LlmClient from ... llm_client import LlmClient
from ... prompts import to_definitions from ... prompts import to_definitions
@ -20,7 +20,7 @@ DEFINITION_VALUE = Value(value=DEFINITION, is_uri=True)
module = ".".join(__name__.split(".")[1:-1]) module = ".".join(__name__.split(".")[1:-1])
default_input_queue = chunk_embeddings_ingest_queue default_input_queue = chunk_embeddings_ingest_queue
default_output_queue = triple_ingest_queue default_output_queue = triples_store_queue
default_subscriber = module default_subscriber = module
class Processor(ConsumerProducer): class Processor(ConsumerProducer):

8
trustgraph/kg/extract_relationships/extract.py
View file

@ -41,14 +41,14 @@ class Processor(ConsumerProducer):
"input_queue": input_queue, "input_queue": input_queue,
"output_queue": output_queue, "output_queue": output_queue,
"subscriber": subscriber, "subscriber": subscriber,
"input_schema": VectorsChunk, "input_schema": ChunkEmbeddings,
"output_schema": Triple, "output_schema": Triple,
} }
) )
self.vec_prod = self.client.create_producer( self.vec_prod = self.client.create_producer(
topic=vector_queue, topic=vector_queue,
schema=JsonSchema(VectorsAssociation), schema=JsonSchema(GraphEmbeddings),
) )
__class__.pubsub_metric.info({ __class__.pubsub_metric.info({
@ -56,9 +56,9 @@ class Processor(ConsumerProducer):
"output_queue": output_queue, "output_queue": output_queue,
"vector_queue": vector_queue, "vector_queue": vector_queue,
"subscriber": subscriber, "subscriber": subscriber,
"input_schema": VectorsChunk.__name__, "input_schema": ChunkEmbeddings.__name__,
"output_schema": Triple.__name__, "output_schema": Triple.__name__,
"vector_schema": VectorsAssociation.__name__, "vector_schema": GraphEmbeddings.__name__,
}) })
self.llm = LlmClient(pulsar_host=self.pulsar_host) self.llm = LlmClient(pulsar_host=self.pulsar_host)

9
trustgraph/llm_client.py
View file

@ -3,10 +3,13 @@
import pulsar import pulsar
import _pulsar import _pulsar
from pulsar.schema import JsonSchema from pulsar.schema import JsonSchema
from trustgraph.schema import TextCompletionRequest, TextCompletionResponse
import hashlib import hashlib
import uuid import uuid
from . schema import TextCompletionRequest, TextCompletionResponse
from . schema import text_completion_request_queue
from . schema import text_completion_response_queue
# Ugly # Ugly
ERROR=_pulsar.LoggerLevel.Error ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn WARN=_pulsar.LoggerLevel.Warn
@ -29,13 +32,13 @@ class LlmClient:
) )
self.producer = self.client.create_producer( self.producer = self.client.create_producer(
topic='llm-complete-text', topic=text_completion_request_queue,
schema=JsonSchema(TextCompletionRequest), schema=JsonSchema(TextCompletionRequest),
chunking_enabled=True, chunking_enabled=True,
) )
self.consumer = self.client.subscribe( self.consumer = self.client.subscribe(
'llm-complete-text-response', client_id, text_completion_response_queue, client_id,
schema=JsonSchema(TextCompletionResponse), schema=JsonSchema(TextCompletionResponse),
) )