apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-05 19:32:11 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix more naming problems

Browse source
This commit is contained in:
Cyber MacGeddon 2024-07-23 21:02:21 +01:00
parent 02fb1e3e77
commit 9a62413d49
11 changed files with 71 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

20
docker-compose-azure.yaml
View file

@ -119,7 +119,7 @@ services:
restart: on-failure:100
pdf-decoder:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "pdf-decoder"
- "-p"
@ -127,7 +127,7 @@ services:
restart: on-failure:100
chunker:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "chunker-recursive"
- "-p"
@ -135,7 +135,7 @@ services:
restart: on-failure:100
vectorize:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "embeddings-vectorize"
- "-p"
@ -143,7 +143,7 @@ services:
restart: on-failure:100
embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "embeddings-hf"
- "-p"
@ -153,7 +153,7 @@ services:
restart: on-failure:100
kg-extract-definitions:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "kg-extract-definitions"
- "-p"
@ -161,7 +161,7 @@ services:
restart: on-failure:100
kg-extract-relationships:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "kg-extract-relationships"
- "-p"
@ -169,7 +169,7 @@ services:
restart: on-failure:100
vector-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "vector-write-milvus"
- "-p"
@ -179,7 +179,7 @@ services:
restart: on-failure:100
graph-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "graph-write-cassandra"
- "-p"
@ -189,7 +189,7 @@ services:
restart: on-failure:100
llm:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "llm-azure-text"
- "-p"
@ -201,7 +201,7 @@ services:
restart: on-failure:100
graph-rag:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "graph-rag"
- "-p"

20
docker-compose-claude.yaml
View file

@ -119,7 +119,7 @@ services:
restart: on-failure:100
pdf-decoder:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "pdf-decoder"
- "-p"
@ -127,7 +127,7 @@ services:
restart: on-failure:100
chunker:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "chunker-recursive"
- "-p"
@ -135,7 +135,7 @@ services:
restart: on-failure:100
vectorize:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "embeddings-vectorize"
- "-p"
@ -143,7 +143,7 @@ services:
restart: on-failure:100
embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "embeddings-hf"
- "-p"
@ -153,7 +153,7 @@ services:
restart: on-failure:100
kg-extract-definitions:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "kg-extract-definitions"
- "-p"
@ -161,7 +161,7 @@ services:
restart: on-failure:100
kg-extract-relationships:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "kg-extract-relationships"
- "-p"
@ -169,7 +169,7 @@ services:
restart: on-failure:100
vector-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "vector-write-milvus"
- "-p"
@ -179,7 +179,7 @@ services:
restart: on-failure:100
graph-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "graph-write-cassandra"
- "-p"
@ -189,7 +189,7 @@ services:
restart: on-failure:100
llm:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "llm-claude-text"
- "-p"
@ -199,7 +199,7 @@ services:
restart: on-failure:100
graph-rag:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "graph-rag"
- "-p"

26
docker-compose-ollama.yaml
View file

@ -119,7 +119,7 @@ services:
restart: on-failure:100
pdf-decoder:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "pdf-decoder"
- "-p"
@ -127,7 +127,7 @@ services:
restart: on-failure:100
chunker:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "chunker-recursive"
- "-p"
@ -135,7 +135,7 @@ services:
restart: on-failure:100
vectorize:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "embeddings-vectorize"
- "-p"
@ -143,7 +143,7 @@ services:
restart: on-failure:100
embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "embeddings-hf"
- "-p"
@ -153,7 +153,7 @@ services:
restart: on-failure:100
kg-extract-definitions:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "kg-extract-definitions"
- "-p"
@ -161,7 +161,7 @@ services:
restart: on-failure:100
kg-extract-relationships:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "kg-extract-relationships"
- "-p"
@ -169,9 +169,9 @@ services:
restart: on-failure:100
vector-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "vector-write-milvus"
- "ge-write-milvus"
- "-p"
- "pulsar://pulsar:6650"
- "-t"
@ -179,9 +179,9 @@ services:
restart: on-failure:100
graph-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "graph-write-cassandra"
- "triples-write-cassandra"
- "-p"
- "pulsar://pulsar:6650"
- "-g"
@ -189,9 +189,9 @@ services:
restart: on-failure:100
llm:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "llm-ollama-text"
- "text-completion-ollama"
- "-p"
- "pulsar://pulsar:6650"
- "-r"
@ -199,7 +199,7 @@ services:
restart: on-failure:100
graph-rag:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "graph-rag"
- "-p"

20
docker-compose-vertexai.yaml
View file

@ -119,7 +119,7 @@ services:
restart: on-failure:100
pdf-decoder:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "pdf-decoder"
- "-p"
@ -127,7 +127,7 @@ services:
restart: on-failure:100
chunker:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "chunker-recursive"
- "-p"
@ -135,7 +135,7 @@ services:
restart: on-failure:100
vectorize:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "embeddings-vectorize"
- "-p"
@ -143,7 +143,7 @@ services:
restart: on-failure:100
embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "embeddings-hf"
- "-p"
@ -153,7 +153,7 @@ services:
restart: on-failure:100
kg-extract-definitions:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "kg-extract-definitions"
- "-p"
@ -161,7 +161,7 @@ services:
restart: on-failure:100
kg-extract-relationships:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "kg-extract-relationships"
- "-p"
@ -169,7 +169,7 @@ services:
restart: on-failure:100
vector-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "vector-write-milvus"
- "-p"
@ -179,7 +179,7 @@ services:
restart: on-failure:100
graph-write:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "graph-write-cassandra"
- "-p"
@ -189,7 +189,7 @@ services:
restart: on-failure:100
llm:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "llm-vertexai-text"
- "-p"
@ -203,7 +203,7 @@ services:
restart: on-failure:100
graph-rag:
image: docker.io/trustgraph/trustgraph-flow:0.4.2
image: docker.io/trustgraph/trustgraph-flow:0.5.0
command:
- "graph-rag"
- "-p"

12
setup.py
View file

@ -50,21 +50,21 @@ setuptools.setup(
"scripts/embeddings-hf",
"scripts/embeddings-ollama",
"scripts/embeddings-vectorize",
"scripts/ge-write-milvus",
"scripts/graph-rag",
"scripts/graph-show",
"scripts/graph-to-turtle",
"scripts/graph-write-cassandra",
"scripts/init-pulsar-manager",
"scripts/kg-extract-definitions",
"scripts/kg-extract-relationships",
"scripts/llm-azure-text",
"scripts/llm-claude-text",
"scripts/llm-ollama-text",
"scripts/llm-vertexai-text",
"scripts/loader",
"scripts/pdf-decoder",
"scripts/query",
"scripts/run-processing",
"scripts/vector-write-milvus",
"scripts/text-completion-azure",
"scripts/text-completion-claude",
"scripts/text-completion-ollama",
"scripts/text-completion-vertexai",
"scripts/triples-write-cassandra",
]
)

2
trustgraph/embeddings/vectorize/vectorize.py
View file

@ -38,7 +38,7 @@ class Processor(ConsumerProducer):
def emit(self, source, chunk, vectors):
r = VectorsChunk(source=source, chunk=chunk, vectors=vectors)
r = ChunkEmbeddings(source=source, chunk=chunk, vectors=vectors)
self.producer.send(r)
def handle(self, msg):

7
trustgraph/embeddings_client.py
View file

@ -3,7 +3,8 @@
import pulsar
import _pulsar
from pulsar.schema import JsonSchema
from trustgraph.schema import EmbeddingsRequest, EmbeddingsResponse
from . schema import EmbeddingsRequest, EmbeddingsResponse
from . schema import embeddings_request_queue, embeddings_response_queue
import hashlib
import uuid
@ -31,13 +32,13 @@ class EmbeddingsClient:
)
self.producer = self.client.create_producer(
topic='embeddings',
topic=embeddings_request_queue,
schema=JsonSchema(EmbeddingsRequest),
chunking_enabled=True,
)
self.consumer = self.client.subscribe(
'embeddings-response', client_id,
embeddings_response_queue, client_id,
schema=JsonSchema(EmbeddingsResponse),
)

8
trustgraph/graph_rag_client.py
View file

@ -3,7 +3,9 @@
import pulsar
import _pulsar
from pulsar.schema import JsonSchema
from trustgraph.schema import GraphRagQuery, GraphRagResponse
from . schema import GraphRagQuery, GraphRagResponse
from . schema import graph_rag_request_queue, graph_rag_response_queue
import hashlib
import uuid
@ -29,13 +31,13 @@ class GraphRagClient:
)
self.producer = self.client.create_producer(
topic='graph-rag-query',
topic=graph_rag_request_queue,
schema=JsonSchema(GraphRagQuery),
chunking_enabled=True,
)
self.consumer = self.client.subscribe(
'graph-rag-response', client_id,
graph_rag_response_queue, client_id,
schema=JsonSchema(GraphRagResponse),
)

4
trustgraph/kg/extract_definitions/extract.py
View file

@ -8,7 +8,7 @@ import urllib.parse
import json
from ... schema import ChunkEmbeddings, Triple, Source, Value
from ... schema import chunk_embeddings_ingest_queue, triple_ingest_queue
from ... schema import chunk_embeddings_ingest_queue, triples_store_queue
from ... log_level import LogLevel
from ... llm_client import LlmClient
from ... prompts import to_definitions
@ -20,7 +20,7 @@ DEFINITION_VALUE = Value(value=DEFINITION, is_uri=True)
module = ".".join(__name__.split(".")[1:-1])
default_input_queue = chunk_embeddings_ingest_queue
default_output_queue = triple_ingest_queue
default_output_queue = triples_store_queue
default_subscriber = module
class Processor(ConsumerProducer):

8
trustgraph/kg/extract_relationships/extract.py
View file

@ -41,14 +41,14 @@ class Processor(ConsumerProducer):
"input_queue": input_queue,
"output_queue": output_queue,
"subscriber": subscriber,
"input_schema": VectorsChunk,
"input_schema": ChunkEmbeddings,
"output_schema": Triple,
}
)
self.vec_prod = self.client.create_producer(
topic=vector_queue,
schema=JsonSchema(VectorsAssociation),
schema=JsonSchema(GraphEmbeddings),
)
__class__.pubsub_metric.info({
@ -56,9 +56,9 @@ class Processor(ConsumerProducer):
"output_queue": output_queue,
"vector_queue": vector_queue,
"subscriber": subscriber,
"input_schema": VectorsChunk.__name__,
"input_schema": ChunkEmbeddings.__name__,
"output_schema": Triple.__name__,
"vector_schema": VectorsAssociation.__name__,
"vector_schema": GraphEmbeddings.__name__,
})
self.llm = LlmClient(pulsar_host=self.pulsar_host)

9
trustgraph/llm_client.py
View file

@ -3,10 +3,13 @@
import pulsar
import _pulsar
from pulsar.schema import JsonSchema
from trustgraph.schema import TextCompletionRequest, TextCompletionResponse
import hashlib
import uuid
from . schema import TextCompletionRequest, TextCompletionResponse
from . schema import text_completion_request_queue
from . schema import text_completion_response_queue
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
@ -29,13 +32,13 @@ class LlmClient:
)
self.producer = self.client.create_producer(
topic='llm-complete-text',
topic=text_completion_request_queue,
schema=JsonSchema(TextCompletionRequest),
chunking_enabled=True,
)
self.consumer = self.client.subscribe(
'llm-complete-text-response', client_id,
text_completion_response_queue, client_id,
schema=JsonSchema(TextCompletionResponse),
)