apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-06 20:02:11 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Refactor subscriber/module names, queue definitions and schema

Browse source
This commit is contained in:
Cyber MacGeddon 2024-07-23 13:04:07 +01:00
parent cbddf197ad
commit d8e32aee82
62 changed files with 612 additions and 485 deletions
Download patch file Download diff file Expand all files Collapse all files

15
trustgraph/embeddings/vectorize/vectorize.py
View file

@ -4,14 +4,17 @@ Vectorizer, calls the embeddings service to get embeddings for a chunk.
Input is text chunk, output is chunk and vectors.
"""
from ... schema import Chunk, VectorsChunk
from ... schema import Chunk, ChunkEmbeddings
from ... schema import chunk_ingest_queue, chunk_embeddings_ingest_queue
from ... embeddings_client import EmbeddingsClient
from ... log_level import LogLevel
from ... base import ConsumerProducer
default_input_queue = 'chunk-load'
default_output_queue = 'vectors-chunk-load'
default_subscriber = 'embeddings-vectorizer'
module = ".".join(__name__.split(".")[1:-1])
default_input_queue = chunk_ingest_queue
default_output_queue = chunk_embeddings_ingest_queue
default_subscriber = module
class Processor(ConsumerProducer):
@ -27,7 +30,7 @@ class Processor(ConsumerProducer):
"output_queue": output_queue,
"subscriber": subscriber,
"input_schema": Chunk,
"output_schema": VectorsChunk,
"output_schema": ChunkEmbeddings,
}
)
@ -70,5 +73,5 @@ class Processor(ConsumerProducer):
def run():
Processor.start("embeddings-vectorize", __doc__)
Processor.start(module, __doc__)