Get fastembed processor working

trustgraph-flow/scripts/embeddings-fastembed

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-from trustgraph.embeddings.ollama import run
+from trustgraph.embeddings.fastembed import run
 
 run()
 

trustgraph-flow/setup.py

@@ -79,6 +79,7 @@ setuptools.setup(
         "scripts/document-embeddings",
         "scripts/document-rag",
         "scripts/embeddings-ollama",
+        "scripts/embeddings-fastembed",
         "scripts/ge-query-milvus",
         "scripts/ge-query-pinecone",
         "scripts/ge-query-qdrant",

trustgraph-flow/trustgraph/embeddings/fastembed/processor.py

@@ -52,11 +52,16 @@ class Processor(ConsumerProducer):
         print(f"Handling input {id}...", flush=True)
 
         text = v.text
-        embeds = self.embeddings.embed([text])
+        vecs = self.embeddings.embed([text])
+
+        vecs = [
+            v.tolist()
+            for v in vecs
+        ]
 
         print("Send response...", flush=True)
         r = EmbeddingsResponse(
-            vectors=embeddings,
+            vectors=list(vecs),
             error=None,
         )
 
@@ -78,12 +83,6 @@ class Processor(ConsumerProducer):
             help=f'Embeddings model (default: {default_model})'
         )
 
-        parser.add_argument(
-            '-r', '--ollama',
-            default=default_ollama,
-            help=f'ollama (default: {default_ollama})'
-        )
-
 def run():
 
     Processor.start(module, __doc__)

trustgraph-flow/trustgraph/embeddings/ollama/processor.py

@@ -1,6 +1,6 @@
 
 """
-Embeddings service, applies an embeddings model selected from HuggingFace.
+Embeddings service, applies an embeddings model hosted on a local Ollama.
 Input is text, output is embeddings vector.
 """