Feature/refactor entity embeddings (#235)

* Make schema changes
* Core entity context flow in place
* extract-def outputs entity contexts
* Refactored qdrant write
* Refactoring of all vector stores in place

trustgraph-base/trustgraph/schema/documents.py

@@ -35,17 +35,6 @@ chunk_ingest_queue = topic('chunk-load')
 
 ############################################################################
 
-# Chunk embeddings are an embeddings associated with a text chunk
-
-class ChunkEmbeddings(Record):
-    metadata = Metadata()
-    vectors = Array(Array(Double()))
-    chunk = Bytes()
-
-chunk_embeddings_ingest_queue = topic('chunk-embeddings-load')
-
-############################################################################
-
 # Doc embeddings query
 
 class DocumentEmbeddingsRequest(Record):
@@ -62,3 +51,4 @@ document_embeddings_request_queue = topic(
 document_embeddings_response_queue = topic(
     'doc-embeddings', kind='non-persistent', namespace='response',
 )
+

trustgraph-base/trustgraph/schema/graph.py

@@ -7,12 +7,31 @@ from . metadata import Metadata
 
 ############################################################################
 
+# Entity context are an entity associated with textual context
+
+class EntityContext(Record):
+    entity = Value()
+    context = String()
+
+# This is a 'batching' mechanism for the above data
+class EntityContexts(Record):
+    metadata = Metadata()
+    entities = Array(EntityContext())
+
+entity_contexts_ingest_queue = topic('entity-contexts-load')
+
+############################################################################
+
 # Graph embeddings are embeddings associated with a graph entity
 
+class EntityEmbeddings(Record):
+    entity = Value()
+    vectors = Array(Array(Double()))
+
+# This is a 'batching' mechanism for the above data
 class GraphEmbeddings(Record):
     metadata = Metadata()
-    vectors = Array(Array(Double()))
-    entity = Value()
+    entities = Array(EntityEmbeddings())
 
 graph_embeddings_store_queue = topic('graph-embeddings-store')