Document chunks not stored in vector store (#665)

- Schema - ChunkEmbeddings now uses chunk_id: str instead of chunk: bytes
- Schema - DocumentEmbeddingsResponse now returns chunk_ids: list[str]
  instead of chunks
- Translators - Updated to serialize/deserialize chunk_id
- Clients - DocumentEmbeddingsClient.query() returns chunk_ids
- SDK/API - flow.py, socket_client.py, bulk_client.py updated
- Document embeddings service - Stores chunk_id (document ID) instead
  of chunk text
- Storage writers - Qdrant, Milvus, Pinecone store chunk_id in payload
- Query services - Return chunk_id from vector store searches
- Gateway dispatchers - Serialize chunk_id in API responses
- Document RAG - Added librarian client to fetch chunk content from
  Garage using chunk_ids
- CLI tools - Updated all three tools:
  - invoke_document_embeddings.py - displays chunk_ids, removed
    max_chunk_length
  - save_doc_embeds.py - exports chunk_id
  - load_doc_embeds.py - imports chunk_id

trustgraph-flow/trustgraph/query/doc_embeddings/milvus/service.py

@@ -1,7 +1,7 @@
 
 """
 Document embeddings query service.  Input is vector, output is an array
-of chunks
+of chunk_ids
 """
 
 import logging
@@ -39,22 +39,22 @@ class Processor(DocumentEmbeddingsQueryService):
             if msg.limit <= 0:
                 return []
 
-            chunks = []
+            chunk_ids = []
 
             for vec in msg.vectors:
 
                 resp = self.vecstore.search(
-                    vec, 
-                    msg.user, 
-                    msg.collection, 
+                    vec,
+                    msg.user,
+                    msg.collection,
                     limit=msg.limit
                 )
 
                 for r in resp:
-                    chunk = r["entity"]["doc"]
-                    chunks.append(chunk)
+                    chunk_id = r["entity"]["chunk_id"]
+                    chunk_ids.append(chunk_id)
 
-            return chunks
+            return chunk_ids
 
         except Exception as e:
 

trustgraph-flow/trustgraph/query/doc_embeddings/pinecone/service.py

@@ -1,7 +1,7 @@
 
 """
 Document embeddings query service.  Input is vector, output is an array
-of chunks.  Pinecone implementation.
+of chunk_ids.  Pinecone implementation.
 """
 
 import logging
@@ -55,7 +55,7 @@ class Processor(DocumentEmbeddingsQueryService):
             if msg.limit <= 0:
                 return []
 
-            chunks = []
+            chunk_ids = []
 
             for vec in msg.vectors:
 
@@ -79,10 +79,10 @@ class Processor(DocumentEmbeddingsQueryService):
                 )
 
                 for r in results.matches:
-                    doc = r.metadata["doc"]
-                    chunks.append(doc)
+                    chunk_id = r.metadata["chunk_id"]
+                    chunk_ids.append(chunk_id)
 
-            return chunks
+            return chunk_ids
 
         except Exception as e:
 

trustgraph-flow/trustgraph/query/doc_embeddings/qdrant/service.py

@@ -1,7 +1,7 @@
 
 """
 Document embeddings query service.  Input is vector, output is an array
-of chunks
+of chunk_ids
 """
 
 import logging
@@ -69,7 +69,7 @@ class Processor(DocumentEmbeddingsQueryService):
 
         try:
 
-            chunks = []
+            chunk_ids = []
 
             for vec in msg.vectors:
 
@@ -90,10 +90,10 @@ class Processor(DocumentEmbeddingsQueryService):
                 ).points
 
                 for r in search_result:
-                    ent = r.payload["doc"]
-                    chunks.append(ent)
+                    chunk_id = r.payload["chunk_id"]
+                    chunk_ids.append(chunk_id)
 
-            return chunks
+            return chunk_ids
 
         except Exception as e: