diff --git a/docs/tech-specs/document-embeddings-chunk-id.md b/docs/tech-specs/document-embeddings-chunk-id.md index 270c7b99..157ecba0 100644 --- a/docs/tech-specs/document-embeddings-chunk-id.md +++ b/docs/tech-specs/document-embeddings-chunk-id.md @@ -23,10 +23,6 @@ Vector store payload: payload={"doc": chunk} # Duplicates Garage content ``` -Document RAG flow: -1. Query embeddings → get chunk text -2. Pass chunk text directly to prompt - ## Design ### Schema Changes @@ -56,51 +52,56 @@ payload={"chunk_id": chunk_id} ### Document RAG Changes -The document RAG processor must fetch chunk content from Garage: +The document RAG processor fetches chunk content from Garage: ```python -# In document_rag.py get_docs(): -async def get_docs(self, query): - vectors = await self.get_vector(query) +# Get chunk_ids from embeddings store +chunk_ids = await self.rag.doc_embeddings_client.query(...) - # Get chunk_ids from embeddings store - chunk_ids = await self.rag.doc_embeddings_client.query( - vectors, limit=self.doc_limit, - user=self.user, collection=self.collection, +# Fetch chunk content from Garage +docs = [] +for chunk_id in chunk_ids: + content = await self.rag.librarian_client.get_document_content( + chunk_id, self.user ) - - # Fetch chunk content from Garage - docs = [] - for chunk_id in chunk_ids: - content = await self.rag.librarian_client.get_document_content( - chunk_id, self.user - ) - docs.append(content) - - return docs + docs.append(content) ``` -### Client Changes +### API/SDK Changes -**DocumentEmbeddingsClient** - return chunk_ids: +**DocumentEmbeddingsClient** returns chunk_ids: ```python -async def query(self, vectors, limit=20, user="trustgraph", - collection="default", timeout=30): - resp = await self.request(...) - if resp.error: - raise RuntimeError(resp.error.message) - return resp.chunk_ids # Changed from resp.chunks +return resp.chunk_ids # Changed from resp.chunks ``` +**Wire format** (DocumentEmbeddingsResponseTranslator): +```python +result["chunk_ids"] = obj.chunk_ids # Changed from chunks +``` + +### CLI Changes + +CLI tool displays chunk_ids (callers can fetch content separately if needed). + ## Files to Modify ### Schema - `trustgraph-base/trustgraph/schema/knowledge/embeddings.py` - ChunkEmbeddings - `trustgraph-base/trustgraph/schema/services/query.py` - DocumentEmbeddingsResponse +### Messaging/Translators +- `trustgraph-base/trustgraph/messaging/translators/embeddings_query.py` - DocumentEmbeddingsResponseTranslator + ### Client - `trustgraph-base/trustgraph/base/document_embeddings_client.py` - return chunk_ids +### Python SDK/API +- `trustgraph-base/trustgraph/api/flow.py` - document_embeddings_query +- `trustgraph-base/trustgraph/api/socket_client.py` - document_embeddings_query +- `trustgraph-base/trustgraph/api/async_flow.py` - if applicable +- `trustgraph-base/trustgraph/api/bulk_client.py` - import/export document embeddings +- `trustgraph-base/trustgraph/api/async_bulk_client.py` - import/export document embeddings + ### Embeddings Service - `trustgraph-flow/trustgraph/embeddings/document_embeddings/embeddings.py` - pass chunk_id @@ -114,10 +115,20 @@ async def query(self, vectors, limit=20, user="trustgraph", - `trustgraph-flow/trustgraph/query/doc_embeddings/milvus/service.py` - `trustgraph-flow/trustgraph/query/doc_embeddings/pinecone/service.py` +### Gateway +- `trustgraph-flow/trustgraph/gateway/dispatch/document_embeddings_query.py` +- `trustgraph-flow/trustgraph/gateway/dispatch/document_embeddings_export.py` +- `trustgraph-flow/trustgraph/gateway/dispatch/document_embeddings_import.py` + ### Document RAG - `trustgraph-flow/trustgraph/retrieval/document_rag/rag.py` - add librarian client - `trustgraph-flow/trustgraph/retrieval/document_rag/document_rag.py` - fetch from Garage +### CLI +- `trustgraph-cli/trustgraph/cli/invoke_document_embeddings.py` +- `trustgraph-cli/trustgraph/cli/save_doc_embeds.py` +- `trustgraph-cli/trustgraph/cli/load_doc_embeds.py` + ## Benefits 1. Single source of truth - chunk text only in Garage