mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-22 03:31:02 +02:00
CLI and API changes
This commit is contained in:
parent
9f90075367
commit
0c62f0e0fa
1 changed files with 41 additions and 30 deletions
|
|
@ -23,10 +23,6 @@ Vector store payload:
|
||||||
payload={"doc": chunk} # Duplicates Garage content
|
payload={"doc": chunk} # Duplicates Garage content
|
||||||
```
|
```
|
||||||
|
|
||||||
Document RAG flow:
|
|
||||||
1. Query embeddings → get chunk text
|
|
||||||
2. Pass chunk text directly to prompt
|
|
||||||
|
|
||||||
## Design
|
## Design
|
||||||
|
|
||||||
### Schema Changes
|
### Schema Changes
|
||||||
|
|
@ -56,51 +52,56 @@ payload={"chunk_id": chunk_id}
|
||||||
|
|
||||||
### Document RAG Changes
|
### Document RAG Changes
|
||||||
|
|
||||||
The document RAG processor must fetch chunk content from Garage:
|
The document RAG processor fetches chunk content from Garage:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
# In document_rag.py get_docs():
|
# Get chunk_ids from embeddings store
|
||||||
async def get_docs(self, query):
|
chunk_ids = await self.rag.doc_embeddings_client.query(...)
|
||||||
vectors = await self.get_vector(query)
|
|
||||||
|
|
||||||
# Get chunk_ids from embeddings store
|
# Fetch chunk content from Garage
|
||||||
chunk_ids = await self.rag.doc_embeddings_client.query(
|
docs = []
|
||||||
vectors, limit=self.doc_limit,
|
for chunk_id in chunk_ids:
|
||||||
user=self.user, collection=self.collection,
|
content = await self.rag.librarian_client.get_document_content(
|
||||||
|
chunk_id, self.user
|
||||||
)
|
)
|
||||||
|
docs.append(content)
|
||||||
# Fetch chunk content from Garage
|
|
||||||
docs = []
|
|
||||||
for chunk_id in chunk_ids:
|
|
||||||
content = await self.rag.librarian_client.get_document_content(
|
|
||||||
chunk_id, self.user
|
|
||||||
)
|
|
||||||
docs.append(content)
|
|
||||||
|
|
||||||
return docs
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Client Changes
|
### API/SDK Changes
|
||||||
|
|
||||||
**DocumentEmbeddingsClient** - return chunk_ids:
|
**DocumentEmbeddingsClient** returns chunk_ids:
|
||||||
```python
|
```python
|
||||||
async def query(self, vectors, limit=20, user="trustgraph",
|
return resp.chunk_ids # Changed from resp.chunks
|
||||||
collection="default", timeout=30):
|
|
||||||
resp = await self.request(...)
|
|
||||||
if resp.error:
|
|
||||||
raise RuntimeError(resp.error.message)
|
|
||||||
return resp.chunk_ids # Changed from resp.chunks
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Wire format** (DocumentEmbeddingsResponseTranslator):
|
||||||
|
```python
|
||||||
|
result["chunk_ids"] = obj.chunk_ids # Changed from chunks
|
||||||
|
```
|
||||||
|
|
||||||
|
### CLI Changes
|
||||||
|
|
||||||
|
CLI tool displays chunk_ids (callers can fetch content separately if needed).
|
||||||
|
|
||||||
## Files to Modify
|
## Files to Modify
|
||||||
|
|
||||||
### Schema
|
### Schema
|
||||||
- `trustgraph-base/trustgraph/schema/knowledge/embeddings.py` - ChunkEmbeddings
|
- `trustgraph-base/trustgraph/schema/knowledge/embeddings.py` - ChunkEmbeddings
|
||||||
- `trustgraph-base/trustgraph/schema/services/query.py` - DocumentEmbeddingsResponse
|
- `trustgraph-base/trustgraph/schema/services/query.py` - DocumentEmbeddingsResponse
|
||||||
|
|
||||||
|
### Messaging/Translators
|
||||||
|
- `trustgraph-base/trustgraph/messaging/translators/embeddings_query.py` - DocumentEmbeddingsResponseTranslator
|
||||||
|
|
||||||
### Client
|
### Client
|
||||||
- `trustgraph-base/trustgraph/base/document_embeddings_client.py` - return chunk_ids
|
- `trustgraph-base/trustgraph/base/document_embeddings_client.py` - return chunk_ids
|
||||||
|
|
||||||
|
### Python SDK/API
|
||||||
|
- `trustgraph-base/trustgraph/api/flow.py` - document_embeddings_query
|
||||||
|
- `trustgraph-base/trustgraph/api/socket_client.py` - document_embeddings_query
|
||||||
|
- `trustgraph-base/trustgraph/api/async_flow.py` - if applicable
|
||||||
|
- `trustgraph-base/trustgraph/api/bulk_client.py` - import/export document embeddings
|
||||||
|
- `trustgraph-base/trustgraph/api/async_bulk_client.py` - import/export document embeddings
|
||||||
|
|
||||||
### Embeddings Service
|
### Embeddings Service
|
||||||
- `trustgraph-flow/trustgraph/embeddings/document_embeddings/embeddings.py` - pass chunk_id
|
- `trustgraph-flow/trustgraph/embeddings/document_embeddings/embeddings.py` - pass chunk_id
|
||||||
|
|
||||||
|
|
@ -114,10 +115,20 @@ async def query(self, vectors, limit=20, user="trustgraph",
|
||||||
- `trustgraph-flow/trustgraph/query/doc_embeddings/milvus/service.py`
|
- `trustgraph-flow/trustgraph/query/doc_embeddings/milvus/service.py`
|
||||||
- `trustgraph-flow/trustgraph/query/doc_embeddings/pinecone/service.py`
|
- `trustgraph-flow/trustgraph/query/doc_embeddings/pinecone/service.py`
|
||||||
|
|
||||||
|
### Gateway
|
||||||
|
- `trustgraph-flow/trustgraph/gateway/dispatch/document_embeddings_query.py`
|
||||||
|
- `trustgraph-flow/trustgraph/gateway/dispatch/document_embeddings_export.py`
|
||||||
|
- `trustgraph-flow/trustgraph/gateway/dispatch/document_embeddings_import.py`
|
||||||
|
|
||||||
### Document RAG
|
### Document RAG
|
||||||
- `trustgraph-flow/trustgraph/retrieval/document_rag/rag.py` - add librarian client
|
- `trustgraph-flow/trustgraph/retrieval/document_rag/rag.py` - add librarian client
|
||||||
- `trustgraph-flow/trustgraph/retrieval/document_rag/document_rag.py` - fetch from Garage
|
- `trustgraph-flow/trustgraph/retrieval/document_rag/document_rag.py` - fetch from Garage
|
||||||
|
|
||||||
|
### CLI
|
||||||
|
- `trustgraph-cli/trustgraph/cli/invoke_document_embeddings.py`
|
||||||
|
- `trustgraph-cli/trustgraph/cli/save_doc_embeds.py`
|
||||||
|
- `trustgraph-cli/trustgraph/cli/load_doc_embeds.py`
|
||||||
|
|
||||||
## Benefits
|
## Benefits
|
||||||
|
|
||||||
1. Single source of truth - chunk text only in Garage
|
1. Single source of truth - chunk text only in Garage
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue