apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-29 02:23:44 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Fix collection management sync prob (#544)

Browse source 
* Address creation/deletion sync problems

* Fix object writer management

* Get Milvus to use ANN
This commit is contained in:
cybermaggedon 2025-09-30 23:04:28 +01:00 • committed by GitHub
parent b7bf7388aa
commit 3e23d3c3ed
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 19 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

11
trustgraph-flow/trustgraph/direct/milvus_doc_embeddings.py
View file

@ -144,14 +144,6 @@ class DocVectors:
coll = self.collections[(dim, user, collection)] coll = self.collections[(dim, user, collection)]
search_params = {
"metric_type": "COSINE",
"params": {
"radius": 0.1,
"range_filter": 0.8
}
}
logger.debug("Loading...") logger.debug("Loading...")
self.client.load_collection( self.client.load_collection(
collection_name=coll, collection_name=coll,
@ -161,10 +153,11 @@ class DocVectors:
res = self.client.search( res = self.client.search(
collection_name=coll, collection_name=coll,
anns_field="vector",
data=[embeds], data=[embeds],
limit=limit, limit=limit,
output_fields=fields, output_fields=fields,
search_params=search_params, search_params={ "metric_type": "COSINE" },
)[0] )[0]

11
trustgraph-flow/trustgraph/direct/milvus_graph_embeddings.py
View file

@ -144,14 +144,6 @@ class EntityVectors:
coll = self.collections[(dim, user, collection)] coll = self.collections[(dim, user, collection)]
search_params = {
"metric_type": "COSINE",
"params": {
"radius": 0.1,
"range_filter": 0.8
}
}
logger.debug("Loading...") logger.debug("Loading...")
self.client.load_collection( self.client.load_collection(
collection_name=coll, collection_name=coll,
@ -161,10 +153,11 @@ class EntityVectors:
res = self.client.search( res = self.client.search(
collection_name=coll, collection_name=coll,
anns_field="vector",
data=[embeds], data=[embeds],
limit=limit, limit=limit,
output_fields=fields, output_fields=fields,
search_params=search_params, search_params={ "metric_type": "COSINE" },
)[0] )[0]

10
trustgraph-flow/trustgraph/librarian/collection_manager.py
View file

@ -88,7 +88,7 @@ class CollectionManager:
logger.info(f"Broadcasting create-collection for {creation_key}") logger.info(f"Broadcasting create-collection for {creation_key}")
self.pending_deletions[creation_key] = { self.pending_deletions[creation_key] = {
"responses_pending": 3, # vector, object, triples "responses_pending": 4, # doc-embeddings, graph-embeddings, object, triples
"responses_received": [], "responses_received": [],
"all_successful": True, "all_successful": True,
"error_messages": [], "error_messages": [],
@ -213,7 +213,7 @@ class CollectionManager:
logger.info(f"Broadcasting create-collection for {creation_key}") logger.info(f"Broadcasting create-collection for {creation_key}")
self.pending_deletions[creation_key] = { self.pending_deletions[creation_key] = {
"responses_pending": 3, # vector, object, triples "responses_pending": 4, # doc-embeddings, graph-embeddings, object, triples
"responses_received": [], "responses_received": [],
"all_successful": True, "all_successful": True,
"error_messages": [], "error_messages": [],
@ -328,7 +328,7 @@ class CollectionManager:
# Track this deletion request # Track this deletion request
self.pending_deletions[deletion_key] = { self.pending_deletions[deletion_key] = {
"responses_pending": 3, # vector, object, triples "responses_pending": 4, # doc-embeddings, graph-embeddings, object, triples
"responses_received": [], "responses_received": [],
"all_successful": True, "all_successful": True,
"error_messages": [], "error_messages": [],
@ -418,9 +418,9 @@ class CollectionManager:
if response.error and response.error.message: if response.error and response.error.message:
info["all_successful"] = False info["all_successful"] = False
info["error_messages"].append(response.error.message) info["error_messages"].append(response.error.message)
logger.warning(f"Storage deletion failed for {deletion_key}: {response.error.message}") logger.warning(f"Storage operation failed for {deletion_key}: {response.error.message}")
else: else:
logger.debug(f"Storage deletion succeeded for {deletion_key}") logger.debug(f"Storage operation succeeded for {deletion_key}")
# If all responses received, signal completion # If all responses received, signal completion
if info["responses_pending"] == 0: if info["responses_pending"] == 0:

19
trustgraph-flow/trustgraph/storage/objects/cassandra/write.py
View file

@ -456,35 +456,36 @@ class Processor(FlowProcessor):
async def on_storage_management(self, msg, consumer, flow): async def on_storage_management(self, msg, consumer, flow):
"""Handle storage management requests for collection operations""" """Handle storage management requests for collection operations"""
logger.info(f"Received storage management request: {msg.operation} for {msg.user}/{msg.collection}") request = msg.value()
logger.info(f"Received storage management request: {request.operation} for {request.user}/{request.collection}")
try: try:
if msg.operation == "create-collection": if request.operation == "create-collection":
await self.create_collection(msg.user, msg.collection) await self.create_collection(request.user, request.collection)
# Send success response # Send success response
response = StorageManagementResponse( response = StorageManagementResponse(
error=None # No error means success error=None # No error means success
) )
await self.storage_response_producer.send(response) await self.storage_response_producer.send(response)
logger.info(f"Successfully created collection {msg.user}/{msg.collection}") logger.info(f"Successfully created collection {request.user}/{request.collection}")
elif msg.operation == "delete-collection": elif request.operation == "delete-collection":
await self.delete_collection(msg.user, msg.collection) await self.delete_collection(request.user, request.collection)
# Send success response # Send success response
response = StorageManagementResponse( response = StorageManagementResponse(
error=None # No error means success error=None # No error means success
) )
await self.storage_response_producer.send(response) await self.storage_response_producer.send(response)
logger.info(f"Successfully deleted collection {msg.user}/{msg.collection}") logger.info(f"Successfully deleted collection {request.user}/{request.collection}")
else: else:
logger.warning(f"Unknown storage management operation: {msg.operation}") logger.warning(f"Unknown storage management operation: {request.operation}")
# Send error response # Send error response
from .... schema import Error from .... schema import Error
response = StorageManagementResponse( response = StorageManagementResponse(
error=Error( error=Error(
type="unknown_operation", type="unknown_operation",
message=f"Unknown operation: {msg.operation}" message=f"Unknown operation: {request.operation}"
) )
) )
await self.storage_response_producer.send(response) await self.storage_response_producer.send(response)