Collection management (#520)

* Tech spec

* Refactored Cassanda knowledge graph for single table

* Collection management, librarian services to manage metadata and collection deletion

trustgraph-base/trustgraph/schema/services/__init__.py

@@ -10,4 +10,6 @@ from .lookup import *
 from .nlp_query import *
 from .structured_query import *
 from .objects_query import *
-from .diagnosis import *
+from .diagnosis import *
+from .collection import *
+from .storage import *

trustgraph-base/trustgraph/schema/services/collection.py

@@ -0,0 +1,60 @@
+from pulsar.schema import Record, String, Integer, Array
+from datetime import datetime
+
+from ..core.primitives import Error
+from ..core.topic import topic
+
+############################################################################
+
+# Collection management operations
+
+# Collection metadata operations (for librarian service)
+
+class CollectionMetadata(Record):
+    """Collection metadata record"""
+    user = String()
+    collection = String()
+    name = String()
+    description = String()
+    tags = Array(String())
+    created_at = String()  # ISO timestamp
+    updated_at = String()  # ISO timestamp
+
+############################################################################
+
+class CollectionManagementRequest(Record):
+    """Request for collection management operations"""
+    operation = String()  # e.g., "delete-collection"
+
+    # For 'list-collections'
+    user = String()
+    collection = String()
+    timestamp = String()  # ISO timestamp
+    name = String()
+    description = String()
+    tags = Array(String())
+    created_at = String()  # ISO timestamp
+    updated_at = String()  # ISO timestamp
+
+    # For list
+    tag_filter = Array(String())  # Optional filter by tags
+    limit = Integer()
+
+class CollectionManagementResponse(Record):
+    """Response for collection management operations"""
+    success = String()  # "true" or "false"
+    error = Error()  # Only populated if success is "false"
+    timestamp = String()  # ISO timestamp
+    collections = Array(CollectionMetadata())
+
+
+############################################################################
+
+# Topics
+
+collection_request_queue = topic(
+    'collection', kind='non-persistent', namespace='request'
+)
+collection_response_queue = topic(
+    'collection', kind='non-persistent', namespace='response'
+)

trustgraph-base/trustgraph/schema/services/storage.py

@@ -0,0 +1,42 @@
+from pulsar.schema import Record, String
+
+from ..core.primitives import Error
+from ..core.topic import topic
+
+############################################################################
+
+# Storage management operations
+
+class StorageManagementRequest(Record):
+    """Request for storage management operations sent to store processors"""
+    operation = String()  # e.g., "delete-collection"
+    user = String()
+    collection = String()
+
+class StorageManagementResponse(Record):
+    """Response from storage processors for management operations"""
+    error = Error()  # Only populated if there's an error, if null success
+
+############################################################################
+
+# Storage management topics
+
+# Topics for sending collection management requests to different storage types
+vector_storage_management_topic = topic(
+    'vector-storage-management', kind='non-persistent', namespace='request'
+)
+
+object_storage_management_topic = topic(
+    'object-storage-management', kind='non-persistent', namespace='request'
+)
+
+triples_storage_management_topic = topic(
+    'triples-storage-management', kind='non-persistent', namespace='request'
+)
+
+# Topic for receiving responses from storage processors
+storage_management_response_topic = topic(
+    'storage-management', kind='non-persistent', namespace='response'
+)
+
+############################################################################