Collection management (#520)

* Tech spec

* Refactored Cassanda knowledge graph for single table

* Collection management, librarian services to manage metadata and collection deletion

trustgraph-flow/trustgraph/tables/library.py

@@ -111,6 +111,21 @@ class LibraryTableStore:
             );
         """);
 
+        logger.debug("collections table...")
+
+        self.cassandra.execute("""
+            CREATE TABLE IF NOT EXISTS collections (
+                user text,
+                collection text,
+                name text,
+                description text,
+                tags set<text>,
+                created_at timestamp,
+                updated_at timestamp,
+                PRIMARY KEY (user, collection)
+            );
+        """);
+
         logger.info("Cassandra schema OK.")
 
     def prepare_statements(self):
@@ -187,6 +202,43 @@ class LibraryTableStore:
             LIMIT 1
         """)
 
+        # Collection management statements
+        self.insert_collection_stmt = self.cassandra.prepare("""
+            INSERT INTO collections
+            (user, collection, name, description, tags, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+        """)
+
+        self.update_collection_stmt = self.cassandra.prepare("""
+            UPDATE collections
+            SET name = ?, description = ?, tags = ?, updated_at = ?
+            WHERE user = ? AND collection = ?
+        """)
+
+        self.get_collection_stmt = self.cassandra.prepare("""
+            SELECT collection, name, description, tags, created_at, updated_at
+            FROM collections
+            WHERE user = ? AND collection = ?
+        """)
+
+        self.list_collections_stmt = self.cassandra.prepare("""
+            SELECT collection, name, description, tags, created_at, updated_at
+            FROM collections
+            WHERE user = ?
+        """)
+
+        self.delete_collection_stmt = self.cassandra.prepare("""
+            DELETE FROM collections
+            WHERE user = ? AND collection = ?
+        """)
+
+        self.collection_exists_stmt = self.cassandra.prepare("""
+            SELECT collection
+            FROM collections
+            WHERE user = ? AND collection = ?
+            LIMIT 1
+        """)
+
         self.list_processing_stmt = self.cassandra.prepare("""
             SELECT
                 id, document_id, time, flow, collection, tags
@@ -521,3 +573,113 @@ class LibraryTableStore:
 
         return lst
 
+
+
+    # Collection management methods
+
+    async def ensure_collection_exists(self, user, collection):
+        """Ensure collection metadata record exists, create if not"""
+        try:
+            resp = await asyncio.get_event_loop().run_in_executor(
+                None, self.cassandra.execute, self.collection_exists_stmt, [user, collection]
+            )
+            if resp:
+                return
+            import datetime
+            now = datetime.datetime.now()
+            await asyncio.get_event_loop().run_in_executor(
+                None, self.cassandra.execute, self.insert_collection_stmt,
+                [user, collection, collection, "", set(), now, now]
+            )
+            logger.debug(f"Created collection metadata for {user}/{collection}")
+        except Exception as e:
+            logger.error(f"Error ensuring collection exists: {e}")
+            raise
+
+    async def list_collections(self, user, tag_filter=None):
+        """List collections for a user, optionally filtered by tags"""
+        try:
+            resp = await asyncio.get_event_loop().run_in_executor(
+                None, self.cassandra.execute, self.list_collections_stmt, [user]
+            )
+            collections = []
+            for row in resp:
+                collection_data = {
+                    "user": user,
+                    "collection": row[0],
+                    "name": row[1] or row[0],
+                    "description": row[2] or "",
+                    "tags": list(row[3]) if row[3] else [],
+                    "created_at": row[4].isoformat() if row[4] else "",
+                    "updated_at": row[5].isoformat() if row[5] else ""
+                }
+                if tag_filter:
+                    collection_tags = set(collection_data["tags"])
+                    filter_tags = set(tag_filter)
+                    if not filter_tags.intersection(collection_tags):
+                        continue
+                collections.append(collection_data)
+            return collections
+        except Exception as e:
+            logger.error(f"Error listing collections: {e}")
+            raise
+
+    async def update_collection(self, user, collection, name=None, description=None, tags=None):
+        """Update collection metadata"""
+        try:
+            resp = await asyncio.get_event_loop().run_in_executor(
+                None, self.cassandra.execute, self.get_collection_stmt, [user, collection]
+            )
+            if not resp:
+                raise RequestError(f"Collection {collection} not found")
+            row = resp.one()
+            current_name = row[1] or collection
+            current_description = row[2] or ""
+            current_tags = set(row[3]) if row[3] else set()
+            new_name = name if name is not None else current_name
+            new_description = description if description is not None else current_description
+            new_tags = set(tags) if tags is not None else current_tags
+            import datetime
+            now = datetime.datetime.now()
+            await asyncio.get_event_loop().run_in_executor(
+                None, self.cassandra.execute, self.update_collection_stmt,
+                [new_name, new_description, new_tags, now, user, collection]
+            )
+            return {
+                "user": user, "collection": collection, "name": new_name,
+                "description": new_description, "tags": list(new_tags),
+                "updated_at": now.isoformat()
+            }
+        except Exception as e:
+            logger.error(f"Error updating collection: {e}")
+            raise
+
+    async def delete_collection_metadata(self, user, collection):
+        """Delete collection metadata record"""
+        try:
+            await asyncio.get_event_loop().run_in_executor(
+                None, self.cassandra.execute, self.delete_collection_stmt, [user, collection]
+            )
+            logger.debug(f"Deleted collection metadata for {user}/{collection}")
+        except Exception as e:
+            logger.error(f"Error deleting collection metadata: {e}")
+            raise
+
+    async def get_collection(self, user, collection):
+        """Get collection metadata"""
+        try:
+            resp = await asyncio.get_event_loop().run_in_executor(
+                None, self.cassandra.execute, self.get_collection_stmt, [user, collection]
+            )
+            if not resp:
+                return None
+            row = resp.one()
+            return {
+                "user": user, "collection": row[0], "name": row[1] or row[0],
+                "description": row[2] or "", "tags": list(row[3]) if row[3] else [],
+                "created_at": row[4].isoformat() if row[4] else "",
+                "updated_at": row[5].isoformat() if row[5] else ""
+            }
+        except Exception as e:
+            logger.error(f"Error getting collection: {e}")
+            raise