diff --git a/trustgraph-base/trustgraph/schema/library.py b/trustgraph-base/trustgraph/schema/library.py index 091d1daf..ed52b2ad 100644 --- a/trustgraph-base/trustgraph/schema/library.py +++ b/trustgraph-base/trustgraph/schema/library.py @@ -27,6 +27,7 @@ from . documents import Document, TextDocument # search([]) : id[] class DocumentPackage(Record): + id = String() document = Bytes() kind = String() user = String() diff --git a/trustgraph-flow/trustgraph/gateway/serialize.py b/trustgraph-flow/trustgraph/gateway/serialize.py index 3121e972..5cc90a78 100644 --- a/trustgraph-flow/trustgraph/gateway/serialize.py +++ b/trustgraph-flow/trustgraph/gateway/serialize.py @@ -84,6 +84,9 @@ def serialize_document_package(message): ret = {} + if message.id: + ret["id"] = message.id + if message.metadata: ret["metadata"] = serialize_subgraph(message.metdata) @@ -137,7 +140,7 @@ def serialize_document_info(message): def to_document_package(x): return DocumentPackage( - metadata = to_subgraph(x["metadata"]), + id = x.get("id", None), kind = x.get("kind", None), user = x.get("user", None), collection = x.get("collection", None), @@ -145,18 +148,20 @@ def to_document_package(x): comments = x.get("comments", None), time = x.get("time", None), document = x.get("document", None), + metadata = to_subgraph(x["metadata"]), ) def to_document_info(x): return DocumentInfo( - metadata = to_subgraph(x["metadata"]), + id = x.get("id", None), kind = x.get("kind", None), user = x.get("user", None), collection = x.get("collection", None), title = x.get("title", None), comments = x.get("comments", None), time = x.get("time", None), + metadata = to_subgraph(x["metadata"]), ) def to_criteria(x): diff --git a/trustgraph-flow/trustgraph/librarian/librarian.py b/trustgraph-flow/trustgraph/librarian/librarian.py index 58511b61..9bccc37a 100644 --- a/trustgraph-flow/trustgraph/librarian/librarian.py +++ b/trustgraph-flow/trustgraph/librarian/librarian.py @@ -26,7 +26,7 @@ class Librarian: self.load_document = load_document self.load_text = load_text - async def add(self, id, document): + async def add(self, document): if document.kind not in ( "text/plain", "application/pdf" @@ -41,9 +41,9 @@ class Librarian: self.table_store.add(object_id, document) if document.kind == "application/pdf": - await self.load_document(id, document) + await self.load_document(document) elif document.kind == "text/plain": - await self.load_text(id, document) + await self.load_text(document) print("Add complete", flush=True) diff --git a/trustgraph-flow/trustgraph/librarian/service.py b/trustgraph-flow/trustgraph/librarian/service.py index 0e4e2303..b42123a5 100755 --- a/trustgraph-flow/trustgraph/librarian/service.py +++ b/trustgraph-flow/trustgraph/librarian/service.py @@ -223,11 +223,11 @@ class Processor(ConsumerProducer): self.librarian.handle_document_embeddings(msg) - async def load_document(self, id, document): + async def load_document(self, document): doc = Document( metadata = Metadata( - id = id, + id = document.id, metadata = document.metadata, user = document.user, collection = document.collection @@ -237,14 +237,14 @@ class Processor(ConsumerProducer): self.document_load.send(None, doc) - async def load_text(self, id, document): + async def load_text(self, document): text = base64.b64decode(document.document) text = text.decode("utf-8") doc = TextDocument( metadata = Metadata( - id = id, + id = document.id, metadata = document.metadata, user = document.user, collection = document.collection @@ -259,20 +259,23 @@ class Processor(ConsumerProducer): if v.operation is None: raise RequestError("Null operation") + print("op", v.operation) + if v.operation == "add": if ( - v.id and v.document and v.document.metadata and + v.document and v.document.id and v.document.metadata and v.document.document and v.document.kind ): return partial( self.librarian.add, - id = v.id, document = v.document, ) else: raise RequestError("Invalid call") if v.operation == "list": + print("list", v) + print(v.user) if v.user: return partial( self.librarian.list, @@ -280,6 +283,7 @@ class Processor(ConsumerProducer): collection = v.collection, ) else: + print("BROK") raise RequestError("Invalid call") raise RequestError("Invalid operation: " + v.operation) diff --git a/trustgraph-flow/trustgraph/librarian/table_store.py b/trustgraph-flow/trustgraph/librarian/table_store.py index ec65d06a..ca10a05e 100644 --- a/trustgraph-flow/trustgraph/librarian/table_store.py +++ b/trustgraph-flow/trustgraph/librarian/table_store.py @@ -62,7 +62,7 @@ class TableStore: CREATE TABLE IF NOT EXISTS document ( user text, collection text, - id uuid, + id text, time timestamp, title text, comments text, @@ -207,10 +207,9 @@ class TableStore: raise RequestError("Invalid document kind: " + document.kind) # Create random doc ID - doc_id = uuid.uuid4() when = int(time.time() * 1000) - print("Adding", object_id, doc_id) + print("Adding", document.id, object_id) metadata = [ ( @@ -220,8 +219,6 @@ class TableStore: for v in document.metadata ] - # FIXME: doc_id should be the user-supplied ID??? - while True: try: @@ -229,7 +226,7 @@ class TableStore: resp = self.cassandra.execute( self.insert_document_stmt, ( - doc_id, document.user, document.collection, + document.id, document.user, document.collection, document.kind, object_id, when, document.title, document.comments, metadata