From 9a47915d05cd7b55a4ba7dd017e21a278ed777b4 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Mon, 5 May 2025 00:03:41 +0100 Subject: [PATCH] Some library CLI --- trustgraph-base/trustgraph/api/api.py | 119 ++++++++++- .../scripts/tg-add-library-document | 201 ++++++++++++++++++ trustgraph-cli/scripts/tg-load-pdf | 3 - .../scripts/tg-show-library-documents | 74 +++++++ trustgraph-cli/setup.py | 2 + 5 files changed, 392 insertions(+), 7 deletions(-) create mode 100755 trustgraph-cli/scripts/tg-add-library-document create mode 100644 trustgraph-cli/scripts/tg-show-library-documents diff --git a/trustgraph-base/trustgraph/api/api.py b/trustgraph-base/trustgraph/api/api.py index cff2b36a..bfe774a4 100644 --- a/trustgraph-base/trustgraph/api/api.py +++ b/trustgraph-base/trustgraph/api/api.py @@ -3,6 +3,9 @@ import requests import json import dataclasses import base64 +from typing import List +import datetime +import time from trustgraph.knowledge import hash, Uri, Literal @@ -29,6 +32,31 @@ class ConfigValue: key : str value : str +@dataclasses.dataclass +class DocumentMetadata: + id : str + time : datetime.datetime + kind : str + title : str + comments : str + metadata : List[Triple] + user : str + tags : List[str] + +@dataclasses.dataclass +class ProcessingMetadata: + id : str + document_id : str + time : datetime.datetime + flow : str + user : str + collection : str + tags : List[str] + +def to_value(x): + if x["e"]: return Uri(x["v"]) + return Literal(x["v"]) + def check_error(response): if "error" in response: @@ -90,6 +118,93 @@ class Api: except: raise ProtocolException(f"Response not formatted correctly") + def library_add_document( + self, document, id, metadata, user, title, comments, + kind="text/plain", tags=[], + ): + + if id is None: + + if metadata is not None: + + # Situation makes no sense. What can the metadata possibly + # mean if the caller doesn't know the document ID. + # Metadata should relate to the document by ID + raise RuntimeError("Can't specify metadata without id") + + id = hash(document) + + if not title: title = "" + if not comments: comments = "" + + triples = [] + + def emit(t): + triples.append(t) + + if metadata: + metadata.emit( + lambda t: triples.append({ + "s": { "v": t["s"], "e": isinstance(t["s"], Uri) }, + "p": { "v": t["p"], "e": isinstance(t["p"], Uri) }, + "o": { "v": t["o"], "e": isinstance(t["o"], Uri) } + }) + ) + + input = { + "operation": "add-document", + "document-metadata": { + "id": id, + "time": int(time.time()), + "kind": kind, + "title": title, + "comments": comments, + "metadata": triples, + "user": user, + "tags": tags + }, + "content": base64.b64encode(document).decode("utf-8"), + } + + return self.request( + f"librarian", + input + ) + + + def library_get_documents(self, user): + + input = { + "operation": "list-documents", + "user": user, + } + + object = self.request("librarian", input) + + try: + return [ + DocumentMetadata( + id = v["id"], + time = datetime.datetime.fromtimestamp(v["time"]), + kind = v["kind"], + title = v["title"], + comments = v.get("comments", ""), + metadata = [ + Triple( + s = to_value(w["s"]), + p = to_value(w["p"]), + o = to_value(w["o"]) + ) + for w in v["metadata"] + ], + user = v["user"], + tags = v["tags"] + ) + for v in object["document-metadatas"] + ] + except: + raise ProtocolException(f"Response not formatted correctly") + def config_get(self, keys): # The input consists of system and prompt strings @@ -386,10 +501,6 @@ class Flow: input ) - def to_value(x): - if x["e"]: return Uri(x["v"]) - return Literal(x["v"]) - return [ Triple( s=to_value(t["s"]), diff --git a/trustgraph-cli/scripts/tg-add-library-document b/trustgraph-cli/scripts/tg-add-library-document new file mode 100755 index 00000000..9bb83487 --- /dev/null +++ b/trustgraph-cli/scripts/tg-add-library-document @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 + +""" +Loads a PDF document into the library +""" + +import hashlib +import argparse +import os +import time +import uuid + +from trustgraph.api import Api +from trustgraph.knowledge import hash, to_uri +from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG +from trustgraph.knowledge import Organization, PublicationEvent +from trustgraph.knowledge import DigitalDocument + +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') +default_user = 'trustgraph' + +class Loader: + + def __init__( + self, url, user, metadata, title, comments, kind, tags + ): + + self.api = Api(url) + + self.user = user + self.metadata = metadata + self.title = title + self.comments = comments + self.kind = kind + + if tags: + self.tags = tags.split(",") + else: + self.tags = [] + + def load(self, files): + + for file in files: + self.load_file(file) + + def load_file(self, file): + + try: + + path = file + data = open(path, "rb").read() + + # Create a SHA256 hash from the data + id = hash(data) + + id = to_uri(PREF_DOC, id) + + self.metadata.id = id + + self.api.library_add_document( + document=data, id=id, metadata=self.metadata, + user=self.user, kind=self.kind, title=self.title, + comments=self.comments, tags=self.tags + ) + + print(f"{file}: Loaded successfully.") + + except Exception as e: + print(f"{file}: Failed: {str(e)}", flush=True) + raise e + +def main(): + + parser = argparse.ArgumentParser( + prog='tg-add-library-document', + description=__doc__, + ) + + parser.add_argument( + '-u', '--url', + default=default_url, + help=f'API URL (default: {default_url})', + ) + + parser.add_argument( + '-U', '--user', + default=default_user, + help=f'User ID (default: {default_user})' + ) + + parser.add_argument( + '--name', help=f'Document name' + ) + + parser.add_argument( + '--description', help=f'Document description' + ) + + parser.add_argument( + '--copyright-notice', help=f'Copyright notice' + ) + + parser.add_argument( + '--copyright-holder', help=f'Copyright holder' + ) + + parser.add_argument( + '--copyright-year', help=f'Copyright year' + ) + + parser.add_argument( + '--license', help=f'Copyright license' + ) + + parser.add_argument( + '--publication-organization', help=f'Publication organization' + ) + + parser.add_argument( + '--publication-description', help=f'Publication description' + ) + + parser.add_argument( + '--publication-date', help=f'Publication date' + ) + + parser.add_argument( + '--document-url', help=f'Document URL' + ) + + parser.add_argument( + '--keyword', nargs='+', help=f'Keyword' + ) + + parser.add_argument( + '--identifier', '--id', help=f'Document ID' + ) + + parser.add_argument( + '-k', '--kind', + required=True, + help=f'Document MIME type' + ) + + parser.add_argument( + '--tags', + help=f'Tags, command separated' + ) + + parser.add_argument( + 'files', nargs='+', + help=f'File to load' + ) + + args = parser.parse_args() + + try: + + document = DigitalDocument( + id, + name=args.name, + description=args.description, + copyright_notice=args.copyright_notice, + copyright_holder=args.copyright_holder, + copyright_year=args.copyright_year, + license=args.license, + url=args.document_url, + keywords=args.keyword, + ) + + if args.publication_organization: + org = Organization( + id=to_uri(PREF_ORG, hash(args.publication_organization)), + name=args.publication_organization, + ) + document.publication = PublicationEvent( + id = to_uri(PREF_PUBEV, str(uuid.uuid4())), + organization=org, + description=args.publication_description, + start_date=args.publication_date, + end_date=args.publication_date, + ) + + p = Loader( + url=args.url, + user=args.user, + metadata=document, + title=args.name, + comments=args.description, + kind=args.kind, + tags=args.tags, + ) + + p.load(args.files) + + except Exception as e: + + print("Exception:", e, flush=True) + +main() + diff --git a/trustgraph-cli/scripts/tg-load-pdf b/trustgraph-cli/scripts/tg-load-pdf index 77105315..e1641593 100755 --- a/trustgraph-cli/scripts/tg-load-pdf +++ b/trustgraph-cli/scripts/tg-load-pdf @@ -4,8 +4,6 @@ Loads a PDF document into TrustGraph processing. """ -import pulsar -from pulsar.schema import JsonSchema import hashlib import argparse import os @@ -31,7 +29,6 @@ class Loader: user, collection, metadata, - pulsar_api_key=None, ): self.api = Api(url).flow(flow_id) diff --git a/trustgraph-cli/scripts/tg-show-library-documents b/trustgraph-cli/scripts/tg-show-library-documents new file mode 100644 index 00000000..63962182 --- /dev/null +++ b/trustgraph-cli/scripts/tg-show-library-documents @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +""" +""" + +import argparse +import os +import tabulate +from trustgraph.api import Api, ConfigKey +import json + +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') +default_user = "trustgraph" + +def show_docs(url, user): + + api = Api(url) + + docs = api.library_get_documents(user=user) + + if len(docs) == 0: + print("No documents.") + return + + for doc in docs: + + table = [] + table.append(("id", doc.id)) + table.append(("time", doc.time)) + table.append(("title", doc.title)) + table.append(("kind", doc.kind)) + table.append(("comments", doc.comments)) + table.append(("tags", ", ".join(doc.tags))) + + print(tabulate.tabulate( + table, + tablefmt="pretty", + stralign="left", + )) + print() + +def main(): + + parser = argparse.ArgumentParser( + prog='tg-show-library-documents', + description=__doc__, + ) + + parser.add_argument( + '-u', '--api-url', + default=default_url, + help=f'API URL (default: {default_url})', + ) + + parser.add_argument( + '-U', '--user', + default=default_user, + help=f'User ID (default: {default_user})' + ) + + args = parser.parse_args() + + try: + + show_docs( + url = args.api_url, user = args.user + ) + + except Exception as e: + + print("Exception:", e, flush=True) + +main() + diff --git a/trustgraph-cli/setup.py b/trustgraph-cli/setup.py index 5e50d06c..63de9450 100644 --- a/trustgraph-cli/setup.py +++ b/trustgraph-cli/setup.py @@ -70,6 +70,8 @@ setuptools.setup( "scripts/tg-show-flow-classes", "scripts/tg-show-flow-state", "scripts/tg-show-flows", + "scripts/tg-show-library-documents", + "scripts/tg-add-library-document", "scripts/tg-show-processor-state", "scripts/tg-show-prompts", "scripts/tg-show-token-costs",