Feature/library cli (#363)

* Major Python client API rework, break down API & colossal class

* Complete rest of library API

* Library CLI support

trustgraph-cli/scripts/tg-add-library-document

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+
+"""
+Loads a PDF document into the library
+"""
+
+import hashlib
+import argparse
+import os
+import time
+import uuid
+
+from trustgraph.api import Api
+from trustgraph.knowledge import hash, to_uri
+from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG
+from trustgraph.knowledge import Organization, PublicationEvent
+from trustgraph.knowledge import DigitalDocument
+
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
+default_user = 'trustgraph'
+
+class Loader:
+
+    def __init__(
+            self, url, user, metadata, title, comments, kind, tags
+    ):
+
+        self.api = Api(url).library()
+
+        self.user = user
+        self.metadata = metadata
+        self.title = title
+        self.comments = comments
+        self.kind = kind
+
+        if tags:
+            self.tags = tags.split(",")
+        else:
+            self.tags = []
+
+    def load(self, files):
+
+        for file in files:
+            self.load_file(file)
+
+    def load_file(self, file):
+
+        try:
+
+            path = file
+            data = open(path, "rb").read()
+
+            # Create a SHA256 hash from the data
+            id = hash(data)
+
+            id = to_uri(PREF_DOC, id)
+
+            self.metadata.id = id
+
+            self.api.add_document(
+                document=data, id=id, metadata=self.metadata, 
+                user=self.user, kind=self.kind, title=self.title,
+                comments=self.comments, tags=self.tags
+            )
+
+            print(f"{file}: Loaded successfully.")
+
+        except Exception as e:
+            print(f"{file}: Failed: {str(e)}", flush=True)
+            raise e
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        prog='tg-add-library-document',
+        description=__doc__,
+    )
+
+    parser.add_argument(
+        '-u', '--url',
+        default=default_url,
+        help=f'API URL (default: {default_url})',
+    )
+
+    parser.add_argument(
+        '-U', '--user',
+        default=default_user,
+        help=f'User ID (default: {default_user})'
+    )
+
+    parser.add_argument(
+        '--name', help=f'Document name'
+    )
+
+    parser.add_argument(
+        '--description', help=f'Document description'
+    )
+
+    parser.add_argument(
+        '--copyright-notice', help=f'Copyright notice'
+    )
+
+    parser.add_argument(
+        '--copyright-holder', help=f'Copyright holder'
+    )
+
+    parser.add_argument(
+        '--copyright-year', help=f'Copyright year'
+    )
+
+    parser.add_argument(
+        '--license', help=f'Copyright license'
+    )
+
+    parser.add_argument(
+        '--publication-organization', help=f'Publication organization'
+    )
+
+    parser.add_argument(
+        '--publication-description', help=f'Publication description'
+    )
+
+    parser.add_argument(
+        '--publication-date', help=f'Publication date'
+    )
+
+    parser.add_argument(
+        '--document-url', help=f'Document URL'
+    )
+
+    parser.add_argument(
+        '--keyword', nargs='+', help=f'Keyword'
+    )
+
+    parser.add_argument(
+        '--identifier', '--id', help=f'Document ID'
+    )
+
+    parser.add_argument(
+        '-k', '--kind',
+        required=True,
+        help=f'Document MIME type'
+    )
+
+    parser.add_argument(
+        '--tags',
+        help=f'Tags, command separated'
+    )
+
+    parser.add_argument(
+        'files', nargs='+',
+        help=f'File to load'
+    )
+
+    args = parser.parse_args()
+
+    try:
+
+        document = DigitalDocument(
+            id,
+            name=args.name,
+            description=args.description,
+            copyright_notice=args.copyright_notice,
+            copyright_holder=args.copyright_holder,
+            copyright_year=args.copyright_year,
+            license=args.license,
+            url=args.document_url,
+            keywords=args.keyword,
+        )
+
+        if args.publication_organization:
+            org = Organization(
+                id=to_uri(PREF_ORG, hash(args.publication_organization)),
+                name=args.publication_organization,
+            )
+            document.publication = PublicationEvent(
+                id = to_uri(PREF_PUBEV, str(uuid.uuid4())),
+                organization=org,
+                description=args.publication_description,
+                start_date=args.publication_date,
+                end_date=args.publication_date,
+            )
+
+        p = Loader(
+            url=args.url,
+            user=args.user,
+            metadata=document,
+            title=args.name,
+            comments=args.description,
+            kind=args.kind,
+            tags=args.tags,
+        )
+
+        p.load(args.files)
+
+    except Exception as e:
+
+        print("Exception:", e, flush=True)
+
+main()
+

trustgraph-cli/scripts/tg-delete-flow-class

@@ -13,9 +13,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def delete_flow_class(url, class_name):
 
-    api = Api(url)
+    api = Api(url).flow()
 
-    class_names = api.flow_delete_class(class_name)
+    class_names = api.delete_class(class_name)
 
 def main():
 

trustgraph-cli/scripts/tg-get-flow-class

@@ -14,9 +14,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def get_flow_class(url, class_name):
 
-    api = Api(url)
+    api = Api(url).flow()
 
-    cls = api.flow_get_class(class_name)
+    cls = api.get_class(class_name)
 
     print(json.dumps(cls, indent=4))
 

trustgraph-cli/scripts/tg-graph-show

@@ -14,9 +14,9 @@ default_collection = 'default'
 
 def show_graph(url, flow_id, user, collection):
 
-    api = Api(url)
+    api = Api(url).flow().id(flow_id)
 
-    rows = api.flow(flow_id).triples_query(
+    rows = api.triples_query(
         user=user, collection=collection,
         s=None, p=None, o=None, limit=10_000,
     )

trustgraph-cli/scripts/tg-graph-to-turtle

@@ -19,9 +19,9 @@ default_collection = 'default'
 
 def show_graph(url, flow_id, user, collection):
 
-    api = Api(url)
+    api = Api(url).flow().id(flow_id)
 
-    rows = api.flow(flow_id).triples_query(
+    rows = api.triples_query(
         s=None, p=None, o=None,
         user=user, collection=collection,
         limit=10_000)

trustgraph-cli/scripts/tg-invoke-document-rag

@@ -15,9 +15,9 @@ default_doc_limit = 10
 
 def question(url, flow_id, question, user, collection, doc_limit):
 
-    api = Api(url)
+    api = Api(url).flow().id(flow_id)
 
-    resp = api.flow(flow_id).document_rag(
+    resp = api.document_rag(
         question=question, user=user, collection=collection,
         doc_limit=doc_limit,
     )

trustgraph-cli/scripts/tg-invoke-graph-rag

@@ -21,9 +21,9 @@ def question(
         max_subgraph_size, max_path_length
 ):
 
-    api = Api(url)
+    api = Api(url).flow().id(flow_id)
 
-    resp = api.flow(flow_id).graph_rag(
+    resp = api.graph_rag(
         question=question, user=user, collection=collection,
         entity_limit=entity_limit, triple_limit=triple_limit,
         max_subgraph_size=max_subgraph_size,

trustgraph-cli/scripts/tg-invoke-llm

@@ -14,9 +14,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def query(url, flow_id, system, prompt):
 
-    api = Api(url)
+    api = Api(url).flow().id(flow_id).
 
-    resp = api.flow(flow_id).text_completion(system=system, prompt=prompt)
+    resp = api.text_completion(system=system, prompt=prompt)
 
     print(resp)
 

trustgraph-cli/scripts/tg-invoke-prompt

@@ -18,9 +18,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def query(url, flow_id, template_id, variables):
 
-    api = Api(url)
+    api = Api(url).flow().id(flow_id)
 
-    resp = api.flow(flow_id).prompt(id=template_id, variables=variables)
+    resp = api.prompt(id=template_id, variables=variables)
 
     if isinstance(resp, str):
         print(resp)

trustgraph-cli/scripts/tg-load-pdf

@@ -4,8 +4,6 @@
 Loads a PDF document into TrustGraph processing.
 """
 
-import pulsar
-from pulsar.schema import JsonSchema
 import hashlib
 import argparse
 import os
@@ -31,10 +29,9 @@ class Loader:
             user,
             collection,
             metadata,
-            pulsar_api_key=None,
     ):
 
-        self.api = Api(url).flow(flow_id)
+        self.api = Api(url).flow().id(flow_id)
 
         self.user = user
         self.collection = collection

trustgraph-cli/scripts/tg-load-text

@@ -33,7 +33,7 @@ class Loader:
             metadata,
     ):
 
-        self.api = Api(url).flow(flow_id)
+        self.api = Api(url).flow().id(flow_id)
 
         self.user = user
         self.collection = collection

trustgraph-cli/scripts/tg-load-turtle

@@ -2,6 +2,8 @@
 
 """
 Loads Graph embeddings into TrustGraph processing.
+
+FIXME: This hasn't been updated following API gateway change.
 """
 
 import pulsar

trustgraph-cli/scripts/tg-put-flow-class

@@ -6,7 +6,6 @@ Dumps out the current configuration
 
 import argparse
 import os
-import tabulate
 from trustgraph.api import Api
 import json
 
@@ -16,7 +15,7 @@ def put_flow_class(url, class_name, config):
 
     api = Api(url)
 
-    class_names = api.flow_put_class(class_name, config)
+    class_names = api.flow().put_class(class_name, config)
 
 def main():
 

trustgraph-cli/scripts/tg-remove-library-document

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+"""
+Remove a PDF document from the library
+"""
+
+import argparse
+import os
+import uuid
+
+from trustgraph.api import Api
+
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
+default_user = 'trustgraph'
+
+
+def remove_doc(url, user, id):
+
+    api = Api(url).library()
+
+    api.remove_document(user=user, id=id)
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        prog='tg-remove-library-document',
+        description=__doc__,
+    )
+
+    parser.add_argument(
+        '-u', '--url',
+        default=default_url,
+        help=f'API URL (default: {default_url})',
+    )
+
+    parser.add_argument(
+        '-U', '--user',
+        default=default_user,
+        help=f'User ID (default: {default_user})'
+    )
+
+    parser.add_argument(
+        '--identifier', '--id',
+        required=True,
+        help=f'Document ID'
+    )
+
+    args = parser.parse_args()
+
+    try:
+
+        remove_doc(args.url, args.user, args.identifier)
+
+    except Exception as e:
+
+        print("Exception:", e, flush=True)
+
+main()
+

trustgraph-cli/scripts/tg-set-prompt

@@ -15,9 +15,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def set_system(url, system):
 
-    api = Api(url)
+    api = Api(url).config()
 
-    api.config_put([
+    api.put([
         ConfigValue(type="prompt", key="system", value=json.dumps(system))
     ])
 

trustgraph-cli/scripts/tg-set-token-costs

@@ -15,9 +15,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def set_costs(api_url, model, input_costs, output_costs):
 
-    api = Api(api_url)
+    api = Api(api_url).config()
 
-    api.config_put([
+    api.put([
         ConfigValue(
             type="token-costs", key=model,
             value=json.dumps({

trustgraph-cli/scripts/tg-show-config

@@ -13,9 +13,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def show_config(url):
 
-    api = Api(url)
+    api = Api(url).config()
 
-    config, version = api.config_all()
+    config, version = api.all()
 
     print("Version:", version)
     print(json.dumps(config, indent=4))

trustgraph-cli/scripts/tg-show-flow-classes

@@ -13,9 +13,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def show_flow_classes(url):
 
-    api = Api(url)
+    api = Api(url).flow()
 
-    class_names = api.flow_list_classes()
+    class_names = api.list_classes()
 
     if len(class_names) == 0:
         print("No flows.")
@@ -24,7 +24,7 @@ def show_flow_classes(url):
     classes = []
 
     for class_name in class_names:
-        cls = api.flow_get_class(class_name)
+        cls = api.get_class(class_name)
         classes.append((
             class_name,
             cls.get("description", ""),

trustgraph-cli/scripts/tg-show-flow-state

@@ -14,9 +14,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def dump_status(metrics_url, api_url, flow_id):
 
-    api = Api(api_url)
+    api = Api(api_url).flow()
 
-    flow = api.flow_get(flow_id)
+    flow = api.get(flow_id)
     class_name = flow["class-name"]
 
     print()

trustgraph-cli/scripts/tg-show-flows

@@ -11,11 +11,11 @@ import json
 
 default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
-def get_interface(api, i):
+def get_interface(config_api, i):
 
     key = ConfigKey("interface-descriptions", i)
 
-    value = api.config_get([key])[0].value
+    value = config_api.get([key])[0].value
 
     return json.loads(value)
 
@@ -49,15 +49,17 @@ def describe_interfaces(intdefs, flow):
 def show_flows(url):
 
     api = Api(url)
+    config_api = api.config()
+    flow_api = api.flow()
 
-    interface_names = api.config_list("interface-descriptions")
+    interface_names = config_api.list("interface-descriptions")
 
     interface_defs = {
-        i: get_interface(api, i)
+        i: get_interface(config_api, i)
         for i in interface_names
     }
 
-    flow_ids = api.flow_list()
+    flow_ids = flow_api.list()
 
     if len(flow_ids) == 0:
         print("No flows.")
@@ -67,7 +69,7 @@ def show_flows(url):
 
     for id in flow_ids:
 
-        flow = api.flow_get(id)
+        flow = flow_api.get(id)
 
         table = []
         table.append(("id", id))

trustgraph-cli/scripts/tg-show-library-documents

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+"""
+"""
+
+import argparse
+import os
+import tabulate
+from trustgraph.api import Api, ConfigKey
+import json
+
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
+default_user = "trustgraph"
+
+def show_docs(url, user):
+
+    api = Api(url).library()
+
+    docs = api.get_documents(user=user)
+
+    if len(docs) == 0:
+        print("No documents.")
+        return
+
+    for doc in docs:
+
+        table = []
+        table.append(("id", doc.id))
+        table.append(("time", doc.time))
+        table.append(("title", doc.title))
+        table.append(("kind", doc.kind))
+        table.append(("comments", doc.comments))
+        table.append(("tags", ", ".join(doc.tags)))
+
+        print(tabulate.tabulate(
+            table,
+            tablefmt="pretty",
+            stralign="left",
+        ))
+        print()
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        prog='tg-show-library-documents',
+        description=__doc__,
+    )
+
+    parser.add_argument(
+        '-u', '--api-url',
+        default=default_url,
+        help=f'API URL (default: {default_url})',
+    )
+
+    parser.add_argument(
+        '-U', '--user',
+        default=default_user,
+        help=f'User ID (default: {default_user})'
+    )
+
+    args = parser.parse_args()
+
+    try:
+
+        show_docs(
+            url = args.api_url, user = args.user
+        )
+
+    except Exception as e:
+
+        print("Exception:", e, flush=True)
+
+main()
+

trustgraph-cli/scripts/tg-show-library-processing

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+"""
+"""
+
+import argparse
+import os
+import tabulate
+from trustgraph.api import Api, ConfigKey
+import json
+
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
+default_user = "trustgraph"
+
+def show_procs(url, user):
+
+    api = Api(url).library()
+
+    procs = api.get_processings(user = user)
+
+    if len(procs) == 0:
+        print("No processing objects.")
+        return
+
+    for proc in procs:
+
+        table = []
+        table.append(("id", proc.id))
+        table.append(("document-id", proc.document_id))
+        table.append(("time", proc.time))
+        table.append(("flow", proc.flow))
+        table.append(("collection", proc.collection))
+        table.append(("tags", ", ".join(proc.tags)))
+
+        print(tabulate.tabulate(
+            table,
+            tablefmt="pretty",
+            stralign="left",
+        ))
+        print()
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        prog='tg-show-library-processing',
+        description=__doc__,
+    )
+
+    parser.add_argument(
+        '-u', '--api-url',
+        default=default_url,
+        help=f'API URL (default: {default_url})',
+    )
+
+    parser.add_argument(
+        '-U', '--user',
+        default=default_user,
+        help=f'User ID (default: {default_user})'
+    )
+
+    args = parser.parse_args()
+
+    try:
+
+        show_procs(
+            url = args.api_url, user = args.user
+        )
+
+    except Exception as e:
+
+        print("Exception:", e, flush=True)
+
+main()
+

trustgraph-cli/scripts/tg-show-prompts

@@ -15,9 +15,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def show_config(url):
 
-    api = Api(url)
+    api = Api(url).config()
 
-    values = api.config_get([
+    values = api.get([
         ConfigKey(type="prompt", key="system"),
         ConfigKey(type="prompt", key="template-index")
     ])
@@ -25,7 +25,7 @@ def show_config(url):
     system = json.loads(values[0].value)
     ix = json.loads(values[1].value)
 
-    values = api.config_get([
+    values = api.get([
         ConfigKey(type="prompt", key=f"template.{v}")
         for v in ix
     ])

trustgraph-cli/scripts/tg-show-token-costs

@@ -17,9 +17,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def show_config(url):
 
-    api = Api(url)
+    api = Api(url).config()
 
-    models = api.config_list("token-costs")
+    models = api.list("token-costs")
 
     costs = []
 
@@ -29,7 +29,7 @@ def show_config(url):
     for model in models:
 
         try:
-            values = json.loads(api.config_get([
+            values = json.loads(api.get([
                 ConfigKey(type="token-costs", key=model),
             ])[0].value)
             costs.append((

trustgraph-cli/scripts/tg-show-tools

@@ -15,15 +15,15 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def show_config(url):
 
-    api = Api(url)
+    api = Api(url).config()
 
-    values = api.config_get([
+    values = api.get([
         ConfigKey(type="agent", key="tool-index")
     ])
 
     ix = json.loads(values[0].value)
 
-    values = api.config_get([
+    values = api.get([
         ConfigKey(type="agent", key=f"tool.{v}")
         for v in ix
     ])

trustgraph-cli/scripts/tg-start-flow

@@ -13,9 +13,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def start_flow(url, class_name, flow_id, description):
 
-    api = Api(url)
+    api = Api(url).flow()
 
-    api.flow_start(
+    api.start(
         class_name = class_name,
         id = flow_id,
         description = description,

trustgraph-cli/scripts/tg-start-library-processing

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+"""
+"""
+
+import argparse
+import os
+import tabulate
+from trustgraph.api import Api, ConfigKey
+import json
+
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
+default_user = "trustgraph"
+
+def start_processing(
+        url, user, document_id, id, flow, collection, tags
+):
+
+    api = Api(url).library()
+
+    if tags:
+        tags = tags.split(",")
+    else:
+        tags = []
+
+    api.start_processing(
+        id = id,
+        document_id = document_id,
+        flow = flow,
+        user = user,
+        collection = collection,
+        tags = tags
+    )
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        prog='tg-start-library-processing',
+        description=__doc__,
+    )
+
+    parser.add_argument(
+        '-u', '--api-url',
+        default=default_url,
+        help=f'API URL (default: {default_url})',
+    )
+
+    parser.add_argument(
+        '-U', '--user',
+        default=default_user,
+        help=f'User ID (default: {default_user})'
+    )
+
+    parser.add_argument(
+        '-i', '--flow-id',
+        default="0000",
+        help=f'Flow ID (default: 0000)',
+    )
+
+    parser.add_argument(
+        '-d', '--document-id',
+        required=True,
+        help=f'Document ID',
+    )
+
+    parser.add_argument(
+        '--id', '--processing-id',
+        required=True,
+        help=f'Processing ID',
+    )
+
+    parser.add_argument(
+        '--collection',
+        default='default',
+        help=f'Collection (default: default)'
+    )
+
+    parser.add_argument(
+        '--tags',
+        help=f'Tags, command separated'
+    )
+
+    args = parser.parse_args()
+
+    try:
+
+        start_processing(
+            url = args.api_url,
+            user = args.user,
+            document_id = args.document_id,
+            id = args.id,
+            flow = args.flow_id,
+            collection = args.collection,
+            tags = args.tags
+        )
+
+    except Exception as e:
+
+        print("Exception:", e, flush=True)
+
+main()
+

trustgraph-cli/scripts/tg-stop-flow

@@ -13,9 +13,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 
 def stop_flow(url, flow_id):
 
-    api = Api(url)
+    api = Api(url).flow()
 
-    api.flow_stop(id = flow_id)
+    api.stop(id = flow_id)
 
 def main():
 

trustgraph-cli/scripts/tg-stop-library-processing

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+"""
+"""
+
+import argparse
+import os
+import tabulate
+from trustgraph.api import Api, ConfigKey
+import json
+
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
+default_user = "trustgraph"
+
+def stop_processing(
+        url, user, id
+):
+
+    api = Api(url).library()
+
+    api.stop_processing(user = user, id = id)
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        prog='tg-stop-library-processing',
+        description=__doc__,
+    )
+
+    parser.add_argument(
+        '-u', '--api-url',
+        default=default_url,
+        help=f'API URL (default: {default_url})',
+    )
+
+    parser.add_argument(
+        '-U', '--user',
+        default=default_user,
+        help=f'User ID (default: {default_user})'
+    )
+
+    parser.add_argument(
+        '--id', '--processing-id',
+        required=True,
+        help=f'Processing ID',
+    )
+
+    args = parser.parse_args()
+
+    try:
+
+        stop_processing(
+            url = args.api_url,
+            user = args.user,
+            id = args.id,
+        )
+
+    except Exception as e:
+
+        print("Exception:", e, flush=True)
+
+main()
+