mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-20 10:41:02 +02:00
Some library CLI
This commit is contained in:
Cyber MacGeddon
parent
8146f0f2ff
commit
9a47915d05
5 changed files with 392 additions and 7 deletions
|
|
@ -3,6 +3,9 @@ import requests
|
|||
import json
|
||||
import dataclasses
|
||||
import base64
|
||||
from typing import List
|
||||
import datetime
|
||||
import time
|
||||
|
||||
from trustgraph.knowledge import hash, Uri, Literal
|
||||
|
||||
|
|
@ -29,6 +32,31 @@ class ConfigValue:
|
|||
key : str
|
||||
value : str
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocumentMetadata:
|
||||
id : str
|
||||
time : datetime.datetime
|
||||
kind : str
|
||||
title : str
|
||||
comments : str
|
||||
metadata : List[Triple]
|
||||
user : str
|
||||
tags : List[str]
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ProcessingMetadata:
|
||||
id : str
|
||||
document_id : str
|
||||
time : datetime.datetime
|
||||
flow : str
|
||||
user : str
|
||||
collection : str
|
||||
tags : List[str]
|
||||
|
||||
def to_value(x):
|
||||
if x["e"]: return Uri(x["v"])
|
||||
return Literal(x["v"])
|
||||
|
||||
def check_error(response):
|
||||
|
||||
if "error" in response:
|
||||
|
|
@ -90,6 +118,93 @@ class Api:
|
|||
except:
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
def library_add_document(
|
||||
self, document, id, metadata, user, title, comments,
|
||||
kind="text/plain", tags=[],
|
||||
):
|
||||
|
||||
if id is None:
|
||||
|
||||
if metadata is not None:
|
||||
|
||||
# Situation makes no sense. What can the metadata possibly
|
||||
# mean if the caller doesn't know the document ID.
|
||||
# Metadata should relate to the document by ID
|
||||
raise RuntimeError("Can't specify metadata without id")
|
||||
|
||||
id = hash(document)
|
||||
|
||||
if not title: title = ""
|
||||
if not comments: comments = ""
|
||||
|
||||
triples = []
|
||||
|
||||
def emit(t):
|
||||
triples.append(t)
|
||||
|
||||
if metadata:
|
||||
metadata.emit(
|
||||
lambda t: triples.append({
|
||||
"s": { "v": t["s"], "e": isinstance(t["s"], Uri) },
|
||||
"p": { "v": t["p"], "e": isinstance(t["p"], Uri) },
|
||||
"o": { "v": t["o"], "e": isinstance(t["o"], Uri) }
|
||||
})
|
||||
)
|
||||
|
||||
input = {
|
||||
"operation": "add-document",
|
||||
"document-metadata": {
|
||||
"id": id,
|
||||
"time": int(time.time()),
|
||||
"kind": kind,
|
||||
"title": title,
|
||||
"comments": comments,
|
||||
"metadata": triples,
|
||||
"user": user,
|
||||
"tags": tags
|
||||
},
|
||||
"content": base64.b64encode(document).decode("utf-8"),
|
||||
}
|
||||
|
||||
return self.request(
|
||||
f"librarian",
|
||||
input
|
||||
)
|
||||
|
||||
|
||||
def library_get_documents(self, user):
|
||||
|
||||
input = {
|
||||
"operation": "list-documents",
|
||||
"user": user,
|
||||
}
|
||||
|
||||
object = self.request("librarian", input)
|
||||
|
||||
try:
|
||||
return [
|
||||
DocumentMetadata(
|
||||
id = v["id"],
|
||||
time = datetime.datetime.fromtimestamp(v["time"]),
|
||||
kind = v["kind"],
|
||||
title = v["title"],
|
||||
comments = v.get("comments", ""),
|
||||
metadata = [
|
||||
Triple(
|
||||
s = to_value(w["s"]),
|
||||
p = to_value(w["p"]),
|
||||
o = to_value(w["o"])
|
||||
)
|
||||
for w in v["metadata"]
|
||||
],
|
||||
user = v["user"],
|
||||
tags = v["tags"]
|
||||
)
|
||||
for v in object["document-metadatas"]
|
||||
]
|
||||
except:
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
def config_get(self, keys):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
|
|
@ -386,10 +501,6 @@ class Flow:
|
|||
input
|
||||
)
|
||||
|
||||
def to_value(x):
|
||||
if x["e"]: return Uri(x["v"])
|
||||
return Literal(x["v"])
|
||||
|
||||
return [
|
||||
Triple(
|
||||
s=to_value(t["s"]),
|
||||
|
|
|
|||
201
trustgraph-cli/scripts/tg-add-library-document
Executable file
201
trustgraph-cli/scripts/tg-add-library-document
Executable file
|
|
@ -0,0 +1,201 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Loads a PDF document into the library
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import argparse
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from trustgraph.api import Api
|
||||
from trustgraph.knowledge import hash, to_uri
|
||||
from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG
|
||||
from trustgraph.knowledge import Organization, PublicationEvent
|
||||
from trustgraph.knowledge import DigitalDocument
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = 'trustgraph'
|
||||
|
||||
class Loader:
|
||||
|
||||
def __init__(
|
||||
self, url, user, metadata, title, comments, kind, tags
|
||||
):
|
||||
|
||||
self.api = Api(url)
|
||||
|
||||
self.user = user
|
||||
self.metadata = metadata
|
||||
self.title = title
|
||||
self.comments = comments
|
||||
self.kind = kind
|
||||
|
||||
if tags:
|
||||
self.tags = tags.split(",")
|
||||
else:
|
||||
self.tags = []
|
||||
|
||||
def load(self, files):
|
||||
|
||||
for file in files:
|
||||
self.load_file(file)
|
||||
|
||||
def load_file(self, file):
|
||||
|
||||
try:
|
||||
|
||||
path = file
|
||||
data = open(path, "rb").read()
|
||||
|
||||
# Create a SHA256 hash from the data
|
||||
id = hash(data)
|
||||
|
||||
id = to_uri(PREF_DOC, id)
|
||||
|
||||
self.metadata.id = id
|
||||
|
||||
self.api.library_add_document(
|
||||
document=data, id=id, metadata=self.metadata,
|
||||
user=self.user, kind=self.kind, title=self.title,
|
||||
comments=self.comments, tags=self.tags
|
||||
)
|
||||
|
||||
print(f"{file}: Loaded successfully.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"{file}: Failed: {str(e)}", flush=True)
|
||||
raise e
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-add-library-document',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--name', help=f'Document name'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--description', help=f'Document description'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--copyright-notice', help=f'Copyright notice'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--copyright-holder', help=f'Copyright holder'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--copyright-year', help=f'Copyright year'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--license', help=f'Copyright license'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--publication-organization', help=f'Publication organization'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--publication-description', help=f'Publication description'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--publication-date', help=f'Publication date'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--document-url', help=f'Document URL'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--keyword', nargs='+', help=f'Keyword'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--identifier', '--id', help=f'Document ID'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-k', '--kind',
|
||||
required=True,
|
||||
help=f'Document MIME type'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--tags',
|
||||
help=f'Tags, command separated'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'files', nargs='+',
|
||||
help=f'File to load'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
document = DigitalDocument(
|
||||
id,
|
||||
name=args.name,
|
||||
description=args.description,
|
||||
copyright_notice=args.copyright_notice,
|
||||
copyright_holder=args.copyright_holder,
|
||||
copyright_year=args.copyright_year,
|
||||
license=args.license,
|
||||
url=args.document_url,
|
||||
keywords=args.keyword,
|
||||
)
|
||||
|
||||
if args.publication_organization:
|
||||
org = Organization(
|
||||
id=to_uri(PREF_ORG, hash(args.publication_organization)),
|
||||
name=args.publication_organization,
|
||||
)
|
||||
document.publication = PublicationEvent(
|
||||
id = to_uri(PREF_PUBEV, str(uuid.uuid4())),
|
||||
organization=org,
|
||||
description=args.publication_description,
|
||||
start_date=args.publication_date,
|
||||
end_date=args.publication_date,
|
||||
)
|
||||
|
||||
p = Loader(
|
||||
url=args.url,
|
||||
user=args.user,
|
||||
metadata=document,
|
||||
title=args.name,
|
||||
comments=args.description,
|
||||
kind=args.kind,
|
||||
tags=args.tags,
|
||||
)
|
||||
|
||||
p.load(args.files)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
|
|
@ -4,8 +4,6 @@
|
|||
Loads a PDF document into TrustGraph processing.
|
||||
"""
|
||||
|
||||
import pulsar
|
||||
from pulsar.schema import JsonSchema
|
||||
import hashlib
|
||||
import argparse
|
||||
import os
|
||||
|
|
@ -31,7 +29,6 @@ class Loader:
|
|||
user,
|
||||
collection,
|
||||
metadata,
|
||||
pulsar_api_key=None,
|
||||
):
|
||||
|
||||
self.api = Api(url).flow(flow_id)
|
||||
|
|
|
|||
74
trustgraph-cli/scripts/tg-show-library-documents
Normal file
74
trustgraph-cli/scripts/tg-show-library-documents
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import tabulate
|
||||
from trustgraph.api import Api, ConfigKey
|
||||
import json
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = "trustgraph"
|
||||
|
||||
def show_docs(url, user):
|
||||
|
||||
api = Api(url)
|
||||
|
||||
docs = api.library_get_documents(user=user)
|
||||
|
||||
if len(docs) == 0:
|
||||
print("No documents.")
|
||||
return
|
||||
|
||||
for doc in docs:
|
||||
|
||||
table = []
|
||||
table.append(("id", doc.id))
|
||||
table.append(("time", doc.time))
|
||||
table.append(("title", doc.title))
|
||||
table.append(("kind", doc.kind))
|
||||
table.append(("comments", doc.comments))
|
||||
table.append(("tags", ", ".join(doc.tags)))
|
||||
|
||||
print(tabulate.tabulate(
|
||||
table,
|
||||
tablefmt="pretty",
|
||||
stralign="left",
|
||||
))
|
||||
print()
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-show-library-documents',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--api-url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
show_docs(
|
||||
url = args.api_url, user = args.user
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
|
|
@ -70,6 +70,8 @@ setuptools.setup(
|
|||
"scripts/tg-show-flow-classes",
|
||||
"scripts/tg-show-flow-state",
|
||||
"scripts/tg-show-flows",
|
||||
"scripts/tg-show-library-documents",
|
||||
"scripts/tg-add-library-document",
|
||||
"scripts/tg-show-processor-state",
|
||||
"scripts/tg-show-prompts",
|
||||
"scripts/tg-show-token-costs",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue