mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-06-09 06:45:13 +02:00
Feature/library cli (#363)
* Major Python client API rework, break down API & colossal class * Complete rest of library API * Library CLI support
This commit is contained in:
cybermaggedon
GitHub
parent
8146f0f2ff
commit
844547ab5f
36 changed files with 1413 additions and 495 deletions
|
|
@ -1,33 +1,14 @@
|
|||
|
||||
import requests
|
||||
import json
|
||||
import dataclasses
|
||||
import base64
|
||||
import time
|
||||
|
||||
from trustgraph.knowledge import hash, Uri, Literal
|
||||
|
||||
class ProtocolException(Exception):
|
||||
pass
|
||||
|
||||
class ApplicationException(Exception):
|
||||
pass
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Triple:
|
||||
s : str
|
||||
p : str
|
||||
o : str
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ConfigKey:
|
||||
type : str
|
||||
key : str
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ConfigValue:
|
||||
type : str
|
||||
key : str
|
||||
value : str
|
||||
from . library import Library
|
||||
from . flow import Flow
|
||||
from . config import Config
|
||||
from . exceptions import *
|
||||
from . types import *
|
||||
|
||||
def check_error(response):
|
||||
|
||||
|
|
@ -52,13 +33,19 @@ class Api:
|
|||
|
||||
self.url += "api/v1/"
|
||||
|
||||
def flow(self, flow="0000"):
|
||||
return Flow(api=self, flow=flow)
|
||||
def flow(self):
|
||||
return Flow(api=self)
|
||||
|
||||
def config(self):
|
||||
return Config(api=self)
|
||||
|
||||
def request(self, path, request):
|
||||
|
||||
url = f"{self.url}{path}"
|
||||
|
||||
# print("uri:", url)
|
||||
# print(json.dumps(request, indent=4))
|
||||
|
||||
# Invoke the API, input is passed as JSON
|
||||
resp = requests.post(url, json=request)
|
||||
|
||||
|
|
@ -66,6 +53,8 @@ class Api:
|
|||
if resp.status_code != 200:
|
||||
raise ProtocolException(f"Status code {resp.status_code}")
|
||||
|
||||
# print(resp.text)
|
||||
|
||||
try:
|
||||
# Parse the response as JSON
|
||||
object = resp.json()
|
||||
|
|
@ -76,418 +65,5 @@ class Api:
|
|||
|
||||
return object
|
||||
|
||||
def config_all(self):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "config"
|
||||
}
|
||||
|
||||
object = self.request("config", input)
|
||||
|
||||
try:
|
||||
return object["config"], object["version"]
|
||||
except:
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
def config_get(self, keys):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "get",
|
||||
"keys": [
|
||||
{ "type": k.type, "key": k.key }
|
||||
for k in keys
|
||||
]
|
||||
}
|
||||
|
||||
object = self.request("config", input)
|
||||
|
||||
try:
|
||||
return [
|
||||
ConfigValue(
|
||||
type = v["type"],
|
||||
key = v["key"],
|
||||
value = v["value"]
|
||||
)
|
||||
for v in object["values"]
|
||||
]
|
||||
except:
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
def config_put(self, values):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "put",
|
||||
"values": [
|
||||
{ "type": v.type, "key": v.key, "value": v.value }
|
||||
for v in values
|
||||
]
|
||||
}
|
||||
|
||||
self.request("config", input)
|
||||
|
||||
def config_list(self, type):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "list",
|
||||
"type": type,
|
||||
}
|
||||
|
||||
return self.request("config", input)["directory"]
|
||||
|
||||
def config_getvalues(self, type):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "getvalues",
|
||||
"type": type,
|
||||
}
|
||||
|
||||
object = self.request("config", input)["directory"]
|
||||
|
||||
try:
|
||||
return [
|
||||
ConfigValue(
|
||||
type = v["type"],
|
||||
key = v["key"],
|
||||
value = v["value"]
|
||||
)
|
||||
for v in object["values"]
|
||||
]
|
||||
except:
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
def flow_list_classes(self):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "list-classes",
|
||||
}
|
||||
|
||||
return self.request("flow", input)["class-names"]
|
||||
|
||||
def flow_get_class(self, class_name):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "get-class",
|
||||
"class-name": class_name,
|
||||
}
|
||||
|
||||
return json.loads(self.request("flow", input)["class-definition"])
|
||||
|
||||
def flow_put_class(self, class_name, definition):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "put-class",
|
||||
"class-name": class_name,
|
||||
"class-definition": json.dumps(definition),
|
||||
}
|
||||
|
||||
self.request("flow", input)
|
||||
|
||||
def flow_delete_class(self, class_name):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "delete-class",
|
||||
"class-name": class_name,
|
||||
}
|
||||
|
||||
self.request("flow", input)
|
||||
|
||||
def flow_list(self):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "list-flows",
|
||||
}
|
||||
|
||||
return self.request("flow", input)["flow-ids"]
|
||||
|
||||
def flow_get(self, id):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "get-flow",
|
||||
"flow-id": id,
|
||||
}
|
||||
|
||||
return json.loads(self.request("flow", input)["flow"])
|
||||
|
||||
def flow_start(self, class_name, id, description):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "start-flow",
|
||||
"flow-id": id,
|
||||
"class-name": class_name,
|
||||
"description": description,
|
||||
}
|
||||
|
||||
self.request("flow", input)
|
||||
|
||||
def flow_stop(self, id):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "stop-flow",
|
||||
"flow-id": id,
|
||||
}
|
||||
|
||||
self.request("flow", input)
|
||||
|
||||
class Flow:
|
||||
|
||||
def __init__(self, api, flow):
|
||||
self.api = api
|
||||
self.flow = flow
|
||||
|
||||
def text_completion(self, system, prompt):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"system": system,
|
||||
"prompt": prompt
|
||||
}
|
||||
|
||||
return self.api.request(
|
||||
f"flow/{self.flow}/service/text-completion",
|
||||
input
|
||||
)["response"]
|
||||
|
||||
def agent(self, question):
|
||||
|
||||
# The input consists of a question
|
||||
input = {
|
||||
"question": question
|
||||
}
|
||||
|
||||
return self.api.request(
|
||||
f"flow/{self.flow}/service/agent",
|
||||
input
|
||||
)["answer"]
|
||||
|
||||
def graph_rag(
|
||||
self, question, user="trustgraph", collection="default",
|
||||
entity_limit=50, triple_limit=30, max_subgraph_size=150,
|
||||
max_path_length=2,
|
||||
):
|
||||
|
||||
# The input consists of a question
|
||||
input = {
|
||||
"query": question,
|
||||
"user": user,
|
||||
"collection": collection,
|
||||
"entity-limit": entity_limit,
|
||||
"triple-limit": triple_limit,
|
||||
"max-subgraph-size": max_subgraph_size,
|
||||
"max-path-length": max_path_length,
|
||||
}
|
||||
|
||||
return self.api.request(
|
||||
f"flow/{self.flow}/service/graph-rag",
|
||||
input
|
||||
)["response"]
|
||||
|
||||
def document_rag(
|
||||
self, question, user="trustgraph", collection="default",
|
||||
doc_limit=10,
|
||||
):
|
||||
|
||||
# The input consists of a question
|
||||
input = {
|
||||
"query": question,
|
||||
"user": user,
|
||||
"collection": collection,
|
||||
"doc-limit": doc_limit,
|
||||
}
|
||||
|
||||
return self.api.request(
|
||||
f"flow/{self.flow}/service/document-rag",
|
||||
input
|
||||
)["response"]
|
||||
|
||||
def embeddings(self, text):
|
||||
|
||||
# The input consists of a text block
|
||||
input = {
|
||||
"text": text
|
||||
}
|
||||
|
||||
return self.api.request(
|
||||
f"flow/{self.flow}/service/embeddings",
|
||||
input
|
||||
)["vectors"]
|
||||
|
||||
def prompt(self, id, variables):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"id": id,
|
||||
"variables": variables
|
||||
}
|
||||
|
||||
object = self.api.request(
|
||||
f"flow/{self.flow}/service/prompt",
|
||||
input
|
||||
)
|
||||
|
||||
if "text" in object:
|
||||
return object["text"]
|
||||
|
||||
if "object" in object:
|
||||
try:
|
||||
return json.loads(object["object"])
|
||||
except Exception as e:
|
||||
raise ProtocolException(
|
||||
"Returned object not well-formed JSON"
|
||||
)
|
||||
|
||||
raise ProtocolException("Response not formatted correctly")
|
||||
|
||||
def triples_query(
|
||||
self, s=None, p=None, o=None,
|
||||
user=None, collection=None, limit=10000
|
||||
):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"limit": limit
|
||||
}
|
||||
|
||||
if user:
|
||||
input["user"] = user
|
||||
|
||||
if collection:
|
||||
input["collection"] = collection
|
||||
|
||||
if s:
|
||||
if not isinstance(s, Uri):
|
||||
raise RuntimeError("s must be Uri")
|
||||
input["s"] = { "v": str(s), "e": isinstance(s, Uri), }
|
||||
|
||||
if p:
|
||||
if not isinstance(p, Uri):
|
||||
raise RuntimeError("p must be Uri")
|
||||
input["p"] = { "v": str(p), "e": isinstance(p, Uri), }
|
||||
|
||||
if o:
|
||||
if not isinstance(o, Uri) and not isinstance(o, Literal):
|
||||
raise RuntimeError("o must be Uri or Literal")
|
||||
input["o"] = { "v": str(o), "e": isinstance(o, Uri), }
|
||||
|
||||
object = self.api.request(
|
||||
f"flow/{self.flow}/service/triples",
|
||||
input
|
||||
)
|
||||
|
||||
def to_value(x):
|
||||
if x["e"]: return Uri(x["v"])
|
||||
return Literal(x["v"])
|
||||
|
||||
return [
|
||||
Triple(
|
||||
s=to_value(t["s"]),
|
||||
p=to_value(t["p"]),
|
||||
o=to_value(t["o"])
|
||||
)
|
||||
for t in object["response"]
|
||||
]
|
||||
|
||||
def load_document(
|
||||
self, document, id=None, metadata=None, user=None,
|
||||
collection=None,
|
||||
):
|
||||
|
||||
if id is None:
|
||||
|
||||
if metadata is not None:
|
||||
|
||||
# Situation makes no sense. What can the metadata possibly
|
||||
# mean if the caller doesn't know the document ID.
|
||||
# Metadata should relate to the document by ID
|
||||
raise RuntimeError("Can't specify metadata without id")
|
||||
|
||||
id = hash(document)
|
||||
|
||||
triples = []
|
||||
|
||||
def emit(t):
|
||||
triples.append(t)
|
||||
|
||||
if metadata:
|
||||
metadata.emit(
|
||||
lambda t: triples.append({
|
||||
"s": { "v": t["s"], "e": isinstance(t["s"], Uri) },
|
||||
"p": { "v": t["p"], "e": isinstance(t["p"], Uri) },
|
||||
"o": { "v": t["o"], "e": isinstance(t["o"], Uri) }
|
||||
})
|
||||
)
|
||||
|
||||
input = {
|
||||
"id": id,
|
||||
"metadata": triples,
|
||||
"data": base64.b64encode(document).decode("utf-8"),
|
||||
}
|
||||
|
||||
if user:
|
||||
input["user"] = user
|
||||
|
||||
if collection:
|
||||
input["collection"] = collection
|
||||
|
||||
return self.api.request(
|
||||
f"flow/{self.flow}/service/document-load",
|
||||
input
|
||||
)
|
||||
|
||||
def load_text(
|
||||
self, text, id=None, metadata=None, charset="utf-8",
|
||||
user=None, collection=None,
|
||||
):
|
||||
|
||||
if id is None:
|
||||
|
||||
if metadata is not None:
|
||||
|
||||
# Situation makes no sense. What can the metadata possibly
|
||||
# mean if the caller doesn't know the document ID.
|
||||
# Metadata should relate to the document by ID
|
||||
raise RuntimeError("Can't specify metadata without id")
|
||||
|
||||
id = hash(text)
|
||||
|
||||
triples = []
|
||||
|
||||
if metadata:
|
||||
metadata.emit(
|
||||
lambda t: triples.append({
|
||||
"s": { "v": t["s"], "e": isinstance(t["s"], Uri) },
|
||||
"p": { "v": t["p"], "e": isinstance(t["p"], Uri) },
|
||||
"o": { "v": t["o"], "e": isinstance(t["o"], Uri) }
|
||||
})
|
||||
)
|
||||
|
||||
input = {
|
||||
"id": id,
|
||||
"metadata": triples,
|
||||
"charset": charset,
|
||||
"text": base64.b64encode(text).decode("utf-8"),
|
||||
}
|
||||
|
||||
if user:
|
||||
input["user"] = user
|
||||
|
||||
if collection:
|
||||
input["collection"] = collection
|
||||
|
||||
return self.api.request(
|
||||
f"flow/{self.flow}/service/text-load",
|
||||
input
|
||||
)
|
||||
|
||||
def library(self):
|
||||
return Library(self)
|
||||
|
|
|
|||
97
trustgraph-base/trustgraph/api/config.py
Normal file
97
trustgraph-base/trustgraph/api/config.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
|
||||
from . exceptions import *
|
||||
from . types import ConfigValue
|
||||
|
||||
class Config:
|
||||
|
||||
def __init__(self, api):
|
||||
self.api = api
|
||||
|
||||
def request(self, request):
|
||||
return self.api.request("config", request)
|
||||
|
||||
def get(self, keys):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "get",
|
||||
"keys": [
|
||||
{ "type": k.type, "key": k.key }
|
||||
for k in keys
|
||||
]
|
||||
}
|
||||
|
||||
object = self.request(input)
|
||||
|
||||
try:
|
||||
return [
|
||||
ConfigValue(
|
||||
type = v["type"],
|
||||
key = v["key"],
|
||||
value = v["value"]
|
||||
)
|
||||
for v in object["values"]
|
||||
]
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise ProtocolException("Response not formatted correctly")
|
||||
|
||||
def put(self, values):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "put",
|
||||
"values": [
|
||||
{ "type": v.type, "key": v.key, "value": v.value }
|
||||
for v in values
|
||||
]
|
||||
}
|
||||
|
||||
self.request(input)
|
||||
|
||||
def list(self, type):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "list",
|
||||
"type": type,
|
||||
}
|
||||
|
||||
return self.request(input)["directory"]
|
||||
|
||||
def get_values(self, type):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "getvalues",
|
||||
"type": type,
|
||||
}
|
||||
|
||||
object = self.request(input)["directory"]
|
||||
|
||||
try:
|
||||
return [
|
||||
ConfigValue(
|
||||
type = v["type"],
|
||||
key = v["key"],
|
||||
value = v["value"]
|
||||
)
|
||||
for v in object["values"]
|
||||
]
|
||||
except:
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
def all(self):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "config"
|
||||
}
|
||||
|
||||
object = self.request(input)
|
||||
|
||||
try:
|
||||
return object["config"], object["version"]
|
||||
except:
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
6
trustgraph-base/trustgraph/api/exceptions.py
Normal file
6
trustgraph-base/trustgraph/api/exceptions.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
|
||||
class ProtocolException(Exception):
|
||||
pass
|
||||
|
||||
class ApplicationException(Exception):
|
||||
pass
|
||||
359
trustgraph-base/trustgraph/api/flow.py
Normal file
359
trustgraph-base/trustgraph/api/flow.py
Normal file
|
|
@ -0,0 +1,359 @@
|
|||
|
||||
import json
|
||||
import base64
|
||||
|
||||
from .. knowledge import hash, Uri, Literal
|
||||
|
||||
def to_value(x):
|
||||
if x["e"]: return Uri(x["v"])
|
||||
return Literal(x["v"])
|
||||
|
||||
class Flow:
|
||||
|
||||
def __init__(self, api):
|
||||
self.api = api
|
||||
|
||||
def request(self, path=None, request=None):
|
||||
|
||||
if request is None:
|
||||
raise RuntimeError("request must be specified")
|
||||
|
||||
if path:
|
||||
return self.api.request(f"flow/{path}", request)
|
||||
else:
|
||||
return self.api.request(f"flow", request)
|
||||
|
||||
def id(self, id="0000"):
|
||||
return FlowInstance(api=self, id=id)
|
||||
|
||||
def list_classes(self):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "list-classes",
|
||||
}
|
||||
|
||||
return self.request(request = input)["class-names"]
|
||||
|
||||
def get_class(self, class_name):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "get-class",
|
||||
"class-name": class_name,
|
||||
}
|
||||
|
||||
return json.loads(self.request(request = input)["class-definition"])
|
||||
|
||||
def put_class(self, class_name, definition):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "put-class",
|
||||
"class-name": class_name,
|
||||
"class-definition": json.dumps(definition),
|
||||
}
|
||||
|
||||
self.request(request = input)
|
||||
|
||||
def delete_class(self, class_name):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "delete-class",
|
||||
"class-name": class_name,
|
||||
}
|
||||
|
||||
self.request(request = input)
|
||||
|
||||
def list(self):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "list-flows",
|
||||
}
|
||||
|
||||
return self.request(request = input)["flow-ids"]
|
||||
|
||||
def get(self, id):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "get-flow",
|
||||
"flow-id": id,
|
||||
}
|
||||
|
||||
return json.loads(self.request(request = input)["flow"])
|
||||
|
||||
def start(self, class_name, id, description):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "start-flow",
|
||||
"flow-id": id,
|
||||
"class-name": class_name,
|
||||
"description": description,
|
||||
}
|
||||
|
||||
self.request(request = input)
|
||||
|
||||
def stop(self, id):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"operation": "stop-flow",
|
||||
"flow-id": id,
|
||||
}
|
||||
|
||||
self.request(request = input)
|
||||
|
||||
class FlowInstance:
|
||||
|
||||
def __init__(self, api, id):
|
||||
self.api = api
|
||||
self.id = id
|
||||
|
||||
def request(self, path, request):
|
||||
|
||||
return self.api.request(path = f"{self.id}/{path}", request = request)
|
||||
|
||||
def text_completion(self, system, prompt):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"system": system,
|
||||
"prompt": prompt
|
||||
}
|
||||
|
||||
return self.request(
|
||||
"service/text-completion",
|
||||
input
|
||||
)["response"]
|
||||
|
||||
def agent(self, question):
|
||||
|
||||
# The input consists of a question
|
||||
input = {
|
||||
"question": question
|
||||
}
|
||||
|
||||
return self.request(
|
||||
"service/agent",
|
||||
input
|
||||
)["answer"]
|
||||
|
||||
def graph_rag(
|
||||
self, question, user="trustgraph", collection="default",
|
||||
entity_limit=50, triple_limit=30, max_subgraph_size=150,
|
||||
max_path_length=2,
|
||||
):
|
||||
|
||||
# The input consists of a question
|
||||
input = {
|
||||
"query": question,
|
||||
"user": user,
|
||||
"collection": collection,
|
||||
"entity-limit": entity_limit,
|
||||
"triple-limit": triple_limit,
|
||||
"max-subgraph-size": max_subgraph_size,
|
||||
"max-path-length": max_path_length,
|
||||
}
|
||||
|
||||
return self.request(
|
||||
"service/graph-rag",
|
||||
input
|
||||
)["response"]
|
||||
|
||||
def document_rag(
|
||||
self, question, user="trustgraph", collection="default",
|
||||
doc_limit=10,
|
||||
):
|
||||
|
||||
# The input consists of a question
|
||||
input = {
|
||||
"query": question,
|
||||
"user": user,
|
||||
"collection": collection,
|
||||
"doc-limit": doc_limit,
|
||||
}
|
||||
|
||||
return self.request(
|
||||
"service/document-rag",
|
||||
input
|
||||
)["response"]
|
||||
|
||||
def embeddings(self, text):
|
||||
|
||||
# The input consists of a text block
|
||||
input = {
|
||||
"text": text
|
||||
}
|
||||
|
||||
return self.request(
|
||||
"service/embeddings",
|
||||
input
|
||||
)["vectors"]
|
||||
|
||||
def prompt(self, id, variables):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"id": id,
|
||||
"variables": variables
|
||||
}
|
||||
|
||||
object = self.request(
|
||||
"service/prompt",
|
||||
input
|
||||
)
|
||||
|
||||
if "text" in object:
|
||||
return object["text"]
|
||||
|
||||
if "object" in object:
|
||||
try:
|
||||
return json.loads(object["object"])
|
||||
except Exception as e:
|
||||
raise ProtocolException(
|
||||
"Returned object not well-formed JSON"
|
||||
)
|
||||
|
||||
raise ProtocolException("Response not formatted correctly")
|
||||
|
||||
def triples_query(
|
||||
self, s=None, p=None, o=None,
|
||||
user=None, collection=None, limit=10000
|
||||
):
|
||||
|
||||
# The input consists of system and prompt strings
|
||||
input = {
|
||||
"limit": limit
|
||||
}
|
||||
|
||||
if user:
|
||||
input["user"] = user
|
||||
|
||||
if collection:
|
||||
input["collection"] = collection
|
||||
|
||||
if s:
|
||||
if not isinstance(s, Uri):
|
||||
raise RuntimeError("s must be Uri")
|
||||
input["s"] = { "v": str(s), "e": isinstance(s, Uri), }
|
||||
|
||||
if p:
|
||||
if not isinstance(p, Uri):
|
||||
raise RuntimeError("p must be Uri")
|
||||
input["p"] = { "v": str(p), "e": isinstance(p, Uri), }
|
||||
|
||||
if o:
|
||||
if not isinstance(o, Uri) and not isinstance(o, Literal):
|
||||
raise RuntimeError("o must be Uri or Literal")
|
||||
input["o"] = { "v": str(o), "e": isinstance(o, Uri), }
|
||||
|
||||
object = self.request(
|
||||
"service/triples",
|
||||
input
|
||||
)
|
||||
|
||||
return [
|
||||
Triple(
|
||||
s=to_value(t["s"]),
|
||||
p=to_value(t["p"]),
|
||||
o=to_value(t["o"])
|
||||
)
|
||||
for t in object["response"]
|
||||
]
|
||||
|
||||
def load_document(
|
||||
self, document, id=None, metadata=None, user=None,
|
||||
collection=None,
|
||||
):
|
||||
|
||||
if id is None:
|
||||
|
||||
if metadata is not None:
|
||||
|
||||
# Situation makes no sense. What can the metadata possibly
|
||||
# mean if the caller doesn't know the document ID.
|
||||
# Metadata should relate to the document by ID
|
||||
raise RuntimeError("Can't specify metadata without id")
|
||||
|
||||
id = hash(document)
|
||||
|
||||
triples = []
|
||||
|
||||
def emit(t):
|
||||
triples.append(t)
|
||||
|
||||
if metadata:
|
||||
metadata.emit(
|
||||
lambda t: triples.append({
|
||||
"s": { "v": t["s"], "e": isinstance(t["s"], Uri) },
|
||||
"p": { "v": t["p"], "e": isinstance(t["p"], Uri) },
|
||||
"o": { "v": t["o"], "e": isinstance(t["o"], Uri) }
|
||||
})
|
||||
)
|
||||
|
||||
input = {
|
||||
"id": id,
|
||||
"metadata": triples,
|
||||
"data": base64.b64encode(document).decode("utf-8"),
|
||||
}
|
||||
|
||||
if user:
|
||||
input["user"] = user
|
||||
|
||||
if collection:
|
||||
input["collection"] = collection
|
||||
|
||||
return self.request(
|
||||
"service/document-load",
|
||||
input
|
||||
)
|
||||
|
||||
def load_text(
|
||||
self, text, id=None, metadata=None, charset="utf-8",
|
||||
user=None, collection=None,
|
||||
):
|
||||
|
||||
if id is None:
|
||||
|
||||
if metadata is not None:
|
||||
|
||||
# Situation makes no sense. What can the metadata possibly
|
||||
# mean if the caller doesn't know the document ID.
|
||||
# Metadata should relate to the document by ID
|
||||
raise RuntimeError("Can't specify metadata without id")
|
||||
|
||||
id = hash(text)
|
||||
|
||||
triples = []
|
||||
|
||||
if metadata:
|
||||
metadata.emit(
|
||||
lambda t: triples.append({
|
||||
"s": { "v": t["s"], "e": isinstance(t["s"], Uri) },
|
||||
"p": { "v": t["p"], "e": isinstance(t["p"], Uri) },
|
||||
"o": { "v": t["o"], "e": isinstance(t["o"], Uri) }
|
||||
})
|
||||
)
|
||||
|
||||
input = {
|
||||
"id": id,
|
||||
"metadata": triples,
|
||||
"charset": charset,
|
||||
"text": base64.b64encode(text).decode("utf-8"),
|
||||
}
|
||||
|
||||
if user:
|
||||
input["user"] = user
|
||||
|
||||
if collection:
|
||||
input["collection"] = collection
|
||||
|
||||
return self.request(
|
||||
"service/text-load",
|
||||
input
|
||||
)
|
||||
|
||||
259
trustgraph-base/trustgraph/api/library.py
Normal file
259
trustgraph-base/trustgraph/api/library.py
Normal file
|
|
@ -0,0 +1,259 @@
|
|||
|
||||
import datetime
|
||||
import time
|
||||
import base64
|
||||
|
||||
from . types import DocumentMetadata, ProcessingMetadata, Triple
|
||||
from .. knowledge import hash, Uri, Literal
|
||||
from . exceptions import *
|
||||
|
||||
def to_value(x):
|
||||
if x["e"]: return Uri(x["v"])
|
||||
return Literal(x["v"])
|
||||
|
||||
class Library:
|
||||
|
||||
def __init__(self, api):
|
||||
self.api = api
|
||||
|
||||
def request(self, request):
|
||||
return self.api.request(f"librarian", request)
|
||||
|
||||
def add_document(
|
||||
self, document, id, metadata, user, title, comments,
|
||||
kind="text/plain", tags=[],
|
||||
):
|
||||
|
||||
if id is None:
|
||||
|
||||
if metadata is not None:
|
||||
|
||||
# Situation makes no sense. What can the metadata possibly
|
||||
# mean if the caller doesn't know the document ID.
|
||||
# Metadata should relate to the document by ID
|
||||
raise RuntimeError("Can't specify metadata without id")
|
||||
|
||||
id = hash(document)
|
||||
|
||||
if not title: title = ""
|
||||
if not comments: comments = ""
|
||||
|
||||
triples = []
|
||||
|
||||
def emit(t):
|
||||
triples.append(t)
|
||||
|
||||
if metadata:
|
||||
metadata.emit(
|
||||
lambda t: triples.append({
|
||||
"s": { "v": t["s"], "e": isinstance(t["s"], Uri) },
|
||||
"p": { "v": t["p"], "e": isinstance(t["p"], Uri) },
|
||||
"o": { "v": t["o"], "e": isinstance(t["o"], Uri) }
|
||||
})
|
||||
)
|
||||
|
||||
input = {
|
||||
"operation": "add-document",
|
||||
"document-metadata": {
|
||||
"id": id,
|
||||
"time": int(time.time()),
|
||||
"kind": kind,
|
||||
"title": title,
|
||||
"comments": comments,
|
||||
"metadata": triples,
|
||||
"user": user,
|
||||
"tags": tags
|
||||
},
|
||||
"content": base64.b64encode(document).decode("utf-8"),
|
||||
}
|
||||
|
||||
return self.request(input)
|
||||
|
||||
def get_documents(self, user):
|
||||
|
||||
input = {
|
||||
"operation": "list-documents",
|
||||
"user": user,
|
||||
}
|
||||
|
||||
object = self.request(input)
|
||||
|
||||
try:
|
||||
return [
|
||||
DocumentMetadata(
|
||||
id = v["id"],
|
||||
time = datetime.datetime.fromtimestamp(v["time"]),
|
||||
kind = v["kind"],
|
||||
title = v["title"],
|
||||
comments = v.get("comments", ""),
|
||||
metadata = [
|
||||
Triple(
|
||||
s = to_value(w["s"]),
|
||||
p = to_value(w["p"]),
|
||||
o = to_value(w["o"])
|
||||
)
|
||||
for w in v["metadata"]
|
||||
],
|
||||
user = v["user"],
|
||||
tags = v["tags"]
|
||||
)
|
||||
for v in object["document-metadatas"]
|
||||
]
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
def get_document(self, user, id):
|
||||
|
||||
input = {
|
||||
"operation": "get-document",
|
||||
"user": user,
|
||||
"document-id": id,
|
||||
}
|
||||
|
||||
object = self.request(input)
|
||||
doc = object["document-metadata"]
|
||||
|
||||
try:
|
||||
DocumentMetadata(
|
||||
id = doc["id"],
|
||||
time = datetime.datetime.fromtimestamp(doc["time"]),
|
||||
kind = doc["kind"],
|
||||
title = doc["title"],
|
||||
comments = doc.get("comments", ""),
|
||||
metadata = [
|
||||
Triple(
|
||||
s = to_value(w["s"]),
|
||||
p = to_value(w["p"]),
|
||||
o = to_value(w["o"])
|
||||
)
|
||||
for w in doc["metadata"]
|
||||
],
|
||||
user = doc["user"],
|
||||
tags = doc["tags"]
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
def update_document(self, user, id, metadata):
|
||||
|
||||
input = {
|
||||
"operation": "update-document",
|
||||
"document-metadata": {
|
||||
"user": user,
|
||||
"document-id": id,
|
||||
"time": metadata.time,
|
||||
"title": metadata.title,
|
||||
"comments": metadata.comments,
|
||||
"metadata": [
|
||||
{
|
||||
"s": { "v": t["s"], "e": isinstance(t["s"], Uri) },
|
||||
"p": { "v": t["p"], "e": isinstance(t["p"], Uri) },
|
||||
"o": { "v": t["o"], "e": isinstance(t["o"], Uri) }
|
||||
}
|
||||
for t in metadata.metadata
|
||||
],
|
||||
"tags": metadata.tags,
|
||||
}
|
||||
}
|
||||
|
||||
object = self.request(input)
|
||||
doc = object["document-metadata"]
|
||||
|
||||
try:
|
||||
DocumentMetadata(
|
||||
id = doc["id"],
|
||||
time = datetime.datetime.fromtimestamp(doc["time"]),
|
||||
kind = doc["kind"],
|
||||
title = doc["title"],
|
||||
comments = doc.get("comments", ""),
|
||||
metadata = [
|
||||
Triple(
|
||||
s = to_value(w["s"]),
|
||||
p = to_value(w["p"]),
|
||||
o = to_value(w["o"])
|
||||
)
|
||||
for w in doc["metadata"]
|
||||
],
|
||||
user = doc["user"],
|
||||
tags = doc["tags"]
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
def remove_document(self, user, id):
|
||||
|
||||
input = {
|
||||
"operation": "remove-document",
|
||||
"user": user,
|
||||
"document-id": id,
|
||||
}
|
||||
|
||||
object = self.request(input)
|
||||
|
||||
return {}
|
||||
|
||||
def start_processing(
|
||||
self, id, document_id, flow="0000",
|
||||
user="trustgraph", collection="default", tags=[],
|
||||
):
|
||||
|
||||
input = {
|
||||
"operation": "add-processing",
|
||||
"processing-metadata": {
|
||||
"id": id,
|
||||
"document-id": document_id,
|
||||
"time": int(time.time()),
|
||||
"flow": flow,
|
||||
"user": user,
|
||||
"collection": collection,
|
||||
"tags": tags,
|
||||
}
|
||||
}
|
||||
|
||||
object = self.request(input)
|
||||
|
||||
return {}
|
||||
|
||||
def stop_processing(
|
||||
self, id, user="trustgraph",
|
||||
):
|
||||
|
||||
input = {
|
||||
"operation": "remove-processing",
|
||||
"processing-id": id,
|
||||
"user": user,
|
||||
}
|
||||
|
||||
object = self.request(input)
|
||||
|
||||
return {}
|
||||
|
||||
def get_processings(self, user="trustgraph"):
|
||||
|
||||
input = {
|
||||
"operation": "list-processing",
|
||||
"user": user,
|
||||
}
|
||||
|
||||
object = self.request(input)
|
||||
|
||||
try:
|
||||
return [
|
||||
ProcessingMetadata(
|
||||
id = v["id"],
|
||||
document_id = v["document-id"],
|
||||
time = datetime.datetime.fromtimestamp(v["time"]),
|
||||
flow = v["flow"],
|
||||
user = v["user"],
|
||||
collection = v["collection"],
|
||||
tags = v["tags"],
|
||||
)
|
||||
for v in object["processing-metadatas"]
|
||||
]
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
42
trustgraph-base/trustgraph/api/types.py
Normal file
42
trustgraph-base/trustgraph/api/types.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
|
||||
import dataclasses
|
||||
import datetime
|
||||
from typing import List
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Triple:
|
||||
s : str
|
||||
p : str
|
||||
o : str
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ConfigKey:
|
||||
type : str
|
||||
key : str
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ConfigValue:
|
||||
type : str
|
||||
key : str
|
||||
value : str
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocumentMetadata:
|
||||
id : str
|
||||
time : datetime.datetime
|
||||
kind : str
|
||||
title : str
|
||||
comments : str
|
||||
metadata : List[Triple]
|
||||
user : str
|
||||
tags : List[str]
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ProcessingMetadata:
|
||||
id : str
|
||||
document_id : str
|
||||
time : datetime.datetime
|
||||
flow : str
|
||||
user : str
|
||||
collection : str
|
||||
tags : List[str]
|
||||
201
trustgraph-cli/scripts/tg-add-library-document
Executable file
201
trustgraph-cli/scripts/tg-add-library-document
Executable file
|
|
@ -0,0 +1,201 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Loads a PDF document into the library
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import argparse
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from trustgraph.api import Api
|
||||
from trustgraph.knowledge import hash, to_uri
|
||||
from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG
|
||||
from trustgraph.knowledge import Organization, PublicationEvent
|
||||
from trustgraph.knowledge import DigitalDocument
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = 'trustgraph'
|
||||
|
||||
class Loader:
|
||||
|
||||
def __init__(
|
||||
self, url, user, metadata, title, comments, kind, tags
|
||||
):
|
||||
|
||||
self.api = Api(url).library()
|
||||
|
||||
self.user = user
|
||||
self.metadata = metadata
|
||||
self.title = title
|
||||
self.comments = comments
|
||||
self.kind = kind
|
||||
|
||||
if tags:
|
||||
self.tags = tags.split(",")
|
||||
else:
|
||||
self.tags = []
|
||||
|
||||
def load(self, files):
|
||||
|
||||
for file in files:
|
||||
self.load_file(file)
|
||||
|
||||
def load_file(self, file):
|
||||
|
||||
try:
|
||||
|
||||
path = file
|
||||
data = open(path, "rb").read()
|
||||
|
||||
# Create a SHA256 hash from the data
|
||||
id = hash(data)
|
||||
|
||||
id = to_uri(PREF_DOC, id)
|
||||
|
||||
self.metadata.id = id
|
||||
|
||||
self.api.add_document(
|
||||
document=data, id=id, metadata=self.metadata,
|
||||
user=self.user, kind=self.kind, title=self.title,
|
||||
comments=self.comments, tags=self.tags
|
||||
)
|
||||
|
||||
print(f"{file}: Loaded successfully.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"{file}: Failed: {str(e)}", flush=True)
|
||||
raise e
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-add-library-document',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--name', help=f'Document name'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--description', help=f'Document description'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--copyright-notice', help=f'Copyright notice'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--copyright-holder', help=f'Copyright holder'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--copyright-year', help=f'Copyright year'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--license', help=f'Copyright license'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--publication-organization', help=f'Publication organization'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--publication-description', help=f'Publication description'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--publication-date', help=f'Publication date'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--document-url', help=f'Document URL'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--keyword', nargs='+', help=f'Keyword'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--identifier', '--id', help=f'Document ID'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-k', '--kind',
|
||||
required=True,
|
||||
help=f'Document MIME type'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--tags',
|
||||
help=f'Tags, command separated'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'files', nargs='+',
|
||||
help=f'File to load'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
document = DigitalDocument(
|
||||
id,
|
||||
name=args.name,
|
||||
description=args.description,
|
||||
copyright_notice=args.copyright_notice,
|
||||
copyright_holder=args.copyright_holder,
|
||||
copyright_year=args.copyright_year,
|
||||
license=args.license,
|
||||
url=args.document_url,
|
||||
keywords=args.keyword,
|
||||
)
|
||||
|
||||
if args.publication_organization:
|
||||
org = Organization(
|
||||
id=to_uri(PREF_ORG, hash(args.publication_organization)),
|
||||
name=args.publication_organization,
|
||||
)
|
||||
document.publication = PublicationEvent(
|
||||
id = to_uri(PREF_PUBEV, str(uuid.uuid4())),
|
||||
organization=org,
|
||||
description=args.publication_description,
|
||||
start_date=args.publication_date,
|
||||
end_date=args.publication_date,
|
||||
)
|
||||
|
||||
p = Loader(
|
||||
url=args.url,
|
||||
user=args.user,
|
||||
metadata=document,
|
||||
title=args.name,
|
||||
comments=args.description,
|
||||
kind=args.kind,
|
||||
tags=args.tags,
|
||||
)
|
||||
|
||||
p.load(args.files)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
|
|
@ -13,9 +13,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def delete_flow_class(url, class_name):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).flow()
|
||||
|
||||
class_names = api.flow_delete_class(class_name)
|
||||
class_names = api.delete_class(class_name)
|
||||
|
||||
def main():
|
||||
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def get_flow_class(url, class_name):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).flow()
|
||||
|
||||
cls = api.flow_get_class(class_name)
|
||||
cls = api.get_class(class_name)
|
||||
|
||||
print(json.dumps(cls, indent=4))
|
||||
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ default_collection = 'default'
|
|||
|
||||
def show_graph(url, flow_id, user, collection):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).flow().id(flow_id)
|
||||
|
||||
rows = api.flow(flow_id).triples_query(
|
||||
rows = api.triples_query(
|
||||
user=user, collection=collection,
|
||||
s=None, p=None, o=None, limit=10_000,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,9 +19,9 @@ default_collection = 'default'
|
|||
|
||||
def show_graph(url, flow_id, user, collection):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).flow().id(flow_id)
|
||||
|
||||
rows = api.flow(flow_id).triples_query(
|
||||
rows = api.triples_query(
|
||||
s=None, p=None, o=None,
|
||||
user=user, collection=collection,
|
||||
limit=10_000)
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@ default_doc_limit = 10
|
|||
|
||||
def question(url, flow_id, question, user, collection, doc_limit):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).flow().id(flow_id)
|
||||
|
||||
resp = api.flow(flow_id).document_rag(
|
||||
resp = api.document_rag(
|
||||
question=question, user=user, collection=collection,
|
||||
doc_limit=doc_limit,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -21,9 +21,9 @@ def question(
|
|||
max_subgraph_size, max_path_length
|
||||
):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).flow().id(flow_id)
|
||||
|
||||
resp = api.flow(flow_id).graph_rag(
|
||||
resp = api.graph_rag(
|
||||
question=question, user=user, collection=collection,
|
||||
entity_limit=entity_limit, triple_limit=triple_limit,
|
||||
max_subgraph_size=max_subgraph_size,
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def query(url, flow_id, system, prompt):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).flow().id(flow_id).
|
||||
|
||||
resp = api.flow(flow_id).text_completion(system=system, prompt=prompt)
|
||||
resp = api.text_completion(system=system, prompt=prompt)
|
||||
|
||||
print(resp)
|
||||
|
||||
|
|
|
|||
|
|
@ -18,9 +18,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def query(url, flow_id, template_id, variables):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).flow().id(flow_id)
|
||||
|
||||
resp = api.flow(flow_id).prompt(id=template_id, variables=variables)
|
||||
resp = api.prompt(id=template_id, variables=variables)
|
||||
|
||||
if isinstance(resp, str):
|
||||
print(resp)
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@
|
|||
Loads a PDF document into TrustGraph processing.
|
||||
"""
|
||||
|
||||
import pulsar
|
||||
from pulsar.schema import JsonSchema
|
||||
import hashlib
|
||||
import argparse
|
||||
import os
|
||||
|
|
@ -31,10 +29,9 @@ class Loader:
|
|||
user,
|
||||
collection,
|
||||
metadata,
|
||||
pulsar_api_key=None,
|
||||
):
|
||||
|
||||
self.api = Api(url).flow(flow_id)
|
||||
self.api = Api(url).flow().id(flow_id)
|
||||
|
||||
self.user = user
|
||||
self.collection = collection
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ class Loader:
|
|||
metadata,
|
||||
):
|
||||
|
||||
self.api = Api(url).flow(flow_id)
|
||||
self.api = Api(url).flow().id(flow_id)
|
||||
|
||||
self.user = user
|
||||
self.collection = collection
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
|
||||
"""
|
||||
Loads Graph embeddings into TrustGraph processing.
|
||||
|
||||
FIXME: This hasn't been updated following API gateway change.
|
||||
"""
|
||||
|
||||
import pulsar
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ Dumps out the current configuration
|
|||
|
||||
import argparse
|
||||
import os
|
||||
import tabulate
|
||||
from trustgraph.api import Api
|
||||
import json
|
||||
|
||||
|
|
@ -16,7 +15,7 @@ def put_flow_class(url, class_name, config):
|
|||
|
||||
api = Api(url)
|
||||
|
||||
class_names = api.flow_put_class(class_name, config)
|
||||
class_names = api.flow().put_class(class_name, config)
|
||||
|
||||
def main():
|
||||
|
||||
|
|
|
|||
59
trustgraph-cli/scripts/tg-remove-library-document
Executable file
59
trustgraph-cli/scripts/tg-remove-library-document
Executable file
|
|
@ -0,0 +1,59 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Remove a PDF document from the library
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import uuid
|
||||
|
||||
from trustgraph.api import Api
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = 'trustgraph'
|
||||
|
||||
|
||||
def remove_doc(url, user, id):
|
||||
|
||||
api = Api(url).library()
|
||||
|
||||
api.remove_document(user=user, id=id)
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-remove-library-document',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--identifier', '--id',
|
||||
required=True,
|
||||
help=f'Document ID'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
remove_doc(args.url, args.user, args.identifier)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
|
|
@ -15,9 +15,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def set_system(url, system):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).config()
|
||||
|
||||
api.config_put([
|
||||
api.put([
|
||||
ConfigValue(type="prompt", key="system", value=json.dumps(system))
|
||||
])
|
||||
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def set_costs(api_url, model, input_costs, output_costs):
|
||||
|
||||
api = Api(api_url)
|
||||
api = Api(api_url).config()
|
||||
|
||||
api.config_put([
|
||||
api.put([
|
||||
ConfigValue(
|
||||
type="token-costs", key=model,
|
||||
value=json.dumps({
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def show_config(url):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).config()
|
||||
|
||||
config, version = api.config_all()
|
||||
config, version = api.all()
|
||||
|
||||
print("Version:", version)
|
||||
print(json.dumps(config, indent=4))
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def show_flow_classes(url):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).flow()
|
||||
|
||||
class_names = api.flow_list_classes()
|
||||
class_names = api.list_classes()
|
||||
|
||||
if len(class_names) == 0:
|
||||
print("No flows.")
|
||||
|
|
@ -24,7 +24,7 @@ def show_flow_classes(url):
|
|||
classes = []
|
||||
|
||||
for class_name in class_names:
|
||||
cls = api.flow_get_class(class_name)
|
||||
cls = api.get_class(class_name)
|
||||
classes.append((
|
||||
class_name,
|
||||
cls.get("description", ""),
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def dump_status(metrics_url, api_url, flow_id):
|
||||
|
||||
api = Api(api_url)
|
||||
api = Api(api_url).flow()
|
||||
|
||||
flow = api.flow_get(flow_id)
|
||||
flow = api.get(flow_id)
|
||||
class_name = flow["class-name"]
|
||||
|
||||
print()
|
||||
|
|
|
|||
|
|
@ -11,11 +11,11 @@ import json
|
|||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
|
||||
def get_interface(api, i):
|
||||
def get_interface(config_api, i):
|
||||
|
||||
key = ConfigKey("interface-descriptions", i)
|
||||
|
||||
value = api.config_get([key])[0].value
|
||||
value = config_api.get([key])[0].value
|
||||
|
||||
return json.loads(value)
|
||||
|
||||
|
|
@ -49,15 +49,17 @@ def describe_interfaces(intdefs, flow):
|
|||
def show_flows(url):
|
||||
|
||||
api = Api(url)
|
||||
config_api = api.config()
|
||||
flow_api = api.flow()
|
||||
|
||||
interface_names = api.config_list("interface-descriptions")
|
||||
interface_names = config_api.list("interface-descriptions")
|
||||
|
||||
interface_defs = {
|
||||
i: get_interface(api, i)
|
||||
i: get_interface(config_api, i)
|
||||
for i in interface_names
|
||||
}
|
||||
|
||||
flow_ids = api.flow_list()
|
||||
flow_ids = flow_api.list()
|
||||
|
||||
if len(flow_ids) == 0:
|
||||
print("No flows.")
|
||||
|
|
@ -67,7 +69,7 @@ def show_flows(url):
|
|||
|
||||
for id in flow_ids:
|
||||
|
||||
flow = api.flow_get(id)
|
||||
flow = flow_api.get(id)
|
||||
|
||||
table = []
|
||||
table.append(("id", id))
|
||||
|
|
|
|||
74
trustgraph-cli/scripts/tg-show-library-documents
Normal file
74
trustgraph-cli/scripts/tg-show-library-documents
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import tabulate
|
||||
from trustgraph.api import Api, ConfigKey
|
||||
import json
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = "trustgraph"
|
||||
|
||||
def show_docs(url, user):
|
||||
|
||||
api = Api(url).library()
|
||||
|
||||
docs = api.get_documents(user=user)
|
||||
|
||||
if len(docs) == 0:
|
||||
print("No documents.")
|
||||
return
|
||||
|
||||
for doc in docs:
|
||||
|
||||
table = []
|
||||
table.append(("id", doc.id))
|
||||
table.append(("time", doc.time))
|
||||
table.append(("title", doc.title))
|
||||
table.append(("kind", doc.kind))
|
||||
table.append(("comments", doc.comments))
|
||||
table.append(("tags", ", ".join(doc.tags)))
|
||||
|
||||
print(tabulate.tabulate(
|
||||
table,
|
||||
tablefmt="pretty",
|
||||
stralign="left",
|
||||
))
|
||||
print()
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-show-library-documents',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--api-url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
show_docs(
|
||||
url = args.api_url, user = args.user
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
74
trustgraph-cli/scripts/tg-show-library-processing
Normal file
74
trustgraph-cli/scripts/tg-show-library-processing
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import tabulate
|
||||
from trustgraph.api import Api, ConfigKey
|
||||
import json
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = "trustgraph"
|
||||
|
||||
def show_procs(url, user):
|
||||
|
||||
api = Api(url).library()
|
||||
|
||||
procs = api.get_processings(user = user)
|
||||
|
||||
if len(procs) == 0:
|
||||
print("No processing objects.")
|
||||
return
|
||||
|
||||
for proc in procs:
|
||||
|
||||
table = []
|
||||
table.append(("id", proc.id))
|
||||
table.append(("document-id", proc.document_id))
|
||||
table.append(("time", proc.time))
|
||||
table.append(("flow", proc.flow))
|
||||
table.append(("collection", proc.collection))
|
||||
table.append(("tags", ", ".join(proc.tags)))
|
||||
|
||||
print(tabulate.tabulate(
|
||||
table,
|
||||
tablefmt="pretty",
|
||||
stralign="left",
|
||||
))
|
||||
print()
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-show-library-processing',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--api-url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
show_procs(
|
||||
url = args.api_url, user = args.user
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
|
|
@ -15,9 +15,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def show_config(url):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).config()
|
||||
|
||||
values = api.config_get([
|
||||
values = api.get([
|
||||
ConfigKey(type="prompt", key="system"),
|
||||
ConfigKey(type="prompt", key="template-index")
|
||||
])
|
||||
|
|
@ -25,7 +25,7 @@ def show_config(url):
|
|||
system = json.loads(values[0].value)
|
||||
ix = json.loads(values[1].value)
|
||||
|
||||
values = api.config_get([
|
||||
values = api.get([
|
||||
ConfigKey(type="prompt", key=f"template.{v}")
|
||||
for v in ix
|
||||
])
|
||||
|
|
|
|||
|
|
@ -17,9 +17,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def show_config(url):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).config()
|
||||
|
||||
models = api.config_list("token-costs")
|
||||
models = api.list("token-costs")
|
||||
|
||||
costs = []
|
||||
|
||||
|
|
@ -29,7 +29,7 @@ def show_config(url):
|
|||
for model in models:
|
||||
|
||||
try:
|
||||
values = json.loads(api.config_get([
|
||||
values = json.loads(api.get([
|
||||
ConfigKey(type="token-costs", key=model),
|
||||
])[0].value)
|
||||
costs.append((
|
||||
|
|
|
|||
|
|
@ -15,15 +15,15 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def show_config(url):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).config()
|
||||
|
||||
values = api.config_get([
|
||||
values = api.get([
|
||||
ConfigKey(type="agent", key="tool-index")
|
||||
])
|
||||
|
||||
ix = json.loads(values[0].value)
|
||||
|
||||
values = api.config_get([
|
||||
values = api.get([
|
||||
ConfigKey(type="agent", key=f"tool.{v}")
|
||||
for v in ix
|
||||
])
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def start_flow(url, class_name, flow_id, description):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).flow()
|
||||
|
||||
api.flow_start(
|
||||
api.start(
|
||||
class_name = class_name,
|
||||
id = flow_id,
|
||||
description = description,
|
||||
|
|
|
|||
102
trustgraph-cli/scripts/tg-start-library-processing
Normal file
102
trustgraph-cli/scripts/tg-start-library-processing
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import tabulate
|
||||
from trustgraph.api import Api, ConfigKey
|
||||
import json
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = "trustgraph"
|
||||
|
||||
def start_processing(
|
||||
url, user, document_id, id, flow, collection, tags
|
||||
):
|
||||
|
||||
api = Api(url).library()
|
||||
|
||||
if tags:
|
||||
tags = tags.split(",")
|
||||
else:
|
||||
tags = []
|
||||
|
||||
api.start_processing(
|
||||
id = id,
|
||||
document_id = document_id,
|
||||
flow = flow,
|
||||
user = user,
|
||||
collection = collection,
|
||||
tags = tags
|
||||
)
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-start-library-processing',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--api-url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-i', '--flow-id',
|
||||
default="0000",
|
||||
help=f'Flow ID (default: 0000)',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-d', '--document-id',
|
||||
required=True,
|
||||
help=f'Document ID',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--id', '--processing-id',
|
||||
required=True,
|
||||
help=f'Processing ID',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--collection',
|
||||
default='default',
|
||||
help=f'Collection (default: default)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--tags',
|
||||
help=f'Tags, command separated'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
start_processing(
|
||||
url = args.api_url,
|
||||
user = args.user,
|
||||
document_id = args.document_id,
|
||||
id = args.id,
|
||||
flow = args.flow_id,
|
||||
collection = args.collection,
|
||||
tags = args.tags
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
|
|
@ -13,9 +13,9 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
|||
|
||||
def stop_flow(url, flow_id):
|
||||
|
||||
api = Api(url)
|
||||
api = Api(url).flow()
|
||||
|
||||
api.flow_stop(id = flow_id)
|
||||
api.stop(id = flow_id)
|
||||
|
||||
def main():
|
||||
|
||||
|
|
|
|||
63
trustgraph-cli/scripts/tg-stop-library-processing
Normal file
63
trustgraph-cli/scripts/tg-stop-library-processing
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import tabulate
|
||||
from trustgraph.api import Api, ConfigKey
|
||||
import json
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = "trustgraph"
|
||||
|
||||
def stop_processing(
|
||||
url, user, id
|
||||
):
|
||||
|
||||
api = Api(url).library()
|
||||
|
||||
api.stop_processing(user = user, id = id)
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-stop-library-processing',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--api-url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--id', '--processing-id',
|
||||
required=True,
|
||||
help=f'Processing ID',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
stop_processing(
|
||||
url = args.api_url,
|
||||
user = args.user,
|
||||
id = args.id,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
|
|
@ -70,6 +70,12 @@ setuptools.setup(
|
|||
"scripts/tg-show-flow-classes",
|
||||
"scripts/tg-show-flow-state",
|
||||
"scripts/tg-show-flows",
|
||||
"scripts/tg-show-library-documents",
|
||||
"scripts/tg-show-library-processing",
|
||||
"scripts/tg-start-library-processing",
|
||||
"scripts/tg-stop-library-processing",
|
||||
"scripts/tg-add-library-document",
|
||||
"scripts/tg-remove-library-document",
|
||||
"scripts/tg-show-processor-state",
|
||||
"scripts/tg-show-prompts",
|
||||
"scripts/tg-show-token-costs",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue