apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-28 18:06:21 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Feature/flow api 3 (#358)

Browse source 
* Working mux socket

* Change API to incorporate flow

* Add Flow ID to all relevant CLIs, not completely implemented

* Change tg-processor-state to use API gateway

* Updated all CLIs

* New tg-show-flow-state command

* tg-show-flow-state shows classes too
This commit is contained in:
cybermaggedon 2025-05-03 10:39:53 +01:00 • committed by GitHub
parent a70ae9793a
commit 3b8b9ea866
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 800 additions and 986 deletions
Download patch file Download diff file Expand all files Collapse all files

19
trustgraph-cli/scripts/tg-graph-show
View file

@ -12,12 +12,12 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
default_user = 'trustgraph'
default_collection = 'default'
def show_graph(url, user, collection):
def show_graph(url, flow_id, user, collection):
api = Api(url)
rows = api.triples_query(
# user=user, collection=collection,
rows = api.flow(flow_id).triples_query(
user=user, collection=collection,
s=None, p=None, o=None, limit=10_000,
)
@ -37,6 +37,12 @@ def main():
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
parser.add_argument(
'-U', '--user',
default=default_user,
@ -54,9 +60,10 @@ def main():
try:
show_graph(
url=args.api_url,
user=args.user,
collection=args.collection,
url = args.api_url,
flow_id = args.flow_id,
user = args.user,
collection = args.collection,
)
except Exception as e:

19
trustgraph-cli/scripts/tg-graph-to-turtle
View file

@ -17,14 +17,14 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
default_user = 'trustgraph'
default_collection = 'default'
def show_graph(url, user, collection):
def show_graph(url, flow_id, user, collection):
api = Api(url)
rows = api.triples_query(
rows = api.flow(flow_id).triples_query(
s=None, p=None, o=None,
user=user, collection=collection,
limit=10_000)
# user=user, collection=collection,
g = rdflib.Graph()
@ -69,6 +69,12 @@ def main():
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
parser.add_argument(
'-U', '--user',
default=default_user,
@ -86,9 +92,10 @@ def main():
try:
show_graph(
url=args.api_url,
user=args.user,
collection=args.collection
url = args.api_url,
flow_id = args.flow_id,
user = args.user,
collection = args.collection,
)
except Exception as e:

33
trustgraph-cli/scripts/tg-invoke-agent
View file

@ -30,7 +30,7 @@ def output(text, prefix="> ", width=78):
print(out)
async def question(
url, question, user, collection,
url, question, flow_id, user, collection,
plan=None, state=None, verbose=False
):
@ -60,6 +60,7 @@ async def question(
req = json.dumps({
"id": mid,
"service": "agent",
"flow": flow_id,
"request": {
"question": question,
}
@ -74,6 +75,9 @@ async def question(
obj = json.loads(msg)
if obj["error"]:
raise RuntimeError(obj["error"])
if obj["id"] != mid:
print("Ignore message")
continue
@ -104,6 +108,12 @@ def main():
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
parser.add_argument(
'-q', '--question',
required=True,
@ -137,12 +147,6 @@ def main():
action="store_true",
help=f'Output thinking/observations'
)
# parser.add_argument(
# '--pulsar-api-key',
# default=default_pulsar_api_key,
# help=f'Pulsar API key',
# )
args = parser.parse_args()
@ -150,13 +154,14 @@ def main():
asyncio.run(
question(
url=args.url,
question=args.question,
user=args.user,
collection=args.collection,
plan=args.plan,
state=args.state,
verbose=args.verbose,
url = args.url,
flow_id = args.flow_id,
question = args.question,
user = args.user,
collection = args.collection,
plan = args.plan,
state = args.state,
verbose = args.verbose,
)
)

17
trustgraph-cli/scripts/tg-invoke-document-rag
View file

@ -13,11 +13,11 @@ default_user = 'trustgraph'
default_collection = 'default'
default_doc_limit = 10
def question(url, question, user, collection, doc_limit):
def question(url, flow_id, question, user, collection, doc_limit):
rag = Api(url)
api = Api(url)
resp = rag.document_rag(
resp = api.flow(flow_id).document_rag(
question=question, user=user, collection=collection,
doc_limit=doc_limit,
)
@ -37,11 +37,11 @@ def main():
help=f'API URL (default: {default_url})',
)
# parser.add_argument(
# '--pulsar-api-key',
# default=default_pulsar_api_key,
# help=f'Pulsar API key',
# )
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
parser.add_argument(
'-q', '--question',
@ -73,6 +73,7 @@ def main():
question(
url=args.url,
flow_id = args.flow_id,
question=args.question,
user=args.user,
collection=args.collection,

13
trustgraph-cli/scripts/tg-invoke-graph-rag
View file

@ -17,13 +17,13 @@ default_max_subgraph_size = 150
default_max_path_length = 2
def question(
url, question, user, collection, entity_limit, triple_limit,
url, flow_id, question, user, collection, entity_limit, triple_limit,
max_subgraph_size, max_path_length
):
rag = Api(url)
api = Api(url)
resp = rag.graph_rag(
resp = api.flow(flow_id).graph_rag(
question=question, user=user, collection=collection,
entity_limit=entity_limit, triple_limit=triple_limit,
max_subgraph_size=max_subgraph_size,
@ -45,6 +45,12 @@ def main():
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
parser.add_argument(
'-q', '--question',
required=True,
@ -93,6 +99,7 @@ def main():
question(
url=args.url,
flow_id = args.flow_id,
question=args.question,
user=args.user,
collection=args.collection,

17
trustgraph-cli/scripts/tg-invoke-llm
View file

@ -12,11 +12,11 @@ from trustgraph.api import Api
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
def query(url, system, prompt):
def query(url, flow_id, system, prompt):
api = Api(url)
resp = api.text_completion(system=system, prompt=prompt)
resp = api.flow(flow_id).text_completion(system=system, prompt=prompt)
print(resp)
@ -44,12 +44,12 @@ def main():
nargs=1,
help='LLM prompt e.g. What is 2 + 2?',
)
# parser.add_argument(
# '--pulsar-api-key',
# default=default_pulsar_api_key,
# help=f'Pulsar API key',
# )
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
args = parser.parse_args()
@ -57,6 +57,7 @@ def main():
query(
url=args.url,
flow = args.flow_id,
system=args.system[0],
prompt=args.prompt[0],
)

11
trustgraph-cli/scripts/tg-invoke-prompt
View file

@ -16,11 +16,11 @@ from trustgraph.api import Api
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
def query(url, template_id, variables):
def query(url, flow_id, template_id, variables):
api = Api(url)
resp = api.prompt(id=template_id, variables=variables)
resp = api.flow(flow_id).prompt(id=template_id, variables=variables)
if isinstance(resp, str):
print(resp)
@ -40,6 +40,12 @@ def main():
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
parser.add_argument(
'id',
metavar='template-id',
@ -77,6 +83,7 @@ specified multiple times''',
query(
url=args.url,
flow_id=args.flow_id,
template_id=args.id[0],
variables=variables,
)

14
trustgraph-cli/scripts/tg-load-doc-embeds
View file

@ -27,7 +27,7 @@ async def load_de(running, queue, url):
async with aiohttp.ClientSession() as session:
async with session.ws_connect(f"{url}load/document-embeddings") as ws:
async with session.ws_connect(url) as ws:
while running.get():
@ -141,6 +141,9 @@ async def run(running, **args):
# grow to eat all memory
de_q = asyncio.Queue(maxsize=10)
url = args["url"]
flow_id = args["flow_id"]
load_task = asyncio.create_task(
loader(
running=running,
@ -154,7 +157,8 @@ async def run(running, **args):
de_task = asyncio.create_task(
load_de(
running=running,
queue=de_q, url=args["url"] + "api/v1/"
queue=de_q,
url = f"{url}api/v1/flow/{flow_id}/import/document-embeddings"
)
)
@ -184,6 +188,12 @@ async def main(running):
help=f'TrustGraph API URL (default: {default_url})',
)
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
parser.add_argument(
'-i', '--input-file',
# Make it mandatory, difficult to over-write an existing file

87
trustgraph-cli/scripts/tg-load-pdf
View file

@ -27,13 +27,14 @@ class Loader:
def __init__(
self,
url,
flow_id,
user,
collection,
metadata,
pulsar_api_key=None,
):
self.api = Api(url)
self.api = Api(url).flow(flow_id)
self.user = user
self.collection = collection
@ -60,14 +61,15 @@ class Loader:
self.api.load_document(
document=data, id=id, metadata=self.metadata,
# user=self.user,
# collection=self.collection,
user=self.user,
collection=self.collection,
)
print(f"{file}: Loaded successfully.")
except Exception as e:
print(f"{file}: Failed: {str(e)}", flush=True)
raise e
def main():
@ -82,6 +84,12 @@ def main():
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
parser.add_argument(
'-U', '--user',
default=default_user,
@ -149,53 +157,46 @@ def main():
args = parser.parse_args()
while True:
try:
try:
document = DigitalDocument(
id,
name=args.name,
description=args.description,
copyright_notice=args.copyright_notice,
copyright_holder=args.copyright_holder,
copyright_year=args.copyright_year,
license=args.license,
url=args.document_url,
keywords=args.keyword,
)
document = DigitalDocument(
id,
name=args.name,
description=args.description,
copyright_notice=args.copyright_notice,
copyright_holder=args.copyright_holder,
copyright_year=args.copyright_year,
license=args.license,
url=args.document_url,
keywords=args.keyword,
if args.publication_organization:
org = Organization(
id=to_uri(PREF_ORG, hash(args.publication_organization)),
name=args.publication_organization,
)
document.publication = PublicationEvent(
id = to_uri(PREF_PUBEV, str(uuid.uuid4())),
organization=org,
description=args.publication_description,
start_date=args.publication_date,
end_date=args.publication_date,
)
if args.publication_organization:
org = Organization(
id=to_uri(PREF_ORG, hash(args.publication_organization)),
name=args.publication_organization,
)
document.publication = PublicationEvent(
id = to_uri(PREF_PUBEV, str(uuid.uuid4())),
organization=org,
description=args.publication_description,
start_date=args.publication_date,
end_date=args.publication_date,
)
p = Loader(
url=args.url,
flow_id = args.flow_id,
user=args.user,
collection=args.collection,
metadata=document,
)
p = Loader(
url=args.url,
user=args.user,
collection=args.collection,
metadata=document,
)
p.load(args.files)
p.load(args.files)
except Exception as e:
print("All done.")
break
except Exception as e:
print("Exception:", e, flush=True)
print("Will retry...", flush=True)
time.sleep(10)
print("Exception:", e, flush=True)
main()

95
trustgraph-cli/scripts/tg-load-text
View file

@ -27,12 +27,13 @@ class Loader:
def __init__(
self,
url,
flow_id,
user,
collection,
metadata,
):
self.api = Api(url)
self.api = Api(url).flow(flow_id)
self.user = user
self.collection = collection
@ -59,14 +60,15 @@ class Loader:
self.api.load_text(
text=data, id=id, metadata=self.metadata,
# user=self.user,
# collection=self.collection,
user=self.user,
collection=self.collection,
)
print(f"{file}: Loaded successfully.")
except Exception as e:
print(f"{file}: Failed: {str(e)}", flush=True)
raise e
def main():
@ -80,12 +82,12 @@ def main():
default=default_url,
help=f'API URL (default: {default_url})',
)
# parser.add_argument(
# '--pulsar-api-key',
# default=default_pulsar_api_key,
# help=f'Pulsar API key',
# )
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
parser.add_argument(
'-U', '--user',
@ -154,53 +156,50 @@ def main():
args = parser.parse_args()
while True:
try:
try:
document = DigitalDocument(
id,
name=args.name,
description=args.description,
copyright_notice=args.copyright_notice,
copyright_holder=args.copyright_holder,
copyright_year=args.copyright_year,
license=args.license,
url=args.document_url,
keywords=args.keyword,
document = DigitalDocument(
id,
name=args.name,
description=args.description,
copyright_notice=args.copyright_notice,
copyright_holder=args.copyright_holder,
copyright_year=args.copyright_year,
license=args.license,
url=args.document_url,
keywords=args.keyword,
)
if args.publication_organization:
org = Organization(
id=to_uri(PREF_ORG, hash(args.publication_organization)),
name=args.publication_organization,
)
document.publication = PublicationEvent(
id = to_uri(PREF_PUBEV, str(uuid.uuid4())),
organization=org,
description=args.publication_description,
start_date=args.publication_date,
end_date=args.publication_date,
)
if args.publication_organization:
org = Organization(
id=to_uri(PREF_ORG, hash(args.publication_organization)),
name=args.publication_organization,
)
document.publication = PublicationEvent(
id = to_uri(PREF_PUBEV, str(uuid.uuid4())),
organization=org,
description=args.publication_description,
start_date=args.publication_date,
end_date=args.publication_date,
)
p = Loader(
url = args.url,
flow_id = args.flow_id,
user = args.user,
collection = args.collection,
metadata = document,
)
p = Loader(
url=args.url,
user=args.user,
collection=args.collection,
metadata=document,
)
p.load(args.files)
p.load(args.files)
print("All done.")
print("All done.")
break
except Exception as e:
except Exception as e:
print("Exception:", e, flush=True)
print("Will retry...", flush=True)
time.sleep(10)
print("Exception:", e, flush=True)
main()

10
trustgraph-cli/scripts/tg-load-turtle
View file

@ -7,7 +7,6 @@ Loads Graph embeddings into TrustGraph processing.
import pulsar
from pulsar.schema import JsonSchema
from trustgraph.schema import Triples, Triple, Value, Metadata
from trustgraph.schema import triples_store_queue
import argparse
import os
import time
@ -109,6 +108,12 @@ def main():
default=default_pulsar_host,
help=f'Pulsar host (default: {default_pulsar_host})',
)
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
parser.add_argument(
'--pulsar-api-key',
@ -174,5 +179,6 @@ def main():
time.sleep(10)
main()
print("Not implemented.")
#main()

59
trustgraph-cli/scripts/tg-processor-state
View file

@ -1,59 +0,0 @@
#!/usr/bin/env python3
"""
Dump out TrustGraph processor states.
"""
import requests
import argparse
import tabulate
default_prometheus_url = "http://localhost:9090"
def dump_status(prom):
url = f"{prom}/api/v1/query?query=processor_state%7Bprocessor_state%3D%22running%22%7D"
resp = requests.get(url)
obj = resp.json()
tbl = [
[
m["metric"]["job"],
"running" if int(m["value"][1]) > 0 else "down"
]
for m in obj["data"]["result"]
]
print(tabulate.tabulate(
tbl, tablefmt="pretty", headers=["processor", "state"],
stralign="left"
))
def main():
parser = argparse.ArgumentParser(
prog='tg-processor-state',
description=__doc__,
)
parser.add_argument(
'-p', '--prometheus-url',
default=default_prometheus_url,
help=f'Prometheus URL (default: {default_prometheus_url})',
)
args = parser.parse_args()
try:
dump_status(args.prometheus_url)
except Exception as e:
print("Exception:", e, flush=True)
main()

15
trustgraph-cli/scripts/tg-save-doc-embeds
View file

@ -27,9 +27,7 @@ async def fetch_de(running, queue, user, collection, url):
async with aiohttp.ClientSession() as session:
de_url = f"{url}stream/document-embeddings"
async with session.ws_connect(de_url) as ws:
async with session.ws_connect(url) as ws:
while running.get():
@ -117,11 +115,14 @@ async def run(running, **args):
q = asyncio.Queue()
url = args["url"]
flow_id = args["flow_id"]
de_task = asyncio.create_task(
fetch_de(
running=running,
queue=q, user=args["user"], collection=args["collection"],
url=args["url"] + "api/v1/"
url = f"{url}api/v1/flow/{flow_id}/export/document-embeddings"
)
)
@ -158,6 +159,12 @@ async def main(running):
help=f'TrustGraph API URL (default: {default_url})',
)
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
parser.add_argument(
'-o', '--output-file',
# Make it mandatory, difficult to over-write an existing file

93
trustgraph-cli/scripts/tg-show-flow-state Executable file
View file

@ -0,0 +1,93 @@
#!/usr/bin/env python3
"""
Dump out TrustGraph processor states.
"""
import requests
import argparse
from trustgraph.api import Api
import os
default_metrics_url = "http://localhost:8088/api/metrics"
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
def dump_status(metrics_url, api_url, flow_id):
api = Api(api_url)
flow = api.flow_get(flow_id)
class_name = flow["class-name"]
print()
print(f"Flow {flow_id}")
show_processors(metrics_url, flow_id)
print()
print(f"Class {class_name}")
show_processors(metrics_url, class_name)
print()
def show_processors(metrics_url, flow_label):
url = f"{metrics_url}/query"
expr = f"consumer_state=\"running\",flow=\"{flow_label}\""
params = {
"query": "consumer_state{" + expr + "}"
}
resp = requests.get(url, params=params)
obj = resp.json()
tbl = [
[
m["metric"]["job"],
"\U0001f49a" if int(m["value"][1]) > 0 else "\U0000274c"
]
for m in obj["data"]["result"]
]
for row in tbl:
print(f"- {row[0]:30} {row[1]}")
def main():
parser = argparse.ArgumentParser(
prog='tg-show-flow-state',
description=__doc__,
)
parser.add_argument(
'-f', '--flow-id',
default="0000",
help=f'Flow ID (default: 0000)'
)
parser.add_argument(
'-u', '--api-url',
default=default_url,
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-m', '--metrics-url',
default=default_metrics_url,
help=f'Metrics URL (default: {default_metrics_url})',
)
args = parser.parse_args()
try:
dump_status(args.metrics_url, args.api_url, args.flow_id)
except Exception as e:
print("Exception:", e, flush=True)
main()