apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-26 17:06:22 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Migrate cli utils to REST API (#239)

Browse source 
* Port a number of commands to use API gateway instead of Pulsar

* Ported tg-invoke-agent to websockets API

* Rename the 2 RAG commands: tg-query-... to tg-invoke-...
This commit is contained in:
cybermaggedon 2025-01-02 19:49:22 +00:00 • committed by GitHub
parent 44c0d6f347
commit 44f8ce8834
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 223 additions and 277 deletions
Download patch file Download diff file Expand all files Collapse all files

29
trustgraph-cli/scripts/tg-graph-show
View file

@ -6,23 +6,23 @@ Connects to the graph query service and dumps all graph edges.
import argparse
import os
from trustgraph.clients.triples_query_client import TriplesQueryClient
from trustgraph.api import Api
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
default_user = 'trustgraph'
default_collection = 'default'
def show_graph(pulsar, user, collection):
def show_graph(url, user, collection):
tq = TriplesQueryClient(pulsar_host=pulsar)
api = Api(url)
rows = tq.request(
user=user, collection=collection,
s=None, p=None, o=None, limit=10_000_000
rows = api.triples_query(
# user=user, collection=collection,
s=None, p=None, o=None, limit=10_000,
)
for row in rows:
print(row.s.value, row.p.value, row.o.value)
print(row.s, row.p, row.o)
def main():
@ -32,19 +32,19 @@ def main():
)
parser.add_argument(
'-p', '--pulsar-host',
default=default_pulsar_host,
help=f'Pulsar host (default: {default_pulsar_host})',
'-u', '--api-url',
default=default_url,
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-u', '--user',
'-U', '--user',
default=default_user,
help=f'User ID (default: {default_user})'
)
parser.add_argument(
'-c', '--collection',
'-C', '--collection',
default=default_collection,
help=f'Collection ID (default: {default_collection})'
)
@ -54,7 +54,8 @@ def main():
try:
show_graph(
pulsar=args.pulsar_host, user=args.user,
url=args.api_url,
user=args.user,
collection=args.collection,
)

58
trustgraph-cli/scripts/tg-graph-to-turtle
View file

@ -5,37 +5,45 @@ Connects to the graph query service and dumps all graph edges in Turtle
format.
"""
import argparse
import os
from trustgraph.clients.triples_query_client import TriplesQueryClient
import rdflib
import io
import sys
import argparse
import os
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
from trustgraph.api import Api, Uri
def show_graph(pulsar):
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
default_user = 'trustgraph'
default_collection = 'default'
tq = TriplesQueryClient(pulsar_host=pulsar)
def show_graph(url, user, collection):
rows = tq.request(None, None, None, limit=10_000_000)
api = Api(url)
rows = api.triples_query(
s=None, p=None, o=None,
limit=10_000)
# user=user, collection=collection,
g = rdflib.Graph()
for row in rows:
sv = rdflib.term.URIRef(row.s.value)
pv = rdflib.term.URIRef(row.p.value)
sv = rdflib.term.URIRef(row.s)
pv = rdflib.term.URIRef(row.p)
if row.o.is_uri:
if isinstance(row.o, Uri):
# Skip malformed URLs with spaces in
if " " in row.o.value:
if " " in row.o:
continue
ov = rdflib.term.URIRef(row.o.value)
ov = rdflib.term.URIRef(row.o)
else:
ov = rdflib.term.Literal(row.o.value)
ov = rdflib.term.Literal(row.o)
g.add((sv, pv, ov))
@ -56,16 +64,32 @@ def main():
)
parser.add_argument(
'-p', '--pulsar-host',
default=default_pulsar_host,
help=f'Pulsar host (default: {default_pulsar_host})',
'-u', '--api-url',
default=default_url,
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-U', '--user',
default=default_user,
help=f'User ID (default: {default_user})'
)
parser.add_argument(
'-C', '--collection',
default=default_collection,
help=f'Collection ID (default: {default_collection})'
)
args = parser.parse_args()
try:
show_graph(args.pulsar_host)
show_graph(
url=args.api_url,
user=args.user,
collection=args.collection
)
except Exception as e:

93
trustgraph-cli/scripts/tg-invoke-agent
View file

@ -1,16 +1,18 @@
#!/usr/bin/env python3
"""
Uses the GraphRAG service to answer a query
Uses the GraphRAG service to answer a question
"""
import argparse
import os
import textwrap
import uuid
import asyncio
import json
from websockets.asyncio.client import connect
from trustgraph.clients.agent_client import AgentClient
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
default_url = os.getenv("TRUSTGRAPH_URL", 'ws://localhost:8088/')
default_user = 'trustgraph'
default_collection = 'default'
@ -27,15 +29,18 @@ def output(text, prefix="> ", width=78):
)
print(out)
def query(
pulsar_host, query, user, collection,
async def question(
url, question, user, collection,
plan=None, state=None, verbose=False
):
am = AgentClient(pulsar_host=pulsar_host)
if not url.endswith("/"):
url += "/"
url = url + "api/v1/socket"
if verbose:
output(wrap(query), "\U00002753 ")
output(wrap(question), "\U00002753 ")
print()
def think(x):
@ -48,11 +53,43 @@ def query(
output(wrap(x), "\U0001f4a1 ")
print()
resp = am.request(
question=query, think=think, observe=observe,
)
mid = str(uuid.uuid4())
print(resp)
async with connect(url) as ws:
req = json.dumps({
"id": mid,
"service": "agent",
"request": {
"question": question,
}
})
await ws.send(req)
while True:
msg = await ws.recv()
obj = json.loads(msg)
if obj["id"] != mid:
print("Ignore message")
continue
if "thought" in obj["response"]:
think(obj["response"]["thought"])
if "observation" in obj["response"]:
observe(obj["response"]["observation"])
if "answer" in obj["response"]:
print(obj["response"]["answer"])
if obj["complete"]: break
await ws.close()
def main():
@ -62,25 +99,25 @@ def main():
)
parser.add_argument(
'-p', '--pulsar-host',
default=default_pulsar_host,
help=f'Pulsar host (default: {default_pulsar_host})',
'-u', '--url',
default=default_url,
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-q', '--query',
'-q', '--question',
required=True,
help=f'Query to execute',
help=f'Question to answer',
)
parser.add_argument(
'-u', '--user',
'-U', '--user',
default=default_user,
help=f'User ID (default: {default_user})'
)
parser.add_argument(
'-c', '--collection',
'-C', '--collection',
default=default_collection,
help=f'Collection ID (default: {default_collection})'
)
@ -105,14 +142,16 @@ def main():
try:
query(
pulsar_host=args.pulsar_host,
query=args.query,
user=args.user,
collection=args.collection,
plan=args.plan,
state=args.state,
verbose=args.verbose,
asyncio.run(
question(
url=args.url,
question=args.question,
user=args.user,
collection=args.collection,
plan=args.plan,
state=args.state,
verbose=args.verbose,
)
)
except Exception as e:

37
trustgraph-cli/scripts/tg-query-graph-rag → trustgraph-cli/scripts/tg-invoke-document-rag
View file

@ -1,50 +1,53 @@
#!/usr/bin/env python3
"""
Uses the GraphRAG service to answer a query
Uses the GraphRAG service to answer a question
"""
import argparse
import os
from trustgraph.clients.graph_rag_client import GraphRagClient
from trustgraph.api import Api
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
default_user = 'trustgraph'
default_collection = 'default'
def query(pulsar_host, query, user, collection):
def question(url, question, user, collection):
rag = Api(url)
# user=user, collection=collection,
resp = rag.document_rag(question=question)
rag = GraphRagClient(pulsar_host=pulsar_host)
resp = rag.request(user=user, collection=collection, query=query)
print(resp)
def main():
parser = argparse.ArgumentParser(
prog='tg-graph-query-rag',
prog='tg-invoke-document-rag',
description=__doc__,
)
parser.add_argument(
'-p', '--pulsar-host',
default=default_pulsar_host,
help=f'Pulsar host (default: {default_pulsar_host})',
'-u', '--url',
default=default_url,
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-q', '--query',
'-q', '--question',
required=True,
help=f'Query to execute',
help=f'Question to answer',
)
parser.add_argument(
'-u', '--user',
'-U', '--user',
default=default_user,
help=f'User ID (default: {default_user})'
)
parser.add_argument(
'-c', '--collection',
'-C', '--collection',
default=default_collection,
help=f'Collection ID (default: {default_collection})'
)
@ -53,9 +56,9 @@ def main():
try:
query(
pulsar_host=args.pulsar_host,
query=args.query,
question(
url=args.url,
question=args.question,
user=args.user,
collection=args.collection,
)

37
trustgraph-cli/scripts/tg-query-document-rag → trustgraph-cli/scripts/tg-invoke-graph-rag
View file

@ -1,50 +1,53 @@
#!/usr/bin/env python3
"""
Uses the Document RAG service to answer a query
Uses the GraphRAG service to answer a question
"""
import argparse
import os
from trustgraph.clients.document_rag_client import DocumentRagClient
from trustgraph.api import Api
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
default_user = 'trustgraph'
default_collection = 'default'
def query(pulsar_host, query, user, collection):
def question(url, question, user, collection):
rag = Api(url)
# user=user, collection=collection,
resp = rag.graph_rag(question=question)
rag = DocumentRagClient(pulsar_host=pulsar)
resp = rag.request(user=user, collection=collection, query=query)
print(resp)
def main():
parser = argparse.ArgumentParser(
prog='tg-query-document-rag',
prog='tg-invoke-graph-rag',
description=__doc__,
)
parser.add_argument(
'-p', '--pulsar-host',
default=default_pulsar_host,
help=f'Pulsar host (default: {default_pulsar_host})',
'-u', '--url',
default=default_url,
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-q', '--query',
'-q', '--question',
required=True,
help=f'Query to execute',
help=f'Question to answer',
)
parser.add_argument(
'-u', '--user',
'-U', '--user',
default=default_user,
help=f'User ID (default: {default_user})'
)
parser.add_argument(
'-c', '--collection',
'-C', '--collection',
default=default_collection,
help=f'Collection ID (default: {default_collection})'
)
@ -53,9 +56,9 @@ def main():
try:
query(
pulsar_host=args.pulsar_host,
query=args.query,
question(
url=args.url,
question=args.question,
user=args.user,
collection=args.collection,
)

18
trustgraph-cli/scripts/tg-invoke-llm
View file

@ -8,15 +8,15 @@ and user prompt. Both arguments are required.
import argparse
import os
import json
from trustgraph.clients.llm_client import LlmClient
from trustgraph.api import Api
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
def query(pulsar_host, system, prompt):
def query(url, system, prompt):
cli = LlmClient(pulsar_host=pulsar_host)
api = Api(url)
resp = cli.request(system=system, prompt=prompt)
resp = api.text_completion(system=system, prompt=prompt)
print(resp)
@ -28,9 +28,9 @@ def main():
)
parser.add_argument(
'-p', '--pulsar-host',
default=default_pulsar_host,
help=f'Pulsar host (default: {default_pulsar_host})',
'-u', '--url',
default=default_url,
help=f'API URL (default: {default_url})',
)
parser.add_argument(
@ -50,7 +50,7 @@ def main():
try:
query(
pulsar_host=args.pulsar_host,
url=args.url,
system=args.system[0],
prompt=args.prompt[0],
)

18
trustgraph-cli/scripts/tg-invoke-prompt
View file

@ -12,15 +12,15 @@ using key=value arguments on the command line, and these replace
import argparse
import os
import json
from trustgraph.clients.prompt_client import PromptClient
from trustgraph.api import Api
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
def query(pulsar_host, template_id, variables):
def query(url, template_id, variables):
cli = PromptClient(pulsar_host=pulsar_host)
api = Api(url)
resp = cli.request(id=template_id, variables=variables)
resp = api.prompt(id=template_id, variables=variables)
if isinstance(resp, str):
print(resp)
@ -35,9 +35,9 @@ def main():
)
parser.add_argument(
'-p', '--pulsar-host',
default=default_pulsar_host,
help=f'Pulsar host (default: {default_pulsar_host})',
'-u', '--url',
default=default_url,
help=f'API URL (default: {default_url})',
)
parser.add_argument(
@ -70,7 +70,7 @@ specified multiple times''',
try:
query(
pulsar_host=args.pulsar_host,
url=args.url,
template_id=args.id[0],
variables=variables,
)

97
trustgraph-cli/scripts/tg-load-pdf
View file

@ -6,21 +6,19 @@ Loads a PDF document into TrustGraph processing.
import pulsar
from pulsar.schema import JsonSchema
import base64
import hashlib
import argparse
import os
import time
import uuid
from trustgraph.schema import Document, document_ingest_queue
from trustgraph.schema import Metadata, Triple, Value
from trustgraph.log_level import LogLevel
from trustgraph.knowledge import hash, to_uri, Uri
from trustgraph.api import Api
from trustgraph.knowledge import hash, to_uri
from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG
from trustgraph.knowledge import Organization, PublicationEvent
from trustgraph.knowledge import DigitalDocument
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
default_user = 'trustgraph'
default_collection = 'default'
@ -28,24 +26,13 @@ class Loader:
def __init__(
self,
pulsar_host,
output_queue,
url,
user,
collection,
log_level,
metadata,
):
self.client = pulsar.Client(
pulsar_host,
logger=pulsar.ConsoleLogger(log_level.to_pulsar())
)
self.producer = self.client.create_producer(
topic=output_queue,
schema=JsonSchema(Document),
chunking_enabled=True,
)
self.api = Api(url)
self.user = user
self.collection = collection
@ -68,49 +55,18 @@ class Loader:
id = to_uri(PREF_DOC, id)
triples = []
def emit(t):
triples.append(t)
self.metadata.id = id
self.metadata.emit(emit)
r = Document(
metadata=Metadata(
id=id,
metadata=[
Triple(
s=Value(
value=t["s"],
is_uri=isinstance(t["s"], Uri)
),
p=Value(
value=t["p"],
is_uri=isinstance(t["p"], Uri)
),
o=Value(
value=t["o"],
is_uri=isinstance(t["o"], Uri)
),
)
for t in triples
],
user=self.user,
collection=self.collection,
),
data=base64.b64encode(data),
self.api.load_document(
document=data, id=id, metadata=self.metadata,
# user=self.user,
# collection=self.collection,
)
self.producer.send(r)
print(f"{file}: Loaded successfully.")
except Exception as e:
print(f"{file}: Failed: {str(e)}", flush=True)
def __del__(self):
self.client.close()
def main():
@ -119,29 +75,20 @@ def main():
description=__doc__,
)
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
default_output_queue = document_ingest_queue
parser.add_argument(
'-p', '--pulsar-host',
default=default_pulsar_host,
help=f'Pulsar host (default: {default_pulsar_host})',
'-u', '--url',
default=default_url,
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-o', '--output-queue',
default=default_output_queue,
help=f'Output queue (default: {default_output_queue})'
)
parser.add_argument(
'-u', '--user',
'-U', '--user',
default=default_user,
help=f'User ID (default: {default_user})'
)
parser.add_argument(
'-c', '--collection',
'-C', '--collection',
default=default_collection,
help=f'Collection ID (default: {default_collection})'
)
@ -183,7 +130,7 @@ def main():
)
parser.add_argument(
'--url', help=f'Document URL'
'--document-url', help=f'Document URL'
)
parser.add_argument(
@ -194,14 +141,6 @@ def main():
'--identifier', '--id', help=f'Document ID'
)
parser.add_argument(
'-l', '--log-level',
type=LogLevel,
default=LogLevel.ERROR,
choices=list(LogLevel),
help=f'Output queue (default: info)'
)
parser.add_argument(
'files', nargs='+',
help=f'File to load'
@ -221,7 +160,7 @@ def main():
copyright_holder=args.copyright_holder,
copyright_year=args.copyright_year,
license=args.license,
url=args.url,
url=args.document_url,
keywords=args.keyword,
)
@ -239,11 +178,9 @@ def main():
)
p = Loader(
pulsar_host=args.pulsar_host,
output_queue=args.output_queue,
url=args.url,
user=args.user,
collection=args.collection,
log_level=args.log_level,
metadata=document,
)

96
trustgraph-cli/scripts/tg-load-text
View file

@ -12,14 +12,13 @@ import os
import time
import uuid
from trustgraph.schema import TextDocument, text_ingest_queue
from trustgraph.schema import Metadata, Triple, Value
from trustgraph.log_level import LogLevel
from trustgraph.knowledge import hash, to_uri, Literal, Uri
from trustgraph.api import Api
from trustgraph.knowledge import hash, to_uri
from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG
from trustgraph.knowledge import Organization, PublicationEvent
from trustgraph.knowledge import DigitalDocument
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
default_user = 'trustgraph'
default_collection = 'default'
@ -27,24 +26,13 @@ class Loader:
def __init__(
self,
pulsar_host,
output_queue,
url,
user,
collection,
log_level,
metadata,
):
self.client = pulsar.Client(
pulsar_host,
logger=pulsar.ConsoleLogger(log_level.to_pulsar())
)
self.producer = self.client.create_producer(
topic=output_queue,
schema=JsonSchema(TextDocument),
chunking_enabled=True,
)
self.api = Api(url)
self.user = user
self.collection = collection
@ -67,49 +55,18 @@ class Loader:
id = to_uri(PREF_DOC, id)
triples = []
def emit(t):
triples.append(t)
self.metadata.id = id
self.metadata.emit(emit)
r = TextDocument(
metadata=Metadata(
id=id,
metadata=[
Triple(
s=Value(
value=t["s"],
is_uri=isinstance(t["s"], Uri)
),
p=Value(
value=t["p"],
is_uri=isinstance(t["p"], Uri)
),
o=Value(
value=t["o"],
is_uri=isinstance(t["o"], Uri)
),
)
for t in triples
],
user=self.user,
collection=self.collection,
),
text=data,
self.api.load_text(
text=data, id=id, metadata=self.metadata,
# user=self.user,
# collection=self.collection,
)
self.producer.send(r)
print(f"{file}: Loaded successfully.")
except Exception as e:
print(f"{file}: Failed: {str(e)}", flush=True)
def __del__(self):
self.client.close()
def main():
@ -118,29 +75,20 @@ def main():
description=__doc__,
)
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
default_output_queue = text_ingest_queue
parser.add_argument(
'-p', '--pulsar-host',
default=default_pulsar_host,
help=f'Pulsar host (default: {default_pulsar_host})',
'-u', '--url',
default=default_url,
help=f'API URL (default: {default_url})',
)
parser.add_argument(
'-o', '--output-queue',
default=default_output_queue,
help=f'Output queue (default: {default_output_queue})'
)
parser.add_argument(
'-u', '--user',
'-U', '--user',
default=default_user,
help=f'User ID (default: {default_user})'
)
parser.add_argument(
'-c', '--collection',
'-C', '--collection',
default=default_collection,
help=f'Collection ID (default: {default_collection})'
)
@ -182,7 +130,7 @@ def main():
)
parser.add_argument(
'--url', help=f'Document URL'
'--document-url', help=f'Document URL'
)
parser.add_argument(
@ -193,14 +141,6 @@ def main():
'--identifier', '--id', help=f'Document ID'
)
parser.add_argument(
'-l', '--log-level',
type=LogLevel,
default=LogLevel.ERROR,
choices=list(LogLevel),
help=f'Output queue (default: info)'
)
parser.add_argument(
'files', nargs='+',
help=f'File to load'
@ -220,7 +160,7 @@ def main():
copyright_holder=args.copyright_holder,
copyright_year=args.copyright_year,
license=args.license,
url=args.url,
url=args.document_url,
keywords=args.keyword,
)
@ -238,11 +178,9 @@ def main():
)
p = Loader(
pulsar_host=args.pulsar_host,
output_queue=args.output_queue,
url=args.url,
user=args.user,
collection=args.collection,
log_level=args.log_level,
metadata=document,
)