diff --git a/trustgraph-cli/scripts/tg-graph-show b/trustgraph-cli/scripts/tg-graph-show index c09266fb..a3d10283 100755 --- a/trustgraph-cli/scripts/tg-graph-show +++ b/trustgraph-cli/scripts/tg-graph-show @@ -6,23 +6,23 @@ Connects to the graph query service and dumps all graph edges. import argparse import os -from trustgraph.clients.triples_query_client import TriplesQueryClient +from trustgraph.api import Api -default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650') +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') default_user = 'trustgraph' default_collection = 'default' -def show_graph(pulsar, user, collection): +def show_graph(url, user, collection): - tq = TriplesQueryClient(pulsar_host=pulsar) + api = Api(url) - rows = tq.request( - user=user, collection=collection, - s=None, p=None, o=None, limit=10_000_000 + rows = api.triples_query( +# user=user, collection=collection, + s=None, p=None, o=None, limit=10_000, ) for row in rows: - print(row.s.value, row.p.value, row.o.value) + print(row.s, row.p, row.o) def main(): @@ -32,19 +32,19 @@ def main(): ) parser.add_argument( - '-p', '--pulsar-host', - default=default_pulsar_host, - help=f'Pulsar host (default: {default_pulsar_host})', + '-u', '--api-url', + default=default_url, + help=f'API URL (default: {default_url})', ) parser.add_argument( - '-u', '--user', + '-U', '--user', default=default_user, help=f'User ID (default: {default_user})' ) parser.add_argument( - '-c', '--collection', + '-C', '--collection', default=default_collection, help=f'Collection ID (default: {default_collection})' ) @@ -54,7 +54,8 @@ def main(): try: show_graph( - pulsar=args.pulsar_host, user=args.user, + url=args.api_url, + user=args.user, collection=args.collection, ) diff --git a/trustgraph-cli/scripts/tg-graph-to-turtle b/trustgraph-cli/scripts/tg-graph-to-turtle index 1d75478e..fc17ddd0 100755 --- a/trustgraph-cli/scripts/tg-graph-to-turtle +++ b/trustgraph-cli/scripts/tg-graph-to-turtle @@ -5,37 +5,45 @@ Connects to the graph query service and dumps all graph edges in Turtle format. """ -import argparse -import os -from trustgraph.clients.triples_query_client import TriplesQueryClient import rdflib import io import sys +import argparse +import os -default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650') +from trustgraph.api import Api, Uri -def show_graph(pulsar): +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') +default_user = 'trustgraph' +default_collection = 'default' - tq = TriplesQueryClient(pulsar_host=pulsar) +def show_graph(url, user, collection): - rows = tq.request(None, None, None, limit=10_000_000) + api = Api(url) + + rows = api.triples_query( + s=None, p=None, o=None, + limit=10_000) +# user=user, collection=collection, g = rdflib.Graph() for row in rows: - sv = rdflib.term.URIRef(row.s.value) - pv = rdflib.term.URIRef(row.p.value) + sv = rdflib.term.URIRef(row.s) + pv = rdflib.term.URIRef(row.p) - if row.o.is_uri: + if isinstance(row.o, Uri): # Skip malformed URLs with spaces in - if " " in row.o.value: + if " " in row.o: continue - ov = rdflib.term.URIRef(row.o.value) + ov = rdflib.term.URIRef(row.o) + else: - ov = rdflib.term.Literal(row.o.value) + + ov = rdflib.term.Literal(row.o) g.add((sv, pv, ov)) @@ -56,16 +64,32 @@ def main(): ) parser.add_argument( - '-p', '--pulsar-host', - default=default_pulsar_host, - help=f'Pulsar host (default: {default_pulsar_host})', + '-u', '--api-url', + default=default_url, + help=f'API URL (default: {default_url})', + ) + + parser.add_argument( + '-U', '--user', + default=default_user, + help=f'User ID (default: {default_user})' + ) + + parser.add_argument( + '-C', '--collection', + default=default_collection, + help=f'Collection ID (default: {default_collection})' ) args = parser.parse_args() try: - show_graph(args.pulsar_host) + show_graph( + url=args.api_url, + user=args.user, + collection=args.collection + ) except Exception as e: diff --git a/trustgraph-cli/scripts/tg-invoke-agent b/trustgraph-cli/scripts/tg-invoke-agent index 3f05071c..bb87c2a6 100755 --- a/trustgraph-cli/scripts/tg-invoke-agent +++ b/trustgraph-cli/scripts/tg-invoke-agent @@ -1,16 +1,18 @@ #!/usr/bin/env python3 """ -Uses the GraphRAG service to answer a query +Uses the GraphRAG service to answer a question """ import argparse import os import textwrap +import uuid +import asyncio +import json +from websockets.asyncio.client import connect -from trustgraph.clients.agent_client import AgentClient - -default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650') +default_url = os.getenv("TRUSTGRAPH_URL", 'ws://localhost:8088/') default_user = 'trustgraph' default_collection = 'default' @@ -27,15 +29,18 @@ def output(text, prefix="> ", width=78): ) print(out) -def query( - pulsar_host, query, user, collection, +async def question( + url, question, user, collection, plan=None, state=None, verbose=False ): - am = AgentClient(pulsar_host=pulsar_host) + if not url.endswith("/"): + url += "/" + + url = url + "api/v1/socket" if verbose: - output(wrap(query), "\U00002753 ") + output(wrap(question), "\U00002753 ") print() def think(x): @@ -48,11 +53,43 @@ def query( output(wrap(x), "\U0001f4a1 ") print() - resp = am.request( - question=query, think=think, observe=observe, - ) + mid = str(uuid.uuid4()) - print(resp) + async with connect(url) as ws: + + req = json.dumps({ + "id": mid, + "service": "agent", + "request": { + "question": question, + } + + }) + + await ws.send(req) + + while True: + + msg = await ws.recv() + + obj = json.loads(msg) + + if obj["id"] != mid: + print("Ignore message") + continue + + if "thought" in obj["response"]: + think(obj["response"]["thought"]) + + if "observation" in obj["response"]: + observe(obj["response"]["observation"]) + + if "answer" in obj["response"]: + print(obj["response"]["answer"]) + + if obj["complete"]: break + + await ws.close() def main(): @@ -62,25 +99,25 @@ def main(): ) parser.add_argument( - '-p', '--pulsar-host', - default=default_pulsar_host, - help=f'Pulsar host (default: {default_pulsar_host})', + '-u', '--url', + default=default_url, + help=f'API URL (default: {default_url})', ) parser.add_argument( - '-q', '--query', + '-q', '--question', required=True, - help=f'Query to execute', + help=f'Question to answer', ) parser.add_argument( - '-u', '--user', + '-U', '--user', default=default_user, help=f'User ID (default: {default_user})' ) parser.add_argument( - '-c', '--collection', + '-C', '--collection', default=default_collection, help=f'Collection ID (default: {default_collection})' ) @@ -105,14 +142,16 @@ def main(): try: - query( - pulsar_host=args.pulsar_host, - query=args.query, - user=args.user, - collection=args.collection, - plan=args.plan, - state=args.state, - verbose=args.verbose, + asyncio.run( + question( + url=args.url, + question=args.question, + user=args.user, + collection=args.collection, + plan=args.plan, + state=args.state, + verbose=args.verbose, + ) ) except Exception as e: diff --git a/trustgraph-cli/scripts/tg-query-graph-rag b/trustgraph-cli/scripts/tg-invoke-document-rag similarity index 51% rename from trustgraph-cli/scripts/tg-query-graph-rag rename to trustgraph-cli/scripts/tg-invoke-document-rag index 8a865eea..3e0a9422 100755 --- a/trustgraph-cli/scripts/tg-query-graph-rag +++ b/trustgraph-cli/scripts/tg-invoke-document-rag @@ -1,50 +1,53 @@ #!/usr/bin/env python3 """ -Uses the GraphRAG service to answer a query +Uses the GraphRAG service to answer a question """ import argparse import os -from trustgraph.clients.graph_rag_client import GraphRagClient +from trustgraph.api import Api -default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650') +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') default_user = 'trustgraph' default_collection = 'default' -def query(pulsar_host, query, user, collection): +def question(url, question, user, collection): + + rag = Api(url) + +# user=user, collection=collection, + resp = rag.document_rag(question=question) - rag = GraphRagClient(pulsar_host=pulsar_host) - resp = rag.request(user=user, collection=collection, query=query) print(resp) def main(): parser = argparse.ArgumentParser( - prog='tg-graph-query-rag', + prog='tg-invoke-document-rag', description=__doc__, ) parser.add_argument( - '-p', '--pulsar-host', - default=default_pulsar_host, - help=f'Pulsar host (default: {default_pulsar_host})', + '-u', '--url', + default=default_url, + help=f'API URL (default: {default_url})', ) parser.add_argument( - '-q', '--query', + '-q', '--question', required=True, - help=f'Query to execute', + help=f'Question to answer', ) parser.add_argument( - '-u', '--user', + '-U', '--user', default=default_user, help=f'User ID (default: {default_user})' ) parser.add_argument( - '-c', '--collection', + '-C', '--collection', default=default_collection, help=f'Collection ID (default: {default_collection})' ) @@ -53,9 +56,9 @@ def main(): try: - query( - pulsar_host=args.pulsar_host, - query=args.query, + question( + url=args.url, + question=args.question, user=args.user, collection=args.collection, ) diff --git a/trustgraph-cli/scripts/tg-query-document-rag b/trustgraph-cli/scripts/tg-invoke-graph-rag similarity index 50% rename from trustgraph-cli/scripts/tg-query-document-rag rename to trustgraph-cli/scripts/tg-invoke-graph-rag index 8d800629..50de5b74 100755 --- a/trustgraph-cli/scripts/tg-query-document-rag +++ b/trustgraph-cli/scripts/tg-invoke-graph-rag @@ -1,50 +1,53 @@ #!/usr/bin/env python3 """ -Uses the Document RAG service to answer a query +Uses the GraphRAG service to answer a question """ import argparse import os -from trustgraph.clients.document_rag_client import DocumentRagClient +from trustgraph.api import Api -default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650') +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') default_user = 'trustgraph' default_collection = 'default' -def query(pulsar_host, query, user, collection): +def question(url, question, user, collection): + + rag = Api(url) + +# user=user, collection=collection, + resp = rag.graph_rag(question=question) - rag = DocumentRagClient(pulsar_host=pulsar) - resp = rag.request(user=user, collection=collection, query=query) print(resp) def main(): parser = argparse.ArgumentParser( - prog='tg-query-document-rag', + prog='tg-invoke-graph-rag', description=__doc__, ) parser.add_argument( - '-p', '--pulsar-host', - default=default_pulsar_host, - help=f'Pulsar host (default: {default_pulsar_host})', + '-u', '--url', + default=default_url, + help=f'API URL (default: {default_url})', ) parser.add_argument( - '-q', '--query', + '-q', '--question', required=True, - help=f'Query to execute', + help=f'Question to answer', ) parser.add_argument( - '-u', '--user', + '-U', '--user', default=default_user, help=f'User ID (default: {default_user})' ) parser.add_argument( - '-c', '--collection', + '-C', '--collection', default=default_collection, help=f'Collection ID (default: {default_collection})' ) @@ -53,9 +56,9 @@ def main(): try: - query( - pulsar_host=args.pulsar_host, - query=args.query, + question( + url=args.url, + question=args.question, user=args.user, collection=args.collection, ) diff --git a/trustgraph-cli/scripts/tg-invoke-llm b/trustgraph-cli/scripts/tg-invoke-llm index d7289b5f..d1a49e87 100755 --- a/trustgraph-cli/scripts/tg-invoke-llm +++ b/trustgraph-cli/scripts/tg-invoke-llm @@ -8,15 +8,15 @@ and user prompt. Both arguments are required. import argparse import os import json -from trustgraph.clients.llm_client import LlmClient +from trustgraph.api import Api -default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650') +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') -def query(pulsar_host, system, prompt): +def query(url, system, prompt): - cli = LlmClient(pulsar_host=pulsar_host) + api = Api(url) - resp = cli.request(system=system, prompt=prompt) + resp = api.text_completion(system=system, prompt=prompt) print(resp) @@ -28,9 +28,9 @@ def main(): ) parser.add_argument( - '-p', '--pulsar-host', - default=default_pulsar_host, - help=f'Pulsar host (default: {default_pulsar_host})', + '-u', '--url', + default=default_url, + help=f'API URL (default: {default_url})', ) parser.add_argument( @@ -50,7 +50,7 @@ def main(): try: query( - pulsar_host=args.pulsar_host, + url=args.url, system=args.system[0], prompt=args.prompt[0], ) diff --git a/trustgraph-cli/scripts/tg-invoke-prompt b/trustgraph-cli/scripts/tg-invoke-prompt index 19f30912..7d54dab0 100755 --- a/trustgraph-cli/scripts/tg-invoke-prompt +++ b/trustgraph-cli/scripts/tg-invoke-prompt @@ -12,15 +12,15 @@ using key=value arguments on the command line, and these replace import argparse import os import json -from trustgraph.clients.prompt_client import PromptClient +from trustgraph.api import Api -default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650') +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') -def query(pulsar_host, template_id, variables): +def query(url, template_id, variables): - cli = PromptClient(pulsar_host=pulsar_host) + api = Api(url) - resp = cli.request(id=template_id, variables=variables) + resp = api.prompt(id=template_id, variables=variables) if isinstance(resp, str): print(resp) @@ -35,9 +35,9 @@ def main(): ) parser.add_argument( - '-p', '--pulsar-host', - default=default_pulsar_host, - help=f'Pulsar host (default: {default_pulsar_host})', + '-u', '--url', + default=default_url, + help=f'API URL (default: {default_url})', ) parser.add_argument( @@ -70,7 +70,7 @@ specified multiple times''', try: query( - pulsar_host=args.pulsar_host, + url=args.url, template_id=args.id[0], variables=variables, ) diff --git a/trustgraph-cli/scripts/tg-load-pdf b/trustgraph-cli/scripts/tg-load-pdf index a0d2b3bc..f27aaf5a 100755 --- a/trustgraph-cli/scripts/tg-load-pdf +++ b/trustgraph-cli/scripts/tg-load-pdf @@ -6,21 +6,19 @@ Loads a PDF document into TrustGraph processing. import pulsar from pulsar.schema import JsonSchema -import base64 import hashlib import argparse import os import time import uuid -from trustgraph.schema import Document, document_ingest_queue -from trustgraph.schema import Metadata, Triple, Value -from trustgraph.log_level import LogLevel -from trustgraph.knowledge import hash, to_uri, Uri +from trustgraph.api import Api +from trustgraph.knowledge import hash, to_uri from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG from trustgraph.knowledge import Organization, PublicationEvent from trustgraph.knowledge import DigitalDocument +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') default_user = 'trustgraph' default_collection = 'default' @@ -28,24 +26,13 @@ class Loader: def __init__( self, - pulsar_host, - output_queue, + url, user, collection, - log_level, metadata, ): - self.client = pulsar.Client( - pulsar_host, - logger=pulsar.ConsoleLogger(log_level.to_pulsar()) - ) - - self.producer = self.client.create_producer( - topic=output_queue, - schema=JsonSchema(Document), - chunking_enabled=True, - ) + self.api = Api(url) self.user = user self.collection = collection @@ -68,49 +55,18 @@ class Loader: id = to_uri(PREF_DOC, id) - triples = [] - - def emit(t): - triples.append(t) - self.metadata.id = id - self.metadata.emit(emit) - r = Document( - metadata=Metadata( - id=id, - metadata=[ - Triple( - s=Value( - value=t["s"], - is_uri=isinstance(t["s"], Uri) - ), - p=Value( - value=t["p"], - is_uri=isinstance(t["p"], Uri) - ), - o=Value( - value=t["o"], - is_uri=isinstance(t["o"], Uri) - ), - ) - for t in triples - ], - user=self.user, - collection=self.collection, - ), - data=base64.b64encode(data), + self.api.load_document( + document=data, id=id, metadata=self.metadata, +# user=self.user, +# collection=self.collection, ) - self.producer.send(r) - print(f"{file}: Loaded successfully.") except Exception as e: print(f"{file}: Failed: {str(e)}", flush=True) - - def __del__(self): - self.client.close() def main(): @@ -119,29 +75,20 @@ def main(): description=__doc__, ) - default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650') - default_output_queue = document_ingest_queue - parser.add_argument( - '-p', '--pulsar-host', - default=default_pulsar_host, - help=f'Pulsar host (default: {default_pulsar_host})', + '-u', '--url', + default=default_url, + help=f'API URL (default: {default_url})', ) parser.add_argument( - '-o', '--output-queue', - default=default_output_queue, - help=f'Output queue (default: {default_output_queue})' - ) - - parser.add_argument( - '-u', '--user', + '-U', '--user', default=default_user, help=f'User ID (default: {default_user})' ) parser.add_argument( - '-c', '--collection', + '-C', '--collection', default=default_collection, help=f'Collection ID (default: {default_collection})' ) @@ -183,7 +130,7 @@ def main(): ) parser.add_argument( - '--url', help=f'Document URL' + '--document-url', help=f'Document URL' ) parser.add_argument( @@ -194,14 +141,6 @@ def main(): '--identifier', '--id', help=f'Document ID' ) - parser.add_argument( - '-l', '--log-level', - type=LogLevel, - default=LogLevel.ERROR, - choices=list(LogLevel), - help=f'Output queue (default: info)' - ) - parser.add_argument( 'files', nargs='+', help=f'File to load' @@ -221,7 +160,7 @@ def main(): copyright_holder=args.copyright_holder, copyright_year=args.copyright_year, license=args.license, - url=args.url, + url=args.document_url, keywords=args.keyword, ) @@ -239,11 +178,9 @@ def main(): ) p = Loader( - pulsar_host=args.pulsar_host, - output_queue=args.output_queue, + url=args.url, user=args.user, collection=args.collection, - log_level=args.log_level, metadata=document, ) diff --git a/trustgraph-cli/scripts/tg-load-text b/trustgraph-cli/scripts/tg-load-text index 51664a1b..634ac3d7 100755 --- a/trustgraph-cli/scripts/tg-load-text +++ b/trustgraph-cli/scripts/tg-load-text @@ -12,14 +12,13 @@ import os import time import uuid -from trustgraph.schema import TextDocument, text_ingest_queue -from trustgraph.schema import Metadata, Triple, Value -from trustgraph.log_level import LogLevel -from trustgraph.knowledge import hash, to_uri, Literal, Uri +from trustgraph.api import Api +from trustgraph.knowledge import hash, to_uri from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG from trustgraph.knowledge import Organization, PublicationEvent from trustgraph.knowledge import DigitalDocument +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') default_user = 'trustgraph' default_collection = 'default' @@ -27,24 +26,13 @@ class Loader: def __init__( self, - pulsar_host, - output_queue, + url, user, collection, - log_level, metadata, ): - self.client = pulsar.Client( - pulsar_host, - logger=pulsar.ConsoleLogger(log_level.to_pulsar()) - ) - - self.producer = self.client.create_producer( - topic=output_queue, - schema=JsonSchema(TextDocument), - chunking_enabled=True, - ) + self.api = Api(url) self.user = user self.collection = collection @@ -67,49 +55,18 @@ class Loader: id = to_uri(PREF_DOC, id) - triples = [] - - def emit(t): - triples.append(t) - self.metadata.id = id - self.metadata.emit(emit) - r = TextDocument( - metadata=Metadata( - id=id, - metadata=[ - Triple( - s=Value( - value=t["s"], - is_uri=isinstance(t["s"], Uri) - ), - p=Value( - value=t["p"], - is_uri=isinstance(t["p"], Uri) - ), - o=Value( - value=t["o"], - is_uri=isinstance(t["o"], Uri) - ), - ) - for t in triples - ], - user=self.user, - collection=self.collection, - ), - text=data, + self.api.load_text( + text=data, id=id, metadata=self.metadata, +# user=self.user, +# collection=self.collection, ) - self.producer.send(r) - print(f"{file}: Loaded successfully.") except Exception as e: print(f"{file}: Failed: {str(e)}", flush=True) - - def __del__(self): - self.client.close() def main(): @@ -118,29 +75,20 @@ def main(): description=__doc__, ) - default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650') - default_output_queue = text_ingest_queue - parser.add_argument( - '-p', '--pulsar-host', - default=default_pulsar_host, - help=f'Pulsar host (default: {default_pulsar_host})', + '-u', '--url', + default=default_url, + help=f'API URL (default: {default_url})', ) parser.add_argument( - '-o', '--output-queue', - default=default_output_queue, - help=f'Output queue (default: {default_output_queue})' - ) - - parser.add_argument( - '-u', '--user', + '-U', '--user', default=default_user, help=f'User ID (default: {default_user})' ) parser.add_argument( - '-c', '--collection', + '-C', '--collection', default=default_collection, help=f'Collection ID (default: {default_collection})' ) @@ -182,7 +130,7 @@ def main(): ) parser.add_argument( - '--url', help=f'Document URL' + '--document-url', help=f'Document URL' ) parser.add_argument( @@ -193,14 +141,6 @@ def main(): '--identifier', '--id', help=f'Document ID' ) - parser.add_argument( - '-l', '--log-level', - type=LogLevel, - default=LogLevel.ERROR, - choices=list(LogLevel), - help=f'Output queue (default: info)' - ) - parser.add_argument( 'files', nargs='+', help=f'File to load' @@ -220,7 +160,7 @@ def main(): copyright_holder=args.copyright_holder, copyright_year=args.copyright_year, license=args.license, - url=args.url, + url=args.document_url, keywords=args.keyword, ) @@ -238,11 +178,9 @@ def main(): ) p = Loader( - pulsar_host=args.pulsar_host, - output_queue=args.output_queue, + url=args.url, user=args.user, collection=args.collection, - log_level=args.log_level, metadata=document, ) diff --git a/trustgraph-cli/setup.py b/trustgraph-cli/setup.py index a810c924..67c70158 100644 --- a/trustgraph-cli/setup.py +++ b/trustgraph-cli/setup.py @@ -40,23 +40,24 @@ setuptools.setup( "rdflib", "tabulate", "msgpack", + "websockets", ], scripts=[ + "scripts/tg-dump-msgpack", "scripts/tg-graph-show", "scripts/tg-graph-to-turtle", + "scripts/tg-init-pulsar", "scripts/tg-init-pulsar-manager", + "scripts/tg-invoke-agent", + "scripts/tg-invoke-document-rag", + "scripts/tg-invoke-graph-rag", + "scripts/tg-invoke-llm", + "scripts/tg-invoke-prompt", + "scripts/tg-load-kg-core", "scripts/tg-load-pdf", "scripts/tg-load-text", "scripts/tg-load-turtle", - "scripts/tg-query-document-rag", - "scripts/tg-query-graph-rag", - "scripts/tg-init-pulsar", "scripts/tg-processor-state", - "scripts/tg-invoke-agent", - "scripts/tg-invoke-prompt", - "scripts/tg-invoke-llm", "scripts/tg-save-kg-core", - "scripts/tg-load-kg-core", - "scripts/tg-dump-msgpack", ] )