diff --git a/trustgraph-base/trustgraph/api/api.py b/trustgraph-base/trustgraph/api/api.py index 85ccf024..cff2b36a 100644 --- a/trustgraph-base/trustgraph/api/api.py +++ b/trustgraph-base/trustgraph/api/api.py @@ -37,9 +37,7 @@ def check_error(response): msg = response["error"]["message"] tp = response["error"]["type"] except: - raise ApplicationException( - "Error, but the error object is broken" - ) + raise ApplicationException(response["error"]) raise ApplicationException(f"{tp}: {msg}") @@ -352,13 +350,22 @@ class Flow: raise ProtocolException("Response not formatted correctly") - def triples_query(self, s=None, p=None, o=None, limit=10000): + def triples_query( + self, s=None, p=None, o=None, + user=None, collection=None, limit=10000 + ): # The input consists of system and prompt strings input = { "limit": limit } + if user: + input["user"] = user + + if collection: + input["collection"] = collection + if s: if not isinstance(s, Uri): raise RuntimeError("s must be Uri") @@ -392,7 +399,10 @@ class Flow: for t in object["response"] ] - def load_document(self, document, id=None, metadata=None): + def load_document( + self, document, id=None, metadata=None, user=None, + collection=None, + ): if id is None: @@ -425,12 +435,21 @@ class Flow: "data": base64.b64encode(document).decode("utf-8"), } + if user: + input["user"] = user + + if collection: + input["collection"] = collection + return self.api.request( f"flow/{self.flow}/service/document-load", input ) - def load_text(self, text, id=None, metadata=None, charset="utf-8"): + def load_text( + self, text, id=None, metadata=None, charset="utf-8", + user=None, collection=None, + ): if id is None: @@ -461,6 +480,12 @@ class Flow: "text": base64.b64encode(text).decode("utf-8"), } + if user: + input["user"] = user + + if collection: + input["collection"] = collection + return self.api.request( f"flow/{self.flow}/service/text-load", input diff --git a/trustgraph-cli/scripts/tg-graph-show b/trustgraph-cli/scripts/tg-graph-show index 0ee09c00..73a67d72 100755 --- a/trustgraph-cli/scripts/tg-graph-show +++ b/trustgraph-cli/scripts/tg-graph-show @@ -12,12 +12,12 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') default_user = 'trustgraph' default_collection = 'default' -def show_graph(url, user, collection): +def show_graph(url, flow_id, user, collection): api = Api(url) - rows = api.triples_query( -# user=user, collection=collection, + rows = api.flow(flow_id).triples_query( + user=user, collection=collection, s=None, p=None, o=None, limit=10_000, ) @@ -60,9 +60,10 @@ def main(): try: show_graph( - url=args.api_url, - user=args.user, - collection=args.collection, + url = args.api_url, + flow_id = args.flow_id, + user = args.user, + collection = args.collection, ) except Exception as e: diff --git a/trustgraph-cli/scripts/tg-graph-to-turtle b/trustgraph-cli/scripts/tg-graph-to-turtle index 85e8abf9..da8a5f56 100755 --- a/trustgraph-cli/scripts/tg-graph-to-turtle +++ b/trustgraph-cli/scripts/tg-graph-to-turtle @@ -17,14 +17,14 @@ default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') default_user = 'trustgraph' default_collection = 'default' -def show_graph(url, user, collection): +def show_graph(url, flow_id, user, collection): api = Api(url) - rows = api.triples_query( + rows = api.flow(flow_id).triples_query( s=None, p=None, o=None, + user=user, collection=collection, limit=10_000) -# user=user, collection=collection, g = rdflib.Graph() @@ -92,9 +92,10 @@ def main(): try: show_graph( - url=args.api_url, - user=args.user, - collection=args.collection + url = args.api_url, + flow_id = args.flow_id, + user = args.user, + collection = args.collection, ) except Exception as e: diff --git a/trustgraph-cli/scripts/tg-invoke-agent b/trustgraph-cli/scripts/tg-invoke-agent index 4019b2d0..79760fec 100755 --- a/trustgraph-cli/scripts/tg-invoke-agent +++ b/trustgraph-cli/scripts/tg-invoke-agent @@ -30,7 +30,7 @@ def output(text, prefix="> ", width=78): print(out) async def question( - url, question, user, collection, + url, question, flow_id, user, collection, plan=None, state=None, verbose=False ): @@ -60,6 +60,7 @@ async def question( req = json.dumps({ "id": mid, "service": "agent", + "flow": flow_id, "request": { "question": question, } @@ -74,6 +75,9 @@ async def question( obj = json.loads(msg) + if obj["error"]: + raise RuntimeError(obj["error"]) + if obj["id"] != mid: print("Ignore message") continue @@ -143,12 +147,6 @@ def main(): action="store_true", help=f'Output thinking/observations' ) - - # parser.add_argument( - # '--pulsar-api-key', - # default=default_pulsar_api_key, - # help=f'Pulsar API key', - # ) args = parser.parse_args() @@ -156,13 +154,14 @@ def main(): asyncio.run( question( - url=args.url, - question=args.question, - user=args.user, - collection=args.collection, - plan=args.plan, - state=args.state, - verbose=args.verbose, + url = args.url, + flow_id = args.flow_id, + question = args.question, + user = args.user, + collection = args.collection, + plan = args.plan, + state = args.state, + verbose = args.verbose, ) ) diff --git a/trustgraph-cli/scripts/tg-invoke-document-rag b/trustgraph-cli/scripts/tg-invoke-document-rag index e900156c..fd8d61ed 100755 --- a/trustgraph-cli/scripts/tg-invoke-document-rag +++ b/trustgraph-cli/scripts/tg-invoke-document-rag @@ -13,11 +13,11 @@ default_user = 'trustgraph' default_collection = 'default' default_doc_limit = 10 -def question(url, question, user, collection, doc_limit): +def question(url, flow_id, question, user, collection, doc_limit): - rag = Api(url) + api = Api(url) - resp = rag.document_rag( + resp = api.flow(flow_id).document_rag( question=question, user=user, collection=collection, doc_limit=doc_limit, ) @@ -43,12 +43,6 @@ def main(): help=f'Flow ID (default: 0000)' ) - # parser.add_argument( - # '--pulsar-api-key', - # default=default_pulsar_api_key, - # help=f'Pulsar API key', - # ) - parser.add_argument( '-q', '--question', required=True, @@ -79,6 +73,7 @@ def main(): question( url=args.url, + flow_id = args.flow_id, question=args.question, user=args.user, collection=args.collection, diff --git a/trustgraph-cli/scripts/tg-invoke-graph-rag b/trustgraph-cli/scripts/tg-invoke-graph-rag index 05d97fc0..9cccf680 100755 --- a/trustgraph-cli/scripts/tg-invoke-graph-rag +++ b/trustgraph-cli/scripts/tg-invoke-graph-rag @@ -17,13 +17,13 @@ default_max_subgraph_size = 150 default_max_path_length = 2 def question( - url, question, user, collection, entity_limit, triple_limit, + url, flow_id, question, user, collection, entity_limit, triple_limit, max_subgraph_size, max_path_length ): - rag = Api(url) + api = Api(url) - resp = rag.graph_rag( + resp = api.flow(flow_id).graph_rag( question=question, user=user, collection=collection, entity_limit=entity_limit, triple_limit=triple_limit, max_subgraph_size=max_subgraph_size, @@ -99,6 +99,7 @@ def main(): question( url=args.url, + flow_id = args.flow_id, question=args.question, user=args.user, collection=args.collection, diff --git a/trustgraph-cli/scripts/tg-invoke-llm b/trustgraph-cli/scripts/tg-invoke-llm index 7e3952cd..21915a00 100755 --- a/trustgraph-cli/scripts/tg-invoke-llm +++ b/trustgraph-cli/scripts/tg-invoke-llm @@ -12,11 +12,11 @@ from trustgraph.api import Api default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') -def query(url, flow, system, prompt): +def query(url, flow_id, system, prompt): api = Api(url) - resp = api.flow(flow).text_completion(system=system, prompt=prompt) + resp = api.flow(flow_id).text_completion(system=system, prompt=prompt) print(resp) diff --git a/trustgraph-cli/scripts/tg-invoke-prompt b/trustgraph-cli/scripts/tg-invoke-prompt index a4fe6fbd..b86ef105 100755 --- a/trustgraph-cli/scripts/tg-invoke-prompt +++ b/trustgraph-cli/scripts/tg-invoke-prompt @@ -16,11 +16,11 @@ from trustgraph.api import Api default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') -def query(url, template_id, variables): +def query(url, flow_id, template_id, variables): api = Api(url) - resp = api.prompt(id=template_id, variables=variables) + resp = api.flow(flow_id).prompt(id=template_id, variables=variables) if isinstance(resp, str): print(resp) @@ -83,6 +83,7 @@ specified multiple times''', query( url=args.url, + flow_id=args.flow_id, template_id=args.id[0], variables=variables, ) diff --git a/trustgraph-cli/scripts/tg-load-doc-embeds b/trustgraph-cli/scripts/tg-load-doc-embeds index 25769864..67e9bfa2 100755 --- a/trustgraph-cli/scripts/tg-load-doc-embeds +++ b/trustgraph-cli/scripts/tg-load-doc-embeds @@ -27,7 +27,7 @@ async def load_de(running, queue, url): async with aiohttp.ClientSession() as session: - async with session.ws_connect(f"{url}load/document-embeddings") as ws: + async with session.ws_connect(url) as ws: while running.get(): @@ -141,6 +141,8 @@ async def run(running, **args): # grow to eat all memory de_q = asyncio.Queue(maxsize=10) + url = args["url"] + load_task = asyncio.create_task( loader( running=running, @@ -154,7 +156,8 @@ async def run(running, **args): de_task = asyncio.create_task( load_de( running=running, - queue=de_q, url=args["url"] + "api/v1/" + queue=de_q, + url = f"{url}api/v1/flow/{flow_id}/import/document-embeddings" ) ) diff --git a/trustgraph-cli/scripts/tg-load-pdf b/trustgraph-cli/scripts/tg-load-pdf index c8da2caf..77105315 100755 --- a/trustgraph-cli/scripts/tg-load-pdf +++ b/trustgraph-cli/scripts/tg-load-pdf @@ -27,13 +27,14 @@ class Loader: def __init__( self, url, + flow_id, user, collection, metadata, pulsar_api_key=None, ): - self.api = Api(url) + self.api = Api(url).flow(flow_id) self.user = user self.collection = collection @@ -60,14 +61,15 @@ class Loader: self.api.load_document( document=data, id=id, metadata=self.metadata, -# user=self.user, -# collection=self.collection, + user=self.user, + collection=self.collection, ) print(f"{file}: Loaded successfully.") except Exception as e: print(f"{file}: Failed: {str(e)}", flush=True) + raise e def main(): @@ -155,53 +157,46 @@ def main(): args = parser.parse_args() - while True: + try: - try: + document = DigitalDocument( + id, + name=args.name, + description=args.description, + copyright_notice=args.copyright_notice, + copyright_holder=args.copyright_holder, + copyright_year=args.copyright_year, + license=args.license, + url=args.document_url, + keywords=args.keyword, + ) - document = DigitalDocument( - id, - name=args.name, - description=args.description, - copyright_notice=args.copyright_notice, - copyright_holder=args.copyright_holder, - copyright_year=args.copyright_year, - license=args.license, - url=args.document_url, - keywords=args.keyword, + if args.publication_organization: + org = Organization( + id=to_uri(PREF_ORG, hash(args.publication_organization)), + name=args.publication_organization, + ) + document.publication = PublicationEvent( + id = to_uri(PREF_PUBEV, str(uuid.uuid4())), + organization=org, + description=args.publication_description, + start_date=args.publication_date, + end_date=args.publication_date, ) - if args.publication_organization: - org = Organization( - id=to_uri(PREF_ORG, hash(args.publication_organization)), - name=args.publication_organization, - ) - document.publication = PublicationEvent( - id = to_uri(PREF_PUBEV, str(uuid.uuid4())), - organization=org, - description=args.publication_description, - start_date=args.publication_date, - end_date=args.publication_date, - ) + p = Loader( + url=args.url, + flow_id = args.flow_id, + user=args.user, + collection=args.collection, + metadata=document, + ) - p = Loader( - url=args.url, - user=args.user, - collection=args.collection, - metadata=document, - ) + p.load(args.files) - p.load(args.files) + except Exception as e: - print("All done.") - break - - except Exception as e: - - print("Exception:", e, flush=True) - print("Will retry...", flush=True) - - time.sleep(10) + print("Exception:", e, flush=True) main() diff --git a/trustgraph-cli/scripts/tg-load-text b/trustgraph-cli/scripts/tg-load-text index 9aa98126..2d1c2a90 100755 --- a/trustgraph-cli/scripts/tg-load-text +++ b/trustgraph-cli/scripts/tg-load-text @@ -27,12 +27,13 @@ class Loader: def __init__( self, url, + flow_id, user, collection, metadata, ): - self.api = Api(url) + self.api = Api(url).flow(flow_id) self.user = user self.collection = collection @@ -59,14 +60,15 @@ class Loader: self.api.load_text( text=data, id=id, metadata=self.metadata, -# user=self.user, -# collection=self.collection, + user=self.user, + collection=self.collection, ) print(f"{file}: Loaded successfully.") except Exception as e: print(f"{file}: Failed: {str(e)}", flush=True) + raise e def main(): @@ -154,53 +156,51 @@ def main(): args = parser.parse_args() - while True: - try: + try: - document = DigitalDocument( - id, - name=args.name, - description=args.description, - copyright_notice=args.copyright_notice, - copyright_holder=args.copyright_holder, - copyright_year=args.copyright_year, - license=args.license, - url=args.document_url, - keywords=args.keyword, + document = DigitalDocument( + id, + name=args.name, + description=args.description, + copyright_notice=args.copyright_notice, + copyright_holder=args.copyright_holder, + copyright_year=args.copyright_year, + license=args.license, + url=args.document_url, + keywords=args.keyword, + ) + + if args.publication_organization: + org = Organization( + id=to_uri(PREF_ORG, hash(args.publication_organization)), + name=args.publication_organization, + ) + document.publication = PublicationEvent( + id = to_uri(PREF_PUBEV, str(uuid.uuid4())), + organization=org, + description=args.publication_description, + start_date=args.publication_date, + end_date=args.publication_date, ) - if args.publication_organization: - org = Organization( - id=to_uri(PREF_ORG, hash(args.publication_organization)), - name=args.publication_organization, - ) - document.publication = PublicationEvent( - id = to_uri(PREF_PUBEV, str(uuid.uuid4())), - organization=org, - description=args.publication_description, - start_date=args.publication_date, - end_date=args.publication_date, - ) + p = Loader( + url = args.url, + flow_id = args.flow_id, + user = args.user, + collection = args.collection, + metadata = document, + ) - p = Loader( - url=args.url, - user=args.user, - collection=args.collection, - metadata=document, - ) + p.load(args.files) - p.load(args.files) + print("All done.") + break - print("All done.") - break + except Exception as e: - except Exception as e: + print("Exception:", e, flush=True) - print("Exception:", e, flush=True) - print("Will retry...", flush=True) - - time.sleep(10) main()