apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-23 20:21:03 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

List works

Browse source
This commit is contained in:
Cyber MacGeddon 2025-05-04 20:52:16 +01:00
parent c13fda0e84
commit ee959e6f17
5 changed files with 179 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

40
test-api/test-library-list-documents Executable file
View file

@ -0,0 +1,40 @@
#!/usr/bin/env python3
import requests
import json
import sys
import base64
import time
url = "http://localhost:8088/api/v1/"
############################################################################
id = "http://trustgraph.ai/doc/9fdee98b-b259-40ac-bcb9-8e82ccedeb04"
input = {
"operation": "list-documents",
"user": "trustgraph",
}
resp = requests.post(
f"{url}librarian",
json=input,
)
print(resp.text)
resp = resp.json()
print(resp)
if "error" in resp:
print(f"Error: {resp['error']}")
sys.exit(1)
# print(resp["response"])
print(resp)
sys.exit(0)
############################################################################

75
test-api/test-library-update-doc Executable file
View file

@ -0,0 +1,75 @@
#!/usr/bin/env python3
import requests
import json
import sys
import base64
import time
url = "http://localhost:8088/api/v1/"
############################################################################
id = "http://trustgraph.ai/doc/9fdee98b-b259-40ac-bcb9-8e82ccedeb04"
input = {
"operation": "update-document",
"document-metadata": {
"id": id,
"time": int(time.time()),
"title": "Mark's cats - a story",
"comments": "Information about Mark's cats",
"metadata": [
{
"s": {
"v": id,
"e": True,
},
"p": {
"v": "http://www.w3.org/2000/01/rdf-schema#label",
"e": True,
},
"o": {
"v": "Mark's pets", "e": False,
},
},
{
"s": {
"v": id,
"e": True,
},
"p": {
"v": 'https://schema.org/keywords',
"e": True,
},
"o": {
"v": "cats", "e": False,
},
},
],
"user": "trustgraph",
"tags": ["mark", "cats", "pets"],
},
}
resp = requests.post(
f"{url}librarian",
json=input,
)
print(resp.text)
resp = resp.json()
print(resp)
if "error" in resp:
print(f"Error: {resp['error']}")
sys.exit(1)
# print(resp["response"])
print(resp)
sys.exit(0)
############################################################################

8
trustgraph-flow/trustgraph/gateway/dispatch/librarian.py
View file

@ -47,7 +47,7 @@ class LibrarianRequestor(ServiceRequestor):
if "content" in body: if "content" in body:
content = base64.b64decode(body["content"].encode("utf-8")) content = base64.b64decode(body["content"].encode("utf-8"))
content = base64.b64encode(content).decode("utf-8"), content = base64.b64encode(content).decode("utf-8")
else: else:
content = None content = None
@ -65,6 +65,8 @@ class LibrarianRequestor(ServiceRequestor):
def from_response(self, message): def from_response(self, message):
print(message)
response = {} response = {}
if message.document_metadata: if message.document_metadata:
@ -75,13 +77,13 @@ class LibrarianRequestor(ServiceRequestor):
if message.content: if message.content:
response["content"] = message.content response["content"] = message.content
if message.document_metadatas: if message.document_metadatas != None:
response["document-metadatas"] = [ response["document-metadatas"] = [
serialize_document_metadata(v) serialize_document_metadata(v)
for v in message.document_metadatas for v in message.document_metadatas
] ]
if message.processing_metadatas: if message.processing_metadatas != None:
response["processing-metadatas"] = [ response["processing-metadatas"] = [
serialize_processing_metadata(v) serialize_processing_metadata(v)
for v in message.processing_metadatas for v in message.processing_metadatas

34
trustgraph-flow/trustgraph/librarian/librarian.py
View file

@ -37,7 +37,8 @@ class Librarian:
"Invalid document kind: " + request.document_metadata.kind "Invalid document kind: " + request.document_metadata.kind
) )
if self.table_store.document_exists( print("Existence test...")
if await self.table_store.document_exists(
request.document_metadata.user, request.document_metadata.user,
request.document_metadata.id request.document_metadata.id
): ):
@ -46,10 +47,22 @@ class Librarian:
# Create object ID for blob # Create object ID for blob
object_id = uuid.uuid4() object_id = uuid.uuid4()
print("OID", object_id)
print(request.content)
print("CONT", base64.b64decode(request.content))
print("Add blob...")
self.blob_store.add(object_id, base64.b64decode(request.content), self.blob_store.add(object_id, base64.b64decode(request.content),
request.document_metadata.kind) request.document_metadata.kind)
self.table_store.add_document(request.document_metadata, object_id) print("Add table...")
await self.table_store.add_document(
request.document_metadata, object_id
)
print("Add complete", flush=True) print("Add complete", flush=True)
@ -70,13 +83,13 @@ class Librarian:
# You can't update the document ID, user or kind. # You can't update the document ID, user or kind.
if not self.table_store.document_exists( if not await self.table_store.document_exists(
request.document_metadata.user, request.document_metadata.user,
request.document_metadata.id request.document_metadata.id
): ):
raise RuntimeError("Document does not exist") raise RuntimeError("Document does not exist")
self.table_store.update_document(request.document_metadata) await self.table_store.update_document(request.document_metadata)
print("Update complete", flush=True) print("Update complete", flush=True)
@ -106,7 +119,18 @@ class Librarian:
raise RuntimeError("Not implemented") raise RuntimeError("Not implemented")
async def list_documents(self, request): async def list_documents(self, request):
raise RuntimeError("Not implemented")
docs = await self.table_store.list_documents(request.user)
print(docs)
return LibrarianResponse(
error = None,
document_metadata = None,
content = None,
document_metadatas = docs,
processing_metadatas = None,
)
async def list_processing(self, request): async def list_processing(self, request):
raise RuntimeError("Not implemented") raise RuntimeError("Not implemented")

78
trustgraph-flow/trustgraph/librarian/table_store.py
View file

@ -244,7 +244,7 @@ class TableStore:
VALUES (?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?)
""") """)
def document_exists(self, user, id): async def document_exists(self, user, id):
resp = self.cassandra.execute( resp = self.cassandra.execute(
self.test_document_exists_stmt, self.test_document_exists_stmt,
@ -259,7 +259,7 @@ class TableStore:
return False return False
def add_document(self, document, object_id): async def add_document(self, document, object_id):
print("Adding document", document.id, object_id) print("Adding document", document.id, object_id)
@ -290,11 +290,11 @@ class TableStore:
print("Exception:", type(e)) print("Exception:", type(e))
print(f"{e}, retry...", flush=True) print(f"{e}, retry...", flush=True)
time.sleep(1) await asyncio.sleep(1)
print("Add complete", flush=True) print("Add complete", flush=True)
def update_document(self, document): async def update_document(self, document):
print("Updating document", document.id) print("Updating document", document.id)
@ -325,11 +325,11 @@ class TableStore:
print("Exception:", type(e)) print("Exception:", type(e))
print(f"{e}, retry...", flush=True) print(f"{e}, retry...", flush=True)
time.sleep(1) await asyncio.sleep(1)
print("Update complete", flush=True) print("Update complete", flush=True)
def add_triples(self, m): async def add_triples(self, m):
when = int(time.time() * 1000) when = int(time.time() * 1000)
@ -371,77 +371,59 @@ class TableStore:
print("Exception:", type(e)) print("Exception:", type(e))
print(f"{e}, retry...", flush=True) print(f"{e}, retry...", flush=True)
time.sleep(1) await asyncio.sleep(1)
def list(self, user, collection=None): async def list_documents(self, user):
print("LIST") print("LIST")
while True: while True:
print("TRY")
print(self.list_document_stmt)
try: try:
if collection: resp = self.cassandra.execute(
resp = self.cassandra.execute( self.list_document_stmt,
self.list_document_by_collection_stmt, (user,)
(user, collection) )
)
else:
resp = self.cassandra.execute(
self.list_document_stmt,
(user,)
)
break
print("OK") print("OK")
break
except Exception as e: except Exception as e:
print("Exception:", type(e)) print("Exception:", type(e))
print(f"{e}, retry...", flush=True) print(f"{e}, retry...", flush=True)
time.sleep(1) await asyncio.sleep(1)
print("OK2")
info = [ lst = [
DocumentMetadata( DocumentMetadata(
id = row[0], id = row[0],
kind = row[1], user = user,
flow = row[2], time = int(time.mktime(row[1].timetuple())),
user = row[3], kind = row[2],
collection = row[4], title = row[3],
title = row[5], comments = row[4],
comments = row[6],
time = int(1000 * row[7].timestamp()),
metadata = [ metadata = [
Triple( Triple(
s=Value(value=m[0], is_uri=m[1]), s=Value(value=m[0], is_uri=m[1]),
p=Value(value=m[2], is_uri=m[3]), p=Value(value=m[2], is_uri=m[3]),
o=Value(value=m[4], is_uri=m[5]) o=Value(value=m[4], is_uri=m[5])
) )
for m in row[8] for m in row[5]
], ],
tags = row[6],
object_id = row[7],
) )
for row in resp for row in resp
] ]
print("OK3") print("OK3")
print(info) print(lst)
# print(info[0].user) return lst
# print(info[0].time)
# print(info[0].kind)
# print(info[0].collection)
# print(info[0].title)
# print(info[0].comments)
# print(info[0].metadata)
# print(info[0].metadata)
return info async def add_graph_embeddings(self, m):
def add_graph_embeddings(self, m):
when = int(time.time() * 1000) when = int(time.time() * 1000)
@ -483,9 +465,9 @@ class TableStore:
print("Exception:", type(e)) print("Exception:", type(e))
print(f"{e}, retry...", flush=True) print(f"{e}, retry...", flush=True)
time.sleep(1) await asyncio.sleep(1)
def add_document_embeddings(self, m): async def add_document_embeddings(self, m):
when = int(time.time() * 1000) when = int(time.time() * 1000)
@ -527,6 +509,6 @@ class TableStore:
print("Exception:", type(e)) print("Exception:", type(e))
print(f"{e}, retry...", flush=True) print(f"{e}, retry...", flush=True)
time.sleep(1) await asyncio.sleep(1)