mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-23 20:21:03 +02:00
Phase 3 started, bulk loading utils
This commit is contained in:
Cyber MacGeddon
parent
561f7f1dda
commit
5a885708df
2 changed files with 168 additions and 150 deletions
|
|
@ -2,18 +2,17 @@
|
||||||
Loads triples and entity contexts into the knowledge graph.
|
Loads triples and entity contexts into the knowledge graph.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
import rdflib
|
import rdflib
|
||||||
import json
|
from typing import Iterator, Tuple
|
||||||
from websockets.asyncio.client import connect
|
|
||||||
from typing import List, Dict, Any
|
|
||||||
|
|
||||||
|
from trustgraph.api import Api, Triple
|
||||||
from trustgraph.log_level import LogLevel
|
from trustgraph.log_level import LogLevel
|
||||||
|
|
||||||
default_url = os.getenv("TRUSTGRAPH_URL", 'ws://localhost:8088/')
|
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||||
|
default_token = os.getenv("TRUSTGRAPH_TOKEN", None)
|
||||||
default_user = 'trustgraph'
|
default_user = 'trustgraph'
|
||||||
default_collection = 'default'
|
default_collection = 'default'
|
||||||
|
|
||||||
|
|
@ -26,108 +25,114 @@ class KnowledgeLoader:
|
||||||
user,
|
user,
|
||||||
collection,
|
collection,
|
||||||
document_id,
|
document_id,
|
||||||
url = default_url,
|
url=default_url,
|
||||||
|
token=None,
|
||||||
):
|
):
|
||||||
|
|
||||||
if not url.endswith("/"):
|
|
||||||
url += "/"
|
|
||||||
|
|
||||||
self.triples_url = url + f"api/v1/flow/{flow}/import/triples"
|
|
||||||
self.entity_contexts_url = url + f"api/v1/flow/{flow}/import/entity-contexts"
|
|
||||||
|
|
||||||
self.files = files
|
self.files = files
|
||||||
|
self.flow = flow
|
||||||
self.user = user
|
self.user = user
|
||||||
self.collection = collection
|
self.collection = collection
|
||||||
self.document_id = document_id
|
self.document_id = document_id
|
||||||
|
self.url = url
|
||||||
|
self.token = token
|
||||||
|
|
||||||
async def run(self):
|
def load_triples_from_file(self, file) -> Iterator[Triple]:
|
||||||
|
"""Generator that yields Triple objects from a Turtle file"""
|
||||||
try:
|
|
||||||
# Load triples first
|
|
||||||
async with connect(self.triples_url) as ws:
|
|
||||||
for file in self.files:
|
|
||||||
await self.load_triples(file, ws)
|
|
||||||
|
|
||||||
# Then load entity contexts
|
|
||||||
async with connect(self.entity_contexts_url) as ws:
|
|
||||||
for file in self.files:
|
|
||||||
await self.load_entity_contexts(file, ws)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(e, flush=True)
|
|
||||||
|
|
||||||
async def load_triples(self, file, ws):
|
|
||||||
|
|
||||||
g = rdflib.Graph()
|
g = rdflib.Graph()
|
||||||
g.parse(file, format="turtle")
|
g.parse(file, format="turtle")
|
||||||
|
|
||||||
def Value(value, is_uri):
|
|
||||||
return { "v": value, "e": is_uri }
|
|
||||||
|
|
||||||
for e in g:
|
for e in g:
|
||||||
s = Value(value=str(e[0]), is_uri=True)
|
# Extract subject, predicate, object
|
||||||
p = Value(value=str(e[1]), is_uri=True)
|
s_value = str(e[0])
|
||||||
if type(e[2]) == rdflib.term.URIRef:
|
p_value = str(e[1])
|
||||||
o = Value(value=str(e[2]), is_uri=True)
|
|
||||||
|
# Check if object is a URI or literal
|
||||||
|
if isinstance(e[2], rdflib.term.URIRef):
|
||||||
|
o_value = str(e[2])
|
||||||
|
o_is_uri = True
|
||||||
else:
|
else:
|
||||||
o = Value(value=str(e[2]), is_uri=False)
|
o_value = str(e[2])
|
||||||
|
o_is_uri = False
|
||||||
|
|
||||||
req = {
|
# Create Triple object
|
||||||
"metadata": {
|
# Note: The Triple dataclass has 's', 'p', 'o' fields as strings
|
||||||
"id": self.document_id,
|
# The API will handle the metadata wrapping
|
||||||
"metadata": [],
|
yield Triple(s=s_value, p=p_value, o=o_value)
|
||||||
"user": self.user,
|
|
||||||
"collection": self.collection
|
|
||||||
},
|
|
||||||
"triples": [
|
|
||||||
{
|
|
||||||
"s": s,
|
|
||||||
"p": p,
|
|
||||||
"o": o,
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
await ws.send(json.dumps(req))
|
def load_entity_contexts_from_file(self, file) -> Iterator[Tuple[str, str]]:
|
||||||
|
"""Generator that yields (entity, context) tuples from a Turtle file"""
|
||||||
async def load_entity_contexts(self, file, ws):
|
|
||||||
"""
|
|
||||||
Load entity contexts by extracting entities from the RDF graph
|
|
||||||
and generating contextual descriptions based on their relationships.
|
|
||||||
"""
|
|
||||||
|
|
||||||
g = rdflib.Graph()
|
g = rdflib.Graph()
|
||||||
g.parse(file, format="turtle")
|
g.parse(file, format="turtle")
|
||||||
|
|
||||||
for s, p, o in g:
|
for s, p, o in g:
|
||||||
# If object is a URI, do nothing
|
# If object is a URI, skip (we only want literal contexts)
|
||||||
if isinstance(o, rdflib.term.URIRef):
|
if isinstance(o, rdflib.term.URIRef):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# If object is a literal, create entity context for subject with literal as context
|
# If object is a literal, create entity context for subject
|
||||||
s_str = str(s)
|
s_str = str(s)
|
||||||
o_str = str(o)
|
o_str = str(o)
|
||||||
|
|
||||||
req = {
|
yield (s_str, o_str)
|
||||||
"metadata": {
|
|
||||||
"id": self.document_id,
|
def run(self):
|
||||||
"metadata": [],
|
"""Load triples and entity contexts using Python API"""
|
||||||
"user": self.user,
|
|
||||||
"collection": self.collection
|
try:
|
||||||
},
|
# Create API client
|
||||||
"entities": [
|
api = Api(url=self.url, token=self.token)
|
||||||
{
|
bulk = api.bulk()
|
||||||
"entity": {
|
|
||||||
"v": s_str,
|
# Load triples from all files
|
||||||
"e": True
|
print("Loading triples...")
|
||||||
},
|
for file in self.files:
|
||||||
"context": o_str
|
print(f" Processing {file}...")
|
||||||
|
triples = self.load_triples_from_file(file)
|
||||||
|
|
||||||
|
bulk.import_triples(
|
||||||
|
flow=self.flow,
|
||||||
|
triples=triples,
|
||||||
|
metadata={
|
||||||
|
"id": self.document_id,
|
||||||
|
"metadata": [],
|
||||||
|
"user": self.user,
|
||||||
|
"collection": self.collection
|
||||||
}
|
}
|
||||||
]
|
)
|
||||||
}
|
|
||||||
|
|
||||||
await ws.send(json.dumps(req))
|
print("Triples loaded.")
|
||||||
|
|
||||||
|
# Load entity contexts from all files
|
||||||
|
print("Loading entity contexts...")
|
||||||
|
for file in self.files:
|
||||||
|
print(f" Processing {file}...")
|
||||||
|
|
||||||
|
# Convert tuples to the format expected by import_entity_contexts
|
||||||
|
def entity_context_generator():
|
||||||
|
for entity, context in self.load_entity_contexts_from_file(file):
|
||||||
|
yield {
|
||||||
|
"entity": {"v": entity, "e": True},
|
||||||
|
"context": context
|
||||||
|
}
|
||||||
|
|
||||||
|
bulk.import_entity_contexts(
|
||||||
|
flow=self.flow,
|
||||||
|
entities=entity_context_generator(),
|
||||||
|
metadata={
|
||||||
|
"id": self.document_id,
|
||||||
|
"metadata": [],
|
||||||
|
"user": self.user,
|
||||||
|
"collection": self.collection
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
print("Entity contexts loaded.")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error: {e}", flush=True)
|
||||||
|
raise
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|
||||||
|
|
@ -142,6 +147,12 @@ def main():
|
||||||
help=f'API URL (default: {default_url})',
|
help=f'API URL (default: {default_url})',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-t', '--token',
|
||||||
|
default=default_token,
|
||||||
|
help='Authentication token (default: $TRUSTGRAPH_TOKEN)',
|
||||||
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'-i', '--document-id',
|
'-i', '--document-id',
|
||||||
required=True,
|
required=True,
|
||||||
|
|
@ -166,7 +177,6 @@ def main():
|
||||||
help=f'Collection ID (default: {default_collection})'
|
help=f'Collection ID (default: {default_collection})'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'files', nargs='+',
|
'files', nargs='+',
|
||||||
help=f'Turtle files to load'
|
help=f'Turtle files to load'
|
||||||
|
|
@ -178,15 +188,16 @@ def main():
|
||||||
|
|
||||||
try:
|
try:
|
||||||
loader = KnowledgeLoader(
|
loader = KnowledgeLoader(
|
||||||
document_id = args.document_id,
|
document_id=args.document_id,
|
||||||
url = args.api_url,
|
url=args.api_url,
|
||||||
flow = args.flow_id,
|
token=args.token,
|
||||||
files = args.files,
|
flow=args.flow_id,
|
||||||
user = args.user,
|
files=args.files,
|
||||||
collection = args.collection,
|
user=args.user,
|
||||||
|
collection=args.collection,
|
||||||
)
|
)
|
||||||
|
|
||||||
asyncio.run(loader.run())
|
loader.run()
|
||||||
|
|
||||||
print("Triples and entity contexts loaded.")
|
print("Triples and entity contexts loaded.")
|
||||||
break
|
break
|
||||||
|
|
@ -199,4 +210,4 @@ def main():
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,18 @@
|
||||||
"""
|
"""
|
||||||
Loads triples into the knowledge graph.
|
Loads triples into the knowledge graph from Turtle files.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
import rdflib
|
import rdflib
|
||||||
import json
|
from typing import Iterator
|
||||||
from websockets.asyncio.client import connect
|
|
||||||
|
|
||||||
|
from trustgraph.api import Api, Triple
|
||||||
from trustgraph.log_level import LogLevel
|
from trustgraph.log_level import LogLevel
|
||||||
|
|
||||||
default_url = os.getenv("TRUSTGRAPH_URL", 'ws://localhost:8088/')
|
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||||
|
default_token = os.getenv("TRUSTGRAPH_TOKEN", None)
|
||||||
default_user = 'trustgraph'
|
default_user = 'trustgraph'
|
||||||
default_collection = 'default'
|
default_collection = 'default'
|
||||||
|
|
||||||
|
|
@ -25,67 +25,67 @@ class Loader:
|
||||||
user,
|
user,
|
||||||
collection,
|
collection,
|
||||||
document_id,
|
document_id,
|
||||||
url = default_url,
|
url=default_url,
|
||||||
|
token=None,
|
||||||
):
|
):
|
||||||
|
|
||||||
if not url.endswith("/"):
|
|
||||||
url += "/"
|
|
||||||
|
|
||||||
url = url + f"api/v1/flow/{flow}/import/triples"
|
|
||||||
|
|
||||||
self.url = url
|
|
||||||
|
|
||||||
self.files = files
|
self.files = files
|
||||||
|
self.flow = flow
|
||||||
self.user = user
|
self.user = user
|
||||||
self.collection = collection
|
self.collection = collection
|
||||||
self.document_id = document_id
|
self.document_id = document_id
|
||||||
|
self.url = url
|
||||||
|
self.token = token
|
||||||
|
|
||||||
async def run(self):
|
def load_triples_from_file(self, file) -> Iterator[Triple]:
|
||||||
|
"""Generator that yields Triple objects from a Turtle file"""
|
||||||
try:
|
|
||||||
|
|
||||||
async with connect(self.url) as ws:
|
|
||||||
for file in self.files:
|
|
||||||
await self.load_file(file, ws)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(e, flush=True)
|
|
||||||
|
|
||||||
async def load_file(self, file, ws):
|
|
||||||
|
|
||||||
g = rdflib.Graph()
|
g = rdflib.Graph()
|
||||||
g.parse(file, format="turtle")
|
g.parse(file, format="turtle")
|
||||||
|
|
||||||
def Value(value, is_uri):
|
|
||||||
return { "v": value, "e": is_uri }
|
|
||||||
|
|
||||||
triples = []
|
|
||||||
|
|
||||||
for e in g:
|
for e in g:
|
||||||
s = Value(value=str(e[0]), is_uri=True)
|
# Extract subject, predicate, object
|
||||||
p = Value(value=str(e[1]), is_uri=True)
|
s_value = str(e[0])
|
||||||
if type(e[2]) == rdflib.term.URIRef:
|
p_value = str(e[1])
|
||||||
o = Value(value=str(e[2]), is_uri=True)
|
|
||||||
|
# Check if object is a URI or literal
|
||||||
|
if isinstance(e[2], rdflib.term.URIRef):
|
||||||
|
o_value = str(e[2])
|
||||||
else:
|
else:
|
||||||
o = Value(value=str(e[2]), is_uri=False)
|
o_value = str(e[2])
|
||||||
|
|
||||||
req = {
|
# Create Triple object
|
||||||
"metadata": {
|
yield Triple(s=s_value, p=p_value, o=o_value)
|
||||||
"id": self.document_id,
|
|
||||||
"metadata": [],
|
def run(self):
|
||||||
"user": self.user,
|
"""Load triples using Python API"""
|
||||||
"collection": self.collection
|
|
||||||
},
|
try:
|
||||||
"triples": [
|
# Create API client
|
||||||
{
|
api = Api(url=self.url, token=self.token)
|
||||||
"s": s,
|
bulk = api.bulk()
|
||||||
"p": p,
|
|
||||||
"o": o,
|
# Load triples from all files
|
||||||
|
print("Loading triples...")
|
||||||
|
for file in self.files:
|
||||||
|
print(f" Processing {file}...")
|
||||||
|
triples = self.load_triples_from_file(file)
|
||||||
|
|
||||||
|
bulk.import_triples(
|
||||||
|
flow=self.flow,
|
||||||
|
triples=triples,
|
||||||
|
metadata={
|
||||||
|
"id": self.document_id,
|
||||||
|
"metadata": [],
|
||||||
|
"user": self.user,
|
||||||
|
"collection": self.collection
|
||||||
}
|
}
|
||||||
]
|
)
|
||||||
}
|
|
||||||
|
|
||||||
await ws.send(json.dumps(req))
|
print("Triples loaded.")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error: {e}", flush=True)
|
||||||
|
raise
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|
||||||
|
|
@ -100,6 +100,12 @@ def main():
|
||||||
help=f'API URL (default: {default_url})',
|
help=f'API URL (default: {default_url})',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-t', '--token',
|
||||||
|
default=default_token,
|
||||||
|
help='Authentication token (default: $TRUSTGRAPH_TOKEN)',
|
||||||
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'-i', '--document-id',
|
'-i', '--document-id',
|
||||||
required=True,
|
required=True,
|
||||||
|
|
@ -134,16 +140,17 @@ def main():
|
||||||
while True:
|
while True:
|
||||||
|
|
||||||
try:
|
try:
|
||||||
p = Loader(
|
loader = Loader(
|
||||||
document_id = args.document_id,
|
document_id=args.document_id,
|
||||||
url = args.api_url,
|
url=args.api_url,
|
||||||
flow = args.flow_id,
|
token=args.token,
|
||||||
files = args.files,
|
flow=args.flow_id,
|
||||||
user = args.user,
|
files=args.files,
|
||||||
collection = args.collection,
|
user=args.user,
|
||||||
|
collection=args.collection,
|
||||||
)
|
)
|
||||||
|
|
||||||
asyncio.run(p.run())
|
loader.run()
|
||||||
|
|
||||||
print("File loaded.")
|
print("File loaded.")
|
||||||
break
|
break
|
||||||
|
|
@ -156,4 +163,4 @@ def main():
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue