apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-21 11:11:03 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Tweak schema

Browse source
This commit is contained in:
Cyber MacGeddon 2025-02-12 22:24:53 +00:00
parent d8abbf3d0b
commit c78c8b6724
2 changed files with 52 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

2
trustgraph-base/trustgraph/schema/library.py
View file

@ -22,6 +22,8 @@ class DocumentPackage(Record):
kind = String() kind = String()
user = String() user = String()
collection = String() collection = String()
title = String()
comments = String()
class DocumentInfo(Record): class DocumentInfo(Record):
metadata = Array(Triple()) metadata = Array(Triple())

71
trustgraph-flow/trustgraph/librarian/table_store.py
View file

@ -58,10 +58,13 @@ class TableStore:
print("document table...", flush=True) print("document table...", flush=True)
self.cassandra.execute(""" self.cassandra.execute("""
create table if not exists document ( CREATE TABLE IF NOT EXISTS document (
user text, user text,
collection text, collection text,
id uuid, id uuid,
time timestamp,
title text,
comments text,
kind text, kind text,
object_id uuid, object_id uuid,
metadata list<tuple< metadata list<tuple<
@ -74,18 +77,19 @@ class TableStore:
print("object index...", flush=True) print("object index...", flush=True)
self.cassandra.execute(""" self.cassandra.execute("""
create index if not exists document_object CREATE INDEX IF NOT EXISTS document_object
on document ( object_id) ON document (object_id)
"""); """);
print("triples table...", flush=True) print("triples table...", flush=True)
self.cassandra.execute(""" self.cassandra.execute("""
create table if not exists triples ( CREATE TABLE IF NOT EXISTS triples (
user text, user text,
collection text, collection text,
document_id text, document_id text,
id uuid, id uuid,
time timestamp,
metadata list<tuple< metadata list<tuple<
text, boolean, text, boolean, text, boolean text, boolean, text, boolean, text, boolean
>>, >>,
@ -104,6 +108,7 @@ class TableStore:
collection text, collection text,
document_id text, document_id text,
id uuid, id uuid,
time timestamp,
metadata list<tuple< metadata list<tuple<
text, boolean, text, boolean, text, boolean text, boolean, text, boolean, text, boolean
>>, >>,
@ -125,6 +130,7 @@ class TableStore:
collection text, collection text,
document_id text, document_id text,
id uuid, id uuid,
time timestamp,
metadata list<tuple< metadata list<tuple<
text, boolean, text, boolean, text, boolean text, boolean, text, boolean, text, boolean
>>, >>,
@ -143,27 +149,39 @@ class TableStore:
def prepare_statements(self): def prepare_statements(self):
self.insert_document_stmt = self.cassandra.prepare(""" self.insert_document_stmt = self.cassandra.prepare("""
insert into document INSERT INTO document
(id, user, collection, kind, object_id, metadata) (
values (?, ?, ?, ?, ?, ?) id, user, collection, kind, object_id, time, title, comments,
metadata
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""") """)
self.insert_triples_stmt = self.cassandra.prepare(""" self.insert_triples_stmt = self.cassandra.prepare("""
insert into triples INSERT INTO triples
(id, user, collection, document_id, metadata, triples) (
values (?, ?, ?, ?, ?, ?) id, user, collection, document_id, time,
metadata, triples
)
VALUES (?, ?, ?, ?, ?, ?, ?)
""") """)
self.insert_graph_embeddings_stmt = self.cassandra.prepare(""" self.insert_graph_embeddings_stmt = self.cassandra.prepare("""
insert into graph_embeddings INSERT INTO graph_embeddings
(id, user, collection, document_id, metadata, entity_embeddings) (
values (?, ?, ?, ?, ?, ?) id, user, collection, document_id, time,
metadata, entity_embeddings
)
VALUES (?, ?, ?, ?, ?, ?, ?)
""") """)
self.insert_document_embeddings_stmt = self.cassandra.prepare(""" self.insert_document_embeddings_stmt = self.cassandra.prepare("""
insert into document_embeddings INSERT INTO document_embeddings
(id, user, collection, document_id, metadata, chunks) (
values (?, ?, ?, ?, ?, ?) id, user, collection, document_id, time,
metadata, chunks
)
VALUES (?, ?, ?, ?, ?, ?, ?)
""") """)
def add(self, object_id, document): def add(self, object_id, document):
@ -175,6 +193,7 @@ class TableStore:
# Create random doc ID # Create random doc ID
doc_id = uuid.uuid4() doc_id = uuid.uuid4()
when = int(time.time() * 1000)
print("Adding", object_id, doc_id) print("Adding", object_id, doc_id)
@ -186,6 +205,8 @@ class TableStore:
for v in document.metadata for v in document.metadata
] ]
# FIXME: doc_id should be the user-supplied ID???
while True: while True:
try: try:
@ -193,8 +214,10 @@ class TableStore:
resp = self.cassandra.execute( resp = self.cassandra.execute(
self.insert_document_stmt, self.insert_document_stmt,
( (
doc_id, document.user, document.collection, doc_id, document.user, document.collection,
document.kind, object_id, metadata document.kind, object_id, when,
document.title, document.comments,
metadata
) )
) )
@ -210,6 +233,8 @@ class TableStore:
def add_triples(self, m): def add_triples(self, m):
when = int(time.time() * 1000)
if m.metadata.metadata: if m.metadata.metadata:
metadata = [ metadata = [
( (
@ -237,7 +262,7 @@ class TableStore:
self.insert_triples_stmt, self.insert_triples_stmt,
( (
uuid.uuid4(), m.metadata.user, uuid.uuid4(), m.metadata.user,
m.metadata.collection, m.metadata.id, m.metadata.collection, m.metadata.id, when,
metadata, triples, metadata, triples,
) )
) )
@ -252,6 +277,8 @@ class TableStore:
def add_graph_embeddings(self, m): def add_graph_embeddings(self, m):
when = int(time.time() * 1000)
if m.metadata.metadata: if m.metadata.metadata:
metadata = [ metadata = [
( (
@ -279,7 +306,7 @@ class TableStore:
self.insert_graph_embeddings_stmt, self.insert_graph_embeddings_stmt,
( (
uuid.uuid4(), m.metadata.user, uuid.uuid4(), m.metadata.user,
m.metadata.collection, m.metadata.id, m.metadata.collection, m.metadata.id, when,
metadata, entities, metadata, entities,
) )
) )
@ -294,6 +321,8 @@ class TableStore:
def add_document_embeddings(self, m): def add_document_embeddings(self, m):
when = int(time.time() * 1000)
if m.metadata.metadata: if m.metadata.metadata:
metadata = [ metadata = [
( (
@ -321,7 +350,7 @@ class TableStore:
self.insert_document_embeddings_stmt, self.insert_document_embeddings_stmt,
( (
uuid.uuid4(), m.metadata.user, uuid.uuid4(), m.metadata.user,
m.metadata.collection, m.metadata.id, m.metadata.collection, m.metadata.id, when,
metadata, chunks, metadata, chunks,
) )
) )