apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-16 16:51:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Update schema defs for source -> metadata

Browse source
This commit is contained in:
Cyber MacGeddon 2024-10-02 10:28:32 +01:00
parent 709221fa10
commit c86ad2d5b8
4 changed files with 21 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

14
trustgraph-base/trustgraph/schema/documents.py
View file

@ -2,17 +2,13 @@
from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double
from . topic import topic
from . types import Error
class Source(Record):
source = String()
id = String()
title = String()
from . metadata import Metadata
############################################################################
# PDF docs etc.
class Document(Record):
source = Source()
metadata = Metadata()
data = Bytes()
document_ingest_queue = topic('document-load')
@ -22,7 +18,7 @@ document_ingest_queue = topic('document-load')
# Text documents / text from PDF
class TextDocument(Record):
source = Source()
metadata = Metadata()
text = Bytes()
text_ingest_queue = topic('text-document-load')
@ -32,7 +28,7 @@ text_ingest_queue = topic('text-document-load')
# Chunks of text
class Chunk(Record):
source = Source()
metadata = Metadata()
chunk = Bytes()
chunk_ingest_queue = topic('chunk-load')
@ -42,7 +38,7 @@ chunk_ingest_queue = topic('chunk-load')
# Chunk embeddings are an embeddings associated with a text chunk
class ChunkEmbeddings(Record):
source = Source()
metadata = Metadata()
vectors = Array(Array(Double()))
chunk = Bytes()

6
trustgraph-base/trustgraph/schema/graph.py
View file

@ -1,16 +1,16 @@
from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double
from . documents import Source
from . types import Error, Value
from . topic import topic
from . metadata import Metadata
############################################################################
# Graph embeddings are embeddings associated with a graph entity
class GraphEmbeddings(Record):
source = Source()
metadata = Metadata()
vectors = Array(Array(Double()))
entity = Value()
@ -40,7 +40,7 @@ graph_embeddings_response_queue = topic(
# Graph triples
class Triple(Record):
source = Source()
metadata = Metadata()
s = Value()
p = Value()
o = Value()

10
trustgraph-base/trustgraph/schema/metadata.py Normal file
View file

@ -0,0 +1,10 @@
from pulsar.schema import Record, String
class Metadata(Record):
source = String()
id = String()
title = String()
user = String()
collection = String()

6
trustgraph-base/trustgraph/schema/object.py
View file

@ -2,7 +2,7 @@
from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array
from pulsar.schema import Double, Map
from . documents import Source
from . metadata import Metadata
from . types import Value, RowSchema
from . topic import topic
@ -12,7 +12,7 @@ from . topic import topic
# object
class ObjectEmbeddings(Record):
source = Source()
metadata = Metadata()
vectors = Array(Array(Double()))
name = String()
key_name = String()
@ -25,7 +25,7 @@ object_embeddings_store_queue = topic('object-embeddings-store')
# Stores rows of information
class Rows(Record):
source = Source()
metadata = Metadata()
row_schema = RowSchema()
rows = Array(Map(String()))