apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-17 01:01:03 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Update schema defs for source -> metadata

Browse source
This commit is contained in:
Cyber MacGeddon 2024-10-02 10:28:32 +01:00
parent 709221fa10
commit c86ad2d5b8
4 changed files with 21 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

14
trustgraph-base/trustgraph/schema/documents.py
View file

@ -2,17 +2,13 @@
from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double
from . topic import topic from . topic import topic
from . types import Error from . types import Error
from . metadata import Metadata
class Source(Record):
source = String()
id = String()
title = String()
############################################################################ ############################################################################
# PDF docs etc. # PDF docs etc.
class Document(Record): class Document(Record):
source = Source() metadata = Metadata()
data = Bytes() data = Bytes()
document_ingest_queue = topic('document-load') document_ingest_queue = topic('document-load')
@ -22,7 +18,7 @@ document_ingest_queue = topic('document-load')
# Text documents / text from PDF # Text documents / text from PDF
class TextDocument(Record): class TextDocument(Record):
source = Source() metadata = Metadata()
text = Bytes() text = Bytes()
text_ingest_queue = topic('text-document-load') text_ingest_queue = topic('text-document-load')
@ -32,7 +28,7 @@ text_ingest_queue = topic('text-document-load')
# Chunks of text # Chunks of text
class Chunk(Record): class Chunk(Record):
source = Source() metadata = Metadata()
chunk = Bytes() chunk = Bytes()
chunk_ingest_queue = topic('chunk-load') chunk_ingest_queue = topic('chunk-load')
@ -42,7 +38,7 @@ chunk_ingest_queue = topic('chunk-load')
# Chunk embeddings are an embeddings associated with a text chunk # Chunk embeddings are an embeddings associated with a text chunk
class ChunkEmbeddings(Record): class ChunkEmbeddings(Record):
source = Source() metadata = Metadata()
vectors = Array(Array(Double())) vectors = Array(Array(Double()))
chunk = Bytes() chunk = Bytes()

6
trustgraph-base/trustgraph/schema/graph.py
View file

@ -1,16 +1,16 @@
from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double
from . documents import Source
from . types import Error, Value from . types import Error, Value
from . topic import topic from . topic import topic
from . metadata import Metadata
############################################################################ ############################################################################
# Graph embeddings are embeddings associated with a graph entity # Graph embeddings are embeddings associated with a graph entity
class GraphEmbeddings(Record): class GraphEmbeddings(Record):
source = Source() metadata = Metadata()
vectors = Array(Array(Double())) vectors = Array(Array(Double()))
entity = Value() entity = Value()
@ -40,7 +40,7 @@ graph_embeddings_response_queue = topic(
# Graph triples # Graph triples
class Triple(Record): class Triple(Record):
source = Source() metadata = Metadata()
s = Value() s = Value()
p = Value() p = Value()
o = Value() o = Value()

10
trustgraph-base/trustgraph/schema/metadata.py Normal file
View file

@ -0,0 +1,10 @@
from pulsar.schema import Record, String
class Metadata(Record):
source = String()
id = String()
title = String()
user = String()
collection = String()

6
trustgraph-base/trustgraph/schema/object.py
View file

@ -2,7 +2,7 @@
from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array
from pulsar.schema import Double, Map from pulsar.schema import Double, Map
from . documents import Source from . metadata import Metadata
from . types import Value, RowSchema from . types import Value, RowSchema
from . topic import topic from . topic import topic
@ -12,7 +12,7 @@ from . topic import topic
# object # object
class ObjectEmbeddings(Record): class ObjectEmbeddings(Record):
source = Source() metadata = Metadata()
vectors = Array(Array(Double())) vectors = Array(Array(Double()))
name = String() name = String()
key_name = String() key_name = String()
@ -25,7 +25,7 @@ object_embeddings_store_queue = topic('object-embeddings-store')
# Stores rows of information # Stores rows of information
class Rows(Record): class Rows(Record):
source = Source() metadata = Metadata()
row_schema = RowSchema() row_schema = RowSchema()
rows = Array(Map(String())) rows = Array(Map(String()))