diff --git a/trustgraph-base/trustgraph/schema/documents.py b/trustgraph-base/trustgraph/schema/documents.py index d80ff38f..59aba287 100644 --- a/trustgraph-base/trustgraph/schema/documents.py +++ b/trustgraph-base/trustgraph/schema/documents.py @@ -2,17 +2,13 @@ from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double from . topic import topic from . types import Error - -class Source(Record): - source = String() - id = String() - title = String() +from . metadata import Metadata ############################################################################ # PDF docs etc. class Document(Record): - source = Source() + metadata = Metadata() data = Bytes() document_ingest_queue = topic('document-load') @@ -22,7 +18,7 @@ document_ingest_queue = topic('document-load') # Text documents / text from PDF class TextDocument(Record): - source = Source() + metadata = Metadata() text = Bytes() text_ingest_queue = topic('text-document-load') @@ -32,7 +28,7 @@ text_ingest_queue = topic('text-document-load') # Chunks of text class Chunk(Record): - source = Source() + metadata = Metadata() chunk = Bytes() chunk_ingest_queue = topic('chunk-load') @@ -42,7 +38,7 @@ chunk_ingest_queue = topic('chunk-load') # Chunk embeddings are an embeddings associated with a text chunk class ChunkEmbeddings(Record): - source = Source() + metadata = Metadata() vectors = Array(Array(Double())) chunk = Bytes() diff --git a/trustgraph-base/trustgraph/schema/graph.py b/trustgraph-base/trustgraph/schema/graph.py index 234a0bed..e4a7fa32 100644 --- a/trustgraph-base/trustgraph/schema/graph.py +++ b/trustgraph-base/trustgraph/schema/graph.py @@ -1,16 +1,16 @@ from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double -from . documents import Source from . types import Error, Value from . topic import topic +from . metadata import Metadata ############################################################################ # Graph embeddings are embeddings associated with a graph entity class GraphEmbeddings(Record): - source = Source() + metadata = Metadata() vectors = Array(Array(Double())) entity = Value() @@ -40,7 +40,7 @@ graph_embeddings_response_queue = topic( # Graph triples class Triple(Record): - source = Source() + metadata = Metadata() s = Value() p = Value() o = Value() diff --git a/trustgraph-base/trustgraph/schema/metadata.py b/trustgraph-base/trustgraph/schema/metadata.py new file mode 100644 index 00000000..c7dbbae6 --- /dev/null +++ b/trustgraph-base/trustgraph/schema/metadata.py @@ -0,0 +1,10 @@ + +from pulsar.schema import Record, String + +class Metadata(Record): + source = String() + id = String() + title = String() + user = String() + collection = String() + diff --git a/trustgraph-base/trustgraph/schema/object.py b/trustgraph-base/trustgraph/schema/object.py index 3377e2df..60c2bdc3 100644 --- a/trustgraph-base/trustgraph/schema/object.py +++ b/trustgraph-base/trustgraph/schema/object.py @@ -2,7 +2,7 @@ from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array from pulsar.schema import Double, Map -from . documents import Source +from . metadata import Metadata from . types import Value, RowSchema from . topic import topic @@ -12,7 +12,7 @@ from . topic import topic # object class ObjectEmbeddings(Record): - source = Source() + metadata = Metadata() vectors = Array(Array(Double())) name = String() key_name = String() @@ -25,7 +25,7 @@ object_embeddings_store_queue = topic('object-embeddings-store') # Stores rows of information class Rows(Record): - source = Source() + metadata = Metadata() row_schema = RowSchema() rows = Array(Map(String()))