Feature / collections (#96)

* Update schema defs for source -> metadata
* Migrate to use metadata part of schema, also add metadata to triples & vecs
* Add user/collection metadata to query
* Use user/collection in RAG
* Write and query working on triples

trustgraph-cli/scripts/tg-load-text

@@ -6,7 +6,7 @@ Loads a text document into TrustGraph processing.
 
 import pulsar
 from pulsar.schema import JsonSchema
-from trustgraph.schema import TextDocument, Source, text_ingest_queue
+from trustgraph.schema import TextDocument, text_ingest_queue, Metadata
 import base64
 import hashlib
 import argparse
@@ -15,12 +15,17 @@ import time
 
 from trustgraph.log_level import LogLevel
 
+default_user = 'trustgraph'
+default_collection = 'default'
+
 class Loader:
 
     def __init__(
             self,
             pulsar_host,
             output_queue,
+            user,
+            collection,
             log_level,
     ):
 
@@ -35,6 +40,9 @@ class Loader:
             chunking_enabled=True,
         )
 
+        self.user = user
+        self.collection = collection
+
     def load(self, files):
 
         for file in files:
@@ -50,10 +58,12 @@ class Loader:
             id = hashlib.sha256(path.encode("utf-8")).hexdigest()[0:8]
 
             r = TextDocument(
-                source=Source(
+                metadata=Metadata(
                     source=path,
                     title=path,
                     id=id,
+                    user=self.user,
+                    collection=self.collection,
                 ),
                 text=data,
             )
@@ -90,6 +100,18 @@ def main():
         help=f'Output queue (default: {default_output_queue})'
     )
 
+    parser.add_argument(
+        '-u', '--user',
+        default=default_user,
+        help=f'User ID (default: {default_user})'
+    )
+
+    parser.add_argument(
+        '-c', '--collection',
+        default=default_collection,
+        help=f'Collection ID (default: {default_collection})'
+    )
+
     parser.add_argument(
         '-l', '--log-level',
         type=LogLevel,
@@ -112,6 +134,8 @@ def main():
             p = Loader(
                 pulsar_host=args.pulsar_host,
                 output_queue=args.output_queue,
+                user=args.user,
+                collection=args.collection,
                 log_level=args.log_level,
             )