Merge branch 'release/v0.21' into pulsar-api-support

trustgraph-cli/scripts/tg-load-text

@@ -12,14 +12,13 @@ import os
 import time
 import uuid
 
-from trustgraph.schema import TextDocument, text_ingest_queue
-from trustgraph.schema import Metadata, Triple, Value
-from trustgraph.log_level import LogLevel
-from trustgraph.knowledge import hash, to_uri, Literal, Uri
+from trustgraph.api import Api
+from trustgraph.knowledge import hash, to_uri
 from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG
 from trustgraph.knowledge import Organization, PublicationEvent
 from trustgraph.knowledge import DigitalDocument
 
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 default_user = 'trustgraph'
 default_collection = 'default'
 
@@ -27,32 +26,14 @@ class Loader:
 
     def __init__(
             self,
-            pulsar_host,
-            output_queue,
+            url,
             user,
             collection,
-            log_level,
             metadata,
             pulsar_api_key=None,
     ):
-        if pulsar_api_key:
-            auth = pulsar.AuthenticationToken(pulsar_api_key)
-            self.client = pulsar.Client(
-                pulsar_host,
-                authentication=auth,
-                logger=pulsar.ConsoleLogger(log_level.to_pulsar())
-            )
-        else:
-            self.client = pulsar.Client(
-                pulsar_host,
-                logger=pulsar.ConsoleLogger(log_level.to_pulsar())
-            )
 
-        self.producer = self.client.create_producer(
-            topic=output_queue,
-            schema=JsonSchema(TextDocument),
-            chunking_enabled=True,
-        )
+        self.api = Api(url)
 
         self.user = user
         self.collection = collection
@@ -75,49 +56,18 @@ class Loader:
 
             id = to_uri(PREF_DOC, id)
 
-            triples = []
-
-            def emit(t):
-                triples.append(t)
-
             self.metadata.id = id
-            self.metadata.emit(emit)
 
-            r = TextDocument(
-                metadata=Metadata(
-                    id=id,
-                    metadata=[
-                        Triple(
-                            s=Value(
-                                value=t["s"],
-                                is_uri=isinstance(t["s"], Uri)
-                            ),
-                            p=Value(
-                                value=t["p"],
-                                is_uri=isinstance(t["p"], Uri)
-                            ),
-                            o=Value(
-                                value=t["o"],
-                                is_uri=isinstance(t["o"], Uri)
-                            ),
-                        )
-                        for t in triples
-                    ],
-                    user=self.user,
-                    collection=self.collection,
-                ),
-                text=data,
+            self.api.load_text(
+                text=data, id=id, metadata=self.metadata, 
+#                user=self.user,
+#                    collection=self.collection,
             )
 
-            self.producer.send(r)
-
             print(f"{file}: Loaded successfully.")
 
         except Exception as e:
             print(f"{file}: Failed: {str(e)}", flush=True)
-            
-    def __del__(self):
-        self.client.close()
 
 def main():
 
@@ -126,37 +76,20 @@ def main():
         description=__doc__,
     )
 
-    default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
-    default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
-
-    default_output_queue = text_ingest_queue
-
     parser.add_argument(
-        '-p', '--pulsar-host',
-        default=default_pulsar_host,
-        help=f'Pulsar host (default: {default_pulsar_host})',
-    )
-    
-    parser.add_argument(
-        '--pulsar-api-key',
-        default=default_pulsar_api_key,
-        help=f'Pulsar API key',
+        '-u', '--url',
+        default=default_url,
+        help=f'API URL (default: {default_url})',
     )
 
     parser.add_argument(
-        '-o', '--output-queue',
-        default=default_output_queue,
-        help=f'Output queue (default: {default_output_queue})'
-    )
-
-    parser.add_argument(
-        '-u', '--user',
+        '-U', '--user',
         default=default_user,
         help=f'User ID (default: {default_user})'
     )
 
     parser.add_argument(
-        '-c', '--collection',
+        '-C', '--collection',
         default=default_collection,
         help=f'Collection ID (default: {default_collection})'
     )
@@ -198,7 +131,7 @@ def main():
     )
 
     parser.add_argument(
-        '--url', help=f'Document URL'
+        '--document-url', help=f'Document URL'
     )
 
     parser.add_argument(
@@ -209,14 +142,6 @@ def main():
         '--identifier', '--id', help=f'Document ID'
     )
 
-    parser.add_argument(
-        '-l', '--log-level',
-        type=LogLevel,
-        default=LogLevel.ERROR,
-        choices=list(LogLevel),
-        help=f'Output queue (default: info)'
-    )
-
     parser.add_argument(
         'files', nargs='+',
         help=f'File to load'
@@ -236,7 +161,7 @@ def main():
                 copyright_holder=args.copyright_holder,
                 copyright_year=args.copyright_year,
                 license=args.license,
-                url=args.url,
+                url=args.document_url,
                 keywords=args.keyword,
             )
 
@@ -254,12 +179,9 @@ def main():
                 )
 
             p = Loader(
-                pulsar_host=args.pulsar_host,
-                pulsar_api_key=args.pulsar_api_key,
-                output_queue=args.output_queue,
+                url=args.url,
                 user=args.user,
                 collection=args.collection,
-                log_level=args.log_level,
                 metadata=document,
             )