apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-06-07 05:45:13 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Replace... (#91)

Browse source 
Replace tg-load-* -f file with tg-load-* file1 file2 ...
This commit is contained in:
cybermaggedon 2024-10-01 19:34:35 +01:00 • committed by GitHub
parent 647010be9f
commit 5985b8612b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 26 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

23
trustgraph-cli/scripts/tg-load-pdf
View file

@ -22,7 +22,6 @@ class Loader:
pulsar_host, pulsar_host,
output_queue, output_queue,
log_level, log_level,
file,
): ):
self.client = pulsar.Client( self.client = pulsar.Client(
@ -36,13 +35,16 @@ class Loader:
chunking_enabled=True, chunking_enabled=True,
) )
self.file = file def load(self, files):
def run(self): for file in files:
self.load_file(file)
def load_file(self, file):
try: try:
path = self.file path = file
data = open(path, "rb").read() data = open(path, "rb").read()
id = hashlib.sha256(path.encode("utf-8")).hexdigest()[0:8] id = hashlib.sha256(path.encode("utf-8")).hexdigest()[0:8]
@ -58,8 +60,10 @@ class Loader:
self.producer.send(r) self.producer.send(r)
print(f"{file}: Loaded successfully.")
except Exception as e: except Exception as e:
print(e, flush=True) print(f"{file}: Failed: {str(e)}", flush=True)
def __del__(self): def __del__(self):
self.client.close() self.client.close()
@ -95,8 +99,7 @@ def main():
) )
parser.add_argument( parser.add_argument(
'-f', '--file', 'files', nargs='+',
required=True,
help=f'File to load' help=f'File to load'
) )
@ -105,16 +108,16 @@ def main():
while True: while True:
try: try:
p = Loader( p = Loader(
pulsar_host=args.pulsar_host, pulsar_host=args.pulsar_host,
output_queue=args.output_queue, output_queue=args.output_queue,
log_level=args.log_level, log_level=args.log_level,
file=args.file,
) )
p.run() p.load(args.files)
print("File loaded.") print("All done.")
break break
except Exception as e: except Exception as e:

23
trustgraph-cli/scripts/tg-load-text
View file

@ -22,7 +22,6 @@ class Loader:
pulsar_host, pulsar_host,
output_queue, output_queue,
log_level, log_level,
file,
): ):
self.client = pulsar.Client( self.client = pulsar.Client(
@ -36,13 +35,16 @@ class Loader:
chunking_enabled=True, chunking_enabled=True,
) )
self.file = file def load(self, files):
def run(self): for file in files:
self.load_file(file)
def load_file(self, file):
try: try:
path = self.file path = file
data = open(path, "rb").read() data = open(path, "rb").read()
id = hashlib.sha256(path.encode("utf-8")).hexdigest()[0:8] id = hashlib.sha256(path.encode("utf-8")).hexdigest()[0:8]
@ -58,8 +60,10 @@ class Loader:
self.producer.send(r) self.producer.send(r)
print(f"{file}: Loaded successfully.")
except Exception as e: except Exception as e:
print(e, flush=True) print(f"{file}: Failed: {str(e)}", flush=True)
def __del__(self): def __del__(self):
self.client.close() self.client.close()
@ -95,8 +99,7 @@ def main():
) )
parser.add_argument( parser.add_argument(
'-f', '--file', 'files', nargs='+',
required=True,
help=f'File to load' help=f'File to load'
) )
@ -105,16 +108,16 @@ def main():
while True: while True:
try: try:
p = Loader( p = Loader(
pulsar_host=args.pulsar_host, pulsar_host=args.pulsar_host,
output_queue=args.output_queue, output_queue=args.output_queue,
log_level=args.log_level, log_level=args.log_level,
file=args.file,
) )
p.run() p.load(args.files)
print("File loaded.") print("All done.")
break break
except Exception as e: except Exception as e: