apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-25 00:16:23 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Replace... (#91)

Browse source 
Replace tg-load-* -f file with tg-load-* file1 file2 ...
This commit is contained in:
cybermaggedon 2024-10-01 19:34:35 +01:00 • committed by GitHub
parent 647010be9f
commit 5985b8612b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 26 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

23
trustgraph-cli/scripts/tg-load-pdf
View file

@ -22,7 +22,6 @@ class Loader:
pulsar_host,
output_queue,
log_level,
file,
):
self.client = pulsar.Client(
@ -36,13 +35,16 @@ class Loader:
chunking_enabled=True,
)
self.file = file
def load(self, files):
def run(self):
for file in files:
self.load_file(file)
def load_file(self, file):
try:
path = self.file
path = file
data = open(path, "rb").read()
id = hashlib.sha256(path.encode("utf-8")).hexdigest()[0:8]
@ -58,8 +60,10 @@ class Loader:
self.producer.send(r)
print(f"{file}: Loaded successfully.")
except Exception as e:
print(e, flush=True)
print(f"{file}: Failed: {str(e)}", flush=True)
def __del__(self):
self.client.close()
@ -95,8 +99,7 @@ def main():
)
parser.add_argument(
'-f', '--file',
required=True,
'files', nargs='+',
help=f'File to load'
)
@ -105,16 +108,16 @@ def main():
while True:
try:
p = Loader(
pulsar_host=args.pulsar_host,
output_queue=args.output_queue,
log_level=args.log_level,
file=args.file,
)
p.run()
p.load(args.files)
print("File loaded.")
print("All done.")
break
except Exception as e:

23
trustgraph-cli/scripts/tg-load-text
View file

@ -22,7 +22,6 @@ class Loader:
pulsar_host,
output_queue,
log_level,
file,
):
self.client = pulsar.Client(
@ -36,13 +35,16 @@ class Loader:
chunking_enabled=True,
)
self.file = file
def load(self, files):
def run(self):
for file in files:
self.load_file(file)
def load_file(self, file):
try:
path = self.file
path = file
data = open(path, "rb").read()
id = hashlib.sha256(path.encode("utf-8")).hexdigest()[0:8]
@ -58,8 +60,10 @@ class Loader:
self.producer.send(r)
print(f"{file}: Loaded successfully.")
except Exception as e:
print(e, flush=True)
print(f"{file}: Failed: {str(e)}", flush=True)
def __del__(self):
self.client.close()
@ -95,8 +99,7 @@ def main():
)
parser.add_argument(
'-f', '--file',
required=True,
'files', nargs='+',
help=f'File to load'
)
@ -105,16 +108,16 @@ def main():
while True:
try:
p = Loader(
pulsar_host=args.pulsar_host,
output_queue=args.output_queue,
log_level=args.log_level,
file=args.file,
)
p.run()
p.load(args.files)
print("File loaded.")
print("All done.")
break
except Exception as e: