diff --git a/trustgraph-cli/scripts/tg-load-pdf b/trustgraph-cli/scripts/tg-load-pdf index 5f585c2a..e0c176db 100755 --- a/trustgraph-cli/scripts/tg-load-pdf +++ b/trustgraph-cli/scripts/tg-load-pdf @@ -19,83 +19,12 @@ from trustgraph.log_level import LogLevel default_user = 'trustgraph' default_collection = 'default' -class MetadataSource: - def __init__(self, args): - self.source = args - - def id_hash(self, data): - - # Create a SHA256 hash from the data - id = hashlib.sha256(data).hexdigest() - - # Convert into a UUID, 64-byte hash becomes 32-byte UUID - id = str(uuid.UUID(id[::2])) - - return id - - def add(self, id, add): - - source = self.source - - add(id, IS_A, DIGITAL_DOCUMENT) - - if source.name: - add(id, LABEL, source.name) - add(id, NAME, source.name) - - if source.description: - add(id, DESCRIPTION, source.description) - - if source.copyright_notice: - add(id, COPYRIGHT_NOTICE, source.copyright_notice) - - if source.copyright_holder: - add(id, COPYRIGHT_HOLDER, source.copyright_holder) - - if source.copyright_year: - add(id, COPYRIGHT_YEAR, source.copyright_year) - - if source.license: - add(id, LICENSE, source.license) - - if source.publication_organization: - - pub_id = self.id_hash( - source.publication_organization.encode("utf-8") - ) - - add(id, IS_A, PUBLICATION_EVENT) - - add(id, PUBLICATION, pub_id) - - if source.publication_organization: - add(pub_id, PUBLISHED_BY, - source.publication_organization) - - if source.publication_description: - add(pub_id, DESCRIPTION, source.publication_description) - - if source.publication_date: - add(pub_id, START_DATE, source.publication_date) - - if source.publication_date: - add(pub_id, END_DATE, source.publication_date) - - if source.url: - add(id, URL, source.url) - - if source.identifier: - add(id, IDENTIFIER, source.identifier) - - if source.keyword: - for k in source.keyword: - add(id, KEYWORD, k) - IS_A = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' LABEL = 'http://www.w3.org/2000/01/rdf-schema#label' DIGITAL_DOCUMENT = 'https://schema.org/DigitalDocument' PUBLICATION_EVENT = 'https://schema.org/PublicationEvent' +ORGANIZATION = 'https://schema.org/Organization' NAME = 'https://schema.org/name' DESCRIPTION = 'https://schema.org/description' @@ -114,6 +43,135 @@ URL = 'https://schema.org/url' IDENTIFIER = 'https://schema.org/identifier' KEYWORD = 'https://schema.org/keywords' +def hash(data): + + if isinstance(data, str): + data = data.encode("utf-8") + + # Create a SHA256 hash from the data + id = hashlib.sha256(data).hexdigest() + + # Convert into a UUID, 64-byte hash becomes 32-byte UUID + id = str(uuid.UUID(id[::2])) + + return id + +class DigitalDocument: + def __init__( + self, id, name=None, description=None, copyright_notice=None, + copyright_holder=None, copyright_year=None, license=None, + identifier=None, + publication=None, url=None, keywords=[] + ): + self.id = id + self.name = name + self.description = description + self.copyright_notice = copyright_notice + self.copyright_holder = copyright_holder + self.copyright_year = copyright_year + self.license = license + self.publication = publication + self.url = url + self.identifier = identifier + self.keywords = keywords + + # def id_hash(self, data): + + # # Create a SHA256 hash from the data + # id = hashlib.sha256(data).hexdigest() + + # # Convert into a UUID, 64-byte hash becomes 32-byte UUID + # id = str(uuid.UUID(id[::2])) + + # return id + + def emit(self, emit): + + emit(self.id, IS_A, DIGITAL_DOCUMENT) + + if self.name: + emit(self.id, LABEL, self.name) + emit(self.id, NAME, self.name) + + if self.identifier: + emit(id, IDENTIFIER, self.identifier) + + if self.description: + emit(self.id, DESCRIPTION, self.description) + + if self.copyright_notice: + emit(self.id, COPYRIGHT_NOTICE, self.copyright_notice) + + if self.copyright_holder: + emit(self.id, COPYRIGHT_HOLDER, self.copyright_holder) + + if self.copyright_year: + emit(self.id, COPYRIGHT_YEAR, self.copyright_year) + + if self.license: + emit(self.id, LICENSE, self.license) + + if self.keywords: + for k in self.keywords: + emit(self.id, KEYWORD, k) + + if self.publication: + emit(self.id, PUBLICATION, self.publication.id) + self.publication.emit(emit) + + if self.url: + emit(self.id, URL, self.url) + +class PublicationEvent: + def __init__( + self, id, organization=None, name=None, description=None, + start_date=None, end_date=None, + ): + self.id = id + self.organization = organization + self.name = name + self.description = description + self.start_date = start_date + self.end_date = end_date + + def emit(self, emit): + + emit(self.id, IS_A, PUBLICATION_EVENT) + + if self.name: + emit(self.id, LABEL, self.name) + emit(self.id, NAME, self.name) + + if self.description: + emit(self.id, DESCRIPTION, self.description) + + if self.organization: + emit(self.id, PUBLISHED_BY, self.organization.id) + self.organization.emit(emit) + + if self.start_date: + emit(self.id, START_DATE, self.start_date) + + if self.end_date: + emit(self.id, END_DATE, self.end_date) + +class Organization: + def __init__(self, id, name=None, description=None): + self.id = id + self.name = name + self.description = description + + def emit(self, emit): + + emit(self.id, IS_A, ORGANIZATION) + + if self.name: + emit(self.id, LABEL, self.name) + emit(self.id, NAME, self.name) + + if self.description: + emit(self.id, DESCRIPTION, self.description) + class Loader: def __init__( @@ -136,10 +194,6 @@ class Loader: # schema=JsonSchema(Document), # chunking_enabled=True, # ) - class Bunch: - def close(self): - print("CLOSE") - self.client = Bunch() self.user = user self.collection = collection @@ -165,10 +219,11 @@ class Loader: triples = [] - def add(s, p, o): + def emit(s, p, o): triples.append((s, p, o)) - self.metadata.add(id, add) + self.metadata.id = id + self.metadata.emit(emit) for t in triples: print(t) @@ -298,13 +353,38 @@ def main(): try: + document = DigitalDocument( + id, + name=args.name, + description=args.description, + copyright_notice=args.copyright_notice, + copyright_holder=args.copyright_holder, + copyright_year=args.copyright_year, + license=args.license, + url=args.url, + keywords=args.keyword, + ) + + if args.publication_organization: + org = Organization( + id=hash(args.publication_organization), + name=args.publication_organization, + ) + document.publication = PublicationEvent( + id = str(uuid.uuid4()), + organization=org, + description=args.publication_description, + start_date=args.publication_date, + end_date=args.publication_date, + ) + p = Loader( pulsar_host=args.pulsar_host, output_queue=args.output_queue, user=args.user, collection=args.collection, log_level=args.log_level, - metadata=MetadataSource(args), + metadata=document, ) p.load(args.files)