From 305ff7ab3d2aec9427f9d0f42ca68b9c92f9b493 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Wed, 23 Oct 2024 12:25:47 +0100 Subject: [PATCH] Refactor for new classes --- trustgraph-cli/scripts/tg-load-pdf | 186 ++--------------------------- 1 file changed, 10 insertions(+), 176 deletions(-) diff --git a/trustgraph-cli/scripts/tg-load-pdf b/trustgraph-cli/scripts/tg-load-pdf index 952f1fe6..2c2e5221 100755 --- a/trustgraph-cli/scripts/tg-load-pdf +++ b/trustgraph-cli/scripts/tg-load-pdf @@ -6,8 +6,6 @@ Loads a PDF document into TrustGraph processing. import pulsar from pulsar.schema import JsonSchema -from trustgraph.schema import Document, document_ingest_queue -from trustgraph.schema import Value, Metadata, Triple import base64 import hashlib import argparse @@ -15,181 +13,17 @@ import os import time import uuid +from trustgraph.schema import Document, document_ingest_queue +from trustgraph.schema import Value, Metadata, Triple from trustgraph.log_level import LogLevel +from trustgraph.knowledge import hash, to_uri +from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG +from trustgraph.knowledge import Organization, PublicationEvent +from trustgraph.knowledge import DigitalDocument default_user = 'trustgraph' default_collection = 'default' -IS_A = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' -LABEL = 'http://www.w3.org/2000/01/rdf-schema#label' - -DIGITAL_DOCUMENT = 'https://schema.org/DigitalDocument' -PUBLICATION_EVENT = 'https://schema.org/PublicationEvent' -ORGANIZATION = 'https://schema.org/Organization' - -NAME = 'https://schema.org/name' -DESCRIPTION = 'https://schema.org/description' -COPYRIGHT_NOTICE = 'https://schema.org/copyrightNotice' -COPYRIGHT_HOLDER = 'https://schema.org/copyrightHolder' -COPYRIGHT_YEAR = 'https://schema.org/copyrightYear' -LICENSE = 'https://schema.org/license' -PUBLICATION = 'https://schema.org/publication' -START_DATE = 'https://schema.org/startDate' -END_DATE = 'https://schema.org/endDate' -PUBLISHED_BY = 'https://schema.org/publishedBy' -DATE_PUBLISHED = 'https://schema.org/datePublished' -PUBLICATION = 'https://schema.org/publication' -DATE_PUBLISHED = 'https://schema.org/datePublished' -URL = 'https://schema.org/url' -IDENTIFIER = 'https://schema.org/identifier' -KEYWORD = 'https://schema.org/keywords' - -def hash(data): - - if isinstance(data, str): - data = data.encode("utf-8") - - # Create a SHA256 hash from the data - id = hashlib.sha256(data).hexdigest() - - # Convert into a UUID, 64-byte hash becomes 32-byte UUID - id = str(uuid.UUID(id[::2])) - - return id - -def curi(pref, id): - return f"https://trustgraph.ai/{pref}/{id}" - -class DigitalDocument: - def __init__( - self, id, name=None, description=None, copyright_notice=None, - copyright_holder=None, copyright_year=None, license=None, - identifier=None, - publication=None, url=None, keywords=[] - ): - self.id = id - self.name = name - self.description = description - self.copyright_notice = copyright_notice - self.copyright_holder = copyright_holder - self.copyright_year = copyright_year - self.license = license - self.publication = publication - self.url = url - self.identifier = identifier - self.keywords = keywords - - def emit(self, emit): - - emit(Triple(Value(self.id), Value(IS_A), Value(DIGITAL_DOCUMENT))) - - if self.name: - emit(Triple(Value(self.id), Value(LABEL), Value(self.name))) - emit(Triple(Value(self.id), Value(NAME), Value(self.name))) - - if self.identifier: - emit(Triple(Value(id), Value(IDENTIFIER), Value(self.identifier))) - - if self.description: - emit(Triple( - Value(self.id), Value(DESCRIPTION), Value(self.description) - )) - - if self.copyright_notice: - emit(Triple( - Value(self.id), Value(COPYRIGHT_NOTICE), - Value(self.copyright_notice) - )) - - if self.copyright_holder: - emit(Triple( - Value(self.id), Value(COPYRIGHT_HOLDER), - Value(self.copyright_holder) - )) - - if self.copyright_year: - emit(Triple( - Value(self.id), Value(COPYRIGHT_YEAR), - Value(self.copyright_year) - )) - - if self.license: - emit(Triple( - Value(self.id), Value(LICENSE), Value(self.license) - )) - - if self.keywords: - for k in self.keywords: - emit(Triple(Value(self.id), Value(KEYWORD), Value(k))) - - if self.publication: - emit(Triple( - Value(self.id), Value(PUBLICATION, Value(self.publication.id)) - )) - self.publication.emit(emit) - - if self.url: - emit(Triple(Value(self.id), Value(URL), Value(self.url))) - -class PublicationEvent: - def __init__( - self, id, organization=None, name=None, description=None, - start_date=None, end_date=None, - ): - self.id = id - self.organization = organization - self.name = name - self.description = description - self.start_date = start_date - self.end_date = end_date - - def emit(self, emit): - - emit(Triple(Value(self.id), Value(IS_A), Value(PUBLICATION_EVENT))) - - if self.name: - emit(Triple(Value(self.id), Value(LABEL), Value(self.name))) - emit(Triple(Value(self.id), Value(NAME), Value(self.name))) - - if self.description: - emit(Triple( - Value(self.id), Value(DESCRIPTION), Value(self.description) - )) - - if self.organization: - emit(Triple( - Value(self.id), Value(PUBLISHED_BY), - Value(self.organization.id) - )) - self.organization.emit(emit) - - if self.start_date: - emit(Triple( - Value(self.id), Value(START_DATE), Value(self.start_date) - )) - - if self.end_date: - emit(Triple(Value(self.id), Value(END_DATE), Value(self.end_date))) - -class Organization: - def __init__(self, id, name=None, description=None): - self.id = id - self.name = name - self.description = description - - def emit(self, emit): - - emit(Triple(Value(self.id), Value(IS_A), Value(ORGANIZATION))) - - if self.name: - emit(Triple(Value(self.id), Value(LABEL), Value(self.name))) - emit(Triple(Value(self.id), Value(NAME), Value(self.name))) - - if self.description: - emit(Triple( - Value(self.id), Value(DESCRIPTION), Value(self.description) - )) - class Loader: def __init__( @@ -232,7 +66,7 @@ class Loader: # Create a SHA256 hash from the data id = hash(data) - id = curi("doc", id) + id = to_uri(PREF_DOC, id) triples = [] @@ -245,7 +79,7 @@ class Loader: r = Document( metadata=Metadata( id=id, - metadata = triples, + metadata=triples, user=self.user, collection=self.collection, ), @@ -377,11 +211,11 @@ def main(): if args.publication_organization: org = Organization( - id=curi("org", hash(args.publication_organization)), + id=to_uri(PREF_ORG, hash(args.publication_organization)), name=args.publication_organization, ) document.publication = PublicationEvent( - id = curi("pubev", str(uuid.uuid4())), + id = to_uri(PREF_PUBEV, str(uuid.uuid4())), organization=org, description=args.publication_description, start_date=args.publication_date,