mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-16 16:51:02 +02:00
Got some triple-like structure
This commit is contained in:
Cyber MacGeddon
parent
866f155bb8
commit
9fd16cbae8
1 changed files with 159 additions and 79 deletions
|
|
@ -19,83 +19,12 @@ from trustgraph.log_level import LogLevel
|
|||
default_user = 'trustgraph'
|
||||
default_collection = 'default'
|
||||
|
||||
class MetadataSource:
|
||||
def __init__(self, args):
|
||||
self.source = args
|
||||
|
||||
def id_hash(self, data):
|
||||
|
||||
# Create a SHA256 hash from the data
|
||||
id = hashlib.sha256(data).hexdigest()
|
||||
|
||||
# Convert into a UUID, 64-byte hash becomes 32-byte UUID
|
||||
id = str(uuid.UUID(id[::2]))
|
||||
|
||||
return id
|
||||
|
||||
def add(self, id, add):
|
||||
|
||||
source = self.source
|
||||
|
||||
add(id, IS_A, DIGITAL_DOCUMENT)
|
||||
|
||||
if source.name:
|
||||
add(id, LABEL, source.name)
|
||||
add(id, NAME, source.name)
|
||||
|
||||
if source.description:
|
||||
add(id, DESCRIPTION, source.description)
|
||||
|
||||
if source.copyright_notice:
|
||||
add(id, COPYRIGHT_NOTICE, source.copyright_notice)
|
||||
|
||||
if source.copyright_holder:
|
||||
add(id, COPYRIGHT_HOLDER, source.copyright_holder)
|
||||
|
||||
if source.copyright_year:
|
||||
add(id, COPYRIGHT_YEAR, source.copyright_year)
|
||||
|
||||
if source.license:
|
||||
add(id, LICENSE, source.license)
|
||||
|
||||
if source.publication_organization:
|
||||
|
||||
pub_id = self.id_hash(
|
||||
source.publication_organization.encode("utf-8")
|
||||
)
|
||||
|
||||
add(id, IS_A, PUBLICATION_EVENT)
|
||||
|
||||
add(id, PUBLICATION, pub_id)
|
||||
|
||||
if source.publication_organization:
|
||||
add(pub_id, PUBLISHED_BY,
|
||||
source.publication_organization)
|
||||
|
||||
if source.publication_description:
|
||||
add(pub_id, DESCRIPTION, source.publication_description)
|
||||
|
||||
if source.publication_date:
|
||||
add(pub_id, START_DATE, source.publication_date)
|
||||
|
||||
if source.publication_date:
|
||||
add(pub_id, END_DATE, source.publication_date)
|
||||
|
||||
if source.url:
|
||||
add(id, URL, source.url)
|
||||
|
||||
if source.identifier:
|
||||
add(id, IDENTIFIER, source.identifier)
|
||||
|
||||
if source.keyword:
|
||||
for k in source.keyword:
|
||||
add(id, KEYWORD, k)
|
||||
|
||||
IS_A = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'
|
||||
LABEL = 'http://www.w3.org/2000/01/rdf-schema#label'
|
||||
|
||||
DIGITAL_DOCUMENT = 'https://schema.org/DigitalDocument'
|
||||
PUBLICATION_EVENT = 'https://schema.org/PublicationEvent'
|
||||
ORGANIZATION = 'https://schema.org/Organization'
|
||||
|
||||
NAME = 'https://schema.org/name'
|
||||
DESCRIPTION = 'https://schema.org/description'
|
||||
|
|
@ -114,6 +43,135 @@ URL = 'https://schema.org/url'
|
|||
IDENTIFIER = 'https://schema.org/identifier'
|
||||
KEYWORD = 'https://schema.org/keywords'
|
||||
|
||||
def hash(data):
|
||||
|
||||
if isinstance(data, str):
|
||||
data = data.encode("utf-8")
|
||||
|
||||
# Create a SHA256 hash from the data
|
||||
id = hashlib.sha256(data).hexdigest()
|
||||
|
||||
# Convert into a UUID, 64-byte hash becomes 32-byte UUID
|
||||
id = str(uuid.UUID(id[::2]))
|
||||
|
||||
return id
|
||||
|
||||
class DigitalDocument:
|
||||
def __init__(
|
||||
self, id, name=None, description=None, copyright_notice=None,
|
||||
copyright_holder=None, copyright_year=None, license=None,
|
||||
identifier=None,
|
||||
publication=None, url=None, keywords=[]
|
||||
):
|
||||
self.id = id
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.copyright_notice = copyright_notice
|
||||
self.copyright_holder = copyright_holder
|
||||
self.copyright_year = copyright_year
|
||||
self.license = license
|
||||
self.publication = publication
|
||||
self.url = url
|
||||
self.identifier = identifier
|
||||
self.keywords = keywords
|
||||
|
||||
# def id_hash(self, data):
|
||||
|
||||
# # Create a SHA256 hash from the data
|
||||
# id = hashlib.sha256(data).hexdigest()
|
||||
|
||||
# # Convert into a UUID, 64-byte hash becomes 32-byte UUID
|
||||
# id = str(uuid.UUID(id[::2]))
|
||||
|
||||
# return id
|
||||
|
||||
def emit(self, emit):
|
||||
|
||||
emit(self.id, IS_A, DIGITAL_DOCUMENT)
|
||||
|
||||
if self.name:
|
||||
emit(self.id, LABEL, self.name)
|
||||
emit(self.id, NAME, self.name)
|
||||
|
||||
if self.identifier:
|
||||
emit(id, IDENTIFIER, self.identifier)
|
||||
|
||||
if self.description:
|
||||
emit(self.id, DESCRIPTION, self.description)
|
||||
|
||||
if self.copyright_notice:
|
||||
emit(self.id, COPYRIGHT_NOTICE, self.copyright_notice)
|
||||
|
||||
if self.copyright_holder:
|
||||
emit(self.id, COPYRIGHT_HOLDER, self.copyright_holder)
|
||||
|
||||
if self.copyright_year:
|
||||
emit(self.id, COPYRIGHT_YEAR, self.copyright_year)
|
||||
|
||||
if self.license:
|
||||
emit(self.id, LICENSE, self.license)
|
||||
|
||||
if self.keywords:
|
||||
for k in self.keywords:
|
||||
emit(self.id, KEYWORD, k)
|
||||
|
||||
if self.publication:
|
||||
emit(self.id, PUBLICATION, self.publication.id)
|
||||
self.publication.emit(emit)
|
||||
|
||||
if self.url:
|
||||
emit(self.id, URL, self.url)
|
||||
|
||||
class PublicationEvent:
|
||||
def __init__(
|
||||
self, id, organization=None, name=None, description=None,
|
||||
start_date=None, end_date=None,
|
||||
):
|
||||
self.id = id
|
||||
self.organization = organization
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.start_date = start_date
|
||||
self.end_date = end_date
|
||||
|
||||
def emit(self, emit):
|
||||
|
||||
emit(self.id, IS_A, PUBLICATION_EVENT)
|
||||
|
||||
if self.name:
|
||||
emit(self.id, LABEL, self.name)
|
||||
emit(self.id, NAME, self.name)
|
||||
|
||||
if self.description:
|
||||
emit(self.id, DESCRIPTION, self.description)
|
||||
|
||||
if self.organization:
|
||||
emit(self.id, PUBLISHED_BY, self.organization.id)
|
||||
self.organization.emit(emit)
|
||||
|
||||
if self.start_date:
|
||||
emit(self.id, START_DATE, self.start_date)
|
||||
|
||||
if self.end_date:
|
||||
emit(self.id, END_DATE, self.end_date)
|
||||
|
||||
class Organization:
|
||||
def __init__(self, id, name=None, description=None):
|
||||
self.id = id
|
||||
self.name = name
|
||||
self.description = description
|
||||
|
||||
def emit(self, emit):
|
||||
|
||||
emit(self.id, IS_A, ORGANIZATION)
|
||||
|
||||
if self.name:
|
||||
emit(self.id, LABEL, self.name)
|
||||
emit(self.id, NAME, self.name)
|
||||
|
||||
if self.description:
|
||||
emit(self.id, DESCRIPTION, self.description)
|
||||
|
||||
class Loader:
|
||||
|
||||
def __init__(
|
||||
|
|
@ -136,10 +194,6 @@ class Loader:
|
|||
# schema=JsonSchema(Document),
|
||||
# chunking_enabled=True,
|
||||
# )
|
||||
class Bunch:
|
||||
def close(self):
|
||||
print("CLOSE")
|
||||
self.client = Bunch()
|
||||
|
||||
self.user = user
|
||||
self.collection = collection
|
||||
|
|
@ -165,10 +219,11 @@ class Loader:
|
|||
|
||||
triples = []
|
||||
|
||||
def add(s, p, o):
|
||||
def emit(s, p, o):
|
||||
triples.append((s, p, o))
|
||||
|
||||
self.metadata.add(id, add)
|
||||
self.metadata.id = id
|
||||
self.metadata.emit(emit)
|
||||
|
||||
for t in triples:
|
||||
print(t)
|
||||
|
|
@ -298,13 +353,38 @@ def main():
|
|||
|
||||
try:
|
||||
|
||||
document = DigitalDocument(
|
||||
id,
|
||||
name=args.name,
|
||||
description=args.description,
|
||||
copyright_notice=args.copyright_notice,
|
||||
copyright_holder=args.copyright_holder,
|
||||
copyright_year=args.copyright_year,
|
||||
license=args.license,
|
||||
url=args.url,
|
||||
keywords=args.keyword,
|
||||
)
|
||||
|
||||
if args.publication_organization:
|
||||
org = Organization(
|
||||
id=hash(args.publication_organization),
|
||||
name=args.publication_organization,
|
||||
)
|
||||
document.publication = PublicationEvent(
|
||||
id = str(uuid.uuid4()),
|
||||
organization=org,
|
||||
description=args.publication_description,
|
||||
start_date=args.publication_date,
|
||||
end_date=args.publication_date,
|
||||
)
|
||||
|
||||
p = Loader(
|
||||
pulsar_host=args.pulsar_host,
|
||||
output_queue=args.output_queue,
|
||||
user=args.user,
|
||||
collection=args.collection,
|
||||
log_level=args.log_level,
|
||||
metadata=MetadataSource(args),
|
||||
metadata=document,
|
||||
)
|
||||
|
||||
p.load(args.files)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue