apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-16 16:51:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Metadata handling complete

Browse source
This commit is contained in:
Cyber MacGeddon 2024-10-22 23:23:24 +01:00
parent 2a4b4d5c18
commit a44ab7a2be
1 changed files with 62 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

109
trustgraph-cli/scripts/tg-load-pdf
View file

@ -6,7 +6,8 @@ Loads a PDF document into TrustGraph processing.
import pulsar import pulsar
from pulsar.schema import JsonSchema from pulsar.schema import JsonSchema
from trustgraph.schema import Document, document_ingest_queue, Metadata from trustgraph.schema import Document, document_ingest_queue
from trustgraph.schema import Value, Metadata, Triple
import base64 import base64
import hashlib import hashlib
import argparse import argparse
@ -80,40 +81,55 @@ class DigitalDocument:
def emit(self, emit): def emit(self, emit):
emit(self.id, IS_A, DIGITAL_DOCUMENT) emit(Triple(Value(self.id), Value(IS_A), Value(DIGITAL_DOCUMENT)))
if self.name: if self.name:
emit(self.id, LABEL, self.name) emit(Triple(Value(self.id), Value(LABEL), Value(self.name)))
emit(self.id, NAME, self.name) emit(Triple(Value(self.id), Value(NAME), Value(self.name)))
if self.identifier: if self.identifier:
emit(id, IDENTIFIER, self.identifier) emit(Triple(Value(id), Value(IDENTIFIER), Value(self.identifier)))
if self.description: if self.description:
emit(self.id, DESCRIPTION, self.description) emit(Triple(
Value(self.id), Value(DESCRIPTION), Value(self.description)
))
if self.copyright_notice: if self.copyright_notice:
emit(self.id, COPYRIGHT_NOTICE, self.copyright_notice) emit(Triple(
Value(self.id), Value(COPYRIGHT_NOTICE),
Value(self.copyright_notice)
))
if self.copyright_holder: if self.copyright_holder:
emit(self.id, COPYRIGHT_HOLDER, self.copyright_holder) emit(Triple(
Value(self.id), Value(COPYRIGHT_HOLDER),
Value(self.copyright_holder)
))
if self.copyright_year: if self.copyright_year:
emit(self.id, COPYRIGHT_YEAR, self.copyright_year) emit(Triple(
Value(self.id), Value(COPYRIGHT_YEAR),
Value(self.copyright_year)
))
if self.license: if self.license:
emit(self.id, LICENSE, self.license) emit(Triple(
Value(self.id), Value(LICENSE), Value(self.license)
))
if self.keywords: if self.keywords:
for k in self.keywords: for k in self.keywords:
emit(self.id, KEYWORD, k) emit(Triple(Value(self.id), Value(KEYWORD), Value(k)))
if self.publication: if self.publication:
emit(self.id, PUBLICATION, self.publication.id) emit(Triple(
Value(self.id), Value(PUBLICATION, Value(self.publication.id))
))
self.publication.emit(emit) self.publication.emit(emit)
if self.url: if self.url:
emit(self.id, URL, self.url) emit(Triple(Value(self.id), Value(URL), Value(self.url)))
class PublicationEvent: class PublicationEvent:
def __init__( def __init__(
@ -129,24 +145,31 @@ class PublicationEvent:
def emit(self, emit): def emit(self, emit):
emit(self.id, IS_A, PUBLICATION_EVENT) emit(Triple(Value(self.id), Value(IS_A), Value(PUBLICATION_EVENT)))
if self.name: if self.name:
emit(self.id, LABEL, self.name) emit(Triple(Value(self.id), Value(LABEL), Value(self.name)))
emit(self.id, NAME, self.name) emit(Triple(Value(self.id), Value(NAME), Value(self.name)))
if self.description: if self.description:
emit(self.id, DESCRIPTION, self.description) emit(Triple(
Value(self.id), Value(DESCRIPTION), Value(self.description)
))
if self.organization: if self.organization:
emit(self.id, PUBLISHED_BY, self.organization.id) emit(Triple(
Value(self.id), Value(PUBLISHED_BY),
Value(self.organization.id)
))
self.organization.emit(emit) self.organization.emit(emit)
if self.start_date: if self.start_date:
emit(self.id, START_DATE, self.start_date) emit(Triple(
Value(self.id), Value(START_DATE), Value(self.start_date)
))
if self.end_date: if self.end_date:
emit(self.id, END_DATE, self.end_date) emit(Triple(Value(self.id), Value(END_DATE), Value(self.end_date)))
class Organization: class Organization:
def __init__(self, id, name=None, description=None): def __init__(self, id, name=None, description=None):
@ -156,14 +179,16 @@ class Organization:
def emit(self, emit): def emit(self, emit):
emit(self.id, IS_A, ORGANIZATION) emit(Triple(Value(self.id), Value(IS_A), Value(ORGANIZATION)))
if self.name: if self.name:
emit(self.id, LABEL, self.name) emit(Triple(Value(self.id), Value(LABEL), Value(self.name)))
emit(self.id, NAME, self.name) emit(Triple(Value(self.id), Value(NAME), Value(self.name)))
if self.description: if self.description:
emit(self.id, DESCRIPTION, self.description) emit(Triple(
Value(self.id), Value(DESCRIPTION), Value(self.description)
))
class Loader: class Loader:
@ -177,16 +202,16 @@ class Loader:
metadata, metadata,
): ):
# self.client = pulsar.Client( self.client = pulsar.Client(
# pulsar_host, pulsar_host,
# logger=pulsar.ConsoleLogger(log_level.to_pulsar()) logger=pulsar.ConsoleLogger(log_level.to_pulsar())
# ) )
# self.producer = self.client.create_producer( self.producer = self.client.create_producer(
# topic=output_queue, topic=output_queue,
# schema=JsonSchema(Document), schema=JsonSchema(Document),
# chunking_enabled=True, chunking_enabled=True,
# ) )
self.user = user self.user = user
self.collection = collection self.collection = collection
@ -205,32 +230,22 @@ class Loader:
data = open(path, "rb").read() data = open(path, "rb").read()
# Create a SHA256 hash from the data # Create a SHA256 hash from the data
id = hashlib.sha256(data).hexdigest() id = hash(data)
# Convert into a UUID, 64-byte hash becomes 32-byte UUID
id = str(uuid.UUID(id[::2]))
id = curi("doc", id) id = curi("doc", id)
triples = [] triples = []
def emit(s, p, o): def emit(t):
triples.append((s, p, o)) triples.append(t)
self.metadata.id = id self.metadata.id = id
self.metadata.emit(emit) self.metadata.emit(emit)
for t in triples:
print(t)
return
r = Document( r = Document(
metadata=Metadata( metadata=Metadata(
source=path,
title=path,
id=id, id=id,
metadata = triples,
user=self.user, user=self.user,
collection=self.collection, collection=self.collection,
), ),