From 127633fe97c7ccfb586d1deeaf8819055523d782 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Wed, 23 Oct 2024 15:07:09 +0100 Subject: [PATCH] Metadata pass-thru working --- trustgraph-base/trustgraph/schema/__init__.py | 2 -- .../trustgraph/chunking/recursive/chunker.py | 10 +--------- trustgraph-flow/trustgraph/chunking/token/chunker.py | 10 +--------- trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py | 9 +-------- 4 files changed, 3 insertions(+), 28 deletions(-) diff --git a/trustgraph-base/trustgraph/schema/__init__.py b/trustgraph-base/trustgraph/schema/__init__.py index f3cc5b60..7f0334be 100644 --- a/trustgraph-base/trustgraph/schema/__init__.py +++ b/trustgraph-base/trustgraph/schema/__init__.py @@ -9,5 +9,3 @@ from . graph import * from . retrieval import * from . metadata import * - - diff --git a/trustgraph-flow/trustgraph/chunking/recursive/chunker.py b/trustgraph-flow/trustgraph/chunking/recursive/chunker.py index bec854bb..694ced70 100755 --- a/trustgraph-flow/trustgraph/chunking/recursive/chunker.py +++ b/trustgraph-flow/trustgraph/chunking/recursive/chunker.py @@ -63,16 +63,8 @@ class Processor(ConsumerProducer): for ix, chunk in enumerate(texts): - id = v.metadata.id + "-c" + str(ix) - r = Chunk( - metadata=Metadata( - source=v.metadata.source, - id=id, - title=v.metadata.title, - user=v.metadata.user, - collection=v.metadata.collection, - ), + metadata=v.metadata, chunk=chunk.page_content.encode("utf-8"), ) diff --git a/trustgraph-flow/trustgraph/chunking/token/chunker.py b/trustgraph-flow/trustgraph/chunking/token/chunker.py index e7bc5667..dccd9c89 100755 --- a/trustgraph-flow/trustgraph/chunking/token/chunker.py +++ b/trustgraph-flow/trustgraph/chunking/token/chunker.py @@ -62,16 +62,8 @@ class Processor(ConsumerProducer): for ix, chunk in enumerate(texts): - id = v.metadata.id + "-c" + str(ix) - r = Chunk( - metadata=Metadata( - source=v.metadata.source, - id=id, - title=v.metadata.title, - user=v.metadata.user, - collection=v.metadata.collection, - ), + metadata=v.metadata, chunk=chunk.page_content.encode("utf-8"), ) diff --git a/trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py b/trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py index 7e9aeafa..38ac9257 100755 --- a/trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py +++ b/trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py @@ -59,15 +59,8 @@ class Processor(ConsumerProducer): for ix, page in enumerate(pages): - id = v.metadata.id + "-p" + str(ix) r = TextDocument( - metadata=Metadata( - source=v.metadata.source, - title=v.metadata.title, - id=id, - user=v.metadata.user, - collection=v.metadata.collection, - ), + metadata=v.metadata, text=page.page_content.encode("utf-8"), )