diff --git a/trustgraph-base/trustgraph/schema/__init__.py b/trustgraph-base/trustgraph/schema/__init__.py index f3cc5b60..7f0334be 100644 --- a/trustgraph-base/trustgraph/schema/__init__.py +++ b/trustgraph-base/trustgraph/schema/__init__.py @@ -9,5 +9,3 @@ from . graph import * from . retrieval import * from . metadata import * - - diff --git a/trustgraph-flow/trustgraph/chunking/recursive/chunker.py b/trustgraph-flow/trustgraph/chunking/recursive/chunker.py index bec854bb..694ced70 100755 --- a/trustgraph-flow/trustgraph/chunking/recursive/chunker.py +++ b/trustgraph-flow/trustgraph/chunking/recursive/chunker.py @@ -63,16 +63,8 @@ class Processor(ConsumerProducer): for ix, chunk in enumerate(texts): - id = v.metadata.id + "-c" + str(ix) - r = Chunk( - metadata=Metadata( - source=v.metadata.source, - id=id, - title=v.metadata.title, - user=v.metadata.user, - collection=v.metadata.collection, - ), + metadata=v.metadata, chunk=chunk.page_content.encode("utf-8"), ) diff --git a/trustgraph-flow/trustgraph/chunking/token/chunker.py b/trustgraph-flow/trustgraph/chunking/token/chunker.py index e7bc5667..dccd9c89 100755 --- a/trustgraph-flow/trustgraph/chunking/token/chunker.py +++ b/trustgraph-flow/trustgraph/chunking/token/chunker.py @@ -62,16 +62,8 @@ class Processor(ConsumerProducer): for ix, chunk in enumerate(texts): - id = v.metadata.id + "-c" + str(ix) - r = Chunk( - metadata=Metadata( - source=v.metadata.source, - id=id, - title=v.metadata.title, - user=v.metadata.user, - collection=v.metadata.collection, - ), + metadata=v.metadata, chunk=chunk.page_content.encode("utf-8"), ) diff --git a/trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py b/trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py index 7e9aeafa..38ac9257 100755 --- a/trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py +++ b/trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py @@ -59,15 +59,8 @@ class Processor(ConsumerProducer): for ix, page in enumerate(pages): - id = v.metadata.id + "-p" + str(ix) r = TextDocument( - metadata=Metadata( - source=v.metadata.source, - title=v.metadata.title, - id=id, - user=v.metadata.user, - collection=v.metadata.collection, - ), + metadata=v.metadata, text=page.page_content.encode("utf-8"), )