apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-16 16:51:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Metadata pass-thru working

Browse source
This commit is contained in:
Cyber MacGeddon 2024-10-23 15:07:09 +01:00
parent ba30c8a08a
commit 127633fe97
4 changed files with 3 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

2
trustgraph-base/trustgraph/schema/__init__.py
View file

@ -9,5 +9,3 @@ from . graph import *
from . retrieval import *
from . metadata import *

10
trustgraph-flow/trustgraph/chunking/recursive/chunker.py
View file

@ -63,16 +63,8 @@ class Processor(ConsumerProducer):
for ix, chunk in enumerate(texts):
id = v.metadata.id + "-c" + str(ix)
r = Chunk(
metadata=Metadata(
source=v.metadata.source,
id=id,
title=v.metadata.title,
user=v.metadata.user,
collection=v.metadata.collection,
),
metadata=v.metadata,
chunk=chunk.page_content.encode("utf-8"),
)

10
trustgraph-flow/trustgraph/chunking/token/chunker.py
View file

@ -62,16 +62,8 @@ class Processor(ConsumerProducer):
for ix, chunk in enumerate(texts):
id = v.metadata.id + "-c" + str(ix)
r = Chunk(
metadata=Metadata(
source=v.metadata.source,
id=id,
title=v.metadata.title,
user=v.metadata.user,
collection=v.metadata.collection,
),
metadata=v.metadata,
chunk=chunk.page_content.encode("utf-8"),
)

9
trustgraph-flow/trustgraph/decoding/pdf/pdf_decoder.py
View file

@ -59,15 +59,8 @@ class Processor(ConsumerProducer):
for ix, page in enumerate(pages):
id = v.metadata.id + "-p" + str(ix)
r = TextDocument(
metadata=Metadata(
source=v.metadata.source,
title=v.metadata.title,
id=id,
user=v.metadata.user,
collection=v.metadata.collection,
),
metadata=v.metadata,
text=page.page_content.encode("utf-8"),
)