apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-05-02 03:42:36 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added chunk_size metrics, and added metrics to dashboard (#16)

Browse source
This commit is contained in:
cybermaggedon 2024-08-22 00:20:24 +01:00 • committed by GitHub
parent d3cdb97528
commit 0043b871ff
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 106 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

11
trustgraph/chunking/recursive/chunker.py
View file

@ -5,7 +5,7 @@ as text as separate output objects.
"""
from langchain_text_splitters import RecursiveCharacterTextSplitter
from prometheus_client import Histogram
from ... schema import TextDocument, Chunk, Source
from ... schema import text_ingest_queue, chunk_ingest_queue
@ -38,6 +38,13 @@ class Processor(ConsumerProducer):
}
)
if not hasattr(__class__, "chunk_metric"):
__class__.chunk_metric = Histogram(
'chunk_size', 'Chunk size',
buckets=[100, 160, 250, 400, 650, 1000, 1600,
2500, 4000, 6400, 10000, 16000]
)
self.text_splitter = RecursiveCharacterTextSplitter(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
@ -67,6 +74,8 @@ class Processor(ConsumerProducer):
chunk=chunk.page_content.encode("utf-8"),
)
__class__.chunk_metric.observe(len(chunk.page_content))
self.send(r)
print("Done.", flush=True)

10
trustgraph/chunking/token/chunker.py
View file

@ -5,6 +5,7 @@ as text as separate output objects.
"""
from langchain_text_splitters import TokenTextSplitter
from prometheus_client import Histogram
from ... schema import TextDocument, Chunk, Source
from ... schema import text_ingest_queue, chunk_ingest_queue
@ -37,6 +38,13 @@ class Processor(ConsumerProducer):
}
)
if not hasattr(__class__, "chunk_metric"):
__class__.chunk_metric = Histogram(
'chunk_size', 'Chunk size',
buckets=[100, 160, 250, 400, 650, 1000, 1600,
2500, 4000, 6400, 10000, 16000]
)
self.text_splitter = TokenTextSplitter(
encoding_name="cl100k_base",
chunk_size=chunk_size,
@ -65,6 +73,8 @@ class Processor(ConsumerProducer):
chunk=chunk.page_content.encode("utf-8"),
)
__class__.chunk_metric.observe(len(chunk.page_content))
self.send(r)
print("Done.", flush=True)