apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-05-02 11:52:37 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix token chunker, broken API invocation (#454)

Browse source
This commit is contained in:
cybermaggedon 2025-08-08 14:41:24 +01:00 • committed by GitHub
parent 6c681967ab
commit a7de175b33
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
trustgraph-flow/trustgraph/chunking/token/chunker.py
View file

@ -9,7 +9,7 @@ from langchain_text_splitters import TokenTextSplitter
from prometheus_client import Histogram from prometheus_client import Histogram
from ... schema import TextDocument, Chunk from ... schema import TextDocument, Chunk
from ... base import FlowProcessor from ... base import FlowProcessor, ConsumerSpec, ProducerSpec
# Module logger # Module logger
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -20,7 +20,7 @@ class Processor(FlowProcessor):
def __init__(self, **params): def __init__(self, **params):
id = params.get("id") id = params.get("id", default_ident)
chunk_size = params.get("chunk_size", 250) chunk_size = params.get("chunk_size", 250)
chunk_overlap = params.get("chunk_overlap", 15) chunk_overlap = params.get("chunk_overlap", 15)