apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-22 03:31:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Fix chunk minimum confusion

Browse source
This commit is contained in:
Cyber MacGeddon 2026-03-05 15:15:12 +00:00
parent 5a64aebc59
commit 285d98aa22
1 changed files with 14 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

23
trustgraph-flow/trustgraph/librarian/librarian.py
View file

@ -16,8 +16,8 @@ import uuid
# Module logger # Module logger
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Default chunk size for multipart uploads (5MB - S3 minimum) # Default chunk size for multipart uploads
DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024 DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024 # 2MB default
class Librarian: class Librarian:
@ -27,6 +27,7 @@ class Librarian:
object_store_endpoint, object_store_access_key, object_store_secret_key, object_store_endpoint, object_store_access_key, object_store_secret_key,
bucket_name, keyspace, load_document, bucket_name, keyspace, load_document,
object_store_use_ssl=False, object_store_region=None, object_store_use_ssl=False, object_store_region=None,
min_chunk_size=1, # Default: no minimum (for Garage)
): ):
self.blob_store = BlobStore( self.blob_store = BlobStore(
@ -39,6 +40,7 @@ class Librarian:
) )
self.load_document = load_document self.load_document = load_document
self.min_chunk_size = min_chunk_size
async def add_document(self, request): async def add_document(self, request):
@ -289,10 +291,12 @@ class Librarian:
if total_size <= 0: if total_size <= 0:
raise RequestError("total_size must be positive") raise RequestError("total_size must be positive")
# Use provided chunk size or default (minimum 5MB for S3) # Use provided chunk size or default
chunk_size = request.chunk_size if request.chunk_size > 0 else DEFAULT_CHUNK_SIZE chunk_size = request.chunk_size if request.chunk_size > 0 else DEFAULT_CHUNK_SIZE
if chunk_size < DEFAULT_CHUNK_SIZE: if chunk_size < self.min_chunk_size:
chunk_size = DEFAULT_CHUNK_SIZE raise RequestError(
f"Chunk size {chunk_size} is below minimum {self.min_chunk_size}"
)
# Calculate total chunks # Calculate total chunks
total_chunks = math.ceil(total_size / chunk_size) total_chunks = math.ceil(total_size / chunk_size)
@ -657,12 +661,13 @@ class Librarian:
""" """
logger.debug(f"Streaming document {request.document_id}, chunk {request.chunk_index}") logger.debug(f"Streaming document {request.document_id}, chunk {request.chunk_index}")
# Server-enforced limits DEFAULT_CHUNK_SIZE = 1024 * 1024 # 1MB default
MAX_CHUNK_SIZE = 10 * 1024 * 1024 # 10MB max
DEFAULT_CHUNK_SIZE = 1024 * 1024 # 1MB default
chunk_size = request.chunk_size if request.chunk_size > 0 else DEFAULT_CHUNK_SIZE chunk_size = request.chunk_size if request.chunk_size > 0 else DEFAULT_CHUNK_SIZE
chunk_size = min(chunk_size, MAX_CHUNK_SIZE) if chunk_size < self.min_chunk_size:
raise RequestError(
f"Chunk size {chunk_size} is below minimum {self.min_chunk_size}"
)
object_id = await self.table_store.get_document_object_id( object_id = await self.table_store.get_document_object_id(
request.user, request.user,