apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-30 19:06:21 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Fix/librarian broken (#674)

Browse source 
* Set end-of-stream cleanly - clean streaming message structures

* Add tg-get-document-content
This commit is contained in:
cybermaggedon 2026-03-09 13:36:24 +00:00 • committed by GitHub
parent df1808768d
commit 3c3e11bef5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 99 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

14
trustgraph-flow/trustgraph/librarian/librarian.py
View file

@ -656,9 +656,8 @@ class Librarian:
This is an async generator that yields document content in smaller chunks,
allowing memory-efficient processing of large documents. Each yielded
response includes chunk information and an end_of_stream flag.
The final chunk will have end_of_stream=True.
response includes chunk_index and total_chunks for tracking progress.
Completion is determined by chunk_index reaching total_chunks - 1.
"""
logger.debug(f"Streaming document {request.document_id}")
@ -688,11 +687,10 @@ class Librarian:
# Fetch only the requested range
chunk_content = await self.blob_store.get_range(object_id, offset, length)
is_last_chunk = (chunk_index == total_chunks - 1)
is_last = (chunk_index == total_chunks - 1)
logger.debug(f"Streaming chunk {chunk_index}/{total_chunks}, "
f"bytes {offset}-{offset + length} of {total_size}, "
f"end_of_stream={is_last_chunk}")
logger.debug(f"Streaming chunk {chunk_index + 1}/{total_chunks}, "
f"bytes {offset}-{offset + length} of {total_size}")
yield LibrarianResponse(
error=None,
@ -702,6 +700,6 @@ class Librarian:
total_chunks=total_chunks,
bytes_received=offset + length,
total_bytes=total_size,
end_of_stream=is_last_chunk,
is_final=is_last,
)

2
trustgraph-flow/trustgraph/librarian/service.py
View file

@ -527,7 +527,6 @@ class Processor(AsyncProcessor):
type = "request-error",
message = str(e),
),
end_of_stream = True,
)
await self.librarian_response_producer.send(
@ -541,7 +540,6 @@ class Processor(AsyncProcessor):
type = "unexpected-error",
message = str(e),
),
end_of_stream = True,
)
await self.librarian_response_producer.send(