Incremental / large document loading (#659)

Tech spec

BlobStore (trustgraph-flow/trustgraph/librarian/blob_store.py):
- get_stream() - yields document content in chunks for streaming retrieval
- create_multipart_upload() - initializes S3 multipart upload, returns
  upload_id
- upload_part() - uploads a single part, returns etag
- complete_multipart_upload() - finalizes upload with part etags
- abort_multipart_upload() - cancels and cleans up

Cassandra schema (trustgraph-flow/trustgraph/tables/library.py):
- New upload_session table with 24-hour TTL
- Index on user for listing sessions
- Prepared statements for all operations
- Methods: create_upload_session(), get_upload_session(),
  update_upload_session_chunk(), delete_upload_session(),
  list_upload_sessions()

- Schema extended with UploadSession, UploadProgress, and new
  request/response fields
- Librarian methods: begin_upload, upload_chunk, complete_upload,
  abort_upload, get_upload_status, list_uploads
- Service routing for all new operations
- Python SDK with transparent chunked upload:
  - add_document() auto-switches to chunked for files > 10MB
  - Progress callback support (on_progress)
  - get_pending_uploads(), get_upload_status(), abort_upload(),
    resume_upload()

- Document table: Added parent_id and document_type columns with index
- Document schema (knowledge/document.py): Added document_id field for
  streaming retrieval
- Librarian operations:
  - add-child-document for extracted PDF pages
  - list-children to get child documents
  - stream-document for chunked content retrieval
  - Cascade delete removes children when parent is deleted
  - list-documents filters children by default
- PDF decoder (decoding/pdf/pdf_decoder.py): Updated to stream large
  documents from librarian API to temp file
- Librarian service (librarian/service.py): Sends document_id instead of
  content for large PDFs (>2MB)
- Deprecated tools (load_pdf.py, load_text.py): Added deprecation
  warnings directing users to tg-add-library-document +
  tg-start-library-processing

Remove load_pdf and load_text utils

Move chunker/librarian comms to base class

Updating tests

tests/unit/test_chunking/test_recursive_chunker.py

@@ -17,13 +17,17 @@ class MockAsyncProcessor:
         self.config_handlers = []
         self.id = params.get('id', 'test-service')
         self.specifications = []
+        self.pubsub = MagicMock()
+        self.taskgroup = params.get('taskgroup', MagicMock())
 
 
 class TestRecursiveChunkerSimple(IsolatedAsyncioTestCase):
     """Test Recursive chunker functionality"""
 
+    @patch('trustgraph.base.chunking_service.Consumer')
+    @patch('trustgraph.base.chunking_service.Producer')
     @patch('trustgraph.base.async_processor.AsyncProcessor', MockAsyncProcessor)
-    def test_processor_initialization_basic(self):
+    def test_processor_initialization_basic(self, mock_producer, mock_consumer):
         """Test basic processor initialization"""
         # Arrange
         config = {
@@ -47,8 +51,10 @@ class TestRecursiveChunkerSimple(IsolatedAsyncioTestCase):
                       if hasattr(spec, 'name') and spec.name in ['chunk-size', 'chunk-overlap']]
         assert len(param_specs) == 2
 
+    @patch('trustgraph.base.chunking_service.Consumer')
+    @patch('trustgraph.base.chunking_service.Producer')
     @patch('trustgraph.base.async_processor.AsyncProcessor', MockAsyncProcessor)
-    async def test_chunk_document_with_chunk_size_override(self):
+    async def test_chunk_document_with_chunk_size_override(self, mock_producer, mock_consumer):
         """Test chunk_document with chunk-size parameter override"""
         # Arrange
         config = {
@@ -79,8 +85,10 @@ class TestRecursiveChunkerSimple(IsolatedAsyncioTestCase):
         assert chunk_size == 2000  # Should use overridden value
         assert chunk_overlap == 100  # Should use default value
 
+    @patch('trustgraph.base.chunking_service.Consumer')
+    @patch('trustgraph.base.chunking_service.Producer')
     @patch('trustgraph.base.async_processor.AsyncProcessor', MockAsyncProcessor)
-    async def test_chunk_document_with_chunk_overlap_override(self):
+    async def test_chunk_document_with_chunk_overlap_override(self, mock_producer, mock_consumer):
         """Test chunk_document with chunk-overlap parameter override"""
         # Arrange
         config = {
@@ -111,8 +119,10 @@ class TestRecursiveChunkerSimple(IsolatedAsyncioTestCase):
         assert chunk_size == 1000  # Should use default value
         assert chunk_overlap == 200  # Should use overridden value
 
+    @patch('trustgraph.base.chunking_service.Consumer')
+    @patch('trustgraph.base.chunking_service.Producer')
     @patch('trustgraph.base.async_processor.AsyncProcessor', MockAsyncProcessor)
-    async def test_chunk_document_with_both_parameters_override(self):
+    async def test_chunk_document_with_both_parameters_override(self, mock_producer, mock_consumer):
         """Test chunk_document with both chunk-size and chunk-overlap overrides"""
         # Arrange
         config = {
@@ -143,9 +153,11 @@ class TestRecursiveChunkerSimple(IsolatedAsyncioTestCase):
         assert chunk_size == 1500   # Should use overridden value
         assert chunk_overlap == 150 # Should use overridden value
 
+    @patch('trustgraph.base.chunking_service.Consumer')
+    @patch('trustgraph.base.chunking_service.Producer')
     @patch('trustgraph.chunking.recursive.chunker.RecursiveCharacterTextSplitter')
     @patch('trustgraph.base.async_processor.AsyncProcessor', MockAsyncProcessor)
-    async def test_on_message_uses_flow_parameters(self, mock_splitter_class):
+    async def test_on_message_uses_flow_parameters(self, mock_splitter_class, mock_producer, mock_consumer):
         """Test that on_message method uses parameters from flow"""
         # Arrange
         mock_splitter = MagicMock()
@@ -174,6 +186,7 @@ class TestRecursiveChunkerSimple(IsolatedAsyncioTestCase):
             collection="test-collection"
         )
         mock_text_doc.text = b"This is test document content"
+        mock_text_doc.document_id = ""  # No librarian fetch needed
         mock_message.value.return_value = mock_text_doc
 
         # Mock consumer and flow with parameter overrides
@@ -202,8 +215,10 @@ class TestRecursiveChunkerSimple(IsolatedAsyncioTestCase):
         sent_chunk = mock_producer.send.call_args[0][0]
         assert isinstance(sent_chunk, Chunk)
 
+    @patch('trustgraph.base.chunking_service.Consumer')
+    @patch('trustgraph.base.chunking_service.Producer')
     @patch('trustgraph.base.async_processor.AsyncProcessor', MockAsyncProcessor)
-    async def test_chunk_document_with_no_overrides(self):
+    async def test_chunk_document_with_no_overrides(self, mock_producer, mock_consumer):
         """Test chunk_document when no parameters are overridden (flow returns None)"""
         # Arrange
         config = {