feat: optimize source_markdown migration by processing documents in batches to reduce memory usage and improve performance

surfsense_backend/alembic/versions/101_add_source_markdown_to_documents.py

@@ -6,7 +6,7 @@ Create Date: 2026-02-17
 
 Adds source_markdown column and converts only documents that have
 blocknote_document data. Uses a pure-Python BlockNote JSON → Markdown
-converter. No external dependencies (no Node.js, no Celery, no HTTP calls).
+converter without external dependencies.
 
 Documents without blocknote_document keep source_markdown = NULL and
 get populated lazily by the editor route when a user first opens them.
@@ -50,22 +50,24 @@ def upgrade() -> None:
     _populate_source_markdown(conn)
 
 
-def _populate_source_markdown(conn) -> None:
+def _populate_source_markdown(conn, batch_size: int = 500) -> None:
-    """Populate source_markdown only for documents that have blocknote_document."""
+    """Populate source_markdown only for documents that have blocknote_document.
+
+    Processes in batches to avoid long-running transactions and high memory usage.
+    """
     from app.utils.blocknote_to_markdown import blocknote_to_markdown
 
-    # Only fetch documents that have blocknote_document content
+    # Get total count first
-    result = conn.execute(
+    count_result = conn.execute(
         sa.text("""
-            SELECT id, title, blocknote_document
+            SELECT count(*)
             FROM documents
             WHERE source_markdown IS NULL
               AND blocknote_document IS NOT NULL
         """)
     )
-    rows = result.fetchall()
+    total = count_result.scalar()
 
-    total = len(rows)
     if total == 0:
         print("✓ No documents with blocknote_document need migration")
         return
@@ -74,6 +76,25 @@ def _populate_source_markdown(conn) -> None:
 
     migrated = 0
     failed = 0
+    offset = 0
+
+    while offset < total:
+        # Fetch one batch at a time
+        result = conn.execute(
+            sa.text("""
+                SELECT id, title, blocknote_document
+                FROM documents
+                WHERE source_markdown IS NULL
+                  AND blocknote_document IS NOT NULL
+                ORDER BY id
+                LIMIT :limit OFFSET :offset
+            """),
+            {"limit": batch_size, "offset": offset},
+        )
+        rows = result.fetchall()
+
+        if not rows:
+            break
 
         for row in rows:
             doc_id = row[0]
@@ -104,6 +125,9 @@ def _populate_source_markdown(conn) -> None:
                 )
                 failed += 1
 
+        print(f"  Batch complete: processed {min(offset + batch_size, total)}/{total}")
+        offset += batch_size
+
     print(
         f"✓ source_markdown migration complete: {migrated} migrated, "
         f"{failed} failed out of {total} total"