diff --git a/surfsense_backend/app/indexing_pipeline/document_persistence.py b/surfsense_backend/app/indexing_pipeline/document_persistence.py index f9be92fe6..d7810e516 100644 --- a/surfsense_backend/app/indexing_pipeline/document_persistence.py +++ b/surfsense_backend/app/indexing_pipeline/document_persistence.py @@ -10,12 +10,22 @@ from app.db import Document, DocumentStatus async def rollback_and_persist_failure( session: AsyncSession, document: Document, message: str ) -> None: - """Roll back the current transaction, refresh the document, and persist a failed status.""" - await session.rollback() - await session.refresh(document) - document.updated_at = datetime.now(UTC) - document.status = DocumentStatus.failed(message) - await session.commit() + """Roll back the current transaction and best-effort persist a failed status. + + Called exclusively from except blocks — must never raise, or the new exception + would chain with the original and mask it entirely. + """ + try: + await session.rollback() + except Exception: + return # Session is completely dead; nothing further we can do. + try: + await session.refresh(document) + document.updated_at = datetime.now(UTC) + document.status = DocumentStatus.failed(message) + await session.commit() + except Exception: + pass # Best-effort; document will be retried on the next sync. def attach_chunks_to_document(document: Document, chunks: list) -> None: diff --git a/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py b/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py index 2fff9358c..70c25fcfe 100644 --- a/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py +++ b/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py @@ -135,7 +135,6 @@ class IndexingPipelineService: return [] except TRANSIENT_DB_ERRORS as e: log_doc_skipped_db(ctx, e) - await self.session.rollback() except Exception as e: log_doc_skipped_unknown(ctx, e) continue @@ -207,14 +206,14 @@ class IndexingPipelineService: log_permanent_llm_error(ctx, e) await rollback_and_persist_failure(self.session, document, llm_permanent_message(e)) - except EMBEDDING_ERRORS as e: - log_embedding_error(ctx, e) - await rollback_and_persist_failure(self.session, document, embedding_message(e)) - except RecursionError as e: log_chunking_overflow(ctx, e) await rollback_and_persist_failure(self.session, document, PipelineMessages.CHUNKING_OVERFLOW) + except EMBEDDING_ERRORS as e: + log_embedding_error(ctx, e) + await rollback_and_persist_failure(self.session, document, embedding_message(e)) + except FATAL_DB_ERRORS as e: log_db_fatal_error(ctx, e) raise diff --git a/surfsense_backend/app/indexing_pipeline/pipeline_logger.py b/surfsense_backend/app/indexing_pipeline/pipeline_logger.py index c0df3f130..771b50645 100644 --- a/surfsense_backend/app/indexing_pipeline/pipeline_logger.py +++ b/surfsense_backend/app/indexing_pipeline/pipeline_logger.py @@ -88,7 +88,7 @@ def log_doc_skipped_unknown(ctx: PipelineLogContext, exc: Exception) -> None: def log_batch_aborted(ctx: PipelineLogContext, exc: Exception) -> None: - _safe_log(logger.critical, LogMessages.BATCH_ABORTED, ctx, error=exc) + _safe_log(logger.critical, LogMessages.BATCH_ABORTED, ctx, exc_info=exc, error=exc) def log_race_condition(ctx: PipelineLogContext) -> None: @@ -110,15 +110,15 @@ def log_retryable_llm_error(ctx: PipelineLogContext, exc: Exception) -> None: def log_permanent_llm_error(ctx: PipelineLogContext, exc: Exception) -> None: - _safe_log(logger.error, LogMessages.LLM_PERMANENT, ctx, error=exc) + _safe_log(logger.error, LogMessages.LLM_PERMANENT, ctx, exc_info=exc, error=exc) def log_embedding_error(ctx: PipelineLogContext, exc: Exception) -> None: - _safe_log(logger.error, LogMessages.EMBEDDING_FAILED, ctx, error=exc) + _safe_log(logger.error, LogMessages.EMBEDDING_FAILED, ctx, exc_info=exc, error=exc) def log_chunking_overflow(ctx: PipelineLogContext, exc: Exception) -> None: - _safe_log(logger.error, LogMessages.CHUNKING_OVERFLOW, ctx, error=exc) + _safe_log(logger.error, LogMessages.CHUNKING_OVERFLOW, ctx, exc_info=exc, error=exc) def log_db_transient_error(ctx: PipelineLogContext, exc: Exception) -> None: @@ -126,7 +126,7 @@ def log_db_transient_error(ctx: PipelineLogContext, exc: Exception) -> None: def log_db_fatal_error(ctx: PipelineLogContext, exc: Exception) -> None: - _safe_log(logger.critical, LogMessages.DB_FATAL, ctx, error=exc) + _safe_log(logger.critical, LogMessages.DB_FATAL, ctx, exc_info=exc, error=exc) def log_unexpected_error(ctx: PipelineLogContext, exc: Exception) -> None: