apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-17 18:35:19 +02:00
Code Issues Projects Releases Packages Wiki Activity

simplify indexing pipeline DB error handling

Browse source
This commit is contained in:
CREDO23 2026-02-25 16:59:09 +02:00
parent 66d7d3da8a
commit 5be58b78ad
3 changed files with 1 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

15
surfsense_backend/app/indexing_pipeline/exceptions.py
View file

@ -12,8 +12,7 @@ from litellm.exceptions import (
Timeout,
UnprocessableEntityError,
)
from sqlalchemy.exc import IntegrityError, InvalidRequestError, OperationalError
from sqlalchemy.orm.exc import DetachedInstanceError
from sqlalchemy.exc import IntegrityError
# Tuples for use directly in except clauses.
RETRYABLE_LLM_ERRORS = (
@ -34,18 +33,6 @@ PERMANENT_LLM_ERRORS = (
APIResponseValidationError,
)
TRANSIENT_DB_ERRORS = (
OperationalError,
IntegrityError,
)
# Session is broken after these — re-raise instead of attempting further DB calls.
FATAL_DB_ERRORS = (
InvalidRequestError,
DetachedInstanceError,
)
# (LiteLLMEmbeddings, CohereEmbeddings, GeminiEmbeddings all normalize to RuntimeError).
EMBEDDING_ERRORS = (
RuntimeError, # local device failure or API backend normalization

21
surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py
View file

@ -12,10 +12,8 @@ from app.indexing_pipeline.document_persistence import attach_chunks_to_document
from app.indexing_pipeline.document_summarizer import summarize_document
from app.indexing_pipeline.exceptions import (
EMBEDDING_ERRORS,
FATAL_DB_ERRORS,
PERMANENT_LLM_ERRORS,
RETRYABLE_LLM_ERRORS,
TRANSIENT_DB_ERRORS,
IntegrityError,
PipelineMessages,
embedding_message,
@ -25,11 +23,7 @@ from app.indexing_pipeline.exceptions import (
)
from app.indexing_pipeline.pipeline_logger import (
PipelineLogContext,
log_batch_aborted,
log_chunking_overflow,
log_db_fatal_error,
log_db_transient_error,
log_doc_skipped_db,
log_doc_skipped_unknown,
log_document_queued,
log_document_requeued,
@ -129,15 +123,8 @@ class IndexingPipelineService:
documents.append(document)
log_document_queued(ctx)
except FATAL_DB_ERRORS as e:
log_batch_aborted(ctx, e)
await self.session.rollback()
return []
except TRANSIENT_DB_ERRORS as e:
log_doc_skipped_db(ctx, e)
except Exception as e:
log_doc_skipped_unknown(ctx, e)
continue
try:
await self.session.commit()
@ -214,14 +201,6 @@ class IndexingPipelineService:
log_embedding_error(ctx, e)
await rollback_and_persist_failure(self.session, document, embedding_message(e))
except FATAL_DB_ERRORS as e:
log_db_fatal_error(ctx, e)
raise
except TRANSIENT_DB_ERRORS as e:
log_db_transient_error(ctx, e)
await rollback_and_persist_failure(self.session, document, PipelineMessages.DB_TRANSIENT)
except Exception as e:
log_unexpected_error(ctx, e)
await rollback_and_persist_failure(self.session, document, safe_exception_message(e))

16
surfsense_backend/app/indexing_pipeline/pipeline_logger.py
View file

@ -79,18 +79,10 @@ def log_document_requeued(ctx: PipelineLogContext) -> None:
_safe_log(logger.info, LogMessages.DOCUMENT_REQUEUED, ctx)
def log_doc_skipped_db(ctx: PipelineLogContext, exc: Exception) -> None:
_safe_log(logger.warning, LogMessages.DOC_SKIPPED_DB, ctx, error=exc)
def log_doc_skipped_unknown(ctx: PipelineLogContext, exc: Exception) -> None:
_safe_log(logger.warning, LogMessages.DOC_SKIPPED_UNKNOWN, ctx, exc_info=exc, error=exc)
def log_batch_aborted(ctx: PipelineLogContext, exc: Exception) -> None:
_safe_log(logger.critical, LogMessages.BATCH_ABORTED, ctx, exc_info=exc, error=exc)
def log_race_condition(ctx: PipelineLogContext) -> None:
_safe_log(logger.warning, LogMessages.RACE_CONDITION, ctx)
@ -121,13 +113,5 @@ def log_chunking_overflow(ctx: PipelineLogContext, exc: Exception) -> None:
_safe_log(logger.error, LogMessages.CHUNKING_OVERFLOW, ctx, exc_info=exc, error=exc)
def log_db_transient_error(ctx: PipelineLogContext, exc: Exception) -> None:
_safe_log(logger.warning, LogMessages.DB_TRANSIENT, ctx, error=exc)
def log_db_fatal_error(ctx: PipelineLogContext, exc: Exception) -> None:
_safe_log(logger.critical, LogMessages.DB_FATAL, ctx, exc_info=exc, error=exc)
def log_unexpected_error(ctx: PipelineLogContext, exc: Exception) -> None:
_safe_log(logger.error, LogMessages.UNEXPECTED, ctx, exc_info=exc, error=exc)