feat: enhance performance logging and caching in various components

- Introduced slow callback logging in FastAPI to identify blocking calls.
- Added performance logging for agent creation and tool loading processes.
- Implemented caching for MCP tools to reduce redundant server calls.
- Enhanced sandbox management with in-process caching for improved efficiency.
- Refactored several functions for better readability and performance tracking.
- Updated tests to ensure proper functionality of new features and optimizations.

surfsense_backend/app/indexing_pipeline/pipeline_logger.py

@@ -8,27 +8,29 @@ logger = logging.getLogger(__name__)
 class PipelineLogContext:
     connector_id: int | None
     search_space_id: int
-    unique_id: str            # always available from ConnectorDocument
-    doc_id: int | None = None # set once the DB row exists (index phase only)
+    unique_id: str  # always available from ConnectorDocument
+    doc_id: int | None = None  # set once the DB row exists (index phase only)
 
 
 class LogMessages:
     # prepare_for_indexing
-    DOCUMENT_QUEUED     = "New document queued for indexing."
-    DOCUMENT_UPDATED    = "Document content changed, re-queued for indexing."
-    DOCUMENT_REQUEUED   = "Stuck document re-queued for indexing."
+    DOCUMENT_QUEUED = "New document queued for indexing."
+    DOCUMENT_UPDATED = "Document content changed, re-queued for indexing."
+    DOCUMENT_REQUEUED = "Stuck document re-queued for indexing."
     DOC_SKIPPED_UNKNOWN = "Unexpected error — document skipped."
-    BATCH_ABORTED       = "Fatal DB error — aborting prepare batch."
-    RACE_CONDITION      = "Concurrent worker beat us to the commit — rolling back batch."
+    BATCH_ABORTED = "Fatal DB error — aborting prepare batch."
+    RACE_CONDITION = "Concurrent worker beat us to the commit — rolling back batch."
 
     # index
-    INDEX_STARTED       = "Document indexing started."
-    INDEX_SUCCESS       = "Document indexed successfully."
-    LLM_RETRYABLE       = "Retryable LLM error — document marked failed, will retry on next sync."
-    LLM_PERMANENT       = "Permanent LLM error — document marked failed."
-    EMBEDDING_FAILED    = "Embedding error — document marked failed."
-    CHUNKING_OVERFLOW   = "Chunking overflow — document marked failed."
-    UNEXPECTED          = "Unexpected error — document marked failed."
+    INDEX_STARTED = "Document indexing started."
+    INDEX_SUCCESS = "Document indexed successfully."
+    LLM_RETRYABLE = (
+        "Retryable LLM error — document marked failed, will retry on next sync."
+    )
+    LLM_PERMANENT = "Permanent LLM error — document marked failed."
+    EMBEDDING_FAILED = "Embedding error — document marked failed."
+    CHUNKING_OVERFLOW = "Chunking overflow — document marked failed."
+    UNEXPECTED = "Unexpected error — document marked failed."
 
 
 def _format_context(ctx: PipelineLogContext) -> str:
@@ -52,7 +54,9 @@ def _build_message(msg: str, ctx: PipelineLogContext, **extra) -> str:
         return msg
 
 
-def _safe_log(level_fn, msg: str, ctx: PipelineLogContext, exc_info=None, **extra) -> None:
+def _safe_log(
+    level_fn, msg: str, ctx: PipelineLogContext, exc_info=None, **extra
+) -> None:
     # Logging must never raise — a broken log call inside an except block would
     # chain with the original exception and mask it entirely.
     try:
@@ -64,6 +68,7 @@ def _safe_log(level_fn, msg: str, ctx: PipelineLogContext, exc_info=None, **extr
 
 # ── prepare_for_indexing ──────────────────────────────────────────────────────
 
+
 def log_document_queued(ctx: PipelineLogContext) -> None:
     _safe_log(logger.info, LogMessages.DOCUMENT_QUEUED, ctx)
 
@@ -77,7 +82,9 @@ def log_document_requeued(ctx: PipelineLogContext) -> None:
 
 
 def log_doc_skipped_unknown(ctx: PipelineLogContext, exc: Exception) -> None:
-    _safe_log(logger.warning, LogMessages.DOC_SKIPPED_UNKNOWN, ctx, exc_info=exc, error=exc)
+    _safe_log(
+        logger.warning, LogMessages.DOC_SKIPPED_UNKNOWN, ctx, exc_info=exc, error=exc
+    )
 
 
 def log_race_condition(ctx: PipelineLogContext) -> None:
@@ -90,6 +97,7 @@ def log_batch_aborted(ctx: PipelineLogContext, exc: Exception) -> None:
 
 # ── index ─────────────────────────────────────────────────────────────────────
 
+
 def log_index_started(ctx: PipelineLogContext) -> None:
     _safe_log(logger.info, LogMessages.INDEX_STARTED, ctx)