feat: enhance performance logging and caching in various components

- Introduced slow callback logging in FastAPI to identify blocking calls.
- Added performance logging for agent creation and tool loading processes.
- Implemented caching for MCP tools to reduce redundant server calls.
- Enhanced sandbox management with in-process caching for improved efficiency.
- Refactored several functions for better readability and performance tracking.
- Updated tests to ensure proper functionality of new features and optimizations.

surfsense_backend/tests/unit/indexing_pipeline/conftest.py

@@ -1,6 +1,7 @@
-import pytest
 from unittest.mock import AsyncMock, MagicMock
 
+import pytest
+
 
 @pytest.fixture
 def patched_summarizer_chain(monkeypatch):
@@ -21,7 +22,9 @@ def patched_summarizer_chain(monkeypatch):
 def patched_chunker_instance(monkeypatch):
     mock = MagicMock()
     mock.chunk.return_value = [MagicMock(text="prose chunk")]
-    monkeypatch.setattr("app.indexing_pipeline.document_chunker.config.chunker_instance", mock)
+    monkeypatch.setattr(
+        "app.indexing_pipeline.document_chunker.config.chunker_instance", mock
+    )
     return mock
 
 
@@ -29,5 +32,7 @@ def patched_chunker_instance(monkeypatch):
 def patched_code_chunker_instance(monkeypatch):
     mock = MagicMock()
     mock.chunk.return_value = [MagicMock(text="code chunk")]
-    monkeypatch.setattr("app.indexing_pipeline.document_chunker.config.code_chunker_instance", mock)
+    monkeypatch.setattr(
+        "app.indexing_pipeline.document_chunker.config.code_chunker_instance", mock
+    )
     return mock

surfsense_backend/tests/unit/indexing_pipeline/test_document_hashing.py

@@ -1,7 +1,10 @@
 import pytest
 
 from app.db import DocumentType
-from app.indexing_pipeline.document_hashing import compute_content_hash, compute_unique_identifier_hash
+from app.indexing_pipeline.document_hashing import (
+    compute_content_hash,
+    compute_unique_identifier_hash,
+)
 
 pytestmark = pytest.mark.unit
 
@@ -10,21 +13,31 @@ def test_different_unique_id_produces_different_hash(make_connector_document):
     """Two documents with different unique_ids produce different identifier hashes."""
     doc_a = make_connector_document(unique_id="id-001")
     doc_b = make_connector_document(unique_id="id-002")
-    assert compute_unique_identifier_hash(doc_a) != compute_unique_identifier_hash(doc_b)
+    assert compute_unique_identifier_hash(doc_a) != compute_unique_identifier_hash(
+        doc_b
+    )
 
 
-def test_different_search_space_produces_different_identifier_hash(make_connector_document):
+def test_different_search_space_produces_different_identifier_hash(
+    make_connector_document,
+):
     """Same document in different search spaces produces different identifier hashes."""
     doc_a = make_connector_document(search_space_id=1)
     doc_b = make_connector_document(search_space_id=2)
-    assert compute_unique_identifier_hash(doc_a) != compute_unique_identifier_hash(doc_b)
+    assert compute_unique_identifier_hash(doc_a) != compute_unique_identifier_hash(
+        doc_b
+    )
 
 
-def test_different_document_type_produces_different_identifier_hash(make_connector_document):
+def test_different_document_type_produces_different_identifier_hash(
+    make_connector_document,
+):
     """Same unique_id with different document types produces different identifier hashes."""
     doc_a = make_connector_document(document_type=DocumentType.CLICKUP_CONNECTOR)
     doc_b = make_connector_document(document_type=DocumentType.NOTION_CONNECTOR)
-    assert compute_unique_identifier_hash(doc_a) != compute_unique_identifier_hash(doc_b)
+    assert compute_unique_identifier_hash(doc_a) != compute_unique_identifier_hash(
+        doc_b
+    )
 
 
 def test_same_content_same_space_produces_same_content_hash(make_connector_document):
@@ -34,7 +47,9 @@ def test_same_content_same_space_produces_same_content_hash(make_connector_docum
     assert compute_content_hash(doc_a) == compute_content_hash(doc_b)
 
 
-def test_same_content_different_space_produces_different_content_hash(make_connector_document):
+def test_same_content_different_space_produces_different_content_hash(
+    make_connector_document,
+):
     """Identical content in different search spaces produces different content hashes."""
     doc_a = make_connector_document(source_markdown="Hello world", search_space_id=1)
     doc_b = make_connector_document(source_markdown="Hello world", search_space_id=2)

surfsense_backend/tests/unit/indexing_pipeline/test_document_summarizer.py

@@ -1,6 +1,7 @@
-import pytest
 from unittest.mock import MagicMock
 
+import pytest
+
 from app.indexing_pipeline.document_summarizer import summarize_document
 
 pytestmark = pytest.mark.unit
@@ -38,5 +39,3 @@ async def test_with_metadata_omits_empty_fields_from_output():
 
     assert "Alice" in result
     assert "description" not in result.lower()
-
-