apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-12 22:42:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

I've added comprehensive unit and integration tests for the recently implemented Slack connector features. This includes tests for backend logic (SlackHistory, indexing tasks, API routes) and frontend components and hooks.

Browse source 
Backend Tests (`surfsense_backend`):
- `test_slack_history.py`:
    - Tests for `SlackHistory` class methods including `get_all_channels`,
      `get_conversation_history`, timestamp conversions, and message
      formatting. Covers various API response scenarios, pagination,
      and error handling.
- `test_connectors_indexing_tasks.py`:
    - Unit tests for `index_slack_messages` task.
    - Scenarios include initial indexing, periodic indexing, on-demand
      re-indexing (targeted channels, force re-index, custom dates),
      channel filtering, and error handling.
    - Mocks `SlackHistory` and database interactions.
- `test_search_source_connectors_routes.py`:
    - Tests for `/slack/{connector_id}/discover-channels` endpoint.
    - Tests for `/slack/{connector_id}/reindex-channels` endpoint,
      verifying parameter passing to the background task.
    - Tests for the main `/search-source-connectors/{connector_id}/index`
      endpoint for Slack connectors, ensuring the `force_full_reindex`
      flag is handled.

Frontend Tests (`surfsense_web`):
- `components/editConnector/EditSlackConnectorConfigForm.test.tsx`:
    - Tests rendering of all Slack configuration fields.
    - Verifies `onConfigChange` callback functionality.
    - Tests conditional rendering of periodic indexing settings.
    - Checks disabled state of the form.
- `app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.test.tsx`:
    - Tests for the "Channel Management" tab specific to Slack connectors.
    - Verifies UI for discovering channels (mocking API calls).
    - Tests selection of channels and updating the connector configuration.
    - Tests UI for triggering on-demand re-indexing with various options.
- `hooks/useSearchSourceConnectors.test.ts`:
    - Unit tests for the `discoverSlackChannels` function, mocking `fetch`.
    - Unit tests for the `reindexSlackChannels` function, mocking `fetch`
      and verifying correct request body construction.
    - Includes testing of the `fetchWithAuth` helper function.
This commit is contained in:
google-labs-jules[bot] 2025-05-28 10:48:28 +00:00
parent d1f11d4fbb
commit 4bf09b8bde
6 changed files with 1804 additions and 387 deletions
Download patch file Download diff file Expand all files Collapse all files

692
surfsense_backend/app/connectors/test_slack_history.py
View file

@ -1,420 +1,338 @@
import unittest
import time # Imported to be available for patching target module
from unittest.mock import patch, Mock, call
from unittest.mock import patch, MagicMock, call
from slack_sdk import WebClient
from slack_sdk.errors import SlackApiError
from datetime import datetime, timezone, timedelta
# Since test_slack_history.py is in the same directory as slack_history.py
from .slack_history import SlackHistory
# Assuming SlackHistory is in app.connectors.slack_history
# Adjust the import path if your structure is different.
# For example, if 'app' is the root of your source:
from app.connectors.slack_history import SlackHistory
class TestSlackHistoryGetAllChannels(unittest.TestCase):
class TestSlackHistory(unittest.TestCase):
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_get_all_channels_pagination_with_delay(self, MockWebClient, mock_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
def setUp(self):
self.dummy_token = "xoxb-test-token"
# We patch WebClient in each test that needs it, so SlackHistory can be instantiated
# without making actual API calls during setup.
self.history = SlackHistory(token=self.dummy_token)
@patch('app.connectors.slack_history.WebClient') # Patch where WebClient is looked up
def test_init_and_set_token(self, MockWebClient):
# Test __init__
mock_client_instance = MockWebClient.return_value # Get the instance returned by MockWebClient()
# Mock API responses now include is_private and is_member
page1_response = {
history_init = SlackHistory(token="test_init_token")
MockWebClient.assert_called_once_with(token="test_init_token")
self.assertEqual(history_init.client, mock_client_instance)
# Reset mock for the next part of the test
MockWebClient.reset_mock()
# Test set_token
history_set_token = SlackHistory() # Initialize without token first
self.assertIsNone(history_set_token.client)
history_set_token.set_token("test_set_token_token")
MockWebClient.assert_called_once_with(token="test_set_token_token")
self.assertEqual(history_set_token.client, mock_client_instance)
@patch.object(WebClient, 'conversations_list')
def test_get_all_channels_success_and_pagination(self, mock_conversations_list):
# Simulate WebClient being already initialized in self.history
self.history.client = mock_conversations_list.MagicMock() # Replace client with a mock that has conversations_list
mock_response_page1 = MagicMock()
mock_response_page1.data = {
"channels": [
{"name": "general", "id": "C1", "is_private": False, "is_member": True},
{"name": "dev", "id": "C0", "is_private": False, "is_member": True}
{"id": "C1", "name": "general", "is_member": True, "is_private": False},
{"id": "C2", "name": "random", "is_member": False, "is_private": False}, # is_member False
],
"response_metadata": {"next_cursor": "cursor123"}
"response_metadata": {"next_cursor": "cursor_page2"}
}
page2_response = {
"channels": [{"name": "random", "id": "C2", "is_private": True, "is_member": True}],
mock_response_page2 = MagicMock()
mock_response_page2.data = {
"channels": [
{"id": "C3", "name": "private-project", "is_member": True, "is_private": True},
],
"response_metadata": {"next_cursor": ""}
}
self.history.client.conversations_list.side_effect = [mock_response_page1, mock_response_page2]
channels = self.history.get_all_channels(include_private=True)
mock_client_instance.conversations_list.side_effect = [
page1_response,
page2_response
]
slack_history = SlackHistory(token="fake_token")
channels_list = slack_history.get_all_channels(include_private=True)
expected_channels_list = [
{"id": "C1", "name": "general", "is_private": False, "is_member": True},
{"id": "C0", "name": "dev", "is_private": False, "is_member": True},
{"id": "C2", "name": "random", "is_private": True, "is_member": True}
]
self.assertEqual(len(channels_list), 3)
self.assertListEqual(channels_list, expected_channels_list) # Assert list equality
self.assertEqual(len(channels), 3)
self.assertEqual(channels[0]['id'], "C1")
self.assertTrue(channels[0]['is_member']) # Verify is_member preserved
self.assertEqual(channels[1]['id'], "C2")
self.assertFalse(channels[1]['is_member']) # Verify is_member preserved
self.assertEqual(channels[2]['id'], "C3")
self.assertTrue(channels[2]['is_private'])
expected_calls = [
call(types="public_channel,private_channel", cursor=None, limit=1000),
call(types="public_channel,private_channel", cursor="cursor123", limit=1000)
call(limit=200, cursor=None, types="public_channel,private_channel"),
call(limit=200, cursor="cursor_page2", types="public_channel,private_channel")
]
mock_client_instance.conversations_list.assert_has_calls(expected_calls)
self.assertEqual(mock_client_instance.conversations_list.call_count, 2)
mock_sleep.assert_called_once_with(3)
mock_logger.info.assert_called_once_with("Paginating for channels, waiting 3 seconds before next call. Cursor: cursor123")
self.history.client.conversations_list.assert_has_calls(expected_calls)
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_get_all_channels_rate_limit_with_retry_after(self, MockWebClient, mock_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
mock_error_response = Mock()
mock_error_response.status_code = 429
mock_error_response.headers = {'Retry-After': '5'}
successful_response = {
"channels": [{"name": "general", "id": "C1", "is_private": False, "is_member": True}],
@patch.object(WebClient, 'conversations_list')
def test_get_all_channels_public_only(self, mock_conversations_list):
self.history.client = mock_conversations_list.MagicMock()
mock_response = MagicMock()
mock_response.data = {
"channels": [{"id": "C1", "name": "general", "is_member": True, "is_private": False}],
"response_metadata": {"next_cursor": ""}
}
mock_client_instance.conversations_list.side_effect = [
SlackApiError(message="ratelimited", response=mock_error_response),
successful_response
]
slack_history = SlackHistory(token="fake_token")
channels_list = slack_history.get_all_channels(include_private=True)
expected_channels_list = [{"id": "C1", "name": "general", "is_private": False, "is_member": True}]
self.assertEqual(len(channels_list), 1)
self.assertListEqual(channels_list, expected_channels_list)
mock_sleep.assert_called_once_with(5)
mock_logger.warning.assert_called_once_with("Slack API rate limit hit while fetching channels. Waiting for 5 seconds. Cursor: None")
expected_calls = [
call(types="public_channel,private_channel", cursor=None, limit=1000),
call(types="public_channel,private_channel", cursor=None, limit=1000)
]
mock_client_instance.conversations_list.assert_has_calls(expected_calls)
self.assertEqual(mock_client_instance.conversations_list.call_count, 2)
self.history.client.conversations_list.return_value = mock_response
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_get_all_channels_rate_limit_no_retry_after_valid_header(self, MockWebClient, mock_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
self.history.get_all_channels(include_private=False)
self.history.client.conversations_list.assert_called_once_with(limit=200, cursor=None, types="public_channel")
@patch.object(WebClient, 'conversations_list')
def test_get_all_channels_empty(self, mock_conversations_list):
self.history.client = mock_conversations_list.MagicMock()
mock_response = MagicMock()
mock_response.data = {"channels": [], "response_metadata": {"next_cursor": ""}}
self.history.client.conversations_list.return_value = mock_response
channels = self.history.get_all_channels()
self.assertEqual(len(channels), 0)
@patch.object(WebClient, 'conversations_list')
def test_get_all_channels_api_error(self, mock_conversations_list):
self.history.client = mock_conversations_list.MagicMock()
# Simulate a SlackApiError
self.history.client.conversations_list.side_effect = SlackApiError("API Error", MagicMock(data={"error": "test_error"}))
mock_error_response = Mock()
mock_error_response.status_code = 429
mock_error_response.headers = {'Retry-After': 'invalid_value'}
successful_response = {
"channels": [{"name": "general", "id": "C1", "is_private": False, "is_member": True}],
"response_metadata": {"next_cursor": ""}
with self.assertRaises(SlackApiError):
self.history.get_all_channels()
@patch.object(WebClient, 'conversations_history')
def test_get_conversation_history_success_and_pagination(self, mock_conversations_history):
self.history.client = mock_conversations_history.MagicMock()
mock_response_page1 = MagicMock()
mock_response_page1.data = {
"messages": [{"ts": "123.001", "text": "Hello"}, {"ts": "123.002", "text": "World"}],
"has_more": True,
"response_metadata": {"next_cursor": "cursor_history2"}
}
mock_client_instance.conversations_list.side_effect = [
SlackApiError(message="ratelimited", response=mock_error_response),
successful_response
]
slack_history = SlackHistory(token="fake_token")
channels_list = slack_history.get_all_channels(include_private=True)
expected_channels_list = [{"id": "C1", "name": "general", "is_private": False, "is_member": True}]
self.assertListEqual(channels_list, expected_channels_list)
mock_sleep.assert_called_once_with(60) # Default fallback
mock_logger.warning.assert_called_once_with("Slack API rate limit hit while fetching channels. Waiting for 60 seconds. Cursor: None")
self.assertEqual(mock_client_instance.conversations_list.call_count, 2)
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_get_all_channels_rate_limit_no_retry_after_header(self, MockWebClient, mock_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
mock_error_response = Mock()
mock_error_response.status_code = 429
mock_error_response.headers = {}
successful_response = {
"channels": [{"name": "general", "id": "C1", "is_private": False, "is_member": True}],
"response_metadata": {"next_cursor": ""}
}
mock_client_instance.conversations_list.side_effect = [
SlackApiError(message="ratelimited", response=mock_error_response),
successful_response
]
slack_history = SlackHistory(token="fake_token")
channels_list = slack_history.get_all_channels(include_private=True)
expected_channels_list = [{"id": "C1", "name": "general", "is_private": False, "is_member": True}]
self.assertListEqual(channels_list, expected_channels_list)
mock_sleep.assert_called_once_with(60) # Default fallback
mock_logger.warning.assert_called_once_with("Slack API rate limit hit while fetching channels. Waiting for 60 seconds. Cursor: None")
self.assertEqual(mock_client_instance.conversations_list.call_count, 2)
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_get_all_channels_other_slack_api_error(self, MockWebClient, mock_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
mock_error_response = Mock()
mock_error_response.status_code = 500
mock_error_response.headers = {}
mock_error_response.data = {"ok": False, "error": "internal_error"}
original_error = SlackApiError(message="server error", response=mock_error_response)
mock_client_instance.conversations_list.side_effect = original_error
slack_history = SlackHistory(token="fake_token")
with self.assertRaises(SlackApiError) as context:
slack_history.get_all_channels(include_private=True)
self.assertEqual(context.exception.response.status_code, 500)
self.assertIn("server error", str(context.exception))
mock_sleep.assert_not_called()
mock_logger.warning.assert_not_called() # Ensure no rate limit log
mock_client_instance.conversations_list.assert_called_once_with(
types="public_channel,private_channel", cursor=None, limit=1000
)
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_get_all_channels_handles_missing_name_id_gracefully(self, MockWebClient, mock_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
response_with_malformed_data = {
"channels": [
{"id": "C1_missing_name", "is_private": False, "is_member": True},
{"name": "channel_missing_id", "is_private": False, "is_member": True},
{"name": "general", "id": "C2_valid", "is_private": False, "is_member": True}
],
"response_metadata": {"next_cursor": ""}
}
mock_client_instance.conversations_list.return_value = response_with_malformed_data
slack_history = SlackHistory(token="fake_token")
channels_list = slack_history.get_all_channels(include_private=True)
expected_channels_list = [
{"id": "C2_valid", "name": "general", "is_private": False, "is_member": True}
]
self.assertEqual(len(channels_list), 1)
self.assertListEqual(channels_list, expected_channels_list)
self.assertEqual(mock_logger.warning.call_count, 2)
mock_logger.warning.assert_any_call("Channel found with missing name or id. Data: {'id': 'C1_missing_name', 'is_private': False, 'is_member': True}")
mock_logger.warning.assert_any_call("Channel found with missing name or id. Data: {'name': 'channel_missing_id', 'is_private': False, 'is_member': True}")
mock_sleep.assert_not_called()
mock_client_instance.conversations_list.assert_called_once_with(
types="public_channel,private_channel", cursor=None, limit=1000
)
if __name__ == '__main__':
unittest.main()
class TestSlackHistoryGetConversationHistory(unittest.TestCase):
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_proactive_delay_single_page(self, MockWebClient, mock_time_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
mock_client_instance.conversations_history.return_value = {
"messages": [{"text": "msg1"}],
mock_response_page2 = MagicMock()
mock_response_page2.data = {
"messages": [{"ts": "123.003", "text": "Again"}],
"has_more": False
}
slack_history = SlackHistory(token="fake_token")
slack_history.get_conversation_history(channel_id="C123")
mock_time_sleep.assert_called_once_with(1.2) # Proactive delay
self.history.client.conversations_history.side_effect = [mock_response_page1, mock_response_page2]
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_proactive_delay_multiple_pages(self, MockWebClient, mock_time_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
mock_client_instance.conversations_history.side_effect = [
{
"messages": [{"text": "msg1"}],
"has_more": True,
"response_metadata": {"next_cursor": "cursor1"}
},
{
"messages": [{"text": "msg2"}],
"has_more": False
}
messages = self.history.get_conversation_history("C123", limit=3) # Request total 3 messages
self.assertEqual(len(messages), 3)
self.assertEqual(messages[0]['text'], "Hello")
self.assertEqual(messages[2]['text'], "Again")
# The internal page_limit in get_conversation_history is 100 (can be smaller than requested limit)
expected_calls = [
call(channel="C123", limit=3, oldest=None, latest=None, cursor=None),
call(channel="C123", limit=1, oldest=None, latest=None, cursor="cursor_history2")
]
slack_history = SlackHistory(token="fake_token")
slack_history.get_conversation_history(channel_id="C123")
# Expected calls: 1.2 (page1), 1.2 (page2)
self.assertEqual(mock_time_sleep.call_count, 2)
mock_time_sleep.assert_has_calls([call(1.2), call(1.2)])
self.history.client.conversations_history.assert_has_calls(expected_calls)
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_retry_after_logic(self, MockWebClient, mock_time_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
mock_error_response = Mock()
mock_error_response.status_code = 429
mock_error_response.headers = {'Retry-After': '5'}
mock_client_instance.conversations_history.side_effect = [
SlackApiError(message="ratelimited", response=mock_error_response),
{"messages": [{"text": "msg1"}], "has_more": False}
]
slack_history = SlackHistory(token="fake_token")
messages = slack_history.get_conversation_history(channel_id="C123")
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]["text"], "msg1")
# Expected sleep calls: 1.2 (proactive for 1st attempt), 5 (rate limit), 1.2 (proactive for 2nd attempt)
mock_time_sleep.assert_has_calls([call(1.2), call(5), call(1.2)], any_order=False)
mock_logger.warning.assert_called_once() # Check that a warning was logged for rate limiting
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_not_in_channel_error(self, MockWebClient, mock_time_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
mock_error_response = Mock()
mock_error_response.status_code = 403 # Typical for not_in_channel, but data matters more
mock_error_response.data = {'ok': False, 'error': 'not_in_channel'}
# This error is now raised by the inner try-except, then caught by the outer one
mock_client_instance.conversations_history.side_effect = SlackApiError(
message="not_in_channel error",
response=mock_error_response
@patch.object(WebClient, 'conversations_history')
def test_get_conversation_history_with_timestamps(self, mock_conversations_history):
self.history.client = mock_conversations_history.MagicMock()
mock_response = MagicMock()
mock_response.data = {"messages": [{"ts": "1609500000.000", "text":"msg1"}], "has_more": False}
self.history.client.conversations_history.return_value = mock_response
oldest_ts_str = str(int(datetime(2021, 1, 1, 0, 0, 0, tzinfo=timezone.utc).timestamp()))
latest_ts_str = str(int(datetime(2021, 1, 2, 0, 0, 0, tzinfo=timezone.utc).timestamp()))
self.history.get_conversation_history("C123", oldest=oldest_ts_str, latest=latest_ts_str, limit=50)
self.history.client.conversations_history.assert_called_once_with(
channel="C123", limit=50, oldest=oldest_ts_str, latest=latest_ts_str, cursor=None
)
slack_history = SlackHistory(token="fake_token")
messages = slack_history.get_conversation_history(channel_id="C123")
self.assertEqual(messages, [])
mock_logger.warning.assert_called_with(
"Bot is not in channel 'C123'. Cannot fetch history. Please add the bot to this channel."
)
mock_time_sleep.assert_called_once_with(1.2) # Proactive delay before the API call
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_other_slack_api_error_propagates(self, MockWebClient, mock_time_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
mock_error_response = Mock()
mock_error_response.status_code = 500
mock_error_response.data = {'ok': False, 'error': 'internal_error'}
original_error = SlackApiError(message="server error", response=mock_error_response)
@patch('logging.warning') # Patch logging where it's used (directly in slack_history module)
@patch.object(WebClient, 'conversations_history')
def test_get_conversation_history_not_in_channel(self, mock_conversations_history, mock_logging_warning):
self.history.client = mock_conversations_history.MagicMock()
error_response = MagicMock()
error_response.data = {"ok": False, "error": "not_in_channel"}
self.history.client.conversations_history.side_effect = SlackApiError("not_in_channel error", error_response)
mock_client_instance.conversations_history.side_effect = original_error
slack_history = SlackHistory(token="fake_token")
with self.assertRaises(SlackApiError) as context:
slack_history.get_conversation_history(channel_id="C123")
self.assertIn("Error retrieving history for channel C123", str(context.exception))
self.assertIs(context.exception.response, mock_error_response)
mock_time_sleep.assert_called_once_with(1.2) # Proactive delay
messages = self.history.get_conversation_history("C123")
self.assertEqual(len(messages), 0) # Should return empty list
mock_logging_warning.assert_called_once()
self.assertIn("Bot is not in channel C123 or history is private.", mock_logging_warning.call_args[0][0])
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_general_exception_propagates(self, MockWebClient, mock_time_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
original_error = Exception("Something broke")
mock_client_instance.conversations_history.side_effect = original_error
slack_history = SlackHistory(token="fake_token")
with self.assertRaises(Exception) as context: # Check for generic Exception
slack_history.get_conversation_history(channel_id="C123")
self.assertIs(context.exception, original_error) # Should re-raise the original error
mock_logger.error.assert_called_once_with("Unexpected error in get_conversation_history for channel C123: Something broke")
mock_time_sleep.assert_called_once_with(1.2) # Proactive delay
@patch.object(WebClient, 'conversations_history')
def test_get_conversation_history_other_api_error(self, mock_conversations_history):
self.history.client = mock_conversations_history.MagicMock()
error_response = MagicMock()
error_response.data = {"ok": False, "error": "some_other_error"}
self.history.client.conversations_history.side_effect = SlackApiError("Some other error", error_response)
class TestSlackHistoryGetUserInfo(unittest.TestCase):
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_retry_after_logic(self, MockWebClient, mock_time_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
mock_error_response = Mock()
mock_error_response.status_code = 429
mock_error_response.headers = {'Retry-After': '3'} # Using 3 seconds for test
successful_user_data = {"id": "U123", "name": "testuser"}
mock_client_instance.users_info.side_effect = [
SlackApiError(message="ratelimited_userinfo", response=mock_error_response),
{"user": successful_user_data}
]
slack_history = SlackHistory(token="fake_token")
user_info = slack_history.get_user_info(user_id="U123")
self.assertEqual(user_info, successful_user_data)
# Assert that time.sleep was called for the rate limit
mock_time_sleep.assert_called_once_with(3)
mock_logger.warning.assert_called_once_with(
"Rate limited by Slack on users.info for user U123. Retrying after 3 seconds."
)
# Assert users_info was called twice (original + retry)
self.assertEqual(mock_client_instance.users_info.call_count, 2)
mock_client_instance.users_info.assert_has_calls([call(user="U123"), call(user="U123")])
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep') # time.sleep might be called by other logic, but not expected here
@patch('slack_sdk.WebClient')
def test_other_slack_api_error_propagates(self, MockWebClient, mock_time_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
mock_error_response = Mock()
mock_error_response.status_code = 500 # Some other error
mock_error_response.data = {'ok': False, 'error': 'internal_server_error'}
original_error = SlackApiError(message="internal server error", response=mock_error_response)
mock_client_instance.users_info.side_effect = original_error
slack_history = SlackHistory(token="fake_token")
with self.assertRaises(SlackApiError) as context:
slack_history.get_user_info(user_id="U123")
# Check that the raised error is the one we expect
self.assertIn("Error retrieving user info for U123", str(context.exception))
self.assertIs(context.exception.response, mock_error_response)
mock_time_sleep.assert_not_called() # No rate limit sleep
@patch('surfsense_backend.app.connectors.slack_history.logger')
@patch('surfsense_backend.app.connectors.slack_history.time.sleep')
@patch('slack_sdk.WebClient')
def test_general_exception_propagates(self, MockWebClient, mock_time_sleep, mock_logger):
mock_client_instance = MockWebClient.return_value
original_error = Exception("A very generic problem")
mock_client_instance.users_info.side_effect = original_error
slack_history = SlackHistory(token="fake_token")
with self.assertRaises(Exception) as context:
slack_history.get_user_info(user_id="U123")
with self.assertRaises(SlackApiError):
self.history.get_conversation_history("C123")
self.assertIs(context.exception, original_error) # Check it's the exact same exception
mock_logger.error.assert_called_once_with(
"Unexpected error in get_user_info for user U123: A very generic problem"
@patch.object(WebClient, 'conversations_history')
def test_get_conversation_history_empty_messages_from_api(self, mock_conversations_history):
self.history.client = mock_conversations_history.MagicMock()
mock_response = MagicMock()
mock_response.data = {"messages": [], "has_more": False}
self.history.client.conversations_history.return_value = mock_response
messages = self.history.get_conversation_history("C123")
self.assertEqual(len(messages), 0)
def test_convert_date_to_timestamp_static(self):
# Valid date
ts = SlackHistory.convert_date_to_timestamp("2023-01-15")
expected_dt = datetime(2023, 1, 15, 0, 0, 0, tzinfo=timezone.utc)
self.assertEqual(ts, str(int(expected_dt.timestamp())))
# Valid date with is_latest = True
ts_latest = SlackHistory.convert_date_to_timestamp("2023-01-15", is_latest=True)
# Should be timestamp for 2023-01-16 00:00:00 UTC to include the whole day of 2023-01-15
expected_dt_latest_inclusive = datetime(2023, 1, 15, 0, 0, 0, tzinfo=timezone.utc) + timedelta(days=1)
self.assertEqual(ts_latest, str(int(expected_dt_latest_inclusive.timestamp())))
# Invalid date format - Expect ValueError as per typical date parsing behavior
with self.assertRaises(ValueError):
SlackHistory.convert_date_to_timestamp("15-01-2023")
# Invalid date value
with self.assertRaises(ValueError):
SlackHistory.convert_date_to_timestamp("2023-13-01") # Month 13
@patch.object(SlackHistory, 'get_conversation_history')
def test_get_history_by_date_range_success(self, mock_get_conv_history):
# Mock convert_date_to_timestamp if it's complex, or test its output directly.
# Here, we assume convert_date_to_timestamp works as tested above.
mock_get_conv_history.return_value = [{"text": "message from range"}]
# Expected timestamps based on convert_date_to_timestamp logic
start_date_str = "2023-01-10"
end_date_str = "2023-01-12"
expected_oldest_ts = str(int(datetime(2023, 1, 10, 0, 0, 0, tzinfo=timezone.utc).timestamp()))
# For latest, it should be the start of the next day to include the entire end_date_str
expected_latest_ts = str(int((datetime(2023, 1, 12, 0, 0, 0, tzinfo=timezone.utc) + timedelta(days=1)).timestamp()))
messages, error = self.history.get_history_by_date_range("C123", start_date_str, end_date_str)
self.assertIsNone(error)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]['text'], "message from range")
mock_get_conv_history.assert_called_once_with(
channel_id="C123",
oldest=expected_oldest_ts,
latest=expected_latest_ts
)
mock_time_sleep.assert_not_called() # No rate limit sleep
def test_get_history_by_date_range_invalid_dates(self):
# Test with invalid start_date format
result, error = self.history.get_history_by_date_range("C123", "invalid-start-date", "2023-01-12")
self.assertIsNone(result)
self.assertIn("Invalid start_date format: invalid-start-date. Use YYYY-MM-DD.", error)
# Test with invalid end_date format
result, error = self.history.get_history_by_date_range("C123", "2023-01-10", "invalid-end-date")
self.assertIsNone(result)
self.assertIn("Invalid end_date format: invalid-end-date. Use YYYY-MM-DD.", error)
@patch.object(WebClient, 'users_info')
def test_get_user_info_success(self, mock_users_info):
self.history.client = mock_users_info.MagicMock()
mock_response = MagicMock()
mock_response.data = {"user": {"id": "U123", "name": "testuser", "real_name": "Test User"}}
self.history.client.users_info.return_value = mock_response
user_info = self.history.get_user_info("U123")
self.assertIsNotNone(user_info)
self.assertEqual(user_info['name'], "testuser")
self.history.client.users_info.assert_called_once_with(user="U123")
@patch.object(WebClient, 'users_info')
def test_get_user_info_api_error(self, mock_users_info):
self.history.client = mock_users_info.MagicMock()
self.history.client.users_info.side_effect = SlackApiError("API Error", MagicMock(data={"error": "user_not_found"}))
user_info = self.history.get_user_info("U123")
self.assertIsNone(user_info)
def test_format_message_basic(self):
raw_msg = {"type": "message", "user": "U123", "text": "Hello world", "ts": "1609459200.000100"} # 2021-01-01 00:00:00 UTC
formatted = self.history.format_message(raw_msg)
self.assertEqual(formatted['user_id'], "U123")
self.assertEqual(formatted['text'], "Hello world")
self.assertEqual(formatted['datetime'], "2021-01-01 00:00:00 UTC")
self.assertNotIn('user_name', formatted) # Not requested
@patch.object(SlackHistory, 'get_user_info')
def test_format_message_with_user_info_and_real_name(self, mock_get_user_info):
mock_get_user_info.return_value = {"real_name": "Test User Real", "name": "testusername"}
raw_msg = {"type": "message", "user": "U123", "text": "Hello with user", "ts": "1609459200.000200"}
formatted = self.history.format_message(raw_msg, include_user_info=True)
mock_get_user_info.assert_called_once_with("U123")
self.assertEqual(formatted['user_id'], "U123")
self.assertEqual(formatted['user_name'], "Test User Real") # Prefers real_name
self.assertEqual(formatted['text'], "Hello with user")
@patch.object(SlackHistory, 'get_user_info')
def test_format_message_with_user_info_name_fallback(self, mock_get_user_info):
mock_get_user_info.return_value = {"name": "testusername_only"} # No real_name
raw_msg = {"type": "message", "user": "U124", "text": "Name fallback", "ts": "1609459200.000300"}
formatted = self.history.format_message(raw_msg, include_user_info=True)
self.assertEqual(formatted['user_name'], "testusername_only")
@patch.object(SlackHistory, 'get_user_info')
def test_format_message_with_user_info_not_found(self, mock_get_user_info):
mock_get_user_info.return_value = None # User info not found
raw_msg = {"type": "message", "user": "U125", "text": "User not found", "ts": "1609459200.000350"}
formatted = self.history.format_message(raw_msg, include_user_info=True)
self.assertEqual(formatted['user_name'], "Unknown User")
def test_format_message_with_files(self):
raw_msg = {
"type": "message", "user": "U123", "text": "Check attachments", "ts": "1609459200.000400",
"files": [
{"name": "file1.txt", "url_private_download": "http://example.com/file1.txt"},
{"name": "image.png", "url_private_download": "http://example.com/image.png", "permalink": "http://slack.com/image.png"}
]
}
formatted = self.history.format_message(raw_msg)
expected_text = (
"Check attachments\n\n"
"Attachments:\n"
"- file1.txt (http://example.com/file1.txt)\n"
"- image.png (http://slack.com/image.png)" # Prefers permalink
)
self.assertEqual(formatted['text'], expected_text)
def test_format_message_with_thread_ts_and_replies(self):
raw_msg = {
"type": "message", "user": "U123", "text": "In a thread", "ts": "1609459200.000500",
"thread_ts": "1609459100.000100",
"reply_count": 2
}
formatted = self.history.format_message(raw_msg)
self.assertEqual(formatted['text'], "In a thread (in reply to thread 1609459100.000100, 2 replies)")
def test_format_message_subtype_handling(self):
raw_msg = {"type": "message", "subtype": "channel_join", "user": "U123", "text": "U123 has joined the channel", "ts": "1609459200.000600"}
formatted = self.history.format_message(raw_msg)
self.assertIn("[channel_join message]", formatted['text']) # Check if subtype is mentioned
if __name__ == '__main__':
unittest.main(argv=['first-arg-is-ignored'], exit=False)

297
surfsense_backend/app/routes/test_search_source_connectors_routes.py Normal file
View file

@ -0,0 +1,297 @@
import pytest
from fastapi.testclient import TestClient
from unittest.mock import patch, MagicMock, AsyncMock
from fastapi import BackgroundTasks, HTTPException
from app.main import app # Assuming your FastAPI app instance is here
from app.db import User, SearchSourceConnector, SearchSourceConnectorType, SearchSpace
from app.schemas.search_source_connector import SearchSourceConnectorRead # For response validation if needed
from app.connectors.slack_history import SlackHistory # To mock its methods
from slack_sdk.errors import SlackApiError
# --- Fixtures ---
@pytest.fixture
def mock_user():
user = User(id="test_user_1", email="test@example.com", hashed_password="hashedpassword", is_active=True, is_superuser=False)
return user
@pytest.fixture
def mock_db_session():
session = AsyncMock()
session.execute = AsyncMock()
session.get = AsyncMock() # For individual record fetching by ID
session.commit = AsyncMock()
session.rollback = AsyncMock()
session.refresh = AsyncMock()
return session
@pytest.fixture
def mock_background_tasks():
tasks = MagicMock(spec=BackgroundTasks)
tasks.add_task = MagicMock()
return tasks
@pytest.fixture
def test_client(mock_user, mock_db_session, mock_background_tasks):
# Override dependencies
app.dependency_overrides[current_active_user_dep] = lambda: mock_user
app.dependency_overrides[get_async_session_dep] = lambda: mock_db_session
# To inject BackgroundTasks, it's usually passed directly to the route function.
# So, we'll mock it where it's used, or ensure it's injectable.
# For testing, the TestClient can handle BackgroundTasks if routes are defined correctly.
# If BackgroundTasks is a dependency, it can be overridden too.
# For this test, we'll often patch where BackgroundTasks.add_task is called.
return TestClient(app)
# --- Helper to get current_active_user dependency ---
# This is needed because current_active_user is likely defined in app.users
# and we need to provide the correct path for overriding.
# Assuming it's from app.users:
from app.users import current_active_user as current_active_user_dep
from app.db import get_async_session as get_async_session_dep
# --- Base Slack Connector Data ---
@pytest.fixture
def base_slack_connector_db():
return SearchSourceConnector(
id=1,
user_id="test_user_1",
name="Test Slack",
connector_type=SearchSourceConnectorType.SLACK_CONNECTOR,
is_indexable=True,
config={"SLACK_BOT_TOKEN": "xoxb-valid-token"},
last_indexed_at=None
)
@pytest.fixture
def mock_search_space_db():
return SearchSpace(id=1, name="Test Space", user_id="test_user_1")
# --- Test Classes ---
class TestDiscoverSlackChannels:
@patch('app.routes.search_source_connectors_routes.SlackHistory')
@patch('app.routes.search_source_connectors_routes.check_ownership', new_callable=AsyncMock)
async def test_discover_channels_success(self, mock_check_ownership, MockSlackHistory, test_client, mock_db_session, base_slack_connector_db):
mock_check_ownership.return_value = base_slack_connector_db
mock_db_session.get.return_value = base_slack_connector_db # For direct session.get calls
mock_slack_instance = MockSlackHistory.return_value
mock_slack_instance.get_all_channels.return_value = [
{"id": "C1", "name": "General", "is_private": False, "is_member": True},
{"id": "C2", "name": "Private", "is_private": True, "is_member": True},
{"id": "C3", "name": "Public-NotMember", "is_private": False, "is_member": False},
]
response = test_client.get("/api/v1/slack/1/discover-channels")
assert response.status_code == 200
data = response.json()
assert len(data["channels"]) == 2 # C3 should be filtered out
assert data["channels"][0]["id"] == "C1"
assert data["channels"][1]["id"] == "C2"
MockSlackHistory.assert_called_once_with(token="xoxb-valid-token")
mock_slack_instance.get_all_channels.assert_called_once_with(include_private=True)
@patch('app.routes.search_source_connectors_routes.check_ownership', new_callable=AsyncMock)
async def test_discover_channels_connector_not_found(self, mock_check_ownership, test_client):
mock_check_ownership.side_effect = HTTPException(status_code=404, detail="Connector not found")
# Or, if session.get is used directly before check_ownership in the route:
# mock_db_session.get.return_value = None
# For this test, assuming check_ownership is the primary guard or session.get is part of it.
response = test_client.get("/api/v1/slack/999/discover-channels")
assert response.status_code == 404
@patch('app.routes.search_source_connectors_routes.check_ownership', new_callable=AsyncMock)
async def test_discover_channels_not_slack_connector(self, mock_check_ownership, test_client, base_slack_connector_db):
base_slack_connector_db.connector_type = SearchSourceConnectorType.NOTION_CONNECTOR
mock_check_ownership.return_value = base_slack_connector_db
# mock_db_session.get.return_value = base_slack_connector_db
response = test_client.get("/api/v1/slack/1/discover-channels")
assert response.status_code == 400
assert "Connector is not a Slack connector" in response.json()["detail"]
@patch('app.routes.search_source_connectors_routes.check_ownership', new_callable=AsyncMock)
async def test_discover_channels_token_not_configured(self, mock_check_ownership, test_client, base_slack_connector_db):
base_slack_connector_db.config = {} # No token
mock_check_ownership.return_value = base_slack_connector_db
# mock_db_session.get.return_value = base_slack_connector_db
response = test_client.get("/api/v1/slack/1/discover-channels")
assert response.status_code == 400
assert "Slack token not configured" in response.json()["detail"]
@patch('app.routes.search_source_connectors_routes.SlackHistory')
@patch('app.routes.search_source_connectors_routes.check_ownership', new_callable=AsyncMock)
async def test_discover_channels_slack_api_error(self, mock_check_ownership, MockSlackHistory, test_client, base_slack_connector_db):
mock_check_ownership.return_value = base_slack_connector_db
# mock_db_session.get.return_value = base_slack_connector_db
mock_slack_instance = MockSlackHistory.return_value
# Simulate SlackApiError with a mock response object that has a 'data' attribute
mock_error_response = MagicMock()
mock_error_response.data = {"error": "test_slack_api_error"}
mock_slack_instance.get_all_channels.side_effect = SlackApiError("Slack API failed", mock_error_response)
response = test_client.get("/api/v1/slack/1/discover-channels")
assert response.status_code == 500 # Based on current route error handling
assert "Slack API error: test_slack_api_error" in response.json()["detail"]
@patch('app.routes.search_source_connectors_routes.SlackHistory')
@patch('app.routes.search_source_connectors_routes.check_ownership', new_callable=AsyncMock)
async def test_discover_channels_value_error(self, mock_check_ownership, MockSlackHistory, test_client, base_slack_connector_db):
mock_check_ownership.return_value = base_slack_connector_db
# mock_db_session.get.return_value = base_slack_connector_db
MockSlackHistory.side_effect = ValueError("Invalid token format") # Error on SlackHistory init
response = test_client.get("/api/v1/slack/1/discover-channels")
assert response.status_code == 400
assert "Invalid token format" in response.json()["detail"]
class TestReindexSlackChannels:
@patch('app.routes.search_source_connectors_routes.run_slack_indexing_with_new_session', new_callable=AsyncMock)
@patch('app.routes.search_source_connectors_routes.check_ownership', new_callable=AsyncMock)
async def test_reindex_channels_success_defaults(self, mock_check_ownership, mock_run_indexing, test_client, mock_db_session, base_slack_connector_db, mock_search_space_db, mock_background_tasks):
mock_check_ownership.return_value = base_slack_connector_db
mock_db_session.get.return_value = base_slack_connector_db
# Mock the search space query within the endpoint
mock_db_session.execute.return_value.scalars.return_value.first.return_value = mock_search_space_db
# Override BackgroundTasks for this specific test or ensure it's injected globally
with patch('fastapi.BackgroundTasks', return_value=mock_background_tasks) as MockedBGTasks:
# Re-apply dependency override if TestClient re-initializes app without it for BG tasks
app.dependency_overrides[BackgroundTasks] = lambda: mock_background_tasks
response = test_client.post("/api/v1/slack/1/reindex-channels", json={"channel_ids": ["C1", "C2"]})
assert response.status_code == 202
assert response.json()["message"] == "Re-indexing task for specific channels has been scheduled."
mock_background_tasks.add_task.assert_called_once_with(
mock_run_indexing, # Direct reference to the mocked task runner
connector_id=1,
search_space_id=mock_search_space_db.id,
target_channel_ids=["C1", "C2"],
force_reindex_all_messages=False, # Default
reindex_start_date_str=None, # Default
reindex_latest_date_str=None # Default
)
# Clean up dependency override if it was local to this test
if BackgroundTasks in app.dependency_overrides:
del app.dependency_overrides[BackgroundTasks]
@patch('app.routes.search_source_connectors_routes.run_slack_indexing_with_new_session', new_callable=AsyncMock)
@patch('app.routes.search_source_connectors_routes.check_ownership', new_callable=AsyncMock)
async def test_reindex_channels_success_with_optional_params(self, mock_check_ownership, mock_run_indexing, test_client, mock_db_session, base_slack_connector_db, mock_search_space_db, mock_background_tasks):
mock_check_ownership.return_value = base_slack_connector_db
mock_db_session.get.return_value = base_slack_connector_db
mock_db_session.execute.return_value.scalars.return_value.first.return_value = mock_search_space_db
with patch('fastapi.BackgroundTasks', return_value=mock_background_tasks):
app.dependency_overrides[BackgroundTasks] = lambda: mock_background_tasks
payload = {
"channel_ids": ["C1"],
"force_reindex_all_messages": True,
"reindex_start_date": "2023-01-01",
"reindex_latest_date": "2023-01-31"
}
response = test_client.post("/api/v1/slack/1/reindex-channels", json=payload)
assert response.status_code == 202
mock_background_tasks.add_task.assert_called_once_with(
mock_run_indexing,
connector_id=1,
search_space_id=mock_search_space_db.id,
target_channel_ids=["C1"],
force_reindex_all_messages=True,
reindex_start_date_str="2023-01-01",
reindex_latest_date_str="2023-01-31"
)
if BackgroundTasks in app.dependency_overrides:
del app.dependency_overrides[BackgroundTasks]
# Error cases for reindex-channels (similar structure to discover-channels errors)
# Connector not found, not Slack, token missing, no channel_ids
@patch('app.routes.search_source_connectors_routes.check_ownership', new_callable=AsyncMock)
async def test_reindex_channels_no_channel_ids(self, mock_check_ownership, test_client, mock_db_session, base_slack_connector_db):
mock_check_ownership.return_value = base_slack_connector_db
mock_db_session.get.return_value = base_slack_connector_db
response = test_client.post("/api/v1/slack/1/reindex-channels", json={"channel_ids": []}) # Empty list
assert response.status_code == 400 # Or 422 depending on Pydantic validation
assert "No channel_ids provided" in response.json()["detail"]
class TestIndexConnectorContentSlack:
@patch('app.routes.search_source_connectors_routes.run_slack_indexing_with_new_session', new_callable=AsyncMock)
@patch('app.routes.search_source_connectors_routes.check_ownership', new_callable=AsyncMock)
async def test_index_slack_connector_default(self, mock_check_ownership, mock_run_indexing, test_client, mock_db_session, base_slack_connector_db, mock_search_space_db, mock_background_tasks):
# Mock check_ownership for connector and search_space
mock_check_ownership.side_effect = [base_slack_connector_db, mock_search_space_db]
with patch('fastapi.BackgroundTasks', return_value=mock_background_tasks):
app.dependency_overrides[BackgroundTasks] = lambda: mock_background_tasks
response = test_client.post("/api/v1/search-source-connectors/1/index?search_space_id=1")
assert response.status_code == 200 # The route itself returns 200, task is background
assert "Slack indexing started" in response.json()["message"]
mock_background_tasks.add_task.assert_called_once_with(
mock_run_indexing,
1, # connector_id
1, # search_space_id
target_channel_ids=None, # From default in run_slack_indexing_with_new_session
force_reindex_all_messages=False, # Default for this endpoint
reindex_start_date_str=None,
reindex_latest_date_str=None
)
if BackgroundTasks in app.dependency_overrides:
del app.dependency_overrides[BackgroundTasks]
@patch('app.routes.search_source_connectors_routes.run_slack_indexing_with_new_session', new_callable=AsyncMock)
@patch('app.routes.search_source_connectors_routes.check_ownership', new_callable=AsyncMock)
async def test_index_slack_connector_force_full_reindex(self, mock_check_ownership, mock_run_indexing, test_client, mock_db_session, base_slack_connector_db, mock_search_space_db, mock_background_tasks):
mock_check_ownership.side_effect = [base_slack_connector_db, mock_search_space_db]
with patch('fastapi.BackgroundTasks', return_value=mock_background_tasks):
app.dependency_overrides[BackgroundTasks] = lambda: mock_background_tasks
response = test_client.post("/api/v1/search-source-connectors/1/index?search_space_id=1&force_full_reindex=true")
assert response.status_code == 200
assert "Full re-index initiated" in response.json()["message"]
mock_background_tasks.add_task.assert_called_once_with(
mock_run_indexing,
1,
1,
target_channel_ids=None,
force_reindex_all_messages=True, # Key check for this test
reindex_start_date_str=None,
reindex_latest_date_str=None
)
if BackgroundTasks in app.dependency_overrides:
del app.dependency_overrides[BackgroundTasks]
@patch('app.routes.search_source_connectors_routes.check_ownership', new_callable=AsyncMock)
async def test_index_connector_not_supported_type(self, mock_check_ownership, test_client, base_slack_connector_db, mock_search_space_db):
base_slack_connector_db.connector_type = "UNSUPPORTED_TYPE" # Simulate an unsupported type
mock_check_ownership.side_effect = [base_slack_connector_db, mock_search_space_db]
response = test_client.post("/api/v1/search-source-connectors/1/index?search_space_id=1")
assert response.status_code == 400
assert "Indexing not supported for connector type: UNSUPPORTED_TYPE" in response.json()["detail"]
# It's important to clear dependency_overrides after tests if they are not scoped to TestClient instance
# However, pytest fixtures usually handle setup/teardown per test.
# If overrides are global to 'app', they might need explicit cleanup in a teardown fixture if not using TestClient's context management fully.
# For TestClient(app), it usually handles this.

427
surfsense_backend/app/tasks/test_connectors_indexing_tasks.py Normal file
View file

@ -0,0 +1,427 @@
import pytest
from unittest.mock import AsyncMock, MagicMock, patch, call
from datetime import datetime, timedelta, timezone
from sqlalchemy.exc import SQLAlchemyError
from app.tasks.connectors_indexing_tasks import index_slack_messages
from app.db import SearchSourceConnector, SearchSourceConnectorType, Document, DocumentType, Chunk
from app.schemas.search_source_connector import SearchSourceConnectorCreate # For creating test connector data
from app.connectors.slack_history import SlackHistory # To mock its methods
# Mock global config object if it's accessed directly
# If it's passed around, then mock where it's used/passed.
# For this example, assuming app.config.config is used.
@pytest.fixture(autouse=True)
def mock_app_config():
mock_llm = MagicMock()
mock_llm.ainvoke = AsyncMock(return_value=MagicMock(content="Test Summary"))
mock_embedding_model = MagicMock()
mock_embedding_model.embed = MagicMock(return_value=[0.1, 0.2, 0.3])
mock_chunker = MagicMock()
# Simulate chunker returning list of objects with a 'text' attribute
mock_chunker.chunk = MagicMock(return_value=[MagicMock(text="chunk1"), MagicMock(text="chunk2")])
with patch('app.config.config.long_context_llm_instance', mock_llm), \
patch('app.config.config.embedding_model_instance', mock_embedding_model), \
patch('app.config.config.chunker_instance', mock_chunker), \
patch('app.config.config.SUMMARY_PROMPT_TEMPLATE', MagicMock()): # Assuming this is a simple object
yield
@pytest.fixture
def mock_session():
session = AsyncMock()
session.execute = AsyncMock()
session.scalars = MagicMock(return_value=MagicMock(first=MagicMock(return_value=None), all=MagicMock(return_value=[])))
session.commit = AsyncMock()
session.rollback = AsyncMock()
session.add = MagicMock()
session.refresh = AsyncMock()
return session
@pytest.fixture
def base_slack_connector():
return SearchSourceConnector(
id=1,
user_id="test_user_1",
search_space_id=1,
name="Test Slack Connector",
connector_type=SearchSourceConnectorType.SLACK_CONNECTOR,
is_indexable=True,
config={
"SLACK_BOT_TOKEN": "xoxb-dummy-token",
"slack_initial_indexing_days": 30,
"slack_initial_max_messages_per_channel": 100,
"slack_max_messages_per_channel_periodic": 50,
"slack_membership_filter_type": "all_member_channels",
"slack_selected_channel_ids": [],
"slack_periodic_indexing_enabled": True, # New field from previous task
"slack_periodic_indexing_frequency": "daily" # New field
},
last_indexed_at=None,
created_at=datetime.now(timezone.utc),
updated_at=datetime.now(timezone.utc)
)
@pytest.fixture
def mock_slack_history_instance():
instance = AsyncMock(spec=SlackHistory)
instance.get_all_channels = AsyncMock(return_value=[])
instance.get_conversation_history = AsyncMock(return_value=[])
# format_message should return a dict, not an AsyncMock
instance.format_message = MagicMock(return_value={
"user_id": "U123", "text": "Formatted message", "datetime": "2023-01-01 10:00:00 UTC"
})
return instance
@pytest.mark.asyncio
async def test_connector_not_found(mock_session):
mock_session.execute.return_value.scalars.return_value.first.return_value = None
count, error = await index_slack_messages(mock_session, 999, 1)
assert count == 0
assert "Connector with ID 999 not found" in error
@pytest.mark.asyncio
async def test_connector_not_slack_type(mock_session, base_slack_connector):
base_slack_connector.connector_type = SearchSourceConnectorType.NOTION_CONNECTOR
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
count, error = await index_slack_messages(mock_session, 1, 1)
assert count == 0
assert "is not a Slack connector" in error
@pytest.mark.asyncio
async def test_slack_token_missing(mock_session, base_slack_connector):
base_slack_connector.config = {} # No token
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
count, error = await index_slack_messages(mock_session, 1, 1)
assert count == 0
assert "Slack token not found" in error
@pytest.mark.asyncio
@patch('app.tasks.connectors_indexing_tasks.SlackHistory')
async def test_no_channels_returned_from_api(MockSlackHistory, mock_session, base_slack_connector, mock_slack_history_instance):
mock_slack_history_instance.get_all_channels = AsyncMock(return_value=[])
MockSlackHistory.return_value = mock_slack_history_instance
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
count, error = await index_slack_messages(mock_session, 1, 1)
assert count == 0
assert error == "No channels to index after filtering." # This is the specific message when no channels are left
# Verify last_indexed_at is updated because the task "ran" successfully even if no channels
assert base_slack_connector.last_indexed_at is not None
@pytest.mark.asyncio
@patch('app.tasks.connectors_indexing_tasks.SlackHistory')
async def test_all_member_channels_filter(MockSlackHistory, mock_session, base_slack_connector, mock_slack_history_instance):
mock_slack_history_instance.get_all_channels.return_value = [
{"id": "C1", "name": "Channel1", "is_member": True, "is_private": False},
{"id": "C2", "name": "Channel2", "is_member": True, "is_private": False}, # Bot is member
]
mock_slack_history_instance.get_conversation_history.return_value = [
{"ts": "1", "user": "U1", "text": "msg1"},
]
MockSlackHistory.return_value = mock_slack_history_instance
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
base_slack_connector.config["slack_membership_filter_type"] = "all_member_channels"
count, _ = await index_slack_messages(mock_session, 1, 1)
assert count == 2 # Two documents created, one for each channel
assert mock_slack_history_instance.get_conversation_history.call_count == 2
mock_slack_history_instance.get_conversation_history.assert_any_call(channel_id="C1", limit=100, oldest=mock_slack_history_instance.convert_date_to_timestamp.return_value, latest=mock_slack_history_instance.convert_date_to_timestamp.return_value)
mock_slack_history_instance.get_conversation_history.assert_any_call(channel_id="C2", limit=100, oldest=mock_slack_history_instance.convert_date_to_timestamp.return_value, latest=mock_slack_history_instance.convert_date_to_timestamp.return_value)
@pytest.mark.asyncio
@patch('app.tasks.connectors_indexing_tasks.SlackHistory')
async def test_selected_member_channels_filter(MockSlackHistory, mock_session, base_slack_connector, mock_slack_history_instance):
mock_slack_history_instance.get_all_channels.return_value = [
{"id": "C1", "name": "Channel1", "is_member": True, "is_private": False},
{"id": "C2", "name": "Channel2", "is_member": True, "is_private": False},
{"id": "C3", "name": "Channel3", "is_member": True, "is_private": False},
]
mock_slack_history_instance.get_conversation_history.return_value = [{"ts": "1", "user": "U1", "text": "msg1"}]
MockSlackHistory.return_value = mock_slack_history_instance
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
base_slack_connector.config["slack_membership_filter_type"] = "selected_member_channels"
base_slack_connector.config["slack_selected_channel_ids"] = ["C1", "C3"]
count, _ = await index_slack_messages(mock_session, 1, 1)
assert count == 2
assert mock_slack_history_instance.get_conversation_history.call_count == 2
mock_slack_history_instance.get_conversation_history.assert_any_call(channel_id="C1", limit=100, oldest=mock_slack_history_instance.convert_date_to_timestamp.return_value, latest=mock_slack_history_instance.convert_date_to_timestamp.return_value)
mock_slack_history_instance.get_conversation_history.assert_any_call(channel_id="C3", limit=100, oldest=mock_slack_history_instance.convert_date_to_timestamp.return_value, latest=mock_slack_history_instance.convert_date_to_timestamp.return_value)
# C2 should not have been called
@pytest.mark.asyncio
@patch('app.tasks.connectors_indexing_tasks.SlackHistory')
async def test_target_channel_ids_interaction(MockSlackHistory, mock_session, base_slack_connector, mock_slack_history_instance):
mock_slack_history_instance.get_all_channels.return_value = [
{"id": "C1", "name": "Channel1", "is_member": True, "is_private": False},
{"id": "C2", "name": "Channel2", "is_member": True, "is_private": False}, # Configured but not targeted
{"id": "C3", "name": "Channel3", "is_member": True, "is_private": False}, # Targeted
]
mock_slack_history_instance.get_conversation_history.return_value = [{"ts": "1", "user": "U1", "text": "msg1"}]
MockSlackHistory.return_value = mock_slack_history_instance
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
base_slack_connector.config["slack_membership_filter_type"] = "all_member_channels" # All are initially valid
count, _ = await index_slack_messages(mock_session, 1, 1, target_channel_ids=["C1", "C3"])
assert count == 2 # Only C1 and C3
mock_slack_history_instance.get_conversation_history.assert_any_call(channel_id="C1", limit=base_slack_connector.config["slack_max_messages_per_channel_periodic"], oldest=mock_slack_history_instance.convert_date_to_timestamp.return_value, latest=mock_slack_history_instance.convert_date_to_timestamp.return_value)
mock_slack_history_instance.get_conversation_history.assert_any_call(channel_id="C3", limit=base_slack_connector.config["slack_max_messages_per_channel_periodic"], oldest=mock_slack_history_instance.convert_date_to_timestamp.return_value, latest=mock_slack_history_instance.convert_date_to_timestamp.return_value)
# Ensure C2 was not called for history
calls = mock_slack_history_instance.get_conversation_history.call_args_list
for c in calls:
assert c.kwargs['channel_id'] != "C2"
@pytest.mark.asyncio
@patch('app.tasks.connectors_indexing_tasks.SlackHistory')
async def test_initial_indexing_days_logic(MockSlackHistory, mock_session, base_slack_connector, mock_slack_history_instance):
base_slack_connector.last_indexed_at = None # Ensure initial run
base_slack_connector.config["slack_initial_indexing_days"] = 7
# Mock convert_date_to_timestamp to check its input
mock_converter = MagicMock()
# Make it return a fixed value so we can check the call to get_conversation_history
fixed_ts = str(int((datetime.now(timezone.utc) - timedelta(days=7)).timestamp()))
mock_converter.return_value = fixed_ts
# Patch the static method on the class, not the instance
with patch.object(SlackHistory, 'convert_date_to_timestamp', mock_converter):
mock_slack_history_instance.get_all_channels.return_value = [{"id": "C1", "name": "Chan1", "is_member": True}]
mock_slack_history_instance.get_conversation_history.return_value = [{"ts": "1"}]
MockSlackHistory.return_value = mock_slack_history_instance
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
await index_slack_messages(mock_session, 1, 1)
# Check if convert_date_to_timestamp was called correctly for the start date
# It's called internally for both oldest and latest. We are interested in the 'oldest' logic.
# The actual call to SlackHistory.convert_date_to_timestamp for 'oldest' happens inside index_slack_messages.
# We can assert that get_conversation_history was called with the 'fixed_ts'
latest_ts_for_call = str(int((datetime.now(timezone.utc) + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0).timestamp()))
mock_slack_history_instance.get_conversation_history.assert_called_once_with(
channel_id="C1",
limit=base_slack_connector.config["slack_initial_max_messages_per_channel"],
oldest=fixed_ts, # This is the key check
latest=latest_ts_for_call
)
@pytest.mark.asyncio
@patch('app.tasks.connectors_indexing_tasks.SlackHistory')
async def test_initial_indexing_all_time(MockSlackHistory, mock_session, base_slack_connector, mock_slack_history_instance):
base_slack_connector.last_indexed_at = None
base_slack_connector.config["slack_initial_indexing_days"] = -1
mock_slack_history_instance.get_all_channels.return_value = [{"id": "C1", "name": "Chan1", "is_member": True}]
mock_slack_history_instance.get_conversation_history.return_value = [{"ts": "1"}]
MockSlackHistory.return_value = mock_slack_history_instance
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
await index_slack_messages(mock_session, 1, 1)
latest_ts_for_call = str(int((datetime.now(timezone.utc) + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0).timestamp()))
mock_slack_history_instance.get_conversation_history.assert_called_once_with(
channel_id="C1",
limit=base_slack_connector.config["slack_initial_max_messages_per_channel"],
oldest="0", # All time
latest=latest_ts_for_call
)
@pytest.mark.asyncio
@patch('app.tasks.connectors_indexing_tasks.SlackHistory')
async def test_periodic_indexing_logic(MockSlackHistory, mock_session, base_slack_connector, mock_slack_history_instance):
last_indexed_time = datetime.now(timezone.utc) - timedelta(days=5)
base_slack_connector.last_indexed_at = last_indexed_time
mock_slack_history_instance.get_all_channels.return_value = [{"id": "C1", "name": "Chan1", "is_member": True}]
mock_slack_history_instance.get_conversation_history.return_value = [{"ts": "1"}]
MockSlackHistory.return_value = mock_slack_history_instance
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
await index_slack_messages(mock_session, 1, 1)
expected_oldest_ts = str(int(last_indexed_time.timestamp()))
latest_ts_for_call = str(int((datetime.now(timezone.utc) + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0).timestamp()))
mock_slack_history_instance.get_conversation_history.assert_called_once_with(
channel_id="C1",
limit=base_slack_connector.config["slack_max_messages_per_channel_periodic"],
oldest=expected_oldest_ts,
latest=latest_ts_for_call
)
@pytest.mark.asyncio
@patch('app.tasks.connectors_indexing_tasks.SlackHistory')
async def test_reindex_force_all_with_dates(MockSlackHistory, mock_session, base_slack_connector, mock_slack_history_instance):
mock_slack_history_instance.get_all_channels.return_value = [{"id": "C1", "name": "Chan1", "is_member": True}]
mock_slack_history_instance.get_conversation_history.return_value = [{"ts": "1"}]
MockSlackHistory.return_value = mock_slack_history_instance
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
start_date_str = "2023-03-01"
latest_date_str = "2023-03-10"
expected_oldest = SlackHistory.convert_date_to_timestamp(start_date_str)
expected_latest = SlackHistory.convert_date_to_timestamp(latest_date_str, is_latest=True)
await index_slack_messages(mock_session, 1, 1,
target_channel_ids=["C1"],
force_reindex_all_messages=True,
reindex_start_date_str=start_date_str,
reindex_latest_date_str=latest_date_str)
mock_slack_history_instance.get_conversation_history.assert_called_once_with(
channel_id="C1",
limit=base_slack_connector.config["slack_initial_max_messages_per_channel"], # Uses initial limit
oldest=expected_oldest,
latest=expected_latest
)
@pytest.mark.asyncio
@patch('app.tasks.connectors_indexing_tasks.SlackHistory')
async def test_document_creation(MockSlackHistory, mock_session, base_slack_connector, mock_slack_history_instance):
mock_slack_history_instance.get_all_channels.return_value = [{"id": "C1", "name": "TestChannel", "is_member": True}]
mock_slack_history_instance.get_conversation_history.return_value = [
{"ts": "1", "user": "U1", "text": "Hello"},
{"ts": "2", "user": "U2", "text": "World", "subtype": "bot_message"}, # Should be skipped
]
# Configure format_message to return what's needed by the task
mock_slack_history_instance.format_message.side_effect = lambda msg, include_user_info=False: {
"user_id": msg["user"], "text": msg["text"], "datetime": "time", "subtype": msg.get("subtype")
}
MockSlackHistory.return_value = mock_slack_history_instance
# Simulate no existing document for this channel
mock_session.execute.return_value.scalars.return_value.all.return_value = []
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector # For connector fetch
count, _ = await index_slack_messages(mock_session, 1, 1)
assert count == 1
# Check Document creation (session.add was called with a Document instance)
added_object = mock_session.add.call_args[0][0]
assert isinstance(added_object, Document)
assert added_object.title == "Slack - TestChannel"
assert added_object.document_type == DocumentType.SLACK_CONNECTOR
assert added_object.document_metadata["channel_id"] == "C1"
assert added_object.document_metadata["message_count"] == 1 # Only one valid message
assert added_object.content == "Test Summary" # From mock_app_config
assert len(added_object.chunks) == 2 # From mock_app_config chunker
# Check that bot message was skipped via format_message calls
format_calls = mock_slack_history_instance.format_message.call_args_list
assert len(format_calls) == 1 # Only called for the non-bot message
assert format_calls[0].args[0]['text'] == "Hello"
assert mock_session.commit.call_count == 1
@pytest.mark.asyncio
@patch('app.tasks.connectors_indexing_tasks.SlackHistory')
async def test_document_update(MockSlackHistory, mock_session, base_slack_connector, mock_slack_history_instance):
mock_slack_history_instance.get_all_channels.return_value = [{"id": "C1", "name": "TestChannel", "is_member": True}]
mock_slack_history_instance.get_conversation_history.return_value = [{"ts": "1", "user": "U1", "text": "Updated Content"}]
mock_slack_history_instance.format_message.return_value = {"user_id": "U1", "text": "Updated Content", "datetime":"time"}
MockSlackHistory.return_value = mock_slack_history_instance
existing_doc = Document(id=101, search_space_id=1, title="Old Title", document_type=DocumentType.SLACK_CONNECTOR, document_metadata={"channel_id": "C1"})
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
# Simulate existing document for this channel_id
mock_session.execute.return_value.scalars.return_value.all.return_value = [existing_doc]
count, _ = await index_slack_messages(mock_session, 1, 1)
assert count == 1 # 0 new, 1 updated
assert existing_doc.content == "Test Summary" # Updated content from summary
assert existing_doc.document_metadata["message_count"] == 1
# Check that session.delete was called for old chunks (implicitly via relationship or explicit call)
# This depends on ORM setup or if task explicitly deletes.
# For this test, we assume the task handles deleting old chunks if needed.
# The provided task code does execute a delete statement for chunks.
delete_chunk_call = None
for call_item in mock_session.execute.call_args_list:
if "DELETE" in str(call_item.args[0]): # Check if a DELETE statement was executed
delete_chunk_call = call_item
break
assert delete_chunk_call is not None
# Verify new chunks are added
num_new_chunks_added = 0
for call_item in mock_session.add.call_args_list:
if isinstance(call_item.args[0], Chunk):
num_new_chunks_added +=1
assert call_item.args[0].document_id == existing_doc.id # Associated with existing doc
assert num_new_chunks_added == 2 # From mock_app_config chunker
assert mock_session.commit.call_count == 1
@pytest.mark.asyncio
async def test_last_indexed_at_update_logic(mock_session, base_slack_connector):
# Scenario 1: update_last_indexed = True, documents processed
mock_slack_history_instance_s1 = MagicMock(spec=SlackHistory)
mock_slack_history_instance_s1.get_all_channels = AsyncMock(return_value=[{"id": "C1", "name": "Chan1", "is_member": True}])
mock_slack_history_instance_s1.get_conversation_history = AsyncMock(return_value=[{"ts": "1", "user": "U1", "text": "msg"}])
mock_slack_history_instance_s1.format_message = MagicMock(return_value={"user_id":"U1", "text":"msg", "datetime":"t"})
with patch('app.tasks.connectors_indexing_tasks.SlackHistory', return_value=mock_slack_history_instance_s1):
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
base_slack_connector.last_indexed_at = None # Reset for this test part
await index_slack_messages(mock_session, 1, 1, update_last_indexed=True)
assert base_slack_connector.last_indexed_at is not None
original_last_indexed_at = base_slack_connector.last_indexed_at
# Scenario 2: update_last_indexed = False
mock_slack_history_instance_s2 = MagicMock(spec=SlackHistory)
mock_slack_history_instance_s2.get_all_channels = AsyncMock(return_value=[{"id": "C1", "name": "Chan1", "is_member": True}])
mock_slack_history_instance_s2.get_conversation_history = AsyncMock(return_value=[{"ts": "1", "user": "U1", "text": "msg"}])
mock_slack_history_instance_s2.format_message = MagicMock(return_value={"user_id":"U1", "text":"msg", "datetime":"t"})
with patch('app.tasks.connectors_indexing_tasks.SlackHistory', return_value=mock_slack_history_instance_s2):
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
# last_indexed_at should remain as it was from scenario 1
await index_slack_messages(mock_session, 1, 1, update_last_indexed=False)
assert base_slack_connector.last_indexed_at == original_last_indexed_at
@pytest.mark.asyncio
@patch('app.tasks.connectors_indexing_tasks.SlackHistory')
async def test_error_get_all_channels_exception(MockSlackHistory, mock_session, base_slack_connector, mock_slack_history_instance):
mock_slack_history_instance.get_all_channels.side_effect = Exception("Failed to get channels")
MockSlackHistory.return_value = mock_slack_history_instance
mock_session.execute.return_value.scalars.return_value.first.return_value = base_slack_connector
count, error = await index_slack_messages(mock_session, 1, 1)
assert count == 0
assert "Failed to get Slack channels: Failed to get channels" in error
assert mock_session.rollback.call_count == 1 # Should rollback on general exception
@pytest.mark.asyncio
async def test_db_error_on_commit(mock_session, base_slack_connector):
# This test needs to allow the initial connector fetch to succeed, then fail on commit
mock_connector_fetch_result = MagicMock()
mock_connector_fetch_result.scalars.return_value.first.return_value = base_slack_connector
mock_docs_fetch_result = MagicMock()
mock_docs_fetch_result.scalars.return_value.all.return_value = [] # No existing docs
# Configure side_effect for session.execute
mock_session.execute.side_effect = [
mock_connector_fetch_result, # First call for getting connector
mock_docs_fetch_result # Second call for getting existing_docs
]
mock_session.commit.side_effect = SQLAlchemyError("DB Commit Failed")
mock_slack_history_instance_db_error = MagicMock(spec=SlackHistory)
mock_slack_history_instance_db_error.get_all_channels = AsyncMock(return_value=[{"id": "C1", "name": "Chan1", "is_member": True}])
mock_slack_history_instance_db_error.get_conversation_history = AsyncMock(return_value=[{"ts": "1", "user":"U1", "text":"msg"}])
mock_slack_history_instance_db_error.format_message = MagicMock(return_value={"user_id":"U1", "text":"msg", "datetime":"t"})
with patch('app.tasks.connectors_indexing_tasks.SlackHistory', return_value=mock_slack_history_instance_db_error):
count, error = await index_slack_messages(mock_session, 1, 1)
assert count == 0
assert "Database error: DB Commit Failed" in error
assert mock_session.rollback.call_count == 1 # Ensure rollback on SQLAlchemyError