diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 57dbdc7b5..fe9a1bf71 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -67,6 +67,10 @@ dependencies = [ [dependency-groups] dev = [ "ruff>=0.12.5", + "pytest>=8.0.0", + "pytest-asyncio>=0.24.0", + "pytest-mock>=3.14.0", + "httpx>=0.28.0", ] [tool.ruff] @@ -158,6 +162,14 @@ known-first-party = ["app"] force-single-line = false combine-as-imports = true +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" + [tool.setuptools.packages.find] where = ["."] include = ["app*", "alembic*"] diff --git a/surfsense_backend/tests/__init__.py b/surfsense_backend/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/conftest.py b/surfsense_backend/tests/conftest.py new file mode 100644 index 000000000..8a3b2e36b --- /dev/null +++ b/surfsense_backend/tests/conftest.py @@ -0,0 +1,104 @@ +"""Shared pytest fixtures for SurfSense backend tests.""" + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.orm import sessionmaker + +from app.db import Base + + +@pytest.fixture +async def async_session(): + """Create an async database session for testing.""" + from sqlalchemy import JSON, ARRAY, event + from sqlalchemy.dialects.postgresql import JSONB + + # Use in-memory SQLite for tests + engine = create_async_engine( + "sqlite+aiosqlite:///:memory:", + echo=False, + ) + + # Replace JSONB and ARRAY with JSON for SQLite compatibility + @event.listens_for(Base.metadata, "before_create") + def _set_json_type(target, connection, **kw): + for table in Base.metadata.tables.values(): + for column in table.columns: + # Convert JSONB to JSON + if isinstance(column.type, type(JSONB())): + column.type = JSON() + # Convert ARRAY to JSON (SQLite doesn't support ARRAY) + elif isinstance(column.type, ARRAY): + column.type = JSON() + + # Create tables + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + # Create session + async_session_maker = sessionmaker( + engine, class_=AsyncSession, expire_on_commit=False + ) + + async with async_session_maker() as session: + yield session + + # Cleanup + await engine.dispose() + + +@pytest.fixture +def mock_connector_config(): + """Mock connector configuration.""" + return { + "tokens": [ + { + "chain": "ethereum", + "address": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + "name": "WETH", + }, + { + "chain": "solana", + "address": "So11111111111111111111111111111111111111112", + "name": "SOL", + }, + ] + } + + +@pytest.fixture +def mock_pair_data(): + """Mock DexScreener API response data.""" + return { + "pairs": [ + { + "chainId": "ethereum", + "dexId": "uniswap", + "url": "https://dexscreener.com/ethereum/0x123", + "pairAddress": "0x123", + "baseToken": { + "address": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + "name": "Wrapped Ether", + "symbol": "WETH", + }, + "quoteToken": { + "address": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + "name": "USD Coin", + "symbol": "USDC", + }, + "priceNative": "1.0", + "priceUsd": "2500.00", + "txns": { + "m5": {"buys": 10, "sells": 5}, + "h1": {"buys": 100, "sells": 50}, + "h6": {"buys": 500, "sells": 250}, + "h24": {"buys": 2000, "sells": 1000}, + }, + "volume": {"h24": 1000000.0, "h6": 250000.0, "h1": 50000.0, "m5": 5000.0}, + "priceChange": {"m5": 0.5, "h1": 1.2, "h6": 2.5, "h24": 5.0}, + "liquidity": {"usd": 5000000.0, "base": 2000.0, "quote": 5000000.0}, + "fdv": 10000000.0, + "pairCreatedAt": 1609459200000, + } + ] + } diff --git a/surfsense_backend/tests/unit/__init__.py b/surfsense_backend/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/connectors/__init__.py b/surfsense_backend/tests/unit/connectors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/connectors/test_dexscreener_connector.py b/surfsense_backend/tests/unit/connectors/test_dexscreener_connector.py new file mode 100644 index 000000000..322b659e1 --- /dev/null +++ b/surfsense_backend/tests/unit/connectors/test_dexscreener_connector.py @@ -0,0 +1,160 @@ +"""Unit tests for DexScreener connector.""" + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch +import httpx + +from app.connectors.dexscreener_connector import DexScreenerConnector + + +class TestDexScreenerConnector: + """Test cases for DexScreenerConnector class.""" + + def test_init_creates_connector(self): + """Test that connector initializes correctly.""" + connector = DexScreenerConnector() + assert connector.base_url == "https://api.dexscreener.com/latest/dex" + assert connector.rate_limit_delay == 0.2 + + @pytest.mark.asyncio + async def test_make_request_success(self, mock_pair_data): + """Test successful API request.""" + connector = DexScreenerConnector() + + with patch("httpx.AsyncClient.get") as mock_get: + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = mock_pair_data + mock_get.return_value = mock_response + + result = await connector.make_request("/dex/tokens/ethereum/0x123") + + assert result == mock_pair_data + mock_get.assert_called_once() + + @pytest.mark.asyncio + async def test_make_request_with_retry_on_429(self, mock_pair_data): + """Test that connector retries on rate limit (429).""" + connector = DexScreenerConnector() + + with patch("httpx.AsyncClient.get") as mock_get: + # First call returns 429, second call succeeds + mock_response_429 = MagicMock() + mock_response_429.status_code = 429 + + mock_response_200 = MagicMock() + mock_response_200.status_code = 200 + mock_response_200.json.return_value = mock_pair_data + + # Mock the responses + mock_get.side_effect = [mock_response_429, mock_response_200] + + with patch("asyncio.sleep", return_value=None): # Skip actual sleep + result = await connector.make_request("/dex/tokens/ethereum/0x123") + + assert result == mock_pair_data + assert mock_get.call_count == 2 + + @pytest.mark.asyncio + async def test_make_request_timeout(self): + """Test that connector handles timeout errors.""" + connector = DexScreenerConnector() + + with patch("httpx.AsyncClient.get") as mock_get: + mock_get.side_effect = httpx.TimeoutException("Request timeout") + + with pytest.raises(Exception, match="Request timeout"): + await connector.make_request("/dex/tokens/ethereum/0x123") + + @pytest.mark.asyncio + async def test_make_request_network_error(self): + """Test that connector handles network errors.""" + connector = DexScreenerConnector() + + with patch("httpx.AsyncClient.get") as mock_get: + mock_get.side_effect = httpx.NetworkError("Network error") + + with pytest.raises(Exception, match="Network error"): + await connector.make_request("/dex/tokens/ethereum/0x123") + + @pytest.mark.asyncio + async def test_get_token_pairs_success(self, mock_pair_data): + """Test successful token pairs retrieval.""" + connector = DexScreenerConnector() + + # get_token_pairs returns a tuple (pairs, error) + with patch.object( + connector, "make_request", return_value=mock_pair_data + ) as mock_request: + pairs, error = await connector.get_token_pairs("ethereum", "0x123") + + assert pairs == mock_pair_data["pairs"] + assert error is None + mock_request.assert_called_once_with("tokens/ethereum/0x123") + + @pytest.mark.asyncio + async def test_get_token_pairs_no_data(self): + """Test handling of empty response.""" + connector = DexScreenerConnector() + + with patch.object(connector, "make_request", return_value={"pairs": None}): + pairs, error = await connector.get_token_pairs("ethereum", "0x123") + + assert pairs == [] + assert error is not None + + def test_format_pair_to_markdown(self, mock_pair_data): + """Test markdown formatting of pair data.""" + connector = DexScreenerConnector() + pair = mock_pair_data["pairs"][0] + + markdown = connector.format_pair_to_markdown(pair, "WETH") + + # Verify key sections are present (actual format is "# WETH/USDC Trading Pair") + assert "# WETH/USDC Trading Pair" in markdown + assert "## Price Information" in markdown + assert "## Trading Volume" in markdown + assert "## Market Metrics" in markdown # New section + assert "## Liquidity" in markdown + assert "## Transactions (24h)" in markdown + assert "WETH" in markdown + assert "USD Coin" in markdown or "USDC" in markdown + assert "$2500.00" in markdown + # Verify new metrics are present + assert "6h Volume" in markdown + assert "1h Volume" in markdown + assert "Market Cap" in markdown + assert "FDV (Fully Diluted Valuation)" in markdown + + def test_format_pair_to_markdown_missing_fields(self): + """Test markdown formatting with missing optional fields.""" + connector = DexScreenerConnector() + minimal_pair = { + "chainId": "ethereum", + "dexId": "uniswap", + "pairAddress": "0x123", + "baseToken": {"symbol": "WETH", "name": "Wrapped Ether"}, + "quoteToken": {"symbol": "USDC", "name": "USD Coin"}, + } + + markdown = connector.format_pair_to_markdown(minimal_pair, "WETH") + + # Should handle missing fields gracefully (actual format is "# WETH/USDC Trading Pair") + assert "# WETH/USDC Trading Pair" in markdown + assert "WETH" in markdown + assert "N/A" in markdown + + @pytest.mark.asyncio + async def test_rate_limit_delay(self): + """Test rate limiting delay calculation.""" + connector = DexScreenerConnector() + + # Simulate rapid requests + import time + + connector.last_request_time = time.time() + + with patch("asyncio.sleep") as mock_sleep: + await connector._rate_limit_delay() + # Should call sleep since last request was recent + assert mock_sleep.called diff --git a/surfsense_backend/tests/unit/routes/__init__.py b/surfsense_backend/tests/unit/routes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/routes/test_dexscreener_routes.py b/surfsense_backend/tests/unit/routes/test_dexscreener_routes.py new file mode 100644 index 000000000..acfd58e40 --- /dev/null +++ b/surfsense_backend/tests/unit/routes/test_dexscreener_routes.py @@ -0,0 +1,298 @@ +"""Unit tests for DexScreener API routes.""" + +import pytest +from fastapi.testclient import TestClient +from unittest.mock import AsyncMock, MagicMock, patch +from sqlalchemy.ext.asyncio import AsyncSession + +from app.app import app +from app.db import SearchSourceConnector, SearchSourceConnectorType, User, get_async_session +from app.users import current_active_user + + +@pytest.fixture +def mock_user(): + """Create a mock user for testing.""" + user = MagicMock(spec=User) + user.id = "test-user-id" + user.email = "test@example.com" + return user + + +@pytest.fixture +def mock_session(): + """Create a mock async database session.""" + session = AsyncMock(spec=AsyncSession) + return session + + +@pytest.fixture +def mock_connector(): + """Create a mock connector for testing.""" + connector = MagicMock(spec=SearchSourceConnector) + connector.id = 1 + connector.name = "DexScreener Connector" + connector.connector_type = SearchSourceConnectorType.DEXSCREENER_CONNECTOR + connector.config = { + "tokens": [ + { + "chain": "ethereum", + "address": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + "name": "WETH", + } + ] + } + connector.search_space_id = 1 + connector.user_id = "test-user-id" + connector.is_indexable = True + return connector + + +@pytest.fixture +def client_with_overrides(mock_user, mock_session): + """Create a test client with dependency overrides.""" + # Override dependencies + app.dependency_overrides[current_active_user] = lambda: mock_user + app.dependency_overrides[get_async_session] = lambda: mock_session + + client = TestClient(app) + yield client + + # Clean up overrides after test + app.dependency_overrides.clear() + + +class TestDexScreenerRoutes: + """Test cases for DexScreener API routes.""" + + @pytest.mark.asyncio + async def test_add_connector_success_new(self, client_with_overrides, mock_session): + """Test successful creation of a new connector.""" + request_data = { + "tokens": [ + { + "chain": "ethereum", + "address": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + "name": "WETH", + } + ], + "space_id": 1, + } + + # Mock the database query to return no existing connector + mock_result = MagicMock() + mock_result.scalars.return_value.first.return_value = None + mock_session.execute.return_value = mock_result + + response = client_with_overrides.post( + "/api/v1/connectors/dexscreener/add", json=request_data + ) + + assert response.status_code == 200 + data = response.json() + assert data["message"] == "DexScreener connector added successfully" + assert data["connector_type"] == "DEXSCREENER_CONNECTOR" + assert data["tokens_count"] == 1 + assert "connector_id" in data + + @pytest.mark.asyncio + async def test_add_connector_success_update_existing( + self, client_with_overrides, mock_session, mock_connector + ): + """Test successful update of an existing connector.""" + request_data = { + "tokens": [ + { + "chain": "ethereum", + "address": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + "name": "WETH", + }, + { + "chain": "bsc", + "address": "0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c", + "name": "WBNB", + }, + ], + "space_id": 1, + } + + # Mock the database query to return an existing connector + mock_result = MagicMock() + mock_result.scalars.return_value.first.return_value = mock_connector + mock_session.execute.return_value = mock_result + + response = client_with_overrides.post( + "/api/v1/connectors/dexscreener/add", json=request_data + ) + + assert response.status_code == 200 + data = response.json() + assert data["message"] == "DexScreener connector updated successfully" + assert data["connector_type"] == "DEXSCREENER_CONNECTOR" + assert data["tokens_count"] == 2 + assert data["connector_id"] == 1 + + def test_add_connector_invalid_tokens_missing_address(self, client_with_overrides): + """Test connector addition with missing address field.""" + request_data = { + "tokens": [{"chain": "ethereum"}], # Missing address + "space_id": 1, + } + + response = client_with_overrides.post( + "/api/v1/connectors/dexscreener/add", json=request_data + ) + + assert response.status_code == 422 # Validation error + + def test_add_connector_invalid_tokens_missing_chain(self, client_with_overrides): + """Test connector addition with missing chain field.""" + request_data = { + "tokens": [ + {"address": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"} + ], # Missing chain + "space_id": 1, + } + + response = client_with_overrides.post( + "/api/v1/connectors/dexscreener/add", json=request_data + ) + + assert response.status_code == 422 # Validation error + + def test_add_connector_empty_tokens_list(self, client_with_overrides): + """Test connector addition with empty tokens list.""" + request_data = { + "tokens": [], # Empty list + "space_id": 1, + } + + response = client_with_overrides.post( + "/api/v1/connectors/dexscreener/add", json=request_data + ) + + assert response.status_code == 422 # Validation error + + @pytest.mark.asyncio + async def test_delete_connector_success( + self, client_with_overrides, mock_session, mock_connector + ): + """Test successful connector deletion.""" + # Mock the database query to return an existing connector + mock_result = MagicMock() + mock_result.scalars.return_value.first.return_value = mock_connector + mock_session.execute.return_value = mock_result + + response = client_with_overrides.delete( + "/api/v1/connectors/dexscreener", + params={"space_id": 1}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["message"] == "DexScreener connector deleted successfully" + mock_session.delete.assert_called_once_with(mock_connector) + + @pytest.mark.asyncio + async def test_delete_connector_not_found(self, client_with_overrides, mock_session): + """Test deletion of non-existent connector.""" + # Mock the database query to return no connector + mock_result = MagicMock() + mock_result.scalars.return_value.first.return_value = None + mock_session.execute.return_value = mock_result + + response = client_with_overrides.delete( + "/api/v1/connectors/dexscreener", + params={"space_id": 999}, + ) + + assert response.status_code == 404 + data = response.json() + assert "not found" in data["detail"].lower() + + @pytest.mark.asyncio + async def test_test_connector_success( + self, client_with_overrides, mock_session, mock_connector, mock_pair_data + ): + """Test successful connector test.""" + # Mock the database query to return a connector + mock_result = MagicMock() + mock_result.scalars.return_value.first.return_value = mock_connector + mock_session.execute.return_value = mock_result + + # Mock the DexScreenerConnector.get_token_pairs method + with patch( + "app.connectors.dexscreener_connector.DexScreenerConnector.get_token_pairs" + ) as mock_get_pairs: + # Return tuple (pairs, None) for success + mock_get_pairs.return_value = (mock_pair_data["pairs"], None) + + response = client_with_overrides.get( + "/api/v1/connectors/dexscreener/test", params={"space_id": 1} + ) + + assert response.status_code == 200 + data = response.json() + assert data["message"] == "DexScreener connector is working correctly" + assert data["tokens_configured"] == 1 + assert data["pairs_found"] == len(mock_pair_data["pairs"]) + assert "sample_pair" in data + + @pytest.mark.asyncio + async def test_test_connector_not_found(self, client_with_overrides, mock_session): + """Test connector test when connector doesn't exist.""" + # Mock the database query to return no connector + mock_result = MagicMock() + mock_result.scalars.return_value.first.return_value = None + mock_session.execute.return_value = mock_result + + response = client_with_overrides.get( + "/api/v1/connectors/dexscreener/test", params={"space_id": 999} + ) + + assert response.status_code == 404 + data = response.json() + assert "not found" in data["detail"].lower() + + @pytest.mark.asyncio + async def test_test_connector_no_tokens(self, client_with_overrides, mock_session): + """Test connector test with no tokens configured.""" + # Create a connector with empty tokens + empty_connector = MagicMock(spec=SearchSourceConnector) + empty_connector.config = {"tokens": []} + + mock_result = MagicMock() + mock_result.scalars.return_value.first.return_value = empty_connector + mock_session.execute.return_value = mock_result + + response = client_with_overrides.get( + "/api/v1/connectors/dexscreener/test", params={"space_id": 1} + ) + + assert response.status_code == 400 + data = response.json() + assert "no tokens" in data["detail"].lower() + + @pytest.mark.asyncio + async def test_test_connector_api_error( + self, client_with_overrides, mock_session, mock_connector + ): + """Test connector test with API error.""" + # Mock the database query to return a connector + mock_result = MagicMock() + mock_result.scalars.return_value.first.return_value = mock_connector + mock_session.execute.return_value = mock_result + + with patch( + "app.connectors.dexscreener_connector.DexScreenerConnector.get_token_pairs" + ) as mock_get_pairs: + # Return tuple ([], error_message) for error + mock_get_pairs.return_value = ([], "API Error: Connection failed") + + response = client_with_overrides.get( + "/api/v1/connectors/dexscreener/test", params={"space_id": 1} + ) + + assert response.status_code == 400 + data = response.json() + assert "failed to connect" in data["detail"].lower() diff --git a/surfsense_backend/tests/unit/tasks/test_dexscreener_indexer.py b/surfsense_backend/tests/unit/tasks/test_dexscreener_indexer.py new file mode 100644 index 000000000..6083c9a11 --- /dev/null +++ b/surfsense_backend/tests/unit/tasks/test_dexscreener_indexer.py @@ -0,0 +1,504 @@ +"""Unit tests for DexScreener indexer.""" + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch +from datetime import datetime + +from app.tasks.connector_indexers.dexscreener_indexer import index_dexscreener_pairs +from app.db import SearchSourceConnectorType, DocumentType + + +class TestDexScreenerIndexer: + """Test cases for DexScreener indexer function.""" + + @pytest.mark.asyncio + async def test_index_pairs_success(self, async_session, mock_connector_config, mock_pair_data): + """Test successful indexing of DexScreener pairs.""" + # Mock connector + mock_connector = MagicMock() + mock_connector.id = 1 + mock_connector.connector_type = SearchSourceConnectorType.DEXSCREENER_CONNECTOR + mock_connector.config = mock_connector_config + mock_connector.last_indexed_at = None + + # Mock dependencies + with patch("app.tasks.connector_indexers.dexscreener_indexer.get_connector_by_id") as mock_get_connector, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.DexScreenerConnector") as mock_dex_client, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.get_user_long_context_llm") as mock_get_llm, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.generate_document_summary") as mock_gen_summary, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.create_document_chunks") as mock_create_chunks, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.update_connector_last_indexed", new_callable=AsyncMock) as mock_update_indexed, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.TaskLoggingService") as mock_task_logger: + + # Setup mocks + mock_get_connector.return_value = mock_connector + + # Mock DexScreener client - use side_effect to return unique markdown for each token + mock_client_instance = MagicMock() + mock_client_instance.get_token_pairs = AsyncMock(return_value=(mock_pair_data["pairs"], None)) + mock_client_instance.format_pair_to_markdown.side_effect = [ + "# Mock Markdown Content 1", + "# Mock Markdown Content 2", + ] + mock_dex_client.return_value = mock_client_instance + + # Mock LLM service + mock_llm = MagicMock() + mock_get_llm.return_value = mock_llm + + # Mock summary generation - use side_effect to return unique summaries for each token + mock_gen_summary.side_effect = [ + (f"Mock summary 1", [0.1] * 384), + (f"Mock summary 2", [0.2] * 384), + ] + + # Mock chunk creation + mock_create_chunks.return_value = [] + + # Mock task logger + mock_logger_instance = MagicMock() + mock_logger_instance.log_task_start = AsyncMock(return_value=MagicMock(id=1)) + mock_logger_instance.log_task_progress = AsyncMock() + mock_logger_instance.log_task_success = AsyncMock() + mock_task_logger.return_value = mock_logger_instance + + # Execute indexer + documents_indexed, error = await index_dexscreener_pairs( + session=async_session, + connector_id=1, + search_space_id=1, + user_id="test-user-id", + ) + + # Assertions + assert error is None + assert documents_indexed == 2 # 2 tokens in mock config + mock_get_connector.assert_called_once() + assert mock_client_instance.get_token_pairs.call_count == 2 # Called for each token + mock_update_indexed.assert_called_once() + + @pytest.mark.asyncio + async def test_index_pairs_connector_not_found(self, async_session): + """Test indexer when connector is not found.""" + with patch("app.tasks.connector_indexers.dexscreener_indexer.get_connector_by_id") as mock_get_connector, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.TaskLoggingService") as mock_task_logger: + + # Setup mocks + mock_get_connector.return_value = None + + # Mock task logger + mock_logger_instance = MagicMock() + mock_logger_instance.log_task_start = AsyncMock(return_value=MagicMock(id=1)) + mock_logger_instance.log_task_progress = AsyncMock() + mock_logger_instance.log_task_failure = AsyncMock() + mock_task_logger.return_value = mock_logger_instance + + # Execute indexer + documents_indexed, error = await index_dexscreener_pairs( + session=async_session, + connector_id=999, + search_space_id=1, + user_id="test-user-id", + ) + + # Assertions + assert documents_indexed == 0 + assert error is not None + assert "not found" in error.lower() + mock_logger_instance.log_task_failure.assert_called_once() + + @pytest.mark.asyncio + async def test_index_pairs_no_tokens_configured(self, async_session): + """Test indexer when no tokens are configured.""" + # Mock connector with empty tokens + mock_connector = MagicMock() + mock_connector.id = 1 + mock_connector.connector_type = SearchSourceConnectorType.DEXSCREENER_CONNECTOR + mock_connector.config = {"tokens": []} + + with patch("app.tasks.connector_indexers.dexscreener_indexer.get_connector_by_id") as mock_get_connector, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.TaskLoggingService") as mock_task_logger: + + # Setup mocks + mock_get_connector.return_value = mock_connector + + # Mock task logger + mock_logger_instance = MagicMock() + mock_logger_instance.log_task_start = AsyncMock(return_value=MagicMock(id=1)) + mock_logger_instance.log_task_progress = AsyncMock() + mock_logger_instance.log_task_failure = AsyncMock() + mock_task_logger.return_value = mock_logger_instance + + # Execute indexer + documents_indexed, error = await index_dexscreener_pairs( + session=async_session, + connector_id=1, + search_space_id=1, + user_id="test-user-id", + ) + + # Assertions + assert documents_indexed == 0 + assert error == "No tokens configured for connector" + mock_logger_instance.log_task_failure.assert_called_once() + + @pytest.mark.asyncio + async def test_index_pairs_api_error(self, async_session, mock_connector_config): + """Test indexer when API returns an error.""" + # Mock connector + mock_connector = MagicMock() + mock_connector.id = 1 + mock_connector.connector_type = SearchSourceConnectorType.DEXSCREENER_CONNECTOR + mock_connector.config = mock_connector_config + mock_connector.last_indexed_at = None + + with patch("app.tasks.connector_indexers.dexscreener_indexer.get_connector_by_id") as mock_get_connector, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.DexScreenerConnector") as mock_dex_client, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.update_connector_last_indexed") as mock_update_indexed, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.TaskLoggingService") as mock_task_logger: + + # Setup mocks + mock_get_connector.return_value = mock_connector + + # Mock DexScreener client with API error + mock_client_instance = MagicMock() + mock_client_instance.get_token_pairs = AsyncMock(return_value=(None, "API Error: Rate limit exceeded")) + mock_dex_client.return_value = mock_client_instance + + # Mock task logger + mock_logger_instance = MagicMock() + mock_logger_instance.log_task_start = AsyncMock(return_value=MagicMock(id=1)) + mock_logger_instance.log_task_progress = AsyncMock() + mock_logger_instance.log_task_success = AsyncMock() + mock_task_logger.return_value = mock_logger_instance + + # Execute indexer + documents_indexed, error = await index_dexscreener_pairs( + session=async_session, + connector_id=1, + search_space_id=1, + user_id="test-user-id", + ) + + # Assertions - should complete successfully but with 0 documents + assert error is None + assert documents_indexed == 0 + mock_update_indexed.assert_called_once() + + @pytest.mark.asyncio + async def test_index_pairs_no_pairs_found(self, async_session, mock_connector_config): + """Test indexer when API returns no pairs.""" + # Mock connector + mock_connector = MagicMock() + mock_connector.id = 1 + mock_connector.connector_type = SearchSourceConnectorType.DEXSCREENER_CONNECTOR + mock_connector.config = mock_connector_config + mock_connector.last_indexed_at = None + + with patch("app.tasks.connector_indexers.dexscreener_indexer.get_connector_by_id") as mock_get_connector, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.DexScreenerConnector") as mock_dex_client, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.update_connector_last_indexed") as mock_update_indexed, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.TaskLoggingService") as mock_task_logger: + + # Setup mocks + mock_get_connector.return_value = mock_connector + + # Mock DexScreener client with empty pairs + mock_client_instance = MagicMock() + mock_client_instance.get_token_pairs = AsyncMock(return_value=([], None)) + mock_dex_client.return_value = mock_client_instance + + # Mock task logger + mock_logger_instance = MagicMock() + mock_logger_instance.log_task_start = AsyncMock(return_value=MagicMock(id=1)) + mock_logger_instance.log_task_progress = AsyncMock() + mock_logger_instance.log_task_success = AsyncMock() + mock_task_logger.return_value = mock_logger_instance + + # Execute indexer + documents_indexed, error = await index_dexscreener_pairs( + session=async_session, + connector_id=1, + search_space_id=1, + user_id="test-user-id", + ) + + # Assertions + assert error is None + assert documents_indexed == 0 + mock_update_indexed.assert_called_once() + + @pytest.mark.asyncio + async def test_index_pairs_skips_invalid_tokens(self, async_session): + """Test indexer skips tokens with missing chain or address.""" + # Mock connector with invalid tokens + mock_connector = MagicMock() + mock_connector.id = 1 + mock_connector.connector_type = SearchSourceConnectorType.DEXSCREENER_CONNECTOR + mock_connector.config = { + "tokens": [ + {"chain": "ethereum"}, # Missing address + {"address": "0x123"}, # Missing chain + {"chain": "solana", "address": "So11111111111111111111111111111111111111112", "name": "SOL"}, + ] + } + mock_connector.last_indexed_at = None + + with patch("app.tasks.connector_indexers.dexscreener_indexer.get_connector_by_id") as mock_get_connector, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.DexScreenerConnector") as mock_dex_client, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.get_user_long_context_llm") as mock_get_llm, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.generate_document_summary") as mock_gen_summary, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.create_document_chunks") as mock_create_chunks, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.update_connector_last_indexed") as mock_update_indexed, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.TaskLoggingService") as mock_task_logger: + + # Setup mocks + mock_get_connector.return_value = mock_connector + + # Mock DexScreener client + mock_client_instance = MagicMock() + mock_pair = { + "pairAddress": "0xabc", + "baseToken": {"symbol": "SOL"}, + "quoteToken": {"symbol": "USDC"}, + "dexId": "raydium", + "priceUsd": "100.0", + "liquidity": {"usd": 1000000}, + "volume": {"h24": 500000}, + "priceChange": {"h24": 2.5}, + } + mock_client_instance.get_token_pairs = AsyncMock(return_value=([mock_pair], None)) + mock_client_instance.format_pair_to_markdown.return_value = "# Mock Markdown" + mock_dex_client.return_value = mock_client_instance + + # Mock LLM and summary + mock_get_llm.return_value = MagicMock() + mock_gen_summary.return_value = ("Mock summary", [0.1] * 384) + mock_create_chunks.return_value = [] + + # Mock task logger + mock_logger_instance = MagicMock() + mock_logger_instance.log_task_start = AsyncMock(return_value=MagicMock(id=1)) + mock_logger_instance.log_task_progress = AsyncMock() + mock_logger_instance.log_task_success = AsyncMock() + mock_task_logger.return_value = mock_logger_instance + + # Execute indexer + documents_indexed, error = await index_dexscreener_pairs( + session=async_session, + connector_id=1, + search_space_id=1, + user_id="test-user-id", + ) + + # Assertions - should only process the valid token + assert error is None + assert documents_indexed == 1 + assert mock_client_instance.get_token_pairs.call_count == 1 # Only called for valid token + + @pytest.mark.asyncio + async def test_index_pairs_skips_pairs_without_address(self, async_session, mock_connector_config, mock_pair_data): + """Test indexer skips pairs without pairAddress.""" + # Mock connector + mock_connector = MagicMock() + mock_connector.id = 1 + mock_connector.connector_type = SearchSourceConnectorType.DEXSCREENER_CONNECTOR + mock_connector.config = mock_connector_config + mock_connector.last_indexed_at = None + + # Create pair without pairAddress + invalid_pair = mock_pair_data["pairs"][0].copy() + del invalid_pair["pairAddress"] + + with patch("app.tasks.connector_indexers.dexscreener_indexer.get_connector_by_id") as mock_get_connector, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.DexScreenerConnector") as mock_dex_client, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.update_connector_last_indexed") as mock_update_indexed, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.TaskLoggingService") as mock_task_logger: + + # Setup mocks + mock_get_connector.return_value = mock_connector + + # Mock DexScreener client with invalid pair + mock_client_instance = MagicMock() + mock_client_instance.get_token_pairs = AsyncMock(return_value=([invalid_pair], None)) + mock_dex_client.return_value = mock_client_instance + + # Mock task logger + mock_logger_instance = MagicMock() + mock_logger_instance.log_task_start = AsyncMock(return_value=MagicMock(id=1)) + mock_logger_instance.log_task_progress = AsyncMock() + mock_logger_instance.log_task_success = AsyncMock() + mock_task_logger.return_value = mock_logger_instance + + # Execute indexer + documents_indexed, error = await index_dexscreener_pairs( + session=async_session, + connector_id=1, + search_space_id=1, + user_id="test-user-id", + ) + + # Assertions - should skip invalid pairs + assert error is None + assert documents_indexed == 0 + + @pytest.mark.asyncio + async def test_index_pairs_without_llm(self, async_session, mock_connector_config, mock_pair_data): + """Test indexer works without LLM service (fallback to basic summary).""" + # Mock connector + mock_connector = MagicMock() + mock_connector.id = 1 + mock_connector.connector_type = SearchSourceConnectorType.DEXSCREENER_CONNECTOR + mock_connector.config = mock_connector_config + mock_connector.last_indexed_at = None + + with patch("app.tasks.connector_indexers.dexscreener_indexer.get_connector_by_id") as mock_get_connector, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.DexScreenerConnector") as mock_dex_client, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.get_user_long_context_llm") as mock_get_llm, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.create_document_chunks") as mock_create_chunks, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.update_connector_last_indexed", new_callable=AsyncMock) as mock_update_indexed, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.config") as mock_config, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.TaskLoggingService") as mock_task_logger: + + # Setup mocks + mock_get_connector.return_value = mock_connector + + # Mock DexScreener client - use side_effect to return unique markdown for each token + mock_client_instance = MagicMock() + mock_client_instance.get_token_pairs = AsyncMock(return_value=(mock_pair_data["pairs"], None)) + mock_client_instance.format_pair_to_markdown.side_effect = [ + "# Mock Markdown 1", + "# Mock Markdown 2", + ] + mock_dex_client.return_value = mock_client_instance + + # Mock LLM service returns None (fallback mode) + mock_get_llm.return_value = None + + # Mock embedding model - use side_effect to return unique embeddings + mock_embedding_instance = MagicMock() + mock_embedding_instance.embed.side_effect = [ + [0.1] * 384, + [0.2] * 384, + ] + mock_config.embedding_model_instance = mock_embedding_instance + + # Mock chunk creation + mock_create_chunks.return_value = [] + + # Mock task logger + mock_logger_instance = MagicMock() + mock_logger_instance.log_task_start = AsyncMock(return_value=MagicMock(id=1)) + mock_logger_instance.log_task_progress = AsyncMock() + mock_logger_instance.log_task_success = AsyncMock() + mock_task_logger.return_value = mock_logger_instance + + # Execute indexer + documents_indexed, error = await index_dexscreener_pairs( + session=async_session, + connector_id=1, + search_space_id=1, + user_id="test-user-id", + ) + + # Assertions - should use fallback summary + assert error is None + assert documents_indexed == 2 + mock_embedding_instance.embed.assert_called() + + @pytest.mark.asyncio + async def test_index_pairs_update_last_indexed_false(self, async_session, mock_connector_config, mock_pair_data): + """Test indexer respects update_last_indexed=False parameter.""" + # Mock connector + mock_connector = MagicMock() + mock_connector.id = 1 + mock_connector.connector_type = SearchSourceConnectorType.DEXSCREENER_CONNECTOR + mock_connector.config = mock_connector_config + mock_connector.last_indexed_at = None + + with patch("app.tasks.connector_indexers.dexscreener_indexer.get_connector_by_id") as mock_get_connector, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.DexScreenerConnector") as mock_dex_client, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.get_user_long_context_llm") as mock_get_llm, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.generate_document_summary") as mock_gen_summary, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.create_document_chunks") as mock_create_chunks, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.update_connector_last_indexed", new_callable=AsyncMock) as mock_update_indexed, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.TaskLoggingService") as mock_task_logger: + + # Setup mocks + mock_get_connector.return_value = mock_connector + + # Mock DexScreener client - use side_effect to return unique markdown for each token + mock_client_instance = MagicMock() + mock_client_instance.get_token_pairs = AsyncMock(return_value=(mock_pair_data["pairs"], None)) + mock_client_instance.format_pair_to_markdown.side_effect = [ + "# Mock Markdown 1", + "# Mock Markdown 2", + ] + mock_dex_client.return_value = mock_client_instance + + # Mock LLM and summary - use side_effect to return unique summaries + mock_get_llm.return_value = MagicMock() + mock_gen_summary.side_effect = [ + (f"Mock summary 1", [0.1] * 384), + (f"Mock summary 2", [0.2] * 384), + ] + mock_create_chunks.return_value = [] + + # Mock task logger + mock_logger_instance = MagicMock() + mock_logger_instance.log_task_start = AsyncMock(return_value=MagicMock(id=1)) + mock_logger_instance.log_task_progress = AsyncMock() + mock_logger_instance.log_task_success = AsyncMock() + mock_task_logger.return_value = mock_logger_instance + + # Execute indexer with update_last_indexed=False + documents_indexed, error = await index_dexscreener_pairs( + session=async_session, + connector_id=1, + search_space_id=1, + user_id="test-user-id", + update_last_indexed=False, + ) + + # Assertions - should NOT update last_indexed_at + assert error is None + assert documents_indexed == 2 + mock_update_indexed.assert_not_called() + + @pytest.mark.asyncio + async def test_index_pairs_database_error(self, async_session, mock_connector_config): + """Test indexer handles database errors gracefully.""" + from sqlalchemy.exc import SQLAlchemyError + + # Mock connector + mock_connector = MagicMock() + mock_connector.id = 1 + mock_connector.connector_type = SearchSourceConnectorType.DEXSCREENER_CONNECTOR + mock_connector.config = mock_connector_config + + with patch("app.tasks.connector_indexers.dexscreener_indexer.get_connector_by_id") as mock_get_connector, \ + patch("app.tasks.connector_indexers.dexscreener_indexer.TaskLoggingService") as mock_task_logger: + + # Setup mocks + mock_get_connector.side_effect = SQLAlchemyError("Database connection failed") + + # Mock task logger + mock_logger_instance = MagicMock() + mock_logger_instance.log_task_start = AsyncMock(return_value=MagicMock(id=1)) + mock_logger_instance.log_task_progress = AsyncMock() + mock_logger_instance.log_task_failure = AsyncMock() + mock_task_logger.return_value = mock_logger_instance + + # Execute indexer + documents_indexed, error = await index_dexscreener_pairs( + session=async_session, + connector_id=1, + search_space_id=1, + user_id="test-user-id", + ) + + # Assertions + assert documents_indexed == 0 + assert error is not None + assert "Database error" in error + mock_logger_instance.log_task_failure.assert_called_once() diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index 16b77a7b2..2bf15b650 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -2578,6 +2578,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/ed/1f1afb2e9e7f38a545d628f864d562a5ae64fe6f7a10e28ffb9b185b4e89/importlib_resources-6.5.2-py3-none-any.whl", hash = "sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec", size = 37461 }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484 }, +] + [[package]] name = "isodate" version = "0.7.2" @@ -5549,6 +5558,47 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/12/a0/d0638470df605ce266991fb04f74c69ab1bed3b90ac3838e9c3c8b69b66a/Pysher-1.0.8.tar.gz", hash = "sha256:7849c56032b208e49df67d7bd8d49029a69042ab0bb45b2ed59fa08f11ac5988", size = 9071 } +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801 }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075 }, +] + +[[package]] +name = "pytest-mock" +version = "3.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095 }, +] + [[package]] name = "python-bidi" version = "0.6.6" @@ -6610,6 +6660,10 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "httpx" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-mock" }, { name = "ruff" }, ] @@ -6676,7 +6730,13 @@ requires-dist = [ ] [package.metadata.requires-dev] -dev = [{ name = "ruff", specifier = ">=0.12.5" }] +dev = [ + { name = "httpx", specifier = ">=0.28.0" }, + { name = "pytest", specifier = ">=8.0.0" }, + { name = "pytest-asyncio", specifier = ">=0.24.0" }, + { name = "pytest-mock", specifier = ">=3.14.0" }, + { name = "ruff", specifier = ">=0.12.5" }, +] [[package]] name = "sympy"