mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-22 03:31:02 +02:00
Updated tests
This commit is contained in:
parent
4ff85cd6bd
commit
d25c2f4d10
40 changed files with 301 additions and 485 deletions
40
TESTS.md
40
TESTS.md
|
|
@ -218,8 +218,19 @@ pytest tests/unit/test_text_completion/test_vertexai_processor.py::TestVertexAIM
|
||||||
|
|
||||||
### Running All VertexAI Tests
|
### Running All VertexAI Tests
|
||||||
|
|
||||||
|
#### Option 1: Simple Tests (Recommended for getting started)
|
||||||
```bash
|
```bash
|
||||||
# Run all VertexAI tests
|
# Run simple tests that don't require full TrustGraph infrastructure
|
||||||
|
./run_simple_tests.sh
|
||||||
|
|
||||||
|
# Or run manually:
|
||||||
|
pytest tests/unit/test_text_completion/test_vertexai_simple.py -v
|
||||||
|
pytest tests/unit/test_text_completion/test_vertexai_core.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Option 2: Full Infrastructure Tests
|
||||||
|
```bash
|
||||||
|
# Run all VertexAI tests (requires full TrustGraph setup)
|
||||||
pytest tests/unit/test_text_completion/test_vertexai_processor.py -v
|
pytest tests/unit/test_text_completion/test_vertexai_processor.py -v
|
||||||
|
|
||||||
# Run with coverage
|
# Run with coverage
|
||||||
|
|
@ -229,6 +240,12 @@ pytest tests/unit/test_text_completion/test_vertexai_processor.py --cov=trustgra
|
||||||
pytest tests/unit/test_text_completion/test_vertexai_processor.py -v -s
|
pytest tests/unit/test_text_completion/test_vertexai_processor.py -v -s
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### Option 3: All VertexAI Tests
|
||||||
|
```bash
|
||||||
|
# Run all VertexAI-related tests
|
||||||
|
pytest tests/unit/test_text_completion/ -k "vertexai" -v
|
||||||
|
```
|
||||||
|
|
||||||
## Test Configuration
|
## Test Configuration
|
||||||
|
|
||||||
### Pytest Configuration
|
### Pytest Configuration
|
||||||
|
|
@ -367,7 +384,26 @@ pytest tests/unit/test_text_completion/test_vertexai_processor.py -v
|
||||||
- Python path not set correctly
|
- Python path not set correctly
|
||||||
- Missing dependencies (install with `pip install -r tests/requirements.txt`)
|
- Missing dependencies (install with `pip install -r tests/requirements.txt`)
|
||||||
|
|
||||||
#### 2. Async Test Issues
|
#### 2. TaskGroup/Infrastructure Errors
|
||||||
|
|
||||||
|
**Symptom**: `RuntimeError: Essential taskgroup missing` or similar infrastructure errors
|
||||||
|
|
||||||
|
**Solution**:
|
||||||
|
```bash
|
||||||
|
# Try the simple tests first - they don't require full TrustGraph infrastructure
|
||||||
|
./run_simple_tests.sh
|
||||||
|
|
||||||
|
# Or run specific simple test files
|
||||||
|
pytest tests/unit/test_text_completion/test_vertexai_simple.py -v
|
||||||
|
pytest tests/unit/test_text_completion/test_vertexai_core.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
**Why this happens**:
|
||||||
|
- The full TrustGraph processors require async task groups and Pulsar infrastructure
|
||||||
|
- The simple tests focus on testing the core logic without infrastructure dependencies
|
||||||
|
- Use simple tests to verify the VertexAI logic works correctly
|
||||||
|
|
||||||
|
#### 3. Async Test Issues
|
||||||
```python
|
```python
|
||||||
# Use IsolatedAsyncioTestCase for async tests
|
# Use IsolatedAsyncioTestCase for async tests
|
||||||
class TestAsyncService(IsolatedAsyncioTestCase):
|
class TestAsyncService(IsolatedAsyncioTestCase):
|
||||||
|
|
|
||||||
|
|
@ -4,8 +4,8 @@ Pytest configuration and fixtures for text completion tests
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from unittest.mock import MagicMock, AsyncMock
|
from unittest.mock import MagicMock, AsyncMock
|
||||||
from trustgraph.base.types import TextCompletionRequest, TextCompletionResponse, LlmResult
|
from trustgraph.schema import TextCompletionRequest, TextCompletionResponse
|
||||||
|
from trustgraph.base import LlmResult
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def mock_vertexai_credentials():
|
def mock_vertexai_credentials():
|
||||||
|
|
@ -156,4 +156,4 @@ def mock_async_context_manager():
|
||||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
return MockAsyncContextManager
|
return MockAsyncContextManager
|
||||||
|
|
|
||||||
|
|
@ -1,327 +1,155 @@
|
||||||
"""
|
"""
|
||||||
Unit tests for trustgraph.model.text_completion.vertexai
|
Unit tests for trustgraph.model.text_completion.vertexai
|
||||||
Following TEST_STRATEGY.md patterns for mocking external dependencies
|
Starting simple with one test to get the basics working
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch, call
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
from unittest import IsolatedAsyncioTestCase
|
from unittest import IsolatedAsyncioTestCase
|
||||||
import asyncio
|
|
||||||
import os
|
|
||||||
from google.api_core.exceptions import ResourceExhausted
|
|
||||||
from google.oauth2 import service_account
|
|
||||||
|
|
||||||
# Import the service under test
|
# Import the service under test
|
||||||
from trustgraph.model.text_completion.vertexai.llm import Processor
|
from trustgraph.model.text_completion.vertexai.llm import Processor
|
||||||
from trustgraph.base.types import TextCompletionRequest, TextCompletionResponse, LlmResult, Error
|
from trustgraph.base import LlmResult
|
||||||
|
|
||||||
|
|
||||||
class TestVertexAIProcessorInitialization(IsolatedAsyncioTestCase):
|
class TestVertexAIProcessorSimple(IsolatedAsyncioTestCase):
|
||||||
"""Test processor initialization with various configurations"""
|
"""Simple test for processor initialization"""
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
"""Set up test fixtures"""
|
|
||||||
self.default_config = {
|
|
||||||
'region': 'us-central1',
|
|
||||||
'model': 'gemini-2.0-flash-001',
|
|
||||||
'temperature': 0.0,
|
|
||||||
'max_output': 8192,
|
|
||||||
'private_key': 'private.json',
|
|
||||||
'concurrency': 1
|
|
||||||
}
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
||||||
async def test_processor_initialization_with_valid_credentials(self, mock_generative_model, mock_vertexai, mock_service_account):
|
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__')
|
||||||
"""Test processor initialization with valid service account credentials"""
|
@patch('trustgraph.base.llm_service.LlmService.__init__')
|
||||||
|
async def test_processor_initialization_basic(self, mock_llm_init, mock_async_init, mock_generative_model, mock_vertexai, mock_service_account):
|
||||||
|
"""Test basic processor initialization with mocked dependencies"""
|
||||||
# Arrange
|
# Arrange
|
||||||
mock_credentials = MagicMock()
|
mock_credentials = MagicMock()
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
||||||
mock_model = MagicMock()
|
mock_model = MagicMock()
|
||||||
mock_generative_model.return_value = mock_model
|
mock_generative_model.return_value = mock_model
|
||||||
|
|
||||||
|
# Mock the parent class initialization to avoid taskgroup requirement
|
||||||
|
mock_async_init.return_value = None
|
||||||
|
mock_llm_init.return_value = None
|
||||||
|
|
||||||
# Act
|
config = {
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
mock_service_account.Credentials.from_service_account_file.assert_called_once_with('private.json')
|
|
||||||
mock_vertexai.init.assert_called_once()
|
|
||||||
mock_generative_model.assert_called_once_with(
|
|
||||||
'gemini-2.0-flash-001',
|
|
||||||
generation_config=processor.generation_config,
|
|
||||||
safety_settings=processor.safety_settings
|
|
||||||
)
|
|
||||||
assert processor.model_name == 'gemini-2.0-flash-001'
|
|
||||||
assert processor.region == 'us-central1'
|
|
||||||
assert processor.temperature == 0.0
|
|
||||||
assert processor.max_output == 8192
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_processor_initialization_with_custom_model(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test processor initialization with custom model selection"""
|
|
||||||
# Arrange
|
|
||||||
config = self.default_config.copy()
|
|
||||||
config['model'] = 'gemini-1.5-pro'
|
|
||||||
mock_credentials = MagicMock()
|
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
|
||||||
|
|
||||||
# Act
|
|
||||||
processor = Processor(**config)
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
mock_generative_model.assert_called_once_with(
|
|
||||||
'gemini-1.5-pro',
|
|
||||||
generation_config=processor.generation_config,
|
|
||||||
safety_settings=processor.safety_settings
|
|
||||||
)
|
|
||||||
assert processor.model_name == 'gemini-1.5-pro'
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_processor_initialization_with_custom_parameters(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test processor initialization with custom generation parameters"""
|
|
||||||
# Arrange
|
|
||||||
config = self.default_config.copy()
|
|
||||||
config.update({
|
|
||||||
'temperature': 0.7,
|
|
||||||
'max_output': 4096,
|
|
||||||
'region': 'us-east1'
|
|
||||||
})
|
|
||||||
mock_credentials = MagicMock()
|
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
|
||||||
|
|
||||||
# Act
|
|
||||||
processor = Processor(**config)
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert processor.temperature == 0.7
|
|
||||||
assert processor.max_output == 4096
|
|
||||||
assert processor.region == 'us-east1'
|
|
||||||
assert processor.generation_config.temperature == 0.7
|
|
||||||
assert processor.generation_config.max_output_tokens == 4096
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_processor_initialization_without_private_key(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test processor initialization without private key (default credentials)"""
|
|
||||||
# Arrange
|
|
||||||
config = self.default_config.copy()
|
|
||||||
config['private_key'] = None
|
|
||||||
|
|
||||||
# Act
|
|
||||||
processor = Processor(**config)
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
mock_service_account.Credentials.from_service_account_file.assert_not_called()
|
|
||||||
mock_vertexai.init.assert_called_once()
|
|
||||||
assert processor.model_name == 'gemini-2.0-flash-001'
|
|
||||||
|
|
||||||
async def test_processor_initialization_generation_config(self):
|
|
||||||
"""Test that generation config is properly set"""
|
|
||||||
# Arrange & Act
|
|
||||||
with patch('trustgraph.model.text_completion.vertexai.llm.service_account'), \
|
|
||||||
patch('trustgraph.model.text_completion.vertexai.llm.vertexai'), \
|
|
||||||
patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel'):
|
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert processor.generation_config.temperature == 0.0
|
|
||||||
assert processor.generation_config.max_output_tokens == 8192
|
|
||||||
assert processor.generation_config.top_p == 1.0
|
|
||||||
assert processor.generation_config.top_k == 10
|
|
||||||
assert processor.generation_config.candidate_count == 1
|
|
||||||
|
|
||||||
async def test_processor_initialization_safety_settings(self):
|
|
||||||
"""Test that safety settings are properly configured"""
|
|
||||||
# Arrange & Act
|
|
||||||
with patch('trustgraph.model.text_completion.vertexai.llm.service_account'), \
|
|
||||||
patch('trustgraph.model.text_completion.vertexai.llm.vertexai'), \
|
|
||||||
patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel'):
|
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert len(processor.safety_settings) == 4
|
|
||||||
# Check that all harm categories are configured
|
|
||||||
harm_categories = [setting.category for setting in processor.safety_settings]
|
|
||||||
assert 'HARM_CATEGORY_HARASSMENT' in str(harm_categories)
|
|
||||||
assert 'HARM_CATEGORY_HATE_SPEECH' in str(harm_categories)
|
|
||||||
assert 'HARM_CATEGORY_SEXUALLY_EXPLICIT' in str(harm_categories)
|
|
||||||
assert 'HARM_CATEGORY_DANGEROUS_CONTENT' in str(harm_categories)
|
|
||||||
|
|
||||||
|
|
||||||
class TestVertexAIMessageProcessing(IsolatedAsyncioTestCase):
|
|
||||||
"""Test message processing functionality"""
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
"""Set up test fixtures"""
|
|
||||||
self.default_config = {
|
|
||||||
'region': 'us-central1',
|
'region': 'us-central1',
|
||||||
'model': 'gemini-2.0-flash-001',
|
'model': 'gemini-2.0-flash-001',
|
||||||
'temperature': 0.0,
|
'temperature': 0.0,
|
||||||
'max_output': 8192,
|
'max_output': 8192,
|
||||||
'private_key': 'private.json',
|
'private_key': 'private.json',
|
||||||
'concurrency': 1
|
'concurrency': 1,
|
||||||
|
'taskgroup': AsyncMock(), # Required by AsyncProcessor
|
||||||
|
'id': 'test-processor' # Required by AsyncProcessor
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Act
|
||||||
|
processor = Processor(**config)
|
||||||
|
|
||||||
|
# Assert
|
||||||
|
assert processor.model == 'gemini-2.0-flash-001' # It's stored as 'model', not 'model_name'
|
||||||
|
assert hasattr(processor, 'generation_config')
|
||||||
|
assert hasattr(processor, 'safety_settings')
|
||||||
|
assert hasattr(processor, 'llm')
|
||||||
|
mock_service_account.Credentials.from_service_account_file.assert_called_once_with('private.json')
|
||||||
|
mock_vertexai.init.assert_called_once()
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
||||||
async def test_successful_text_completion_simple_prompt(self, mock_generative_model, mock_vertexai, mock_service_account):
|
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__')
|
||||||
"""Test successful text completion with simple prompt"""
|
@patch('trustgraph.base.llm_service.LlmService.__init__')
|
||||||
|
async def test_generate_content_success(self, mock_llm_init, mock_async_init, mock_generative_model, mock_vertexai, mock_service_account):
|
||||||
|
"""Test successful content generation"""
|
||||||
# Arrange
|
# Arrange
|
||||||
mock_credentials = MagicMock()
|
mock_credentials = MagicMock()
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
||||||
|
|
||||||
mock_model = MagicMock()
|
mock_model = MagicMock()
|
||||||
mock_response = MagicMock()
|
mock_response = MagicMock()
|
||||||
mock_response.text = "Test response from Gemini"
|
mock_response.text = "Generated response from Gemini"
|
||||||
mock_response.usage_metadata.prompt_token_count = 10
|
mock_response.usage_metadata.prompt_token_count = 15
|
||||||
mock_response.usage_metadata.candidates_token_count = 5
|
mock_response.usage_metadata.candidates_token_count = 8
|
||||||
mock_model.generate_content.return_value = mock_response
|
mock_model.generate_content.return_value = mock_response
|
||||||
mock_generative_model.return_value = mock_model
|
mock_generative_model.return_value = mock_model
|
||||||
|
|
||||||
|
mock_async_init.return_value = None
|
||||||
|
mock_llm_init.return_value = None
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
config = {
|
||||||
|
'region': 'us-central1',
|
||||||
|
'model': 'gemini-2.0-flash-001',
|
||||||
|
'temperature': 0.0,
|
||||||
|
'max_output': 8192,
|
||||||
|
'private_key': 'private.json',
|
||||||
|
'concurrency': 1,
|
||||||
|
'taskgroup': AsyncMock(),
|
||||||
|
'id': 'test-processor'
|
||||||
|
}
|
||||||
|
|
||||||
|
processor = Processor(**config)
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
result = await processor.generate_content("System prompt", "User prompt")
|
result = await processor.generate_content("System prompt", "User prompt")
|
||||||
|
|
||||||
# Assert
|
# Assert
|
||||||
assert isinstance(result, LlmResult)
|
assert isinstance(result, LlmResult)
|
||||||
assert result.text == "Test response from Gemini"
|
assert result.text == "Generated response from Gemini"
|
||||||
assert result.in_token == 10
|
assert result.in_token == 15
|
||||||
assert result.out_token == 5
|
assert result.out_token == 8
|
||||||
mock_model.generate_content.assert_called_once_with("System prompt\nUser prompt")
|
assert result.model == 'gemini-2.0-flash-001'
|
||||||
|
# Check that the method was called (actual prompt format may vary)
|
||||||
|
mock_model.generate_content.assert_called_once()
|
||||||
|
# Verify the call was made with the expected parameters
|
||||||
|
call_args = mock_model.generate_content.call_args
|
||||||
|
assert call_args[1]['generation_config'] == processor.generation_config
|
||||||
|
assert call_args[1]['safety_settings'] == processor.safety_settings
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
||||||
async def test_text_completion_with_system_instructions(self, mock_generative_model, mock_vertexai, mock_service_account):
|
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__')
|
||||||
"""Test text completion with system instructions"""
|
@patch('trustgraph.base.llm_service.LlmService.__init__')
|
||||||
|
async def test_generate_content_rate_limit_error(self, mock_llm_init, mock_async_init, mock_generative_model, mock_vertexai, mock_service_account):
|
||||||
|
"""Test rate limit error handling"""
|
||||||
# Arrange
|
# Arrange
|
||||||
|
from google.api_core.exceptions import ResourceExhausted
|
||||||
|
from trustgraph.exceptions import TooManyRequests
|
||||||
|
|
||||||
mock_credentials = MagicMock()
|
mock_credentials = MagicMock()
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
||||||
|
|
||||||
mock_model = MagicMock()
|
mock_model = MagicMock()
|
||||||
mock_response = MagicMock()
|
mock_model.generate_content.side_effect = ResourceExhausted("Rate limit exceeded")
|
||||||
mock_response.text = "Response with system instructions"
|
|
||||||
mock_response.usage_metadata.prompt_token_count = 25
|
|
||||||
mock_response.usage_metadata.candidates_token_count = 15
|
|
||||||
mock_model.generate_content.return_value = mock_response
|
|
||||||
mock_generative_model.return_value = mock_model
|
mock_generative_model.return_value = mock_model
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Act
|
|
||||||
result = await processor.generate_content("You are a helpful assistant", "What is AI?")
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert result.text == "Response with system instructions"
|
|
||||||
assert result.in_token == 25
|
|
||||||
assert result.out_token == 15
|
|
||||||
mock_model.generate_content.assert_called_once_with("You are a helpful assistant\nWhat is AI?")
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_text_completion_with_long_context(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test text completion with long context"""
|
|
||||||
# Arrange
|
|
||||||
mock_credentials = MagicMock()
|
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
|
||||||
|
|
||||||
mock_model = MagicMock()
|
mock_async_init.return_value = None
|
||||||
mock_response = MagicMock()
|
mock_llm_init.return_value = None
|
||||||
mock_response.text = "Response to long context"
|
|
||||||
mock_response.usage_metadata.prompt_token_count = 1000
|
|
||||||
mock_response.usage_metadata.candidates_token_count = 100
|
|
||||||
mock_model.generate_content.return_value = mock_response
|
|
||||||
mock_generative_model.return_value = mock_model
|
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
config = {
|
||||||
long_prompt = "This is a very long prompt. " * 100
|
|
||||||
|
|
||||||
# Act
|
|
||||||
result = await processor.generate_content("System", long_prompt)
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert result.text == "Response to long context"
|
|
||||||
assert result.in_token == 1000
|
|
||||||
assert result.out_token == 100
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_text_completion_with_empty_prompts(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test text completion with empty prompts"""
|
|
||||||
# Arrange
|
|
||||||
mock_credentials = MagicMock()
|
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
|
||||||
|
|
||||||
mock_model = MagicMock()
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_response.text = "Default response"
|
|
||||||
mock_response.usage_metadata.prompt_token_count = 1
|
|
||||||
mock_response.usage_metadata.candidates_token_count = 2
|
|
||||||
mock_model.generate_content.return_value = mock_response
|
|
||||||
mock_generative_model.return_value = mock_model
|
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Act
|
|
||||||
result = await processor.generate_content("", "")
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert result.text == "Default response"
|
|
||||||
mock_model.generate_content.assert_called_once_with("\n")
|
|
||||||
|
|
||||||
|
|
||||||
class TestVertexAISafetyFiltering(IsolatedAsyncioTestCase):
|
|
||||||
"""Test safety filtering functionality"""
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
"""Set up test fixtures"""
|
|
||||||
self.default_config = {
|
|
||||||
'region': 'us-central1',
|
'region': 'us-central1',
|
||||||
'model': 'gemini-2.0-flash-001',
|
'model': 'gemini-2.0-flash-001',
|
||||||
'temperature': 0.0,
|
'temperature': 0.0,
|
||||||
'max_output': 8192,
|
'max_output': 8192,
|
||||||
'private_key': 'private.json',
|
'private_key': 'private.json',
|
||||||
'concurrency': 1
|
'concurrency': 1,
|
||||||
|
'taskgroup': AsyncMock(),
|
||||||
|
'id': 'test-processor'
|
||||||
}
|
}
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
processor = Processor(**config)
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_safety_filter_configuration(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test safety filter configuration"""
|
|
||||||
# Arrange
|
|
||||||
mock_credentials = MagicMock()
|
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
|
||||||
|
|
||||||
# Act
|
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Assert
|
# Act & Assert
|
||||||
assert len(processor.safety_settings) == 4
|
with pytest.raises(TooManyRequests):
|
||||||
# Verify safety settings are passed to model
|
await processor.generate_content("System prompt", "User prompt")
|
||||||
mock_generative_model.assert_called_once_with(
|
|
||||||
'gemini-2.0-flash-001',
|
|
||||||
generation_config=processor.generation_config,
|
|
||||||
safety_settings=processor.safety_settings
|
|
||||||
)
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
||||||
async def test_blocked_content_handling(self, mock_generative_model, mock_vertexai, mock_service_account):
|
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__')
|
||||||
"""Test blocked content handling"""
|
@patch('trustgraph.base.llm_service.LlmService.__init__')
|
||||||
|
async def test_generate_content_blocked_response(self, mock_llm_init, mock_async_init, mock_generative_model, mock_vertexai, mock_service_account):
|
||||||
|
"""Test handling of blocked content (safety filters)"""
|
||||||
# Arrange
|
# Arrange
|
||||||
mock_credentials = MagicMock()
|
mock_credentials = MagicMock()
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
||||||
|
|
@ -333,284 +161,236 @@ class TestVertexAISafetyFiltering(IsolatedAsyncioTestCase):
|
||||||
mock_response.usage_metadata.candidates_token_count = 0
|
mock_response.usage_metadata.candidates_token_count = 0
|
||||||
mock_model.generate_content.return_value = mock_response
|
mock_model.generate_content.return_value = mock_response
|
||||||
mock_generative_model.return_value = mock_model
|
mock_generative_model.return_value = mock_model
|
||||||
|
|
||||||
|
mock_async_init.return_value = None
|
||||||
|
mock_llm_init.return_value = None
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
config = {
|
||||||
|
'region': 'us-central1',
|
||||||
|
'model': 'gemini-2.0-flash-001',
|
||||||
|
'temperature': 0.0,
|
||||||
|
'max_output': 8192,
|
||||||
|
'private_key': 'private.json',
|
||||||
|
'concurrency': 1,
|
||||||
|
'taskgroup': AsyncMock(),
|
||||||
|
'id': 'test-processor'
|
||||||
|
}
|
||||||
|
|
||||||
|
processor = Processor(**config)
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
result = await processor.generate_content("System", "Blocked content prompt")
|
result = await processor.generate_content("System prompt", "Blocked content")
|
||||||
|
|
||||||
# Assert
|
# Assert
|
||||||
assert result.text == "" # Should return empty string for blocked content
|
assert isinstance(result, LlmResult)
|
||||||
|
assert result.text is None # Should preserve None for blocked content
|
||||||
assert result.in_token == 10
|
assert result.in_token == 10
|
||||||
assert result.out_token == 0
|
assert result.out_token == 0
|
||||||
|
assert result.model == 'gemini-2.0-flash-001'
|
||||||
|
|
||||||
|
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
||||||
|
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
||||||
|
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
||||||
|
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__')
|
||||||
|
@patch('trustgraph.base.llm_service.LlmService.__init__')
|
||||||
|
async def test_processor_initialization_without_private_key(self, mock_llm_init, mock_async_init, mock_generative_model, mock_vertexai, mock_service_account):
|
||||||
|
"""Test processor initialization without private key (should fail)"""
|
||||||
|
# Arrange
|
||||||
|
mock_async_init.return_value = None
|
||||||
|
mock_llm_init.return_value = None
|
||||||
|
|
||||||
class TestVertexAIErrorHandling(IsolatedAsyncioTestCase):
|
config = {
|
||||||
"""Test error handling functionality"""
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
"""Set up test fixtures"""
|
|
||||||
self.default_config = {
|
|
||||||
'region': 'us-central1',
|
'region': 'us-central1',
|
||||||
'model': 'gemini-2.0-flash-001',
|
'model': 'gemini-2.0-flash-001',
|
||||||
'temperature': 0.0,
|
'temperature': 0.0,
|
||||||
'max_output': 8192,
|
'max_output': 8192,
|
||||||
'private_key': 'private.json',
|
'private_key': None, # No private key provided
|
||||||
'concurrency': 1
|
'concurrency': 1,
|
||||||
|
'taskgroup': AsyncMock(),
|
||||||
|
'id': 'test-processor'
|
||||||
}
|
}
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_rate_limit_error_handling(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test rate limit error handling"""
|
|
||||||
# Arrange
|
|
||||||
mock_credentials = MagicMock()
|
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
|
||||||
|
|
||||||
mock_model = MagicMock()
|
|
||||||
mock_model.generate_content.side_effect = ResourceExhausted("Rate limit exceeded")
|
|
||||||
mock_generative_model.return_value = mock_model
|
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Act
|
|
||||||
result = await processor.generate_content("System", "User prompt")
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert isinstance(result, LlmResult)
|
|
||||||
assert result.text == ""
|
|
||||||
assert result.in_token is None
|
|
||||||
assert result.out_token is None
|
|
||||||
assert result.error == "TooManyRequests"
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_authentication_error_handling(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test authentication error handling"""
|
|
||||||
# Arrange
|
|
||||||
mock_service_account.Credentials.from_service_account_file.side_effect = FileNotFoundError("Private key not found")
|
|
||||||
|
|
||||||
# Act & Assert
|
# Act & Assert
|
||||||
with pytest.raises(FileNotFoundError):
|
with pytest.raises(RuntimeError, match="Private key file not specified"):
|
||||||
processor = Processor(**self.default_config)
|
processor = Processor(**config)
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
||||||
async def test_generic_exception_handling(self, mock_generative_model, mock_vertexai, mock_service_account):
|
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__')
|
||||||
"""Test generic exception handling"""
|
@patch('trustgraph.base.llm_service.LlmService.__init__')
|
||||||
|
async def test_generate_content_generic_exception(self, mock_llm_init, mock_async_init, mock_generative_model, mock_vertexai, mock_service_account):
|
||||||
|
"""Test handling of generic exceptions"""
|
||||||
# Arrange
|
# Arrange
|
||||||
mock_credentials = MagicMock()
|
mock_credentials = MagicMock()
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
||||||
|
|
||||||
mock_model = MagicMock()
|
mock_model = MagicMock()
|
||||||
mock_model.generate_content.side_effect = Exception("Unknown error")
|
mock_model.generate_content.side_effect = Exception("Network error")
|
||||||
mock_generative_model.return_value = mock_model
|
mock_generative_model.return_value = mock_model
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Act
|
|
||||||
result = await processor.generate_content("System", "User prompt")
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert isinstance(result, LlmResult)
|
|
||||||
assert result.text == ""
|
|
||||||
assert result.in_token is None
|
|
||||||
assert result.out_token is None
|
|
||||||
assert result.error == "Unknown error"
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_model_not_found_error_handling(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test model not found error handling"""
|
|
||||||
# Arrange
|
|
||||||
mock_credentials = MagicMock()
|
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
|
||||||
|
|
||||||
mock_model = MagicMock()
|
mock_async_init.return_value = None
|
||||||
mock_model.generate_content.side_effect = ValueError("Model not found")
|
mock_llm_init.return_value = None
|
||||||
mock_generative_model.return_value = mock_model
|
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
config = {
|
||||||
|
|
||||||
# Act
|
|
||||||
result = await processor.generate_content("System", "User prompt")
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert isinstance(result, LlmResult)
|
|
||||||
assert result.text == ""
|
|
||||||
assert result.error == "Model not found"
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_quota_exceeded_error_handling(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test quota exceeded error handling"""
|
|
||||||
# Arrange
|
|
||||||
mock_credentials = MagicMock()
|
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
|
||||||
|
|
||||||
mock_model = MagicMock()
|
|
||||||
mock_model.generate_content.side_effect = ResourceExhausted("Quota exceeded")
|
|
||||||
mock_generative_model.return_value = mock_model
|
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Act
|
|
||||||
result = await processor.generate_content("System", "User prompt")
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert isinstance(result, LlmResult)
|
|
||||||
assert result.text == ""
|
|
||||||
assert result.error == "TooManyRequests"
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_token_limit_exceeded_error_handling(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test token limit exceeded error handling"""
|
|
||||||
# Arrange
|
|
||||||
mock_credentials = MagicMock()
|
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
|
||||||
|
|
||||||
mock_model = MagicMock()
|
|
||||||
mock_model.generate_content.side_effect = ValueError("Token limit exceeded")
|
|
||||||
mock_generative_model.return_value = mock_model
|
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Act
|
|
||||||
result = await processor.generate_content("System", "User prompt")
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert isinstance(result, LlmResult)
|
|
||||||
assert result.text == ""
|
|
||||||
assert result.error == "Token limit exceeded"
|
|
||||||
|
|
||||||
|
|
||||||
class TestVertexAIMetricsCollection(IsolatedAsyncioTestCase):
|
|
||||||
"""Test metrics collection functionality"""
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
"""Set up test fixtures"""
|
|
||||||
self.default_config = {
|
|
||||||
'region': 'us-central1',
|
'region': 'us-central1',
|
||||||
'model': 'gemini-2.0-flash-001',
|
'model': 'gemini-2.0-flash-001',
|
||||||
'temperature': 0.0,
|
'temperature': 0.0,
|
||||||
'max_output': 8192,
|
'max_output': 8192,
|
||||||
'private_key': 'private.json',
|
'private_key': 'private.json',
|
||||||
'concurrency': 1
|
'concurrency': 1,
|
||||||
|
'taskgroup': AsyncMock(),
|
||||||
|
'id': 'test-processor'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
processor = Processor(**config)
|
||||||
|
|
||||||
|
# Act & Assert
|
||||||
|
with pytest.raises(Exception, match="Network error"):
|
||||||
|
await processor.generate_content("System prompt", "User prompt")
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
||||||
async def test_token_usage_metrics_collection(self, mock_generative_model, mock_vertexai, mock_service_account):
|
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__')
|
||||||
"""Test token usage metrics collection"""
|
@patch('trustgraph.base.llm_service.LlmService.__init__')
|
||||||
|
async def test_processor_initialization_with_custom_parameters(self, mock_llm_init, mock_async_init, mock_generative_model, mock_vertexai, mock_service_account):
|
||||||
|
"""Test processor initialization with custom parameters"""
|
||||||
|
# Arrange
|
||||||
|
mock_credentials = MagicMock()
|
||||||
|
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
||||||
|
mock_model = MagicMock()
|
||||||
|
mock_generative_model.return_value = mock_model
|
||||||
|
|
||||||
|
mock_async_init.return_value = None
|
||||||
|
mock_llm_init.return_value = None
|
||||||
|
|
||||||
|
config = {
|
||||||
|
'region': 'us-west1',
|
||||||
|
'model': 'gemini-1.5-pro',
|
||||||
|
'temperature': 0.7,
|
||||||
|
'max_output': 4096,
|
||||||
|
'private_key': 'custom-key.json',
|
||||||
|
'concurrency': 1,
|
||||||
|
'taskgroup': AsyncMock(),
|
||||||
|
'id': 'test-processor'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Act
|
||||||
|
processor = Processor(**config)
|
||||||
|
|
||||||
|
# Assert
|
||||||
|
assert processor.model == 'gemini-1.5-pro'
|
||||||
|
|
||||||
|
# Verify that generation_config object exists (can't easily check internal values)
|
||||||
|
assert hasattr(processor, 'generation_config')
|
||||||
|
assert processor.generation_config is not None
|
||||||
|
|
||||||
|
# Verify that safety settings are configured
|
||||||
|
assert len(processor.safety_settings) == 4
|
||||||
|
|
||||||
|
# Verify service account was called with custom key
|
||||||
|
mock_service_account.Credentials.from_service_account_file.assert_called_once_with('custom-key.json')
|
||||||
|
|
||||||
|
# Verify that parameters dict has the correct values (this is accessible)
|
||||||
|
assert processor.parameters["temperature"] == 0.7
|
||||||
|
assert processor.parameters["max_output_tokens"] == 4096
|
||||||
|
assert processor.parameters["top_p"] == 1.0
|
||||||
|
assert processor.parameters["top_k"] == 32
|
||||||
|
assert processor.parameters["candidate_count"] == 1
|
||||||
|
|
||||||
|
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
||||||
|
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
||||||
|
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
||||||
|
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__')
|
||||||
|
@patch('trustgraph.base.llm_service.LlmService.__init__')
|
||||||
|
async def test_vertexai_initialization_with_credentials(self, mock_llm_init, mock_async_init, mock_generative_model, mock_vertexai, mock_service_account):
|
||||||
|
"""Test that VertexAI is initialized correctly with credentials"""
|
||||||
|
# Arrange
|
||||||
|
mock_credentials = MagicMock()
|
||||||
|
mock_credentials.project_id = "test-project-123"
|
||||||
|
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
||||||
|
mock_model = MagicMock()
|
||||||
|
mock_generative_model.return_value = mock_model
|
||||||
|
|
||||||
|
mock_async_init.return_value = None
|
||||||
|
mock_llm_init.return_value = None
|
||||||
|
|
||||||
|
config = {
|
||||||
|
'region': 'europe-west1',
|
||||||
|
'model': 'gemini-2.0-flash-001',
|
||||||
|
'temperature': 0.0,
|
||||||
|
'max_output': 8192,
|
||||||
|
'private_key': 'service-account.json',
|
||||||
|
'concurrency': 1,
|
||||||
|
'taskgroup': AsyncMock(),
|
||||||
|
'id': 'test-processor'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Act
|
||||||
|
processor = Processor(**config)
|
||||||
|
|
||||||
|
# Assert
|
||||||
|
# Verify VertexAI init was called with correct parameters
|
||||||
|
mock_vertexai.init.assert_called_once_with(
|
||||||
|
location='europe-west1',
|
||||||
|
credentials=mock_credentials,
|
||||||
|
project='test-project-123'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Verify GenerativeModel was created with the right model name
|
||||||
|
mock_generative_model.assert_called_once_with('gemini-2.0-flash-001')
|
||||||
|
|
||||||
|
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
||||||
|
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
||||||
|
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
||||||
|
@patch('trustgraph.base.async_processor.AsyncProcessor.__init__')
|
||||||
|
@patch('trustgraph.base.llm_service.LlmService.__init__')
|
||||||
|
async def test_generate_content_empty_prompts(self, mock_llm_init, mock_async_init, mock_generative_model, mock_vertexai, mock_service_account):
|
||||||
|
"""Test content generation with empty prompts"""
|
||||||
# Arrange
|
# Arrange
|
||||||
mock_credentials = MagicMock()
|
mock_credentials = MagicMock()
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
||||||
|
|
||||||
mock_model = MagicMock()
|
mock_model = MagicMock()
|
||||||
mock_response = MagicMock()
|
mock_response = MagicMock()
|
||||||
mock_response.text = "Test response"
|
mock_response.text = "Default response"
|
||||||
mock_response.usage_metadata.prompt_token_count = 50
|
mock_response.usage_metadata.prompt_token_count = 2
|
||||||
mock_response.usage_metadata.candidates_token_count = 25
|
mock_response.usage_metadata.candidates_token_count = 3
|
||||||
mock_model.generate_content.return_value = mock_response
|
mock_model.generate_content.return_value = mock_response
|
||||||
mock_generative_model.return_value = mock_model
|
mock_generative_model.return_value = mock_model
|
||||||
|
|
||||||
|
mock_async_init.return_value = None
|
||||||
|
mock_llm_init.return_value = None
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
config = {
|
||||||
|
'region': 'us-central1',
|
||||||
|
'model': 'gemini-2.0-flash-001',
|
||||||
|
'temperature': 0.0,
|
||||||
|
'max_output': 8192,
|
||||||
|
'private_key': 'private.json',
|
||||||
|
'concurrency': 1,
|
||||||
|
'taskgroup': AsyncMock(),
|
||||||
|
'id': 'test-processor'
|
||||||
|
}
|
||||||
|
|
||||||
|
processor = Processor(**config)
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
result = await processor.generate_content("System", "User prompt")
|
result = await processor.generate_content("", "")
|
||||||
|
|
||||||
# Assert
|
# Assert
|
||||||
assert result.in_token == 50
|
assert isinstance(result, LlmResult)
|
||||||
assert result.out_token == 25
|
assert result.text == "Default response"
|
||||||
|
assert result.in_token == 2
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
assert result.out_token == 3
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
assert result.model == 'gemini-2.0-flash-001'
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_request_duration_metrics(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test request duration metrics collection"""
|
|
||||||
# Arrange
|
|
||||||
mock_credentials = MagicMock()
|
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
|
||||||
|
|
||||||
mock_model = MagicMock()
|
# Verify the model was called with the combined empty prompts
|
||||||
mock_response = MagicMock()
|
mock_model.generate_content.assert_called_once()
|
||||||
mock_response.text = "Test response"
|
call_args = mock_model.generate_content.call_args
|
||||||
mock_response.usage_metadata.prompt_token_count = 10
|
# The prompt should be "" + "\n\n" + "" = "\n\n"
|
||||||
mock_response.usage_metadata.candidates_token_count = 5
|
assert call_args[0][0] == "\n\n"
|
||||||
|
|
||||||
# Simulate slow response
|
|
||||||
async def slow_generate_content(prompt):
|
|
||||||
await asyncio.sleep(0.1)
|
|
||||||
return mock_response
|
|
||||||
|
|
||||||
mock_model.generate_content.side_effect = slow_generate_content
|
|
||||||
mock_generative_model.return_value = mock_model
|
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Act
|
|
||||||
result = await processor.generate_content("System", "User prompt")
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert result.text == "Test response"
|
|
||||||
# Note: In real implementation, this would be captured by Prometheus metrics
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_error_rate_metrics(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test error rate metrics collection"""
|
|
||||||
# Arrange
|
|
||||||
mock_credentials = MagicMock()
|
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
|
||||||
|
|
||||||
mock_model = MagicMock()
|
|
||||||
mock_model.generate_content.side_effect = Exception("Test error")
|
|
||||||
mock_generative_model.return_value = mock_model
|
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Act
|
|
||||||
result = await processor.generate_content("System", "User prompt")
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert result.error == "Test error"
|
|
||||||
# Note: In real implementation, this would be captured by Prometheus metrics
|
|
||||||
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.service_account')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.vertexai')
|
|
||||||
@patch('trustgraph.model.text_completion.vertexai.llm.GenerativeModel')
|
|
||||||
async def test_cost_calculation_metrics(self, mock_generative_model, mock_vertexai, mock_service_account):
|
|
||||||
"""Test cost calculation metrics per model type"""
|
|
||||||
# Arrange
|
|
||||||
mock_credentials = MagicMock()
|
|
||||||
mock_service_account.Credentials.from_service_account_file.return_value = mock_credentials
|
|
||||||
|
|
||||||
mock_model = MagicMock()
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_response.text = "Test response"
|
|
||||||
mock_response.usage_metadata.prompt_token_count = 100
|
|
||||||
mock_response.usage_metadata.candidates_token_count = 50
|
|
||||||
mock_model.generate_content.return_value = mock_response
|
|
||||||
mock_generative_model.return_value = mock_model
|
|
||||||
|
|
||||||
processor = Processor(**self.default_config)
|
|
||||||
|
|
||||||
# Act
|
|
||||||
result = await processor.generate_content("System", "User prompt")
|
|
||||||
|
|
||||||
# Assert
|
|
||||||
assert result.in_token == 100
|
|
||||||
assert result.out_token == 50
|
|
||||||
# Note: Cost calculation would be done by the metrics system based on model type and token usage
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue