From d2303ce1f6aee0f9ab5d5be5c9f0f601e04f17c6 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Wed, 3 Sep 2025 16:16:49 +0100 Subject: [PATCH] Tool group tests --- .../test_tool_group_integration.py | 454 ++++++++++++++++++ tests/unit/test_agent/test_tool_filter.py | 321 +++++++++++++ 2 files changed, 775 insertions(+) create mode 100644 tests/integration/test_tool_group_integration.py create mode 100644 tests/unit/test_agent/test_tool_filter.py diff --git a/tests/integration/test_tool_group_integration.py b/tests/integration/test_tool_group_integration.py new file mode 100644 index 00000000..9b8673f7 --- /dev/null +++ b/tests/integration/test_tool_group_integration.py @@ -0,0 +1,454 @@ +""" +Integration tests for the tool group system. + +Tests the complete workflow from AgentRequest processing through +tool filtering and execution in the ReAct agent service. +""" + +import pytest +import json +from unittest.mock import Mock, AsyncMock, patch + +from trustgraph.schema import AgentRequest, AgentResponse, AgentStep +from trustgraph.agent.react.service import Processor +from trustgraph.agent.react.types import Tool, Argument + + +@pytest.fixture +def sample_tools(): + """Sample tools with different groups and states for testing.""" + return { + 'knowledge_query': Tool( + name='knowledge_query', + description='Query knowledge graph', + implementation=Mock(), + config={ + 'group': ['read-only', 'knowledge', 'basic'], + 'state': 'analysis', + 'available_in_states': ['undefined', 'research'] + }, + arguments=[] + ), + 'graph_update': Tool( + name='graph_update', + description='Update knowledge graph', + implementation=Mock(), + config={ + 'group': ['write', 'knowledge', 'admin'], + 'available_in_states': ['analysis', 'modification'] + }, + arguments=[] + ), + 'text_completion': Tool( + name='text_completion', + description='Generate text', + implementation=Mock(), + config={ + 'group': ['read-only', 'text', 'basic'], + 'state': 'undefined' + # No available_in_states = available in all states + }, + arguments=[] + ), + 'complex_analysis': Tool( + name='complex_analysis', + description='Complex analysis tool', + implementation=Mock(), + config={ + 'group': ['advanced', 'compute', 'expensive'], + 'state': 'results', + 'available_in_states': ['analysis'] + }, + arguments=[] + ) + } + + +@pytest.fixture +def agent_processor(): + """Create agent processor for testing.""" + return Processor(id="test-agent", max_iterations=5) + + +class TestAgentRequestProcessing: + """Test AgentRequest processing with tool groups and states.""" + + @pytest.mark.asyncio + async def test_basic_group_filtering(self, agent_processor, sample_tools): + """Test that agent only sees tools matching requested groups.""" + + # Setup agent with sample tools + agent_processor.agent.tools = sample_tools + + # Mock the agent's react method to return a Final response + from trustgraph.agent.react.types import Final + mock_final = Final(final="Test response") + agent_processor.agent.react = AsyncMock(return_value=mock_final) + + # Create request with read-only group + request = AgentRequest( + question="Test question", + state="undefined", + group=["read-only", "knowledge"], + history=[] + ) + + responses = [] + + async def mock_respond(response): + responses.append(response) + + async def mock_next(next_request): + pass # Not needed for this test + + # Process request + await agent_processor.agent_request(request, mock_respond, mock_next, {}) + + # Verify agent was called with filtered tools + agent_processor.agent.react.assert_called_once() + call_kwargs = agent_processor.agent.react.call_args.kwargs + + # The agent should have been created with filtered tools + # We can't directly inspect the temp agent, but we can verify the filtering worked + # by checking that only appropriate tools would have been available + + # Verify final response was sent + assert len(responses) == 1 + assert responses[0].answer == "Test response" + + @pytest.mark.asyncio + async def test_state_based_filtering(self, agent_processor, sample_tools): + """Test filtering based on current state.""" + + agent_processor.agent.tools = sample_tools + + from trustgraph.agent.react.types import Final + mock_final = Final(final="Analysis complete") + agent_processor.agent.react = AsyncMock(return_value=mock_final) + + # Create request in 'analysis' state + request = AgentRequest( + question="Perform analysis", + state="analysis", + group=["advanced", "compute"], + history=[] + ) + + responses = [] + + async def mock_respond(response): + responses.append(response) + + # Process request + await agent_processor.agent_request(request, mock_respond, lambda x: None, {}) + + # Verify response + assert len(responses) == 1 + assert responses[0].answer == "Analysis complete" + + @pytest.mark.asyncio + async def test_state_transition_handling(self, agent_processor, sample_tools): + """Test state transitions after tool execution.""" + + agent_processor.agent.tools = sample_tools + + # Mock agent to return an action that uses knowledge_query + from trustgraph.agent.react.types import Action + mock_action = Action( + thought="I need to query knowledge", + name="knowledge_query", + arguments={}, + observation="Found information" + ) + agent_processor.agent.react = AsyncMock(return_value=mock_action) + + # Create initial request + request = AgentRequest( + question="Research question", + state="undefined", + group=["read-only", "knowledge"], + history=[] + ) + + next_requests = [] + + async def mock_next(next_request): + next_requests.append(next_request) + + # Process request + await agent_processor.agent_request(request, lambda x: None, mock_next, {}) + + # Verify state transition occurred + assert len(next_requests) == 1 + next_req = next_requests[0] + assert next_req.state == "analysis" # knowledge_query transitions to analysis + assert next_req.group == ["read-only", "knowledge"] + + @pytest.mark.asyncio + async def test_wildcard_group_access(self, agent_processor, sample_tools): + """Test wildcard group grants access to all tools.""" + + agent_processor.agent.tools = sample_tools + + from trustgraph.agent.react.types import Final + mock_final = Final(final="All tools available") + agent_processor.agent.react = AsyncMock(return_value=mock_final) + + # Create request with wildcard group + request = AgentRequest( + question="Administrative task", + state="undefined", + group=["*"], # Wildcard access + history=[] + ) + + responses = [] + + async def mock_respond(response): + responses.append(response) + + # Process request + await agent_processor.agent_request(request, mock_respond, lambda x: None, {}) + + # All tools should have been available to the agent + assert len(responses) == 1 + assert responses[0].answer == "All tools available" + + @pytest.mark.asyncio + async def test_no_matching_tools(self, agent_processor, sample_tools): + """Test behavior when no tools match the requested groups.""" + + agent_processor.agent.tools = sample_tools + + from trustgraph.agent.react.types import Final + mock_final = Final(final="No tools available") + agent_processor.agent.react = AsyncMock(return_value=mock_final) + + # Create request with non-matching group + request = AgentRequest( + question="Some task", + state="undefined", + group=["nonexistent-group"], + history=[] + ) + + responses = [] + + async def mock_respond(response): + responses.append(response) + + # Process request + await agent_processor.agent_request(request, mock_respond, lambda x: None, {}) + + # Agent should still work but with empty tool set + assert len(responses) == 1 + + @pytest.mark.asyncio + async def test_default_group_behavior(self, agent_processor): + """Test default group behavior when no group is specified.""" + + # Create tools with and without explicit groups + tools = { + 'default_tool': Tool( + name='default_tool', + description='Default tool', + implementation=Mock(), + config={}, # No group = default group + arguments=[] + ), + 'admin_tool': Tool( + name='admin_tool', + description='Admin tool', + implementation=Mock(), + config={'group': ['admin']}, + arguments=[] + ) + } + + agent_processor.agent.tools = tools + + from trustgraph.agent.react.types import Final + mock_final = Final(final="Default tools only") + agent_processor.agent.react = AsyncMock(return_value=mock_final) + + # Create request without specifying group (should default to ["default"]) + request = AgentRequest( + question="Basic task", + state="undefined", + group=None, # Should default to ["default"] + history=[] + ) + + responses = [] + + async def mock_respond(response): + responses.append(response) + + # Process request + await agent_processor.agent_request(request, mock_respond, lambda x: None, {}) + + # Only default_tool should have been available + assert len(responses) == 1 + + +class TestToolConfigurationLoading: + """Test tool configuration loading with group metadata.""" + + @pytest.mark.asyncio + async def test_tool_config_validation(self, agent_processor): + """Test that invalid tool configurations are rejected.""" + + # Mock configuration with invalid group field + invalid_config = { + "tool": { + "invalid-tool": json.dumps({ + "name": "invalid_tool", + "description": "Invalid tool", + "type": "text-completion", + "group": "not-a-list" # Should be list + }) + } + } + + # Should raise validation error + with pytest.raises(ValueError, match="'group' field must be a list"): + await agent_processor.on_tools_config(invalid_config, 1) + + @pytest.mark.asyncio + async def test_valid_tool_config_loading(self, agent_processor): + """Test that valid tool configurations load successfully.""" + + valid_config = { + "tool": { + "valid-tool": json.dumps({ + "name": "valid_tool", + "description": "Valid tool", + "type": "text-completion", + "group": ["read-only", "text"], + "state": "analysis", + "available_in_states": ["undefined", "research"] + }) + } + } + + # Should not raise any exception + await agent_processor.on_tools_config(valid_config, 1) + + # Verify tool was loaded with correct config + assert "valid_tool" in agent_processor.agent.tools + tool = agent_processor.agent.tools["valid_tool"] + assert tool.config["group"] == ["read-only", "text"] + assert tool.config["state"] == "analysis" + + +class TestCompleteWorkflow: + """Test complete multi-step workflows with state transitions.""" + + @pytest.mark.asyncio + async def test_research_analysis_workflow(self, agent_processor, sample_tools): + """Test complete research -> analysis -> results workflow.""" + + agent_processor.agent.tools = sample_tools + + # Step 1: Initial research request + from trustgraph.agent.react.types import Action + research_action = Action( + thought="I should query the knowledge base", + name="knowledge_query", + arguments={"query": "test"}, + observation="Found relevant information" + ) + + agent_processor.agent.react = AsyncMock(return_value=research_action) + + request1 = AgentRequest( + question="Research topic X", + state="undefined", + group=["read-only", "knowledge"], + history=[] + ) + + next_requests = [] + + async def capture_next(req): + next_requests.append(req) + + # Execute step 1 + await agent_processor.agent_request(request1, lambda x: None, capture_next, {}) + + # Verify state transition to analysis + assert len(next_requests) == 1 + step2_request = next_requests[0] + assert step2_request.state == "analysis" + assert len(step2_request.history) == 1 + + # Step 2: Analysis phase + analysis_action = Action( + thought="Now I can perform complex analysis", + name="complex_analysis", + arguments={"data": "research results"}, + observation="Analysis completed" + ) + + agent_processor.agent.react = AsyncMock(return_value=analysis_action) + + # Update request groups for analysis phase + step2_request.group = ["advanced", "compute"] + + next_requests_2 = [] + + # Execute step 2 + await agent_processor.agent_request(step2_request, lambda x: None, + lambda req: next_requests_2.append(req), {}) + + # Verify final state transition + assert len(next_requests_2) == 1 + final_request = next_requests_2[0] + assert final_request.state == "results" + assert len(final_request.history) == 2 + + @pytest.mark.asyncio + async def test_multi_tenant_scenario(self, agent_processor, sample_tools): + """Test different users with different permissions.""" + + agent_processor.agent.tools = sample_tools + + from trustgraph.agent.react.types import Final + + # User A: Read-only permissions + user_a_final = Final(final="Read-only operations completed") + agent_processor.agent.react = AsyncMock(return_value=user_a_final) + + request_a = AgentRequest( + question="What information is available about X?", + state="undefined", + group=["read-only"], + history=[] + ) + + responses_a = [] + await agent_processor.agent_request(request_a, + lambda r: responses_a.append(r), + lambda x: None, {}) + + # User B: Admin permissions + user_b_final = Final(final="Administrative tasks completed") + agent_processor.agent.react = AsyncMock(return_value=user_b_final) + + request_b = AgentRequest( + question="Update the knowledge base", + state="analysis", + group=["write", "admin"], + history=[] + ) + + responses_b = [] + await agent_processor.agent_request(request_b, + lambda r: responses_b.append(r), + lambda x: None, {}) + + # Verify both users got appropriate responses + assert len(responses_a) == 1 + assert len(responses_b) == 1 + assert "Read-only" in responses_a[0].answer + assert "Administrative" in responses_b[0].answer \ No newline at end of file diff --git a/tests/unit/test_agent/test_tool_filter.py b/tests/unit/test_agent/test_tool_filter.py new file mode 100644 index 00000000..d053be99 --- /dev/null +++ b/tests/unit/test_agent/test_tool_filter.py @@ -0,0 +1,321 @@ +""" +Unit tests for the tool filtering logic in the tool group system. +""" + +import pytest +import sys +import os +from unittest.mock import Mock + +# Add trustgraph-flow to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..', 'trustgraph-flow')) + +from trustgraph.agent.tool_filter import ( + filter_tools_by_group_and_state, + get_next_state, + validate_tool_config, + _is_tool_available +) + + +class TestToolFiltering: + """Test tool filtering based on groups and states.""" + + def test_filter_tools_default_group(self): + """Tools without groups should belong to 'default' group.""" + tools = { + 'tool1': Mock(config={}), + 'tool2': Mock(config={'group': ['read-only']}) + } + + # Request default group (implicit) + filtered = filter_tools_by_group_and_state(tools, None, None) + + # Only tool1 should be available (no group = default group) + assert 'tool1' in filtered + assert 'tool2' not in filtered + + def test_filter_tools_explicit_groups(self): + """Test filtering with explicit group membership.""" + tools = { + 'read_tool': Mock(config={'group': ['read-only', 'basic']}), + 'write_tool': Mock(config={'group': ['write', 'admin']}), + 'mixed_tool': Mock(config={'group': ['read-only', 'write']}) + } + + # Request read-only tools + filtered = filter_tools_by_group_and_state(tools, ['read-only'], None) + + assert 'read_tool' in filtered + assert 'write_tool' not in filtered + assert 'mixed_tool' in filtered # Has read-only in its groups + + def test_filter_tools_multiple_requested_groups(self): + """Test filtering with multiple requested groups.""" + tools = { + 'tool1': Mock(config={'group': ['read-only']}), + 'tool2': Mock(config={'group': ['write']}), + 'tool3': Mock(config={'group': ['admin']}) + } + + # Request read-only and write tools + filtered = filter_tools_by_group_and_state(tools, ['read-only', 'write'], None) + + assert 'tool1' in filtered + assert 'tool2' in filtered + assert 'tool3' not in filtered + + def test_filter_tools_wildcard_group(self): + """Test wildcard group grants access to all tools.""" + tools = { + 'tool1': Mock(config={'group': ['read-only']}), + 'tool2': Mock(config={'group': ['admin']}), + 'tool3': Mock(config={}) # default group + } + + # Request wildcard access + filtered = filter_tools_by_group_and_state(tools, ['*'], None) + + assert len(filtered) == 3 + assert all(tool in filtered for tool in tools) + + def test_filter_tools_by_state(self): + """Test filtering based on available_in_states.""" + tools = { + 'init_tool': Mock(config={'available_in_states': ['undefined']}), + 'analysis_tool': Mock(config={'available_in_states': ['analysis']}), + 'any_state_tool': Mock(config={}) # available in all states + } + + # Filter for 'analysis' state + filtered = filter_tools_by_group_and_state(tools, ['default'], 'analysis') + + assert 'init_tool' not in filtered + assert 'analysis_tool' in filtered + assert 'any_state_tool' in filtered + + def test_filter_tools_state_wildcard(self): + """Test tools with '*' in available_in_states are always available.""" + tools = { + 'wildcard_tool': Mock(config={'available_in_states': ['*']}), + 'specific_tool': Mock(config={'available_in_states': ['research']}) + } + + # Filter for 'analysis' state + filtered = filter_tools_by_group_and_state(tools, ['default'], 'analysis') + + assert 'wildcard_tool' in filtered + assert 'specific_tool' not in filtered + + def test_filter_tools_combined_group_and_state(self): + """Test combined group and state filtering.""" + tools = { + 'valid_tool': Mock(config={ + 'group': ['read-only'], + 'available_in_states': ['analysis'] + }), + 'wrong_group': Mock(config={ + 'group': ['admin'], + 'available_in_states': ['analysis'] + }), + 'wrong_state': Mock(config={ + 'group': ['read-only'], + 'available_in_states': ['research'] + }), + 'wrong_both': Mock(config={ + 'group': ['admin'], + 'available_in_states': ['research'] + }) + } + + filtered = filter_tools_by_group_and_state( + tools, ['read-only'], 'analysis' + ) + + assert 'valid_tool' in filtered + assert 'wrong_group' not in filtered + assert 'wrong_state' not in filtered + assert 'wrong_both' not in filtered + + def test_filter_tools_empty_request_groups(self): + """Test that empty group list results in no available tools.""" + tools = { + 'tool1': Mock(config={'group': ['read-only']}), + 'tool2': Mock(config={}) + } + + filtered = filter_tools_by_group_and_state(tools, [], None) + + assert len(filtered) == 0 + + +class TestStateTransitions: + """Test state transition logic.""" + + def test_get_next_state_with_transition(self): + """Test state transition when tool defines next state.""" + tool = Mock(config={'state': 'analysis'}) + + next_state = get_next_state(tool, 'undefined') + + assert next_state == 'analysis' + + def test_get_next_state_no_transition(self): + """Test no state change when tool doesn't define next state.""" + tool = Mock(config={}) + + next_state = get_next_state(tool, 'research') + + assert next_state == 'research' + + def test_get_next_state_empty_config(self): + """Test with tool that has no config.""" + tool = Mock(config=None) + tool.config = None + + next_state = get_next_state(tool, 'initial') + + assert next_state == 'initial' + + +class TestConfigValidation: + """Test tool configuration validation.""" + + def test_validate_valid_config(self): + """Test validation of valid configuration.""" + config = { + 'group': ['read-only', 'basic'], + 'state': 'analysis', + 'available_in_states': ['undefined', 'research'] + } + + # Should not raise an exception + validate_tool_config(config) + + def test_validate_group_not_list(self): + """Test validation fails when group is not a list.""" + config = {'group': 'read-only'} # Should be list + + with pytest.raises(ValueError, match="'group' field must be a list"): + validate_tool_config(config) + + def test_validate_group_non_string_elements(self): + """Test validation fails when group contains non-strings.""" + config = {'group': ['read-only', 123]} # 123 is not string + + with pytest.raises(ValueError, match="All group names must be strings"): + validate_tool_config(config) + + def test_validate_state_not_string(self): + """Test validation fails when state is not a string.""" + config = {'state': 123} # Should be string + + with pytest.raises(ValueError, match="'state' field must be a string"): + validate_tool_config(config) + + def test_validate_available_in_states_not_list(self): + """Test validation fails when available_in_states is not a list.""" + config = {'available_in_states': 'undefined'} # Should be list + + with pytest.raises(ValueError, match="'available_in_states' field must be a list"): + validate_tool_config(config) + + def test_validate_available_in_states_non_string_elements(self): + """Test validation fails when available_in_states contains non-strings.""" + config = {'available_in_states': ['undefined', 123]} + + with pytest.raises(ValueError, match="All state names must be strings"): + validate_tool_config(config) + + def test_validate_minimal_config(self): + """Test validation of minimal valid configuration.""" + config = {'name': 'test', 'description': 'Test tool'} + + # Should not raise an exception + validate_tool_config(config) + + +class TestToolAvailability: + """Test the internal _is_tool_available function.""" + + def test_tool_available_default_groups_and_states(self): + """Test tool with default groups and states.""" + tool = Mock(config={}) + + # Default group request, default state + assert _is_tool_available(tool, ['default'], 'undefined') + + # Non-default group request should fail + assert not _is_tool_available(tool, ['admin'], 'undefined') + + def test_tool_available_string_group_conversion(self): + """Test that single group string is converted to list.""" + tool = Mock(config={'group': 'read-only'}) # Single string + + assert _is_tool_available(tool, ['read-only'], 'undefined') + assert not _is_tool_available(tool, ['admin'], 'undefined') + + def test_tool_available_string_state_conversion(self): + """Test that single state string is converted to list.""" + tool = Mock(config={'available_in_states': 'analysis'}) # Single string + + assert _is_tool_available(tool, ['default'], 'analysis') + assert not _is_tool_available(tool, ['default'], 'research') + + def test_tool_no_config_attribute(self): + """Test tool without config attribute.""" + tool = Mock() + del tool.config # Remove config attribute + + # Should use defaults and be available for default group/state + assert _is_tool_available(tool, ['default'], 'undefined') + assert not _is_tool_available(tool, ['admin'], 'undefined') + + +class TestWorkflowScenarios: + """Test complete workflow scenarios from the tech spec.""" + + def test_research_to_analysis_workflow(self): + """Test the research -> analysis workflow from tech spec.""" + tools = { + 'knowledge_query': Mock(config={ + 'group': ['read-only', 'knowledge'], + 'state': 'analysis', + 'available_in_states': ['undefined', 'research'] + }), + 'complex_analysis': Mock(config={ + 'group': ['advanced', 'compute'], + 'state': 'results', + 'available_in_states': ['analysis'] + }), + 'text_completion': Mock(config={ + 'group': ['read-only', 'text', 'basic'] + # No available_in_states = available in all states + }) + } + + # Phase 1: Initial research (undefined state) + phase1_filtered = filter_tools_by_group_and_state( + tools, ['read-only', 'knowledge'], 'undefined' + ) + assert 'knowledge_query' in phase1_filtered + assert 'text_completion' in phase1_filtered + assert 'complex_analysis' not in phase1_filtered + + # Simulate tool execution and state transition + executed_tool = phase1_filtered['knowledge_query'] + next_state = get_next_state(executed_tool, 'undefined') + assert next_state == 'analysis' + + # Phase 2: Analysis state (include basic group for text_completion) + phase2_filtered = filter_tools_by_group_and_state( + tools, ['advanced', 'compute', 'basic'], 'analysis' + ) + assert 'knowledge_query' not in phase2_filtered # Not available in analysis + assert 'complex_analysis' in phase2_filtered + assert 'text_completion' in phase2_filtered # Always available + + # Simulate complex analysis execution + executed_tool = phase2_filtered['complex_analysis'] + final_state = get_next_state(executed_tool, 'analysis') + assert final_state == 'results' \ No newline at end of file