Merge branch 'mgx_ops' into basic_ability

tests/metagpt/actions/test_action_node.py

@@ -91,10 +91,10 @@ async def test_action_node_two_layer():
     assert node_b in root.children.values()
 
     # FIXME: ADD MARKDOWN SUPPORT. NEED TO TUNE MARKDOWN SYMBOL FIRST.
-    answer1 = await root.fill(context="what's the answer to 123+456?", schema="json", strgy="simple", llm=LLM())
+    answer1 = await root.fill(req="what's the answer to 123+456?", schema="json", strgy="simple", llm=LLM())
     assert "579" in answer1.content
 
-    answer2 = await root.fill(context="what's the answer to 123+456?", schema="json", strgy="complex", llm=LLM())
+    answer2 = await root.fill(req="what's the answer to 123+456?", schema="json", strgy="complex", llm=LLM())
     assert "579" in answer2.content
 
 
@@ -112,7 +112,7 @@ async def test_action_node_review():
     with pytest.raises(RuntimeError):
         _ = await node_a.review()
 
-    _ = await node_a.fill(context=None, llm=LLM())
+    _ = await node_a.fill(req=None, llm=LLM())
     setattr(node_a.instruct_content, key, "game snake")  # wrong content to review
 
     review_comments = await node_a.review(review_mode=ReviewMode.AUTO)
@@ -126,7 +126,7 @@ async def test_action_node_review():
     with pytest.raises(RuntimeError):
         _ = await node.review()
 
-    _ = await node.fill(context=None, llm=LLM())
+    _ = await node.fill(req=None, llm=LLM())
 
     review_comments = await node.review(review_mode=ReviewMode.AUTO)
     assert len(review_comments) == 1
@@ -151,7 +151,7 @@ async def test_action_node_revise():
     with pytest.raises(RuntimeError):
         _ = await node_a.review()
 
-    _ = await node_a.fill(context=None, llm=LLM())
+    _ = await node_a.fill(req=None, llm=LLM())
     setattr(node_a.instruct_content, key, "game snake")  # wrong content to revise
     revise_contents = await node_a.revise(revise_mode=ReviseMode.AUTO)
     assert len(revise_contents) == 1
@@ -164,7 +164,7 @@ async def test_action_node_revise():
     with pytest.raises(RuntimeError):
         _ = await node.revise()
 
-    _ = await node.fill(context=None, llm=LLM())
+    _ = await node.fill(req=None, llm=LLM())
     setattr(node.instruct_content, key, "game snake")
     revise_contents = await node.revise(revise_mode=ReviseMode.AUTO)
     assert len(revise_contents) == 1
@@ -257,7 +257,7 @@ async def test_action_node_with_image(mocker):
     invoice_path = Path(__file__).parent.joinpath("..", "..", "data", "invoices", "invoice-2.png")
     img_base64 = encode_image(invoice_path)
     mocker.patch("metagpt.provider.openai_api.OpenAILLM._cons_kwargs", _cons_kwargs)
-    node = await invoice.fill(context="", llm=LLM(), images=[img_base64])
+    node = await invoice.fill(req="", llm=LLM(), images=[img_base64])
     assert node.instruct_content.invoice
 
 

tests/metagpt/actions/test_design_api_an.py

@@ -38,7 +38,7 @@ async def test_write_design_an(mocker):
     mocker.patch("metagpt.actions.design_api_an.REFINED_DESIGN_NODE.fill", return_value=root)
 
     prompt = NEW_REQ_TEMPLATE.format(old_design=DESIGN_SAMPLE, context=dict_to_markdown(REFINED_PRD_JSON))
-    node = await REFINED_DESIGN_NODE.fill(prompt, llm)
+    node = await REFINED_DESIGN_NODE.fill(req=prompt, llm=llm)
 
     assert "Refined Implementation Approach" in node.instruct_content.model_dump()
     assert "Refined File list" in node.instruct_content.model_dump()

tests/metagpt/actions/test_project_management_an.py

@@ -42,7 +42,7 @@ async def test_project_management_an(mocker):
     root.instruct_content.model_dump = mock_task_json
     mocker.patch("metagpt.actions.project_management_an.PM_NODE.fill", return_value=root)
 
-    node = await PM_NODE.fill(dict_to_markdown(REFINED_DESIGN_JSON), llm)
+    node = await PM_NODE.fill(req=dict_to_markdown(REFINED_DESIGN_JSON), llm=llm)
 
     assert "Logic Analysis" in node.instruct_content.model_dump()
     assert "Task list" in node.instruct_content.model_dump()
@@ -59,7 +59,7 @@ async def test_project_management_an_inc(mocker):
     mocker.patch("metagpt.actions.project_management_an.REFINED_PM_NODE.fill", return_value=root)
 
     prompt = NEW_REQ_TEMPLATE.format(old_task=TASK_SAMPLE, context=dict_to_markdown(REFINED_DESIGN_JSON))
-    node = await REFINED_PM_NODE.fill(prompt, llm)
+    node = await REFINED_PM_NODE.fill(req=prompt, llm=llm)
 
     assert "Refined Logic Analysis" in node.instruct_content.model_dump()
     assert "Refined Task list" in node.instruct_content.model_dump()

tests/metagpt/actions/test_write_prd_an.py

@@ -39,7 +39,7 @@ async def test_write_prd_an(mocker):
         requirements=NEW_REQUIREMENT_SAMPLE,
         old_prd=PRD_SAMPLE,
     )
-    node = await REFINED_PRD_NODE.fill(prompt, llm)
+    node = await REFINED_PRD_NODE.fill(req=prompt, llm=llm)
 
     assert "Refined Requirements" in node.instruct_content.model_dump()
     assert "Refined Product Goals" in node.instruct_content.model_dump()

tests/metagpt/exp_pool/test_context_builders/test_base_context_builder.py

@@ -0,0 +1,32 @@
+import pytest
+
+from metagpt.exp_pool.context_builders.base import (
+    EXP_TEMPLATE,
+    BaseContextBuilder,
+    Experience,
+)
+from metagpt.exp_pool.schema import Metric, Score
+
+
+class TestBaseContextBuilder:
+    class ConcreteContextBuilder(BaseContextBuilder):
+        async def build(self, *args, **kwargs):
+            pass
+
+    @pytest.fixture
+    def context_builder(self):
+        return self.ConcreteContextBuilder()
+
+    def test_format_exps(self, context_builder):
+        exp1 = Experience(req="req1", resp="resp1", metric=Metric(score=Score(val=8)))
+        exp2 = Experience(req="req2", resp="resp2", metric=Metric(score=Score(val=9)))
+        context_builder.exps = [exp1, exp2]
+
+        result = context_builder.format_exps()
+        expected = "\n".join(
+            [
+                f"1. {EXP_TEMPLATE.format(req='req1', resp='resp1', score=8)}",
+                f"2. {EXP_TEMPLATE.format(req='req2', resp='resp2', score=9)}",
+            ]
+        )
+        assert result == expected

tests/metagpt/exp_pool/test_context_builders/test_rolezero_context_builder.py

@@ -0,0 +1,51 @@
+import pytest
+
+from metagpt.exp_pool.context_builders.base import BaseContextBuilder
+from metagpt.exp_pool.context_builders.role_zero import RoleZeroContextBuilder
+
+
+class TestRoleZeroContextBuilder:
+    @pytest.fixture
+    def context_builder(self):
+        return RoleZeroContextBuilder()
+
+    @pytest.mark.asyncio
+    async def test_build_empty_req(self, context_builder):
+        result = await context_builder.build(req=[])
+        assert result == []
+
+    @pytest.mark.asyncio
+    async def test_build_no_experiences(self, context_builder, mocker):
+        mocker.patch.object(BaseContextBuilder, "format_exps", return_value="")
+        req = [{"content": "Original content"}]
+        result = await context_builder.build(req=req)
+        assert result == req
+
+    @pytest.mark.asyncio
+    async def test_build_with_experiences(self, context_builder, mocker):
+        mocker.patch.object(BaseContextBuilder, "format_exps", return_value="Formatted experiences")
+        mocker.patch.object(RoleZeroContextBuilder, "replace_example_content", return_value="Updated content")
+        req = [{"content": "Original content 1"}]
+        result = await context_builder.build(req=req)
+        assert result == [{"content": "Updated content"}]
+
+    def test_replace_example_content(self, context_builder, mocker):
+        mocker.patch.object(RoleZeroContextBuilder, "replace_content_between_markers", return_value="Replaced content")
+        result = context_builder.replace_example_content("Original text", "New example content")
+        assert result == "Replaced content"
+        context_builder.replace_content_between_markers.assert_called_once_with(
+            "Original text", "# Example", "# Instruction", "New example content"
+        )
+
+    def test_replace_content_between_markers(self):
+        text = "Start\n# Example\nOld content\n# Instruction\nEnd"
+        new_content = "New content"
+        result = RoleZeroContextBuilder.replace_content_between_markers(text, "# Example", "# Instruction", new_content)
+        expected = "Start\n# Example\nNew content\n\n# Instruction\nEnd"
+        assert result == expected
+
+    def test_replace_content_between_markers_no_match(self):
+        text = "Start\nNo markers\nEnd"
+        new_content = "New content"
+        result = RoleZeroContextBuilder.replace_content_between_markers(text, "# Example", "# Instruction", new_content)
+        assert result == text

tests/metagpt/exp_pool/test_context_builders/test_simple_context_builder.py

@@ -0,0 +1,47 @@
+import pytest
+
+from metagpt.exp_pool.context_builders.base import BaseContextBuilder
+from metagpt.exp_pool.context_builders.simple import (
+    SIMPLE_CONTEXT_TEMPLATE,
+    SimpleContextBuilder,
+)
+
+
+class TestSimpleContextBuilder:
+    @pytest.fixture
+    def context_builder(self):
+        return SimpleContextBuilder()
+
+    @pytest.mark.asyncio
+    async def test_build_with_experiences(self, mocker, context_builder: SimpleContextBuilder):
+        # Mock the format_exps method
+        mock_exps = "Mocked experiences"
+        mocker.patch.object(BaseContextBuilder, "format_exps", return_value=mock_exps)
+
+        req = "Test request"
+        result = await context_builder.build(req=req)
+
+        expected = SIMPLE_CONTEXT_TEMPLATE.format(req=req, exps=mock_exps)
+        assert result == expected
+
+    @pytest.mark.asyncio
+    async def test_build_without_experiences(self, mocker, context_builder: SimpleContextBuilder):
+        # Mock the format_exps method to return an empty string
+        mocker.patch.object(BaseContextBuilder, "format_exps", return_value="")
+
+        req = "Test request"
+        result = await context_builder.build(req=req)
+
+        expected = SIMPLE_CONTEXT_TEMPLATE.format(req=req, exps="")
+        assert result == expected
+
+    @pytest.mark.asyncio
+    async def test_build_without_req(self, mocker, context_builder: SimpleContextBuilder):
+        # Mock the format_exps method
+        mock_exps = "Mocked experiences"
+        mocker.patch.object(BaseContextBuilder, "format_exps", return_value=mock_exps)
+
+        result = await context_builder.build(req="")
+
+        expected = SIMPLE_CONTEXT_TEMPLATE.format(req="", exps=mock_exps)
+        assert result == expected

tests/metagpt/exp_pool/test_decorator.py

@@ -0,0 +1,200 @@
+import asyncio
+
+import pytest
+
+from metagpt.config2 import Config
+from metagpt.configs.exp_pool_config import ExperiencePoolConfig
+from metagpt.exp_pool.context_builders import SimpleContextBuilder
+from metagpt.exp_pool.decorator import ExpCacheHandler, exp_cache
+from metagpt.exp_pool.manager import ExperienceManager
+from metagpt.exp_pool.perfect_judges import SimplePerfectJudge
+from metagpt.exp_pool.schema import Experience, QueryType, Score
+from metagpt.exp_pool.scorers import SimpleScorer
+from metagpt.rag.engines import SimpleEngine
+
+
+class TestExpCacheHandler:
+    @pytest.fixture
+    def mock_func(self, mocker):
+        return mocker.AsyncMock()
+
+    @pytest.fixture
+    def mock_exp_manager(self, mocker):
+        manager = mocker.MagicMock(spec=ExperienceManager)
+        manager.storage = mocker.MagicMock(spec=SimpleEngine)
+        manager.config = mocker.MagicMock(spec=Config)
+        manager.config.exp_pool = ExperiencePoolConfig()
+        manager.query_exps = mocker.AsyncMock()
+        manager.create_exp = mocker.MagicMock()
+        return manager
+
+    @pytest.fixture
+    def mock_scorer(self, mocker):
+        scorer = mocker.MagicMock(spec=SimpleScorer)
+        scorer.evaluate = mocker.AsyncMock()
+        return scorer
+
+    @pytest.fixture
+    def mock_perfect_judge(self, mocker):
+        return mocker.MagicMock(spec=SimplePerfectJudge)
+
+    @pytest.fixture
+    def mock_context_builder(self, mocker):
+        return mocker.MagicMock(spec=SimpleContextBuilder)
+
+    @pytest.fixture
+    def exp_cache_handler(self, mock_func, mock_exp_manager, mock_scorer, mock_perfect_judge, mock_context_builder):
+        return ExpCacheHandler(
+            func=mock_func,
+            args=(),
+            kwargs={"req": "test_req"},
+            exp_manager=mock_exp_manager,
+            exp_scorer=mock_scorer,
+            exp_perfect_judge=mock_perfect_judge,
+            context_builder=mock_context_builder,
+        )
+
+    @pytest.mark.asyncio
+    async def test_fetch_experiences(self, exp_cache_handler, mock_exp_manager):
+        mock_exp_manager.query_exps.return_value = [Experience(req="test_req", resp="test_resp")]
+        await exp_cache_handler.fetch_experiences()
+        mock_exp_manager.query_exps.assert_called_once_with(
+            "test_req", query_type=QueryType.SEMANTIC, tag=exp_cache_handler.tag
+        )
+        assert len(exp_cache_handler._exps) == 1
+
+    @pytest.mark.asyncio
+    async def test_get_one_perfect_exp(self, exp_cache_handler, mock_perfect_judge):
+        exp = Experience(req="test_req", resp="perfect_resp")
+        exp_cache_handler._exps = [exp]
+        mock_perfect_judge.is_perfect_exp.return_value = True
+        result = await exp_cache_handler.get_one_perfect_exp()
+        assert result == "perfect_resp"
+
+    @pytest.mark.asyncio
+    async def test_execute_function(self, exp_cache_handler, mock_func, mock_context_builder):
+        mock_context_builder.build.return_value = "built_context"
+        mock_func.return_value = "function_result"
+        await exp_cache_handler.execute_function()
+        mock_context_builder.build.assert_called_once()
+        mock_func.assert_called_once_with(req="built_context")
+        assert exp_cache_handler._raw_resp == "function_result"
+        assert exp_cache_handler._resp == "function_result"
+
+    @pytest.mark.asyncio
+    async def test_process_experience(self, exp_cache_handler, mock_scorer, mock_exp_manager):
+        exp_cache_handler._resp = "test_resp"
+        mock_scorer.evaluate.return_value = Score(val=8)
+        await exp_cache_handler.process_experience()
+        mock_scorer.evaluate.assert_called_once()
+        mock_exp_manager.create_exp.assert_called_once()
+
+    @pytest.mark.asyncio
+    async def test_evaluate_experience(self, exp_cache_handler, mock_scorer):
+        exp_cache_handler._resp = "test_resp"
+        mock_scorer.evaluate.return_value = Score(val=9)
+        await exp_cache_handler.evaluate_experience()
+        assert exp_cache_handler._score.val == 9
+
+    def test_save_experience(self, exp_cache_handler, mock_exp_manager):
+        exp_cache_handler._req = "test_req"
+        exp_cache_handler._resp = "test_resp"
+        exp_cache_handler._score = Score(val=7)
+        exp_cache_handler.save_experience()
+        mock_exp_manager.create_exp.assert_called_once()
+
+    def test_choose_wrapper_async(self, mocker):
+        async def async_func():
+            pass
+
+        wrapper = ExpCacheHandler.choose_wrapper(async_func, mocker.AsyncMock())
+        assert asyncio.iscoroutinefunction(wrapper)
+
+    def test_choose_wrapper_sync(self, mocker):
+        def sync_func():
+            pass
+
+        wrapper = ExpCacheHandler.choose_wrapper(sync_func, mocker.AsyncMock())
+        assert not asyncio.iscoroutinefunction(wrapper)
+
+    def test_validate_params(self):
+        with pytest.raises(ValueError):
+            ExpCacheHandler(func=lambda x: x, args=(), kwargs={})
+
+    def test_generate_tag(self):
+        class TestClass:
+            def test_method(self):
+                pass
+
+        handler = ExpCacheHandler(func=TestClass().test_method, args=(TestClass(),), kwargs={"req": "test"})
+        assert handler._generate_tag() == "TestClass.test_method"
+
+        handler = ExpCacheHandler(func=lambda x: x, args=(), kwargs={"req": "test"})
+        assert handler._generate_tag() == "<lambda>"
+
+
+class TestExpCache:
+    @pytest.fixture
+    def mock_exp_manager(self, mocker, mock_config):
+        manager = mocker.MagicMock(spec=ExperienceManager)
+        manager.storage = mocker.MagicMock(spec=SimpleEngine)
+        manager.config = mock_config
+        manager.query_exps = mocker.AsyncMock()
+        manager.create_exp = mocker.MagicMock()
+        return manager
+
+    @pytest.fixture
+    def mock_scorer(self, mocker):
+        scorer = mocker.MagicMock(spec=SimpleScorer)
+        scorer.evaluate = mocker.AsyncMock(return_value=Score())
+        return scorer
+
+    @pytest.fixture
+    def mock_perfect_judge(self, mocker):
+        return mocker.MagicMock(spec=SimplePerfectJudge)
+
+    @pytest.fixture
+    def mock_config(self, mocker):
+        return mocker.patch("metagpt.exp_pool.decorator.config")
+
+    @pytest.mark.asyncio
+    async def test_exp_cache_disabled(self, mock_config, mock_exp_manager):
+        mock_config.exp_pool.enable_read = False
+
+        @exp_cache(manager=mock_exp_manager)
+        async def test_func(req):
+            return "result"
+
+        result = await test_func(req="test")
+        assert result == "result"
+        mock_exp_manager.query_exps.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_exp_cache_enabled_no_perfect_exp(self, mock_config, mock_exp_manager, mock_scorer):
+        mock_config.exp_pool.enable_read = True
+        mock_exp_manager.query_exps.return_value = []
+
+        @exp_cache(manager=mock_exp_manager, scorer=mock_scorer)
+        async def test_func(req):
+            return "computed_result"
+
+        result = await test_func(req="test")
+        assert result == "computed_result"
+        mock_exp_manager.query_exps.assert_called()
+        mock_exp_manager.create_exp.assert_called()
+
+    @pytest.mark.asyncio
+    async def test_exp_cache_enabled_with_perfect_exp(self, mock_config, mock_exp_manager, mock_perfect_judge):
+        mock_config.exp_pool.enable_read = True
+        perfect_exp = Experience(req="test", resp="perfect_result")
+        mock_exp_manager.query_exps.return_value = [perfect_exp]
+        mock_perfect_judge.is_perfect_exp.return_value = True
+
+        @exp_cache(manager=mock_exp_manager, perfect_judge=mock_perfect_judge)
+        async def test_func(req):
+            return "should_not_be_called"
+
+        result = await test_func(req="test")
+        assert result == "perfect_result"
+        mock_exp_manager.query_exps.assert_called_once()
+        mock_exp_manager.create_exp.assert_not_called()

tests/metagpt/exp_pool/test_manager.py

@@ -0,0 +1,78 @@
+import pytest
+
+from metagpt.config2 import Config
+from metagpt.configs.exp_pool_config import ExperiencePoolConfig
+from metagpt.configs.llm_config import LLMConfig
+from metagpt.exp_pool.manager import Experience, ExperienceManager
+from metagpt.exp_pool.schema import QueryType
+
+
+class TestExperienceManager:
+    @pytest.fixture
+    def mock_config(self):
+        return Config(llm=LLMConfig(), exp_pool=ExperiencePoolConfig(enable_write=True, enable_read=True))
+
+    @pytest.fixture
+    def mock_storage(self, mocker):
+        engine = mocker.MagicMock()
+        engine.add_objs = mocker.MagicMock()
+        engine.aretrieve = mocker.AsyncMock(return_value=[])
+        engine._retriever = mocker.MagicMock()
+        engine._retriever._vector_store = mocker.MagicMock()
+        engine._retriever._vector_store._collection = mocker.MagicMock()
+        engine._retriever._vector_store._collection.count = mocker.MagicMock(return_value=10)
+        return engine
+
+    @pytest.fixture
+    def exp_manager(self, mock_config, mock_storage):
+        manager = ExperienceManager(config=mock_config)
+        manager._storage = mock_storage
+        return manager
+
+    def test_vector_store_property(self, exp_manager):
+        assert exp_manager.vector_store == exp_manager.storage._retriever._vector_store
+
+    @pytest.mark.asyncio
+    async def test_query_exps_with_exact_match(self, exp_manager, mocker):
+        req = "exact query"
+        exp1 = Experience(req=req, resp="response1")
+        exp2 = Experience(req="different query", resp="response2")
+
+        mock_node1 = mocker.MagicMock(metadata={"obj": exp1})
+        mock_node2 = mocker.MagicMock(metadata={"obj": exp2})
+
+        exp_manager.storage.aretrieve.return_value = [mock_node1, mock_node2]
+
+        result = await exp_manager.query_exps(req, query_type=QueryType.EXACT)
+        assert len(result) == 1
+        assert result[0].req == req
+
+    @pytest.mark.asyncio
+    async def test_query_exps_with_tag_filter(self, exp_manager, mocker):
+        tag = "test_tag"
+        exp1 = Experience(req="query1", resp="response1", tag=tag)
+        exp2 = Experience(req="query2", resp="response2", tag="other_tag")
+
+        mock_node1 = mocker.MagicMock(metadata={"obj": exp1})
+        mock_node2 = mocker.MagicMock(metadata={"obj": exp2})
+
+        exp_manager.storage.aretrieve.return_value = [mock_node1, mock_node2]
+
+        result = await exp_manager.query_exps("query", tag=tag)
+        assert len(result) == 1
+        assert result[0].tag == tag
+
+    def test_get_exps_count(self, exp_manager):
+        assert exp_manager.get_exps_count() == 10
+
+    def test_create_exp_write_disabled(self, exp_manager, mock_config):
+        mock_config.exp_pool.enable_write = False
+        exp = Experience(req="test", resp="response")
+        exp_manager.create_exp(exp)
+        exp_manager.storage.add_objs.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_query_exps_read_disabled(self, exp_manager, mock_config):
+        mock_config.exp_pool.enable_read = False
+        result = await exp_manager.query_exps("query")
+        assert result == []

tests/metagpt/exp_pool/test_perfect_judges/test_simple_perfect_judge.py

@@ -0,0 +1,40 @@
+import pytest
+
+from metagpt.exp_pool.perfect_judges import SimplePerfectJudge
+from metagpt.exp_pool.schema import MAX_SCORE, Experience, Metric, Score
+
+
+class TestSimplePerfectJudge:
+    @pytest.fixture
+    def simple_perfect_judge(self):
+        return SimplePerfectJudge()
+
+    @pytest.mark.asyncio
+    async def test_is_perfect_exp_perfect_match(self, simple_perfect_judge):
+        exp = Experience(req="test_request", resp="resp", metric=Metric(score=Score(val=MAX_SCORE)))
+        result = await simple_perfect_judge.is_perfect_exp(exp, "test_request")
+        assert result is True
+
+    @pytest.mark.asyncio
+    async def test_is_perfect_exp_imperfect_score(self, simple_perfect_judge):
+        exp = Experience(req="test_request", resp="resp", metric=Metric(score=Score(val=MAX_SCORE - 1)))
+        result = await simple_perfect_judge.is_perfect_exp(exp, "test_request")
+        assert result is False
+
+    @pytest.mark.asyncio
+    async def test_is_perfect_exp_mismatched_request(self, simple_perfect_judge):
+        exp = Experience(req="test_request", resp="resp", metric=Metric(score=Score(val=MAX_SCORE)))
+        result = await simple_perfect_judge.is_perfect_exp(exp, "different_request")
+        assert result is False
+
+    @pytest.mark.asyncio
+    async def test_is_perfect_exp_no_metric(self, simple_perfect_judge):
+        exp = Experience(req="test_request", resp="resp")
+        result = await simple_perfect_judge.is_perfect_exp(exp, "test_request")
+        assert result is False
+
+    @pytest.mark.asyncio
+    async def test_is_perfect_exp_no_score(self, simple_perfect_judge):
+        exp = Experience(req="test_request", resp="resp", metric=Metric())
+        result = await simple_perfect_judge.is_perfect_exp(exp, "test_request")
+        assert result is False

tests/metagpt/exp_pool/test_scorers/test_simple_scorer.py

@@ -0,0 +1,64 @@
+import json
+
+import pytest
+
+from metagpt.exp_pool.schema import Score
+from metagpt.exp_pool.scorers.simple import SIMPLE_SCORER_TEMPLATE, SimpleScorer
+from metagpt.llm import BaseLLM
+
+
+class TestSimpleScorer:
+    @pytest.fixture
+    def mock_llm(self, mocker):
+        mock_llm = mocker.MagicMock(spec=BaseLLM)
+        return mock_llm
+
+    @pytest.fixture
+    def simple_scorer(self, mock_llm):
+        return SimpleScorer(llm=mock_llm)
+
+    def test_init(self, mock_llm):
+        scorer = SimpleScorer(llm=mock_llm)
+        assert isinstance(scorer.llm, BaseLLM)
+
+    @pytest.mark.asyncio
+    async def test_evaluate(self, simple_scorer, mock_llm, mocker):
+        # Mock request and response
+        req = "What is the capital of France?"
+        resp = "The capital of France is Paris."
+
+        # Mock LLM response
+        mock_llm_response = '{"val": 9, "reason": "Accurate and concise answer"}'
+        mock_llm.aask.return_value = f"```json\n{mock_llm_response}\n```"
+
+        # Mock CodeParser.parse_code
+        mocker.patch("metagpt.utils.common.CodeParser.parse_code", return_value=mock_llm_response)
+
+        # Test evaluate method
+        result = await simple_scorer.evaluate(req, resp)
+
+        # Assert LLM was called with correct prompt
+        expected_prompt = SIMPLE_SCORER_TEMPLATE.format(req=req, resp=resp)
+        mock_llm.aask.assert_called_once_with(expected_prompt)
+
+        # Assert the result is correct
+        assert isinstance(result, Score)
+        assert result.val == 9
+        assert result.reason == "Accurate and concise answer"
+
+    @pytest.mark.asyncio
+    async def test_evaluate_invalid_response(self, simple_scorer, mock_llm, mocker):
+        # Mock request and response
+        req = "What is the capital of France?"
+        resp = "The capital of France is Paris."
+
+        # Mock LLM response with invalid JSON
+        mock_llm_response = "Invalid JSON"
+        mock_llm.aask.return_value = f"```json\n{mock_llm_response}\n```"
+
+        # Mock CodeParser.parse_code
+        mocker.patch("metagpt.utils.common.CodeParser.parse_code", return_value=mock_llm_response)
+
+        # Test evaluate method with invalid response
+        with pytest.raises(json.JSONDecodeError):
+            await simple_scorer.evaluate(req, resp)

tests/metagpt/exp_pool/test_serializers/test_action_node.py

@@ -0,0 +1,35 @@
+from typing import Type
+
+import pytest
+
+from metagpt.actions.action_node import ActionNode
+from metagpt.exp_pool.serializers.action_node import ActionNodeSerializer
+
+
+class TestActionNodeSerializer:
+    @pytest.fixture
+    def serializer(self):
+        return ActionNodeSerializer()
+
+    @pytest.fixture
+    def action_node(self):
+        class InstructContent:
+            def __init__(self, json_data):
+                self.json_data = json_data
+
+            def model_dump_json(self):
+                return self.json_data
+
+        action_node = ActionNode(key="", expected_type=Type[str], instruction="", example="")
+        action_node.instruct_content = InstructContent('{"key": "value"}')
+
+        return action_node
+
+    def test_serialize_resp(self, serializer: ActionNodeSerializer, action_node: ActionNode):
+        serialized = serializer.serialize_resp(action_node)
+        assert serialized == '{"key": "value"}'
+
+    def test_deserialize_resp(self, serializer: ActionNodeSerializer):
+        deserialized = serializer.deserialize_resp('{"key": "value"}')
+        assert isinstance(deserialized, ActionNode)
+        assert deserialized.instruct_content.model_dump_json() == '{"key": "value"}'

tests/metagpt/exp_pool/test_serializers/test_role_zero.py

@@ -0,0 +1,46 @@
+import json
+
+import pytest
+
+from metagpt.exp_pool.serializers import RoleZeroSerializer
+
+
+class TestRoleZeroSerializer:
+    @pytest.fixture
+    def serializer(self) -> RoleZeroSerializer:
+        return RoleZeroSerializer()
+
+    @pytest.fixture
+    def last_item(self) -> dict:
+        return {
+            "role": "user",
+            "content": "# Current Plan\nsome plan\n# Current Plan\nsome plan\n# Instruction\nsome instruction",
+        }
+
+    @pytest.fixture
+    def sample_req(self):
+        return [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
+
+    def test_serialize_req_empty_input(self, serializer: RoleZeroSerializer):
+        assert serializer.serialize_req(req=[]) == ""
+
+    def test_serialize_req_with_content(self, serializer: RoleZeroSerializer, last_item: dict):
+        req = [
+            {"role": "user", "content": "Command Editor.read executed: file_path=test.py"},
+            {"role": "assistant", "content": "Some other content"},
+            last_item,
+        ]
+        expected_output = json.dumps([{"role": "user", "content": "Command Editor.read executed: file_path=test.py"}])
+        assert serializer.serialize_req(req=req) == expected_output
+
+    def test_filter_req(self, serializer: RoleZeroSerializer):
+        req = [
+            {"role": "user", "content": "Command Editor.read executed: file_path=test1.py"},
+            {"role": "assistant", "content": "Some other content"},
+            {"role": "user", "content": "Command Editor.read executed: file_path=test2.py"},
+            {"role": "assistant", "content": "Final content"},
+        ]
+        filtered_req = serializer._filter_req(req)
+        assert len(filtered_req) == 2
+        assert filtered_req[0]["content"] == "Command Editor.read executed: file_path=test1.py"
+        assert filtered_req[1]["content"] == "Command Editor.read executed: file_path=test2.py"

tests/metagpt/exp_pool/test_serializers/test_simple.py

@@ -0,0 +1,44 @@
+import pytest
+
+from metagpt.exp_pool.serializers.simple import SimpleSerializer
+
+
+class TestSimpleSerializer:
+    @pytest.fixture
+    def serializer(self):
+        return SimpleSerializer()
+
+    def test_serialize_req(self, serializer: SimpleSerializer):
+        # Test with different types of input
+        assert serializer.serialize_req(req=123) == "123"
+        assert serializer.serialize_req(req="test") == "test"
+        assert serializer.serialize_req(req=[1, 2, 3]) == "[1, 2, 3]"
+        assert serializer.serialize_req(req={"a": 1}) == "{'a': 1}"
+
+    def test_serialize_resp(self, serializer: SimpleSerializer):
+        # Test with different types of input
+        assert serializer.serialize_resp(456) == "456"
+        assert serializer.serialize_resp("response") == "response"
+        assert serializer.serialize_resp([4, 5, 6]) == "[4, 5, 6]"
+        assert serializer.serialize_resp({"b": 2}) == "{'b': 2}"
+
+    def test_deserialize_resp(self, serializer: SimpleSerializer):
+        # Test with different types of input
+        assert serializer.deserialize_resp("789") == "789"
+        assert serializer.deserialize_resp("test_response") == "test_response"
+        assert serializer.deserialize_resp("[7, 8, 9]") == "[7, 8, 9]"
+        assert serializer.deserialize_resp("{'c': 3}") == "{'c': 3}"
+
+    def test_roundtrip(self, serializer: SimpleSerializer):
+        # Test serialization and deserialization roundtrip
+        original = "test_roundtrip"
+        serialized = serializer.serialize_resp(original)
+        deserialized = serializer.deserialize_resp(serialized)
+        assert deserialized == original
+
+    @pytest.mark.parametrize("input_value", [123, "test", [1, 2, 3], {"a": 1}, None])
+    def test_serialize_req_types(self, serializer: SimpleSerializer, input_value):
+        # Test serialize_req with various input types
+        result = serializer.serialize_req(req=input_value)
+        assert isinstance(result, str)
+        assert result == str(input_value)

tests/metagpt/roles/di/run_data_analyst.py

@@ -0,0 +1,54 @@
+from metagpt.roles.di.data_analyst import DataAnalyst
+
+HOUSE_PRICE_TRAIN_PATH = '/data/house-prices-advanced-regression-techniques/split_train.csv'
+HOUSE_PRICE_EVAL_PATH = '/data/house-prices-advanced-regression-techniques/split_eval.csv'
+HOUSE_PRICE_REQ = f"""
+This is a house price dataset, your goal is to predict the sale price of a property based on its features. The target column is SalePrice. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report RMSE between the logarithm of the predicted value and the logarithm of the observed sales price on the eval data. Train data path: '{HOUSE_PRICE_TRAIN_PATH}', eval data path: '{HOUSE_PRICE_EVAL_PATH}'.
+"""
+
+CALIFORNIA_HOUSING_REQ = """
+Analyze the 'Canifornia-housing-dataset' using https://scikit-learn.org/stable/modules/generated/sklearn.datasets.fetch_california_housing.html#sklearn.datasets.fetch_california_housing to predict the median house value. you need to perfrom data preprocessing, feature engineering and finally modeling to predict the target. Use machine learning techniques such as linear regression (including ridge regression and lasso regression), random forest, CatBoost, LightGBM, XGBoost or other appropriate method. You also need to report the MSE on the test dataset
+"""
+
+# For web scraping task, please provide url begin with `https://` or `http://`
+PAPER_LIST_REQ = """"
+Get data from `paperlist` table in https://papercopilot.com/statistics/iclr-statistics/iclr-2024-statistics/,
+and save it to a csv file. paper title must include `multiagent` or `large language model`.
+**Notice: view the page element before writing scraping code**
+"""
+
+ECOMMERCE_REQ = """
+Get products data from website https://scrapeme.live/shop/ and save it as a csv file.
+The first page product name, price, product URL, and image URL must be saved in the csv.
+**Notice: view the page element before writing scraping code**
+"""
+
+NEWS_36KR_REQ = """从36kr创投平台https://pitchhub.36kr.com/financing-flash 所有初创企业融资的信息, **注意: 这是一个中文网站**;
+下面是一个大致流程, 你会根据每一步的运行结果对当前计划中的任务做出适当调整:
+1. 爬取并本地保存html结构;
+2. 直接打印第7个*`快讯`*关键词后2000个字符的html内容, 作为*快讯的html内容示例*;
+3. 反思*快讯的html内容示例*中的规律, 设计正则匹配表达式来获取*`快讯`*的标题、链接、时间;
+4. 筛选最近3天的初创企业融资*`快讯`*, 以list[dict]形式打印前5个。
+5. 将全部结果存在本地csv中
+**Notice: view the page element before writing scraping code**
+"""
+
+WIKIPEDIA_SEARCH_REQ = """
+Search for `LLM` on https://www.wikipedia.org/ and print all the meaningful significances of the entry.
+"""
+
+STACKOVERFLOW_CLICK_REQ = """
+Click the Questions tag in https://stackoverflow.com/ and scrap question name, votes, answers and views num to csv in the first result page.
+"""
+
+
+async def main():
+    di = DataAnalyst()
+    await di.browser.start()
+    await di.run(STACKOVERFLOW_CLICK_REQ)
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

tests/metagpt/roles/di/run_swe_agent_for_benchmark.py

@@ -55,15 +55,15 @@ async def run(instance, swe_result_dir):
         logger.info(f"Instance {instance['instance_id']} already exists, skipping execution.")
         return
 
-    repo_path = TEST_REPO_DIR / instance["repo"].replace("-", "_").replace("/", "__") + "_" + instance["version"]
+    repo_path = TEST_REPO_DIR / (instance["repo"].replace("-", "_").replace("/", "__") + "_" + instance["version"])
 
     # 前处理
     terminal = Terminal()
-    terminal.run_command(f"cd {repo_path} && git reset --hard && git clean -n -d && git clean -f -d")
-    terminal.run_command("BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}')")
-    logger.info(terminal.run_command("echo $BRANCH"))
-    logger.info(terminal.run_command('git checkout "$BRANCH"'))
-    logger.info(terminal.run_command("git branch"))
+    await terminal.run_command(f"cd {repo_path} && git reset --hard && git clean -n -d && git clean -f -d")
+    await terminal.run_command("BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}')")
+    logger.info(await terminal.run_command("echo $BRANCH"))
+    logger.info(await terminal.run_command('git checkout "$BRANCH"'))
+    logger.info(await terminal.run_command("git branch"))
 
     user_requirement_and_issue = INSTANCE_TEMPLATE.format(
         issue=instance["problem_statement"],

tests/metagpt/tools/libs/test_terminal.py

@@ -4,16 +4,17 @@ from metagpt.const import DATA_PATH, METAGPT_ROOT
 from metagpt.tools.libs.terminal import Terminal
 
 
-def test_terminal():
+@pytest.mark.asyncio
+async def test_terminal():
     terminal = Terminal()
 
-    terminal.run_command(f"cd {METAGPT_ROOT}")
-    output = terminal.run_command("pwd")
+    await terminal.run_command(f"cd {METAGPT_ROOT}")
+    output = await terminal.run_command("pwd")
     assert output.strip() == str(METAGPT_ROOT)
 
     # pwd now should be METAGPT_ROOT, cd data should land in DATA_PATH
-    terminal.run_command("cd data")
-    output = terminal.run_command("pwd")
+    await terminal.run_command("cd data")
+    output = await terminal.run_command("pwd")
     assert output.strip() == str(DATA_PATH)