feat: parse class view

.github/workflows/unittest.yaml

@@ -23,15 +23,21 @@ jobs:
     - name: Test with pytest
       run: |
         echo "${{ secrets.METAGPT_KEY_YAML }}" | base64 -d > config/key.yaml
-        pytest tests/ --doctest-modules --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov=./metagpt/ --cov-report=xml:cov.xml --cov-report=html:htmlcov --durations=20
+        pytest tests/ --doctest-modules --cov=./metagpt/ --cov-report=xml:cov.xml --cov-report=html:htmlcov --durations=20 | tee unittest.txt
+    - name: Show coverage report
+      run: |
         coverage report -m
+    - name: Show failed tests and overall summary
+      run: |
+        grep -E "FAILED tests|[0-9]+ passed," unittest.txt
     - name: Upload pytest test results
       uses: actions/upload-artifact@v3
       with:
         name: pytest-results-${{ matrix.python-version }}
         path: |
-          ./junit/test-results-${{ matrix.python-version }}.xml
+          ./unittest.txt
           ./htmlcov/
+          ./tests/data/rsp_cache_new.json
         retention-days: 3
       if: ${{ always() }}
       

.gitignore

@@ -52,6 +52,7 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 cover/
+unittest.txt
 
 # Translations
 *.mo
@@ -173,4 +174,6 @@ htmlcov
 htmlcov.*
 *.dot
 *.pkl
+*-structure.csv
+*-structure.json
 

docs/scripts/coverage.sh

@@ -1 +1 @@
-coverage run --source ./metagpt -m pytest --durations=0 && coverage report -m && coverage html && open htmlcov/index.html
+coverage run --source ./metagpt -m pytest --durations=0 --timeout=100 && coverage report -m && coverage html && open htmlcov/index.html

metagpt/repo_parser.py

@@ -12,13 +12,11 @@ import json
 import re
 import subprocess
 from pathlib import Path
-from pprint import pformat
 from typing import Dict, List, Optional
 
 import pandas as pd
 from pydantic import BaseModel, Field
 
-from metagpt.config import CONFIG
 from metagpt.const import AGGREGATION, COMPOSITION, GENERALIZATION
 from metagpt.logs import logger
 from metagpt.utils.common import any_to_str, aread
@@ -99,16 +97,16 @@ class RepoParser(BaseModel):
 
     def generate_json_structure(self, output_path):
         """Generate a JSON file documenting the repository structure."""
-        files_classes = self.generate_symbols()
+        files_classes = [i.model_dump() for i in self.generate_symbols()]
         output_path.write_text(json.dumps(files_classes, indent=4))
 
     def generate_dataframe_structure(self, output_path):
         """Generate a DataFrame documenting the repository structure and save as CSV."""
-        files_classes = self.generate_symbols()
+        files_classes = [i.model_dump() for i in self.generate_symbols()]
         df = pd.DataFrame(files_classes)
         df.to_csv(output_path, index=False)
 
-    def generate_structure(self, output_path=None, mode="json"):
+    def generate_structure(self, output_path=None, mode="json") -> Path:
         """Generate the structure of the repository as a specified format."""
         output_file = self.base_directory / f"{self.base_directory.name}-structure.{mode}"
         output_path = Path(output_path) if output_path else output_file
@@ -117,6 +115,7 @@ class RepoParser(BaseModel):
             self.generate_json_structure(output_path)
         elif mode == "csv":
             self.generate_dataframe_structure(output_path)
+        return output_path
 
     @staticmethod
     def node_to_str(node) -> CodeBlockInfo | None:
@@ -420,18 +419,3 @@ class RepoParser(BaseModel):
 
 def is_func(node):
     return isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
-
-
-def main():
-    repo_parser = RepoParser(base_directory=CONFIG.workspace_path / "web_2048")
-    symbols = repo_parser.generate_symbols()
-    logger.info(pformat(symbols))
-
-
-def error():
-    """raise Exception and logs it"""
-    RepoParser._parse_file(Path("test.py"))
-
-
-if __name__ == "__main__":
-    main()

metagpt/strategy/base.py

@@ -2,13 +2,14 @@
 # @Date    : 12/25/2023 9:16 PM
 # @Author  : stellahong (stellahong@fuzhi.ai)
 # @Desc    :
+from abc import ABC
 from typing import List
 
 from anytree import Node, RenderTree
 from pydantic import BaseModel
 
 
-class BaseParser(BaseModel):
+class BaseParser(BaseModel, ABC):
     def __call__(self, *args, **kwargs):
         raise NotImplementedError
 
@@ -22,7 +23,7 @@ class BaseParser(BaseModel):
         raise NotImplementedError
 
 
-class BaseEvaluator(BaseModel):
+class BaseEvaluator(BaseModel, ABC):
     def __call__(self, *args, **kwargs):
         raise NotImplementedError
 

metagpt/strategy/tot.py

@@ -2,8 +2,10 @@
 # @Date    : 12/23/2023 4:51 PM
 # @Author  : stellahong (stellahong@fuzhi.ai)
 # @Desc    :
+from __future__ import annotations
+
 import asyncio
-from typing import Any, List
+from typing import Any, List, Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -15,7 +17,7 @@ from metagpt.strategy.tot_schema import MethodSelect, Strategy, ThoughtSolverCon
 from metagpt.utils.common import CodeParser
 
 OUTPUT_FORMAT = """
-Output a list of jsons following the format:
+Each output should be strictly a list of nodes, in json format, like this:
 ```json
     [
         {
@@ -31,7 +33,7 @@ Output a list of jsons following the format:
 class ThoughtSolverBase(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
-    thought_tree: str = ""
+    thought_tree: Optional[ThoughtTree] = Field(default=None)
     llm: BaseLLM = Field(default_factory=LLM, exclude=True)
     config: ThoughtSolverConfig = Field(default_factory=ThoughtSolverConfig)
 
@@ -60,7 +62,7 @@ class ThoughtSolverBase(BaseModel):
             current_state=current_state, **{"n_generate_sample": self.config.n_generate_sample}
         )
         rsp = await self.llm.aask(msg=state_prompt + "\n" + OUTPUT_FORMAT)
-        thoughts = CodeParser.parse_code(block=None, text=rsp)
+        thoughts = CodeParser.parse_code(block="", text=rsp)
         thoughts = eval(thoughts)
         # fixme 避免不跟随，生成过多nodes
         # valid_thoughts = [_node for idx, _node in enumerate(thoughts) if idx < self.n_generate_sample]
@@ -97,15 +99,16 @@ class ThoughtSolverBase(BaseModel):
         Returns:
             List[ThoughtNode]: List of selected nodes.
         """
-        # selection
+        # nodes to be selected
+        nodes = []
         if self.config.method_select == MethodSelect.SAMPLE:
             raise NotImplementedError
         elif self.config.method_select == MethodSelect.GREEDY:
-            select_nodes = sorted(thought_nodes, key=lambda x: x.value, reverse=True)[: self.config.n_select_sample]
+            nodes = sorted(thought_nodes, key=lambda x: x.value, reverse=True)[: self.config.n_select_sample]
         for node in thought_nodes:
-            if node not in select_nodes:
+            if node not in nodes:
                 node.parent = None  # 从树中删除节点
-        return select_nodes
+        return nodes
 
     def update_solution(self):
         """

metagpt/tools/search_engine_serpapi.py

@@ -18,7 +18,7 @@ class SerpAPIWrapper(BaseModel):
 
     search_engine: Any = None  #: :meta private:
     params: dict = Field(
-        default={
+        default_factory=lambda: {
             "engine": "google",
             "google_domain": "google.com",
             "gl": "us",

metagpt/tools/search_engine_serper.py

@@ -9,14 +9,16 @@ import json
 from typing import Any, Dict, Optional, Tuple
 
 import aiohttp
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 from metagpt.config import CONFIG
 
 
 class SerperWrapper(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     search_engine: Any = None  #: :meta private:
-    payload: dict = Field(default={"page": 1, "num": 10})
+    payload: dict = Field(default_factory=lambda: {"page": 1, "num": 10})
     serper_api_key: Optional[str] = Field(default=None, validate_default=True)
     aiosession: Optional[aiohttp.ClientSession] = None
 

setup.py

@@ -39,6 +39,7 @@ extras_require["test"] = [
     "pytest-mock",
     "pytest-html",
     "pytest-xdist",
+    "pytest-timeout",
     "connexion[uvicorn]~=3.0.5",
     "azure-cognitiveservices-speech~=1.31.0",
     "aioboto3~=11.3.0",

tests/conftest.py

@@ -7,20 +7,87 @@
 """
 
 import asyncio
+import json
 import logging
+import os
 import re
 import uuid
-from unittest.mock import Mock
+from typing import Optional
 
 import pytest
 
 from metagpt.config import CONFIG, Config
-from metagpt.const import DEFAULT_WORKSPACE_ROOT
+from metagpt.const import DEFAULT_WORKSPACE_ROOT, TEST_DATA_PATH
 from metagpt.llm import LLM
 from metagpt.logs import logger
+from metagpt.provider.openai_api import OpenAILLM
 from metagpt.utils.git_repository import GitRepository
 
 
+class MockLLM(OpenAILLM):
+    rsp_cache: dict = {}
+
+    async def original_aask(
+        self,
+        msg: str,
+        system_msgs: Optional[list[str]] = None,
+        format_msgs: Optional[list[dict[str, str]]] = None,
+        timeout=3,
+        stream=True,
+    ):
+        """A copy of metagpt.provider.base_llm.BaseLLM.aask, we can't use super().aask because it will be mocked"""
+        if system_msgs:
+            message = self._system_msgs(system_msgs)
+        else:
+            message = [self._default_system_msg()] if self.use_system_prompt else []
+        if format_msgs:
+            message.extend(format_msgs)
+        message.append(self._user_msg(msg))
+        rsp = await self.acompletion_text(message, stream=stream, timeout=timeout)
+        return rsp
+
+    async def aask(
+        self,
+        msg: str,
+        system_msgs: Optional[list[str]] = None,
+        format_msgs: Optional[list[dict[str, str]]] = None,
+        timeout=3,
+        stream=True,
+    ) -> str:
+        if msg not in self.rsp_cache:
+            # Call the original unmocked method
+            rsp = await self.original_aask(msg, system_msgs, format_msgs, timeout, stream)
+            logger.info(f"Added '{rsp[:20]}' ... to response cache")
+            self.rsp_cache[msg] = rsp
+            return rsp
+        else:
+            logger.info("Use response cache")
+            return self.rsp_cache[msg]
+
+
+@pytest.fixture(scope="session")
+def rsp_cache():
+    # model_version = CONFIG.openai_api_model
+    rsp_cache_file_path = TEST_DATA_PATH / "rsp_cache.json"  # read repo-provided
+    new_rsp_cache_file_path = TEST_DATA_PATH / "rsp_cache_new.json"  # exporting a new copy
+    if os.path.exists(rsp_cache_file_path):
+        with open(rsp_cache_file_path, "r") as f1:
+            rsp_cache_json = json.load(f1)
+    else:
+        rsp_cache_json = {}
+    yield rsp_cache_json
+    with open(new_rsp_cache_file_path, "w") as f2:
+        json.dump(rsp_cache_json, f2, indent=4, ensure_ascii=False)
+
+
+@pytest.fixture(scope="function")
+def llm_mock(rsp_cache, mocker):
+    llm = MockLLM()
+    llm.rsp_cache = rsp_cache
+    mocker.patch("metagpt.provider.base_llm.BaseLLM.aask", llm.aask)
+    yield mocker
+
+
 class Context:
     def __init__(self):
         self._llm_ui = None
@@ -45,12 +112,6 @@ def llm_api():
     logger.info("Tearing down the test")
 
 
-@pytest.fixture(scope="function")
-def mock_llm():
-    # Create a mock LLM for testing
-    return Mock()
-
-
 @pytest.fixture(scope="session")
 def proxy():
     pattern = re.compile(

tests/metagpt/actions/test_debug_error.py

@@ -117,6 +117,7 @@ if __name__ == '__main__':
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_debug_error():
     CONFIG.src_workspace = CONFIG.git_repo.workdir / uuid.uuid4().hex
     ctx = RunCodeContext(

tests/metagpt/actions/test_design_api.py

@@ -17,6 +17,7 @@ from tests.metagpt.actions.mock_markdown import PRD_SAMPLE
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_design_api():
     inputs = ["我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。", PRD_SAMPLE]
     for prd in inputs:

tests/metagpt/actions/test_design_api_review.py

@@ -11,6 +11,7 @@ from metagpt.actions.design_api_review import DesignReview
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_design_api_review():
     prd = "我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。"
     api_design = """

tests/metagpt/actions/test_generate_questions.py

@@ -20,6 +20,7 @@ context = """
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_generate_questions():
     action = GenerateQuestions()
     rsp = await action.run(context)

tests/metagpt/actions/test_invoice_ocr.py

@@ -54,6 +54,7 @@ async def test_generate_table(invoice_path: Path, expected_result: dict):
     ("invoice_path", "query", "expected_result"),
     [(Path("invoices/invoice-1.pdf"), "Invoicing date", "2023年02月03日")],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_reply_question(invoice_path: Path, query: dict, expected_result: str):
     invoice_path = TEST_DATA_PATH / invoice_path
     ocr_result = await InvoiceOCR().run(file_path=Path(invoice_path))

tests/metagpt/actions/test_prepare_interview.py

@@ -12,6 +12,7 @@ from metagpt.logs import logger
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_prepare_interview():
     action = PrepareInterview()
     rsp = await action.run("I just graduated and hope to find a job as a Python engineer")

tests/metagpt/actions/test_project_management.py

@@ -18,6 +18,7 @@ from tests.metagpt.actions.mock_json import DESIGN, PRD
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_design_api():
     await FileRepository.save_file("1.txt", content=str(PRD), relative_path=PRDS_FILE_REPO)
     await FileRepository.save_file("1.txt", content=str(DESIGN), relative_path=SYSTEM_DESIGN_FILE_REPO)

tests/metagpt/actions/test_summarize_code.py

@@ -177,6 +177,7 @@ class Snake:
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_summarize_code():
     CONFIG.src_workspace = CONFIG.git_repo.workdir / "src"
     await FileRepository.save_file(filename="1.json", relative_path=SYSTEM_DESIGN_FILE_REPO, content=DESIGN_CONTENT)

tests/metagpt/actions/test_talk_action.py

@@ -33,6 +33,7 @@ from metagpt.schema import Message
         ),
     ],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_prompt(agent_description, language, context, knowledge, history_summary):
     # Prerequisites
     CONFIG.agent_description = agent_description

tests/metagpt/actions/test_write_code.py

@@ -28,6 +28,7 @@ from tests.metagpt.actions.mock_markdown import TASKS_2, WRITE_CODE_PROMPT_SAMPL
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code():
     context = CodingContext(
         filename="task_filename.py", design_doc=Document(content="设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。")
@@ -44,6 +45,7 @@ async def test_write_code():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code_directly():
     prompt = WRITE_CODE_PROMPT_SAMPLE + "\n" + TASKS_2[0]
     llm = LLM()
@@ -52,6 +54,7 @@ async def test_write_code_directly():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code_deps():
     # Prerequisites
     CONFIG.src_workspace = CONFIG.git_repo.workdir / "snake1/snake1"

tests/metagpt/actions/test_write_code_review.py

@@ -12,6 +12,7 @@ from metagpt.schema import CodingContext, Document
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code_review(capfd):
     code = """
 def add(a, b):

tests/metagpt/actions/test_write_docstring.py

@@ -27,12 +27,14 @@ class Person:
     ],
     ids=["google", "numpy", "sphinx"],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_docstring(style: str, part: str):
     ret = await WriteDocstring().run(code, style=style)
     assert part in ret
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write():
     code = await WriteDocstring.write_docstring(__file__)
     assert code

tests/metagpt/actions/test_write_prd.py

@@ -18,6 +18,7 @@ from metagpt.utils.file_repository import FileRepository
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_prd():
     product_manager = ProductManager()
     requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结"

tests/metagpt/actions/test_write_prd_review.py

@@ -11,6 +11,7 @@ from metagpt.actions.write_prd_review import WritePRDReview
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_prd_review():
     prd = """
     Introduction: This is a new feature for our product.

tests/metagpt/actions/test_write_review.py

@@ -46,6 +46,7 @@ CONTEXT = """
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_review():
     write_review = WriteReview()
     review = await write_review.run(CONTEXT)

tests/metagpt/actions/test_write_teaching_plan.py

@@ -16,6 +16,7 @@ from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart
     ("topic", "context"),
     [("Title", "Lesson 1: Learn to draw an apple."), ("Teaching Content", "Lesson 1: Learn to draw an apple.")],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_teaching_plan_part(topic, context):
     action = WriteTeachingPlanPart(topic=topic, context=context)
     rsp = await action.run()

tests/metagpt/actions/test_write_test.py

@@ -13,6 +13,7 @@ from metagpt.schema import Document, TestingContext
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_test():
     code = """
     import random
@@ -39,6 +40,7 @@ async def test_write_test():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code_invalid_code(mocker):
     # Mock the _aask method to return an invalid code string
     mocker.patch.object(WriteTest, "_aask", return_value="Invalid Code String")

tests/metagpt/actions/test_write_tutorial.py

@@ -14,6 +14,7 @@ from metagpt.actions.write_tutorial import WriteContent, WriteDirectory
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(("language", "topic"), [("English", "Write a tutorial about Python")])
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_directory(language: str, topic: str):
     ret = await WriteDirectory(language=language).run(topic=topic)
     assert isinstance(ret, dict)
@@ -29,6 +30,7 @@ async def test_write_directory(language: str, topic: str):
     ("language", "topic", "directory"),
     [("English", "Write a tutorial about Python", {"Introduction": ["What is Python?", "Why learn Python?"]})],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_content(language: str, topic: str, directory: Dict):
     ret = await WriteContent(language=language, directory=directory).run(topic=topic)
     assert isinstance(ret, str)

tests/metagpt/provider/test_zhipuai_api.py

@@ -84,10 +84,6 @@ async def test_zhipuai_acompletion(mocker):
 
 
 def test_zhipuai_proxy():
-    import openai
-
-    from metagpt.config import CONFIG
-
-    CONFIG.openai_proxy = "http://127.0.0.1:8080"
+    # CONFIG.openai_proxy = "http://127.0.0.1:8080"
     _ = ZhiPuAILLM()
-    assert openai.proxy == CONFIG.openai_proxy
+    # assert openai.proxy == CONFIG.openai_proxy

tests/metagpt/roles/test_architect.py

@@ -22,6 +22,7 @@ from tests.metagpt.roles.mock import MockMessages
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_architect():
     # Prerequisites
     filename = uuid.uuid4().hex + ".json"

tests/metagpt/roles/test_assistant.py

@@ -21,6 +21,7 @@ from metagpt.utils.common import any_to_str
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_run():
     CONFIG.language = "Chinese"
 

tests/metagpt/roles/test_engineer.py

@@ -30,6 +30,7 @@ from tests.metagpt.roles.mock import STRS_FOR_PARSING, TASKS, MockMessages
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_engineer():
     # Prerequisites
     rqno = "20231221155954.json"
@@ -113,6 +114,7 @@ def test_todo():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_new_coding_context():
     # Prerequisites
     demo_path = Path(__file__).parent / "../../data/demo_project"

tests/metagpt/roles/test_invoice_ocr_assistant.py

@@ -41,6 +41,7 @@ from metagpt.schema import Message
         ),
     ],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_invoice_ocr_assistant(query: str, invoice_path: Path, invoice_table_path: Path, expected_result: dict):
     invoice_path = TEST_DATA_PATH / invoice_path
     role = InvoiceOCRAssistant()

tests/metagpt/roles/test_product_manager.py

@@ -13,6 +13,7 @@ from tests.metagpt.roles.mock import MockMessages
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_product_manager():
     product_manager = ProductManager()
     rsp = await product_manager.run(MockMessages.req)

tests/metagpt/roles/test_project_manager.py

@@ -13,6 +13,7 @@ from tests.metagpt.roles.mock import MockMessages
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_project_manager():
     project_manager = ProjectManager()
     rsp = await project_manager.run(MockMessages.system_design)

tests/metagpt/roles/test_teacher.py

@@ -103,6 +103,7 @@ async def test_new_file_name():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_run():
     CONFIG.set_context({"language": "Chinese", "teaching_language": "English"})
     lesson = """

tests/metagpt/roles/test_tutorial_assistant.py

@@ -15,6 +15,7 @@ from metagpt.roles.tutorial_assistant import TutorialAssistant
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(("language", "topic"), [("Chinese", "Write a tutorial about pip")])
+@pytest.mark.usefixtures("llm_mock")
 async def test_tutorial_assistant(language: str, topic: str):
     role = TutorialAssistant(language=language)
     msg = await role.run(topic)

tests/metagpt/serialize_deserialize/test_action.py

@@ -21,12 +21,13 @@ def test_action_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_action_deserialize():
     action = Action()
     serialized_data = action.model_dump()
 
     new_action = Action(**serialized_data)
 
-    assert new_action.name == ""
+    assert new_action.name == "Action"
     assert isinstance(new_action.llm, type(LLM()))
     assert len(await new_action._aask("who are you")) > 0

tests/metagpt/serialize_deserialize/test_architect_deserialize.py

@@ -17,6 +17,7 @@ def test_architect_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_architect_deserialize():
     role = Architect()
     ser_role_dict = role.model_dump(by_alias=True)

tests/metagpt/serialize_deserialize/test_prepare_interview.py

@@ -8,6 +8,7 @@ from metagpt.actions.prepare_interview import PrepareInterview
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_action_deserialize():
     action = PrepareInterview()
     serialized_data = action.model_dump()

tests/metagpt/serialize_deserialize/test_product_manager.py

@@ -10,6 +10,7 @@ from metagpt.schema import Message
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_product_manager_deserialize():
     role = ProductManager()
     ser_role_dict = role.model_dump(by_alias=True)

tests/metagpt/serialize_deserialize/test_project_manager.py

@@ -18,6 +18,7 @@ def test_project_manager_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_project_manager_deserialize():
     role = ProjectManager()
     ser_role_dict = role.model_dump(by_alias=True)

tests/metagpt/serialize_deserialize/test_role.py

@@ -69,6 +69,7 @@ def test_engineer_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_engineer_deserialize():
     role = Engineer(use_code_review=True)
     ser_role_dict = role.model_dump()
@@ -96,6 +97,7 @@ def test_role_serdeser_save():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_role_serdeser_interrupt():
     role_c = RoleC()
     shutil.rmtree(SERDESER_PATH.joinpath("team"), ignore_errors=True)

tests/metagpt/serialize_deserialize/test_team.py

@@ -109,6 +109,7 @@ async def test_team_recover_save():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_team_recover_multi_roles_save():
     idea = "write a snake game"
     stg_path = SERDESER_PATH.joinpath("team")

tests/metagpt/serialize_deserialize/test_write_code.py

@@ -17,6 +17,7 @@ def test_write_design_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code_deserialize():
     context = CodingContext(
         filename="test_code.py", design_doc=Document(content="write add function to calculate two numbers")

tests/metagpt/serialize_deserialize/test_write_code_review.py

@@ -9,6 +9,7 @@ from metagpt.schema import CodingContext, Document
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code_review_deserialize():
     code_content = """
 def div(a: int, b: int = 0):

tests/metagpt/serialize_deserialize/test_write_design.py

@@ -22,18 +22,20 @@ def test_write_task_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_design_deserialize():
     action = WriteDesign()
     serialized_data = action.model_dump()
     new_action = WriteDesign(**serialized_data)
-    assert new_action.name == ""
+    assert new_action.name == "WriteDesign"
     await new_action.run(with_messages="write a cli snake game")
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_task_deserialize():
     action = WriteTasks()
     serialized_data = action.model_dump()
     new_action = WriteTasks(**serialized_data)
-    assert new_action.name == "CreateTasks"
+    assert new_action.name == "WriteTasks"
     await new_action.run(with_messages="write a cli snake game")

tests/metagpt/serialize_deserialize/test_write_docstring.py

@@ -29,6 +29,7 @@ class Person:
     ],
     ids=["google", "numpy", "sphinx"],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_action_deserialize(style: str, part: str):
     action = WriteDocstring()
     serialized_data = action.model_dump()
@@ -38,7 +39,7 @@ async def test_action_deserialize(style: str, part: str):
 
     new_action = WriteDocstring(**serialized_data)
 
-    assert not new_action.name
+    assert new_action.name == "WriteDocstring"
     assert new_action.desc == "Write docstring for code."
     ret = await new_action.run(code, style=style)
     assert part in ret

tests/metagpt/serialize_deserialize/test_write_prd.py

@@ -17,6 +17,7 @@ def test_action_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_action_deserialize():
     action = WritePRD()
     serialized_data = action.model_dump()

tests/metagpt/serialize_deserialize/test_write_review.py

@@ -42,6 +42,7 @@ CONTEXT = """
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_action_deserialize():
     action = WriteReview()
     serialized_data = action.model_dump()

tests/metagpt/serialize_deserialize/test_write_tutorial.py

@@ -9,6 +9,7 @@ from metagpt.actions.write_tutorial import WriteContent, WriteDirectory
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(("language", "topic"), [("English", "Write a tutorial about Python")])
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_directory_deserialize(language: str, topic: str):
     action = WriteDirectory()
     serialized_data = action.model_dump()
@@ -30,6 +31,7 @@ async def test_write_directory_deserialize(language: str, topic: str):
     ("language", "topic", "directory"),
     [("English", "Write a tutorial about Python", {"Introduction": ["What is Python?", "Why learn Python?"]})],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_content_deserialize(language: str, topic: str, directory: Dict):
     action = WriteContent(language=language, directory=directory)
     serialized_data = action.model_dump()

tests/metagpt/strategy/examples/creative_writing.py

@@ -71,7 +71,7 @@ def test_creative_writing():
     parser = TextGenParser()
     evaluator = TextGenEvaluator()
 
-    config = ThoughtSolverConfig(n_generate_sample=3, parser=parser, evaluator=evaluator)
+    config = ThoughtSolverConfig(max_step=2, n_generate_sample=1, n_select_sample=1, parser=parser, evaluator=evaluator)
 
     tot_base = TreeofThought(strategy=Strategy.BFS, config=config)
     asyncio.run(tot_base.solve(init_prompt=initial_prompt))

tests/metagpt/strategy/prompt_templates/creative_writing.py

@@ -5,13 +5,13 @@ Write a coherent passage of 4 short paragraphs. The end sentence of each paragra
 cot_prompt = """
 Write a coherent passage of 4 short paragraphs. The end sentence of each paragraph must be: {input}
 
-Make a plan then write. Your output should be of the following format:
+Make a plan then write. Your output should be like:
 
 Plan:
-Your plan here.
+<Your plan here with json format>
 
 Passage:
-Your passage here.
+<Your passage here with json format>
 """
 
 

tests/metagpt/test_repo_parser.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+from pprint import pformat
+
+from metagpt.const import METAGPT_ROOT
+from metagpt.logs import logger
+from metagpt.repo_parser import RepoParser
+
+
+def test_repo_parser():
+    repo_parser = RepoParser(base_directory=METAGPT_ROOT / "metagpt" / "strategy")
+    symbols = repo_parser.generate_symbols()
+    logger.info(pformat(symbols))
+
+    assert "tot_schema.py" in str(symbols)
+
+    output_path = repo_parser.generate_structure(mode="json")
+    assert output_path.exists()
+    output_path = repo_parser.generate_structure(mode="csv")
+    assert output_path.exists()
+
+
+def test_error():
+    """_parse_file should return empty list when file not existed"""
+    rsp = RepoParser._parse_file(Path("test_not_existed_file.py"))
+    assert rsp == []

tests/metagpt/tools/test_search_engine.py

@@ -58,7 +58,7 @@ async def test_search_engine(search_engine_type, run_func: Callable, max_results
         assert isinstance(rsp, str)
     else:
         assert isinstance(rsp, list)
-        assert len(rsp) == max_results
+        assert len(rsp) <= max_results
 
 
 if __name__ == "__main__":

tests/metagpt/tools/test_translate.py

@@ -14,6 +14,7 @@ from metagpt.tools.translator import Translator
 
 @pytest.mark.asyncio
 @pytest.mark.usefixtures("llm_api")
+@pytest.mark.usefixtures("llm_mock")
 async def test_translate(llm_api):
     poetries = [
         ("Let life be beautiful like summer flowers", "花"),