Merge pull request #669 from garylin2099/llm_cache

Add llm response cache to speed up testing

tests/metagpt/actions/test_debug_error.py

@@ -117,6 +117,7 @@ if __name__ == '__main__':
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_debug_error():
     CONFIG.src_workspace = CONFIG.git_repo.workdir / uuid.uuid4().hex
     ctx = RunCodeContext(

tests/metagpt/actions/test_design_api.py

@@ -17,6 +17,7 @@ from tests.metagpt.actions.mock_markdown import PRD_SAMPLE
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_design_api():
     inputs = ["我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。", PRD_SAMPLE]
     for prd in inputs:

tests/metagpt/actions/test_design_api_review.py

@@ -11,6 +11,7 @@ from metagpt.actions.design_api_review import DesignReview
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_design_api_review():
     prd = "我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。"
     api_design = """

tests/metagpt/actions/test_generate_questions.py

@@ -20,6 +20,7 @@ context = """
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_generate_questions():
     action = GenerateQuestions()
     rsp = await action.run(context)

tests/metagpt/actions/test_invoice_ocr.py

@@ -54,6 +54,7 @@ async def test_generate_table(invoice_path: Path, expected_result: dict):
     ("invoice_path", "query", "expected_result"),
     [(Path("invoices/invoice-1.pdf"), "Invoicing date", "2023年02月03日")],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_reply_question(invoice_path: Path, query: dict, expected_result: str):
     invoice_path = TEST_DATA_PATH / invoice_path
     ocr_result = await InvoiceOCR().run(file_path=Path(invoice_path))

tests/metagpt/actions/test_prepare_interview.py

@@ -12,6 +12,7 @@ from metagpt.logs import logger
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_prepare_interview():
     action = PrepareInterview()
     rsp = await action.run("I just graduated and hope to find a job as a Python engineer")

tests/metagpt/actions/test_project_management.py

@@ -18,6 +18,7 @@ from tests.metagpt.actions.mock_json import DESIGN, PRD
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_design_api():
     await FileRepository.save_file("1.txt", content=str(PRD), relative_path=PRDS_FILE_REPO)
     await FileRepository.save_file("1.txt", content=str(DESIGN), relative_path=SYSTEM_DESIGN_FILE_REPO)

tests/metagpt/actions/test_summarize_code.py

@@ -177,6 +177,7 @@ class Snake:
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_summarize_code():
     CONFIG.src_workspace = CONFIG.git_repo.workdir / "src"
     await FileRepository.save_file(filename="1.json", relative_path=SYSTEM_DESIGN_FILE_REPO, content=DESIGN_CONTENT)

tests/metagpt/actions/test_talk_action.py

@@ -33,6 +33,7 @@ from metagpt.schema import Message
         ),
     ],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_prompt(agent_description, language, context, knowledge, history_summary):
     # Prerequisites
     CONFIG.agent_description = agent_description

tests/metagpt/actions/test_write_code.py

@@ -28,6 +28,7 @@ from tests.metagpt.actions.mock_markdown import TASKS_2, WRITE_CODE_PROMPT_SAMPL
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code():
     context = CodingContext(
         filename="task_filename.py", design_doc=Document(content="设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。")
@@ -44,6 +45,7 @@ async def test_write_code():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code_directly():
     prompt = WRITE_CODE_PROMPT_SAMPLE + "\n" + TASKS_2[0]
     llm = LLM()
@@ -52,6 +54,7 @@ async def test_write_code_directly():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code_deps():
     # Prerequisites
     CONFIG.src_workspace = CONFIG.git_repo.workdir / "snake1/snake1"

tests/metagpt/actions/test_write_code_review.py

@@ -12,6 +12,7 @@ from metagpt.schema import CodingContext, Document
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code_review(capfd):
     code = """
 def add(a, b):

tests/metagpt/actions/test_write_docstring.py

@@ -27,12 +27,14 @@ class Person:
     ],
     ids=["google", "numpy", "sphinx"],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_docstring(style: str, part: str):
     ret = await WriteDocstring().run(code, style=style)
     assert part in ret
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write():
     code = await WriteDocstring.write_docstring(__file__)
     assert code

tests/metagpt/actions/test_write_prd.py

@@ -18,6 +18,7 @@ from metagpt.utils.file_repository import FileRepository
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_prd():
     product_manager = ProductManager()
     requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结"

tests/metagpt/actions/test_write_prd_review.py

@@ -11,6 +11,7 @@ from metagpt.actions.write_prd_review import WritePRDReview
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_prd_review():
     prd = """
     Introduction: This is a new feature for our product.

tests/metagpt/actions/test_write_review.py

@@ -46,6 +46,7 @@ CONTEXT = """
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_review():
     write_review = WriteReview()
     review = await write_review.run(CONTEXT)

tests/metagpt/actions/test_write_teaching_plan.py

@@ -16,6 +16,7 @@ from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart
     ("topic", "context"),
     [("Title", "Lesson 1: Learn to draw an apple."), ("Teaching Content", "Lesson 1: Learn to draw an apple.")],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_teaching_plan_part(topic, context):
     action = WriteTeachingPlanPart(topic=topic, context=context)
     rsp = await action.run()

tests/metagpt/actions/test_write_test.py

@@ -13,6 +13,7 @@ from metagpt.schema import Document, TestingContext
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_test():
     code = """
     import random
@@ -39,6 +40,7 @@ async def test_write_test():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code_invalid_code(mocker):
     # Mock the _aask method to return an invalid code string
     mocker.patch.object(WriteTest, "_aask", return_value="Invalid Code String")

tests/metagpt/actions/test_write_tutorial.py

@@ -14,6 +14,7 @@ from metagpt.actions.write_tutorial import WriteContent, WriteDirectory
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(("language", "topic"), [("English", "Write a tutorial about Python")])
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_directory(language: str, topic: str):
     ret = await WriteDirectory(language=language).run(topic=topic)
     assert isinstance(ret, dict)
@@ -29,6 +30,7 @@ async def test_write_directory(language: str, topic: str):
     ("language", "topic", "directory"),
     [("English", "Write a tutorial about Python", {"Introduction": ["What is Python?", "Why learn Python?"]})],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_content(language: str, topic: str, directory: Dict):
     ret = await WriteContent(language=language, directory=directory).run(topic=topic)
     assert isinstance(ret, str)

tests/metagpt/roles/test_architect.py

@@ -22,6 +22,7 @@ from tests.metagpt.roles.mock import MockMessages
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_architect():
     # Prerequisites
     filename = uuid.uuid4().hex + ".json"

tests/metagpt/roles/test_assistant.py

@@ -21,6 +21,7 @@ from metagpt.utils.common import any_to_str
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_run():
     CONFIG.language = "Chinese"
 

tests/metagpt/roles/test_engineer.py

@@ -30,6 +30,7 @@ from tests.metagpt.roles.mock import STRS_FOR_PARSING, TASKS, MockMessages
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_engineer():
     # Prerequisites
     rqno = "20231221155954.json"
@@ -113,6 +114,7 @@ def test_todo():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_new_coding_context():
     # Prerequisites
     demo_path = Path(__file__).parent / "../../data/demo_project"

tests/metagpt/roles/test_invoice_ocr_assistant.py

@@ -41,6 +41,7 @@ from metagpt.schema import Message
         ),
     ],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_invoice_ocr_assistant(query: str, invoice_path: Path, invoice_table_path: Path, expected_result: dict):
     invoice_path = TEST_DATA_PATH / invoice_path
     role = InvoiceOCRAssistant()

tests/metagpt/roles/test_product_manager.py

@@ -13,6 +13,7 @@ from tests.metagpt.roles.mock import MockMessages
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_product_manager():
     product_manager = ProductManager()
     rsp = await product_manager.run(MockMessages.req)

tests/metagpt/roles/test_project_manager.py

@@ -13,6 +13,7 @@ from tests.metagpt.roles.mock import MockMessages
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_project_manager():
     project_manager = ProjectManager()
     rsp = await project_manager.run(MockMessages.system_design)

tests/metagpt/roles/test_teacher.py

@@ -103,6 +103,7 @@ async def test_new_file_name():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_run():
     CONFIG.set_context({"language": "Chinese", "teaching_language": "English"})
     lesson = """

tests/metagpt/roles/test_tutorial_assistant.py

@@ -15,6 +15,7 @@ from metagpt.roles.tutorial_assistant import TutorialAssistant
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(("language", "topic"), [("Chinese", "Write a tutorial about pip")])
+@pytest.mark.usefixtures("llm_mock")
 async def test_tutorial_assistant(language: str, topic: str):
     role = TutorialAssistant(language=language)
     msg = await role.run(topic)

tests/metagpt/serialize_deserialize/test_action.py

@@ -21,6 +21,7 @@ def test_action_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_action_deserialize():
     action = Action()
     serialized_data = action.model_dump()

tests/metagpt/serialize_deserialize/test_architect_deserialize.py

@@ -17,6 +17,7 @@ def test_architect_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_architect_deserialize():
     role = Architect()
     ser_role_dict = role.model_dump(by_alias=True)

tests/metagpt/serialize_deserialize/test_prepare_interview.py

@@ -8,6 +8,7 @@ from metagpt.actions.prepare_interview import PrepareInterview
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_action_deserialize():
     action = PrepareInterview()
     serialized_data = action.model_dump()

tests/metagpt/serialize_deserialize/test_product_manager.py

@@ -10,6 +10,7 @@ from metagpt.schema import Message
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_product_manager_deserialize():
     role = ProductManager()
     ser_role_dict = role.model_dump(by_alias=True)

tests/metagpt/serialize_deserialize/test_project_manager.py

@@ -18,6 +18,7 @@ def test_project_manager_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_project_manager_deserialize():
     role = ProjectManager()
     ser_role_dict = role.model_dump(by_alias=True)

tests/metagpt/serialize_deserialize/test_role.py

@@ -69,6 +69,7 @@ def test_engineer_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_engineer_deserialize():
     role = Engineer(use_code_review=True)
     ser_role_dict = role.model_dump()
@@ -96,6 +97,7 @@ def test_role_serdeser_save():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_role_serdeser_interrupt():
     role_c = RoleC()
     shutil.rmtree(SERDESER_PATH.joinpath("team"), ignore_errors=True)

tests/metagpt/serialize_deserialize/test_team.py

@@ -109,6 +109,7 @@ async def test_team_recover_save():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_team_recover_multi_roles_save():
     idea = "write a snake game"
     stg_path = SERDESER_PATH.joinpath("team")

tests/metagpt/serialize_deserialize/test_write_code.py

@@ -17,6 +17,7 @@ def test_write_design_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code_deserialize():
     context = CodingContext(
         filename="test_code.py", design_doc=Document(content="write add function to calculate two numbers")

tests/metagpt/serialize_deserialize/test_write_code_review.py

@@ -9,6 +9,7 @@ from metagpt.schema import CodingContext, Document
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_code_review_deserialize():
     code_content = """
 def div(a: int, b: int = 0):

tests/metagpt/serialize_deserialize/test_write_design.py

@@ -22,6 +22,7 @@ def test_write_task_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_design_deserialize():
     action = WriteDesign()
     serialized_data = action.model_dump()
@@ -31,6 +32,7 @@ async def test_write_design_deserialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_task_deserialize():
     action = WriteTasks()
     serialized_data = action.model_dump()

tests/metagpt/serialize_deserialize/test_write_docstring.py

@@ -29,6 +29,7 @@ class Person:
     ],
     ids=["google", "numpy", "sphinx"],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_action_deserialize(style: str, part: str):
     action = WriteDocstring()
     serialized_data = action.model_dump()

tests/metagpt/serialize_deserialize/test_write_prd.py

@@ -17,6 +17,7 @@ def test_action_serialize():
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_action_deserialize():
     action = WritePRD()
     serialized_data = action.model_dump()

tests/metagpt/serialize_deserialize/test_write_review.py

@@ -42,6 +42,7 @@ CONTEXT = """
 
 
 @pytest.mark.asyncio
+@pytest.mark.usefixtures("llm_mock")
 async def test_action_deserialize():
     action = WriteReview()
     serialized_data = action.model_dump()

tests/metagpt/serialize_deserialize/test_write_tutorial.py

@@ -9,6 +9,7 @@ from metagpt.actions.write_tutorial import WriteContent, WriteDirectory
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(("language", "topic"), [("English", "Write a tutorial about Python")])
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_directory_deserialize(language: str, topic: str):
     action = WriteDirectory()
     serialized_data = action.model_dump()
@@ -30,6 +31,7 @@ async def test_write_directory_deserialize(language: str, topic: str):
     ("language", "topic", "directory"),
     [("English", "Write a tutorial about Python", {"Introduction": ["What is Python?", "Why learn Python?"]})],
 )
+@pytest.mark.usefixtures("llm_mock")
 async def test_write_content_deserialize(language: str, topic: str, directory: Dict):
     action = WriteContent(language=language, directory=directory)
     serialized_data = action.model_dump()

tests/metagpt/tools/test_translate.py

@@ -14,6 +14,7 @@ from metagpt.tools.translator import Translator
 
 @pytest.mark.asyncio
 @pytest.mark.usefixtures("llm_api")
+@pytest.mark.usefixtures("llm_mock")
 async def test_translate(llm_api):
     poetries = [
         ("Let life be beautiful like summer flowers", "花"),