From fcf254a6918ed949c0d02e9c1504b462921161ab Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Fri, 7 Jun 2024 19:10:58 +0800 Subject: [PATCH 01/41] save doc in folder by default, change WriteTasks.run, add run examples --- metagpt/actions/design_api.py | 84 ++++++------------- metagpt/actions/project_management.py | 62 ++++++++++---- metagpt/roles/di/role_zero.py | 2 +- tests/metagpt/roles/di/run_architect.py | 5 +- tests/metagpt/roles/di/run_product_manager.py | 18 ++++ tests/metagpt/roles/di/run_project_manager.py | 36 ++++++++ 6 files changed, 127 insertions(+), 80 deletions(-) create mode 100644 tests/metagpt/roles/di/run_product_manager.py create mode 100644 tests/metagpt/roles/di/run_project_manager.py diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 566e96efe..cc88171ff 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,7 +11,6 @@ @Modified By: mashenquan, 2024/5/31. Implement Chapter 3 of RFC 236. """ import json -import uuid from pathlib import Path from typing import List, Optional, Union @@ -87,84 +86,48 @@ class WriteDesign(Action): output_pathname (str, optional): The output path name of file that the system design should be saved to. Returns: - AIMessage: An AIMessage object containing the system design. + str: The file path of the generated system design. Example: - # Write a new system design. - >>> user_requirement = "Your user requirements" - >>> extra_info = "Your extra information" - >>> action = WriteDesign() - >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info) - >>> print(result) - System Design filename: "/path/to/design/filename" - - # Modify an exists system design. - >>> user_requirement = "Your user requirements" - >>> extra_info = "Your extra information" - >>> legacy_design_filename = "/path/to/exists/design/filename" - >>> action = WriteDesign() - >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename) - >>> print(result) - System Design filename: "/path/to/design/filename" - - # Write a new system design with the given PRD(Product Requirement Document). - >>> user_requirement = "Your user requirements" - >>> extra_info = "Your extra information" - >>> prd_filename = "/path/to/prd/filename" - >>> action = WriteDesign() - >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, prd_filename=prd_filename) - >>> print(result) - System Design filename: "/path/to/design/filename" - - # Modify an exists system design with the given PRD(Product Requirement Document). - >>> user_requirement = "Your user requirements" - >>> extra_info = "Your extra information" - >>> prd_filename = "/path/to/prd/filename" - >>> legacy_design_filename = "/path/to/exists/design/filename" - >>> action = WriteDesign() - >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename, prd_filename=prd_filename) - >>> print(result) - TSystem Design filename: "/path/to/design/filename" - # Write a new system design and save to the path name. - >>> user_requirement = "Your user requirements" + >>> user_requirement = "Write system design for a snake game" >>> extra_info = "Your extra information" - >>> output_pathname = "/path/to/design/filename" + >>> output_pathname = "snake_game/docs/system_design.json" >>> action = WriteDesign() >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, output_pathname=output_pathname) >>> print(result) - System Design filename: "/path/to/design/filename" + System Design filename: "/absolute/path/to/snake_game/docs/system_design.json" - # Modify an exists system design and save to the path name. - >>> user_requirement = "Your user requirements" + # Rewrite an existing system design and save to the path name. + >>> user_requirement = "Write system design for a snake game, include new features such as a web UI" >>> extra_info = "Your extra information" - >>> legacy_design_filename = "/path/to/exists/design/filename" - >>> output_pathname = "/path/to/design/filename" + >>> legacy_design_filename = "/absolute/path/to/snake_game/docs/system_design.json" + >>> output_pathname = "/absolute/path/to/snake_game/docs/system_design_new.json" >>> action = WriteDesign() >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename, output_pathname=output_pathname) >>> print(result) - System Design filename: "/path/to/design/filename" + System Design filename: "/absolute/path/to/snake_game/docs/system_design_new.json" # Write a new system design with the given PRD(Product Requirement Document) and save to the path name. - >>> user_requirement = "Your user requirements" + >>> user_requirement = "Write system design for a snake game based on the PRD at /absolute/path/to/snake_game/docs/prd.json" >>> extra_info = "Your extra information" - >>> prd_filename = "/path/to/prd/filename" - >>> output_pathname = "/path/to/design/filename" + >>> prd_filename = "/absolute/path/to/snake_game/docs/prd.json" + >>> output_pathname = "/absolute/path/to/snake_game/docs/sytem_design.json" >>> action = WriteDesign() >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, prd_filename=prd_filename, output_pathname=output_pathname) >>> print(result) - System Design filename: "/path/to/design/filename" + System Design filename: "/absolute/path/to/snake_game/docs/sytem_design.json" - # Modify an exists system design with the given PRD(Product Requirement Document) and save to the path name. - >>> user_requirement = "Your user requirements" + # Rewrite an existing system design with the given PRD(Product Requirement Document) and save to the path name. + >>> user_requirement = "Write system design for a snake game, include new features such as a web UI" >>> extra_info = "Your extra information" - >>> prd_filename = "/path/to/prd/filename" - >>> legacy_design_filename = "/path/to/exists/design/filename" - >>> output_pathname = "/path/to/design/filename" + >>> prd_filename = "/absolute/path/to/snake_game/docs/prd.json" + >>> legacy_design_filename = "/absolute/path/to/snake_game/docs/system_design.json" + >>> output_pathname = "/absolute/path/to/snake_game/docs/system_design_new.json" >>> action = WriteDesign() - >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename, prd_filename=prd_filename, output_pathname=output_pathname) + >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, prd_filename=prd_filename, legacy_design_filename=legacy_design_filename, output_pathname=output_pathname) >>> print(result) - System Design filename: "/path/to/design/filename" + System Design filename: "/absolute/path/to/snake_game/docs/system_design_new.json" """ if not with_messages: return await self._execute_api( @@ -301,9 +264,10 @@ class WriteDesign(Action): ) if not output_pathname: - output_path = DEFAULT_WORKSPACE_ROOT - output_path.mkdir(parents=True, exist_ok=True) - output_pathname = Path(output_path) / f"{uuid.uuid4().hex}.json" + output_pathname = Path(output_pathname) / "docs" / "sytem_design.json" + output_pathname.mkdir(parents=True, exist_ok=True) + elif not Path(output_pathname).is_absolute(): + output_pathname = DEFAULT_WORKSPACE_ROOT / output_pathname output_pathname = Path(output_pathname) await awrite(filename=output_pathname, data=design.content) output_filename = output_pathname.parent / f"{output_pathname.stem}-class-diagram" diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 38234cfb7..a39840bf1 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -19,11 +19,16 @@ from pydantic import BaseModel, Field from metagpt.actions.action import Action from metagpt.actions.project_management_an import PM_NODE, REFINED_PM_NODE -from metagpt.const import PACKAGE_REQUIREMENTS_FILENAME +from metagpt.const import DEFAULT_WORKSPACE_ROOT, PACKAGE_REQUIREMENTS_FILENAME from metagpt.logs import logger from metagpt.schema import AIMessage, Document, Documents, Message from metagpt.tools.tool_registry import register_tool -from metagpt.utils.common import aread, to_markdown_code_block +from metagpt.utils.common import ( + aread, + awrite, + save_json_to_markdown, + to_markdown_code_block, +) from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.report import DocsReporter @@ -44,7 +49,13 @@ class WriteTasks(Action): input_args: Optional[BaseModel] = Field(default=None, exclude=True) async def run( - self, with_messages: List[Message] = None, *, user_requirement: str = "", design_filename: str = "", **kwargs + self, + with_messages: List[Message] = None, + *, + user_requirement: str = "", + design_filename: str = "", + output_pathname: str = "", + **kwargs, ) -> Union[AIMessage, str]: """ Write a project schedule given a project system design file. @@ -52,29 +63,33 @@ class WriteTasks(Action): Args: user_requirement (str, optional): A string specifying the user's requirements. Defaults to an empty string. design_filename (str): The filename of the project system design file. Defaults to an empty string. + output_pathname (str, optional): The output path name of file that the project schedule should be saved to. **kwargs: Additional keyword arguments. Returns: - AIMessage: The generated project schedule. + str: Path to the generated project schedule. Example: - # Write a new project schedule. - >>> design_filename = "/path/to/design/filename" + # Write a project schedule with a given system design. + >>> design_filename = "/absolute/path/to/snake_game/docs/system_design.json" + >>> output_pathname = "/absolute/path/to/snake_game/docs/project_schedule.json" >>> action = WriteTasks() - >>> result = await action.run(design_filename=design_filename) + >>> result = await action.run(design_filename=design_filename, output_pathname=output_pathname) >>> print(result) - The project schedule is balabala... + The project schedule is at /absolute/path/to/snake_game/docs/project_schedule.json - # Write a new project schedule with the user requirement. - >>> design_filename = "/path/to/design/filename" - >>> user_requirement = "Your user requirements" + # Write a project schedule with a user requirement. + >>> user_requirement = "Write project schedule for a snake game following these requirements: ..." + >>> output_pathname = "/absolute/path/to/snake_game/docs/project_schedule.json" >>> action = WriteTasks() - >>> result = await action.run(design_filename=design_filename, user_requirement=user_requirement) + >>> result = await action.run(user_requirement=user_requirement, output_pathname=output_pathname) >>> print(result) - The project schedule is balabala... + The project schedule is at /absolute/path/to/snake_game/docs/project_schedule.json """ if not with_messages: - return await self._execute_api(user_requirement=user_requirement, design_filename=design_filename) + return await self._execute_api( + user_requirement=user_requirement, design_filename=design_filename, output_pathname=output_pathname + ) self.input_args = with_messages[-1].instruct_content self.repo = ProjectRepo(self.input_args.project_path) @@ -158,10 +173,23 @@ class WriteTasks(Action): packages.add(pkg) await self.repo.save(filename=PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages)) - async def _execute_api(self, user_requirement: str = "", design_filename: str = "") -> str: + async def _execute_api( + self, user_requirement: str = "", design_filename: str = "", output_pathname: str = "" + ) -> str: context = to_markdown_code_block(user_requirement) - if not design_filename: + if design_filename: content = await aread(filename=design_filename) context += to_markdown_code_block(content) node = await self._run_new_tasks(context) - return node.instruct_content.model_dump_json() + file_content = node.instruct_content.model_dump_json() + + if not output_pathname: + output_pathname = Path(output_pathname) / "docs" / "project_schedule.json" + output_pathname.mkdir(parents=True, exist_ok=True) + elif not Path(output_pathname).is_absolute(): + output_pathname = DEFAULT_WORKSPACE_ROOT / output_pathname + output_pathname = Path(output_pathname) + await awrite(filename=output_pathname, data=file_content) + await save_json_to_markdown(content=file_content, output_filename=output_pathname.with_suffix(".md")) + + return f'Project Schedule filename: "{str(output_pathname)}"' diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index 9c678c600..dc28e53a6 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -195,7 +195,7 @@ class RoleZero(Role): outputs.append(output) except Exception as e: tb = traceback.format_exc() - logger.exception(e + tb) + logger.exception(str(e) + tb) outputs.append(output + f": {tb}") break # Stop executing if any command fails else: diff --git a/tests/metagpt/roles/di/run_architect.py b/tests/metagpt/roles/di/run_architect.py index e615af4eb..455b60d92 100644 --- a/tests/metagpt/roles/di/run_architect.py +++ b/tests/metagpt/roles/di/run_architect.py @@ -2,6 +2,7 @@ import asyncio import os from metagpt.roles.architect import Architect +from metagpt.schema import Message DESIGN_DOC_SNAKE = """ { @@ -30,9 +31,9 @@ async def main(requirement): with open("temp_design.json", "w") as f: f.write(DESIGN_DOC_SNAKE) architect = Architect() - await architect.run(requirement) + await architect.run(Message(content=requirement, send_to="Bob")) os.remove("temp_design.json") if __name__ == "__main__": - asyncio.run(main(WRITE_SNAKE)) + asyncio.run(main(REWRITE_SNAKE)) diff --git a/tests/metagpt/roles/di/run_product_manager.py b/tests/metagpt/roles/di/run_product_manager.py new file mode 100644 index 000000000..3ab1e9bab --- /dev/null +++ b/tests/metagpt/roles/di/run_product_manager.py @@ -0,0 +1,18 @@ +import asyncio + +from metagpt.roles.product_manager import ProductManager + +WRITE_2048 = """Write a PRD for a cli 2048 game""" + +REWRITE_2048 = """Rewrite the prd at /Users/gary/Files/temp/workspace/2048_game/docs/prd.json, add a web UI""" + +CASUAL_CHAT = """What's your name?""" + + +async def main(requirement): + product_manager = ProductManager() + await product_manager.run(requirement) + + +if __name__ == "__main__": + asyncio.run(main(WRITE_2048)) diff --git a/tests/metagpt/roles/di/run_project_manager.py b/tests/metagpt/roles/di/run_project_manager.py new file mode 100644 index 000000000..30889c59c --- /dev/null +++ b/tests/metagpt/roles/di/run_project_manager.py @@ -0,0 +1,36 @@ +import asyncio +import os + +from metagpt.roles.project_manager import ProjectManager +from metagpt.schema import Message + +DESIGN_DOC_2048 = '{"Implementation approach":"We will use the Pygame library to implement the 2048 game logic and user interface. Pygame is a set of Python modules designed for writing video games, which will help us create a responsive and visually appealing UI. For the mobile responsiveness, we will ensure that the game scales appropriately on different screen sizes. We will also use the Pygame GUI library to create buttons for restarting the game and choosing difficulty levels.","File list":["main.py","game.py","ui.py"],"Data structures and interfaces":"\\nclassDiagram\\n class Game {\\n -grid: list[list[int]]\\n -score: int\\n +__init__()\\n +move(direction: str) bool\\n +merge() bool\\n +spawn_tile() None\\n +is_game_over() bool\\n +reset() None\\n }\\n class UI {\\n -game: Game\\n +__init__(game: Game)\\n +draw_grid() None\\n +draw_score() None\\n +draw_buttons() None\\n +handle_input() None\\n }\\n class Main {\\n -ui: UI\\n +main() None\\n }\\n Main --> UI\\n UI --> Game\\n","Program call flow":"\\nsequenceDiagram\\n participant M as Main\\n participant U as UI\\n participant G as Game\\n M->>U: __init__(game)\\n U->>G: __init__()\\n M->>U: draw_grid()\\n U->>G: move(direction)\\n G-->>U: return bool\\n U->>G: merge()\\n G-->>U: return bool\\n U->>G: spawn_tile()\\n G-->>U: return None\\n U->>G: is_game_over()\\n G-->>U: return bool\\n U->>G: reset()\\n G-->>U: return None\\n M->>U: draw_score()\\n M->>U: draw_buttons()\\n M->>U: handle_input()\\n","Anything UNCLEAR":"Clarification needed on the specific design elements for the UI to ensure it meets the \'beautiful\' requirement. Additionally, we need to confirm the exact difficulty levels and how they should affect the game mechanics."}' +DESIGN_DOC_SNAKE = """ +{ + "Implementation approach": "We will use the Pygame library to create the CLI-based snake game. Pygame is a set of Python modules designed for writing video games, which will help us handle graphics, sound, and input. The game will be structured into different modules to handle the main game loop, snake movement, food generation, collision detection, and user interface. We will ensure the game is engaging and responsive by optimizing the game loop and input handling. The score display and different speed levels will be implemented to enhance the user experience.", + "File list": [ + "main.py", + "game.py", + "snake.py", + "food.py", + "ui.py" + ], + "Data structures and interfaces": "\nclassDiagram\n class Main {\n +main() void\n }\n class Game {\n -Snake snake\n -Food food\n -int score\n -int speed\n +__init__(speed: int)\n +run() void\n +restart() void\n +update_score() void\n }\n class Snake {\n -list body\n -str direction\n +__init__()\n +move() void\n +change_direction(new_direction: str) void\n +check_collision() bool\n +grow() void\n }\n class Food {\n -tuple position\n +__init__()\n +generate_new_position() void\n }\n class UI {\n +display_score(score: int) void\n +display_game_over() void\n +display_game(snake: Snake, food: Food) void\n }\n Main --> Game\n Game --> Snake\n Game --> Food\n Game --> UI\n", + "Program call flow": "\nsequenceDiagram\n participant M as Main\n participant G as Game\n participant S as Snake\n participant F as Food\n participant U as UI\n M->>G: __init__(speed)\n M->>G: run()\n G->>S: __init__()\n G->>F: __init__()\n loop Game Loop\n G->>S: move()\n G->>S: check_collision()\n alt Collision Detected\n G->>G: restart()\n G->>U: display_game_over()\n else No Collision\n G->>F: generate_new_position()\n G->>S: grow()\n G->>G: update_score()\n G->>U: display_score(score)\n end\n G->>U: display_game(snake, food)\n end\n", + "Anything UNCLEAR": "Currently, all aspects of the project are clear." +} +""" +REQ = """Write a project schedule based on the design at temp_design.json""" +CASUAL_CHAT = """what's your name?""" + + +async def main(requirement): + with open("temp_design.json", "w") as f: + f.write(DESIGN_DOC_2048) + project_manager = ProjectManager() + await project_manager.run(Message(content=requirement, send_to="Eve")) + os.remove("temp_design.json") + + +if __name__ == "__main__": + asyncio.run(main(REQ)) From e742d9d619ada7b7ad439194236fa457b4998c67 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Fri, 7 Jun 2024 19:11:43 +0800 Subject: [PATCH 02/41] fix set_state bug --- metagpt/roles/di/data_analyst.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py index fc298ea4c..e489c9d98 100644 --- a/metagpt/roles/di/data_analyst.py +++ b/metagpt/roles/di/data_analyst.py @@ -54,7 +54,6 @@ class DataAnalyst(DataInterpreter): if self.tools and not self.tool_recommender: self.tool_recommender = BM25ToolRecommender(tools=self.tools) self.set_actions([WriteAnalysisCode]) - self._set_state(0) # HACK: Init Planner, control it through dynamic thinking; Consider formalizing as a react mode self.planner = Planner(goal="", working_memory=self.rc.working_memory, auto_run=True) @@ -112,11 +111,15 @@ class DataAnalyst(DataInterpreter): return Message(content="Task completed", role="assistant", sent_from=self._setting, cause_by=WriteAnalysisCode) async def _react(self) -> Message: + # NOTE: Diff 1: Each time landing here means observing news, set todo to allow news processing in _think + self._set_state(0) + actions_taken = 0 rsp = Message(content="No actions taken yet", cause_by=Action) # will be overwritten after Role _act while actions_taken < self.rc.max_react_loop: - # NOTE: difference here, keep observing within react + # NOTE: Diff 2: Keep observing within _react, news will go into memory, allowing adapting to new info await self._observe() + # think has_todo = await self._think() if not has_todo: From 521ba88425388d665784c30c2fe0cb77362d6283 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Fri, 7 Jun 2024 19:14:23 +0800 Subject: [PATCH 03/41] prd saving --- metagpt/actions/write_prd.py | 54 ++++++++++++------------------------ 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index b6062c36c..7199ec415 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -15,7 +15,6 @@ from __future__ import annotations import json -import uuid from pathlib import Path from typing import List, Optional, Union @@ -105,45 +104,27 @@ class WritePRD(Action): **kwargs: Additional keyword arguments. Returns: - str: The resulting message after generating the Product Requirement Document. + str: The file path of the generated Product Requirement Document. Example: - # Write a new PRD(Product Requirement Document) - >>> user_requirement = "YOUR REQUIREMENTS" - >>> extra_info = "YOUR EXTRA INFO" + # Write a new PRD (Product Requirement Document) + >>> user_requirement = "Write PRD for a snake game" + >>> output_pathname = "snake_game/docs/prd.json" + >>> extra_info = "YOUR EXTRA INFO, if any" >>> write_prd = WritePRD() - >>> result = await write_prd.run(user_requirement=user_requirement, extra_info=extra_info) + >>> result = await write_prd.run(user_requirement=user_requirement, output_pathname=output_pathname, extra_info=extra_info) >>> print(result) - PRD filename: "/path/to/prd/directory/213434ad.json" + PRD filename: "/absolute/path/to/snake_game/docs/prd.json" - # Modify a exists PRD(Product Requirement Document) - >>> user_requirement = "YOUR REQUIREMENTS" - >>> extra_info = "YOUR EXTRA INFO" - >>> legacy_prd_filename = "/path/to/exists/prd_filename" + # Rewrite an existing PRD (Product Requirement Document) and save to a new path. + >>> user_requirement = "Write PRD for a snake game, include new features such as a web UI" + >>> legacy_prd_filename = "/absolute/path/to/snake_game/docs/prd.json" + >>> output_pathname = "/absolute/path/to/snake_game/docs/prd_new.json" + >>> extra_info = "YOUR EXTRA INFO, if any" >>> write_prd = WritePRD() - >>> result = await write_prd.run(user_requirement=user_requirement, extra_info=extra_info, legacy_prd_filename=legacy_prd_filename) + >>> result = await write_prd.run(user_requirement=user_requirement, legacy_prd_filename=legacy_prd_filename, extra_info=extra_info) >>> print(result) - PRD filename: "/path/to/prd/directory/213434ad.json" - - # Write and save a new PRD(Product Requirement Document) to the path name. - >>> user_requirement = "YOUR REQUIREMENTS" - >>> extra_info = "YOUR EXTRA INFO" - >>> output_pathname = "/path/to/prd/directory/213434ad.json" - >>> write_prd = WritePRD() - >>> result = await write_prd.run(user_requirement=user_requirement, extra_info=extra_info, output_pathname=output_pathname) - >>> print(result) - PRD filename: "/path/to/prd/directory/213434ad.json" - - # Modify a exists PRD(Product Requirement Document) and save to the path name. - >>> user_requirement = "YOUR REQUIREMENTS" - >>> extra_info = "YOUR EXTRA INFO" - >>> legacy_prd_filename = "/path/to/exists/prd_filename" - >>> output_pathname = "/path/to/prd/directory/213434ad.json" - >>> write_prd = WritePRD() - >>> result = await write_prd.run(user_requirement=user_requirement, extra_info=extra_info, legacy_prd_filename=legacy_prd_filename, output_pathname=output_pathname) - >>> print(result) - PRD filename: "/path/to/prd/directory/213434ad.json" - + PRD filename: "/absolute/path/to/snake_game/docs/prd_new.json" """ if not with_messages: return await self._execute_api( @@ -329,9 +310,10 @@ class WritePRD(Action): new_prd = await self._merge(req=req, related_doc=old_prd) if not output_pathname: - output_path = DEFAULT_WORKSPACE_ROOT - output_path.mkdir(parents=True, exist_ok=True) - output_pathname = Path(output_path) / f"{uuid.uuid4().hex}.json" + output_pathname = DEFAULT_WORKSPACE_ROOT / "docs" / "prd.json" + output_pathname.mkdir(parents=True, exist_ok=True) + elif not Path(output_pathname).is_absolute(): + output_pathname = DEFAULT_WORKSPACE_ROOT / output_pathname output_pathname = Path(output_pathname) await awrite(filename=output_pathname, data=new_prd.content) competitive_analysis_filename = output_pathname.parent / f"{output_pathname.stem}-competitive-analysis" From 1f9a66ea604b22b247d3c87fbbf0dec5d7836bb6 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Fri, 7 Jun 2024 20:24:07 +0800 Subject: [PATCH 04/41] add instruction for tl and engineer2 --- metagpt/prompts/di/engineer2.py | 11 +++++++---- metagpt/prompts/di/team_leader.py | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 346f2fc5a..4fd52e320 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -1,6 +1,9 @@ from metagpt.prompts.di.role_zero import ROLE_INSTRUCTION -ENGINEER2_INSTRUCTION = ( - ROLE_INSTRUCTION - + "4. Each time you write a code in your response, write with the Editor directly without preparing a repetitive code block beforehand." -) +EXTRA_INSTRUCTION = """ +4. Each time you write a code in your response, write with the Editor directly without preparing a repetitive code block beforehand. +5. Take on ONE task and write ONE code file in each response. DON'T attempt all tasks in one response. +6. When not specified, you should write files in a folder named "src". If you know the project path, then write in a "src" folder under the project path. +7. When provided system design or project schedule, read them first, then adhere to them in your implementation. +""" +ENGINEER2_INSTRUCTION = ROLE_INSTRUCTION + EXTRA_INSTRUCTION.strip() diff --git a/metagpt/prompts/di/team_leader.py b/metagpt/prompts/di/team_leader.py index 14297b026..2e4a17932 100644 --- a/metagpt/prompts/di/team_leader.py +++ b/metagpt/prompts/di/team_leader.py @@ -11,13 +11,14 @@ Your team member: You should NOT assign consecutive tasks to the same team member, instead, assign an aggregated task (or the complete requirement) and let the team member to decompose it. When creating a new plan involving multiple members, create all tasks at once. If plan is created, you should track the progress based on team member feedback message, and update plan accordingly, such as Plan.finish_current_task, Plan.reset_task, Plan.replace_task, etc. -You should use TeamLeader.publish_team_message to team members, asking them to start their task. +You should use TeamLeader.publish_team_message to team members, asking them to start their task. DONT omit any necessary info such as path, link, environment, programming language, framework, requirement, constraint from original content to team members because you are their sole info source. Pay close attention to new user message, review the conversation history, use RoleZero.reply_to_human to respond to the user directly, DON'T ask your team members. Note: 1. If the requirement is a pure DATA-RELATED requirement, such as bug fixes, issue reporting, environment setup, terminal operations, pip install, web browsing, web scraping, web searching, web imitation, data science, data analysis, machine learning, deep learning, text-to-image etc. DON'T decompose it, assign a single task with the original user requirement as instruction directly to Data Analyst. 2. If the requirement is developing a software, game, app, or website, excluding the above data-related tasks, you should decompose the requirement into multiple tasks and assign them to different team members based on their expertise, usually the sequence of Product Manager -> Architect -> Project Manager -> Engineer -> (optional: QaEngine if present) -> (optional: DataAnalyst if user requests deployment), each assigned ONE task. When publishing message to Product Manager, you should directly copy the full original user requirement. 3. If the requirement contains both DATA-RELATED part mentioned in 1 and software development part mentioned in 2, you should decompose the software development part and assign them to different team members based on their expertise, and assign the DATA-RELATED part to Data Analyst David directly. +4. It it helpful for Engineer to have both the system design and the project schedule for writing the code, so include paths of both files (if available) when publishing message to Engineer. """ FINISH_CURRENT_TASK_CMD = """ From a7733fb4b27ab7cd8974747aae33ce8bd45bc0d6 Mon Sep 17 00:00:00 2001 From: zhanglei Date: Tue, 11 Jun 2024 16:23:36 +0800 Subject: [PATCH 05/41] =?UTF-8?q?update:=20terminal=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=8C=96=E7=9A=84=E6=97=B6=E5=80=99=E6=8A=8A=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E7=BB=99=E8=BF=87=E5=8E=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/tools/libs/terminal.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index e6d059636..7b0f124c0 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -1,3 +1,4 @@ +import os import subprocess import threading from queue import Queue @@ -19,6 +20,7 @@ class Terminal: self.shell_command = ["bash"] # FIXME: should consider windows support later self.command_terminator = "\n" + env = dict(os.environ) # Start a persistent shell process self.process = subprocess.Popen( self.shell_command, @@ -27,6 +29,7 @@ class Terminal: stdout=subprocess.PIPE, stderr=subprocess.STDOUT, executable="/bin/bash", + env=env ) self.stdout_queue = Queue() self.observer = TerminalReporter() From 3de62c543342eb48287f8ef7c5a4dfac4ee9db49 Mon Sep 17 00:00:00 2001 From: zhanglei Date: Tue, 11 Jun 2024 18:04:33 +0800 Subject: [PATCH 06/41] =?UTF-8?q?update:=20=E7=8E=AF=E5=A2=83=E5=8F=98?= =?UTF-8?q?=E9=87=8F=E5=8E=BB=E6=8E=89=EF=BC=8Cmgx=20backend=E5=B7=B2?= =?UTF-8?q?=E7=BB=8F=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/tools/libs/terminal.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index 7b0f124c0..0cc586fc1 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -20,7 +20,6 @@ class Terminal: self.shell_command = ["bash"] # FIXME: should consider windows support later self.command_terminator = "\n" - env = dict(os.environ) # Start a persistent shell process self.process = subprocess.Popen( self.shell_command, @@ -28,8 +27,7 @@ class Terminal: stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - executable="/bin/bash", - env=env + executable="/bin/bash" ) self.stdout_queue = Queue() self.observer = TerminalReporter() From 20ce56ce10b627cbc8586ecf0fd4d9d967057503 Mon Sep 17 00:00:00 2001 From: zhanglei Date: Tue, 11 Jun 2024 18:05:35 +0800 Subject: [PATCH 07/41] =?UTF-8?q?update:=20=E7=8E=AF=E5=A2=83=E5=8F=98?= =?UTF-8?q?=E9=87=8F=E5=8E=BB=E6=8E=89=EF=BC=8Cmgx=20backend=E5=B7=B2?= =?UTF-8?q?=E7=BB=8F=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/tools/libs/terminal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index 0cc586fc1..faf2893a7 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -1,4 +1,3 @@ -import os import subprocess import threading from queue import Queue From ad2530aff98b9f8c7a78ac332e414538a754a4c4 Mon Sep 17 00:00:00 2001 From: zhanglei Date: Thu, 13 Jun 2024 11:17:16 +0800 Subject: [PATCH 08/41] update: git tool --- metagpt/tools/libs/git.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metagpt/tools/libs/git.py b/metagpt/tools/libs/git.py index b4d759bf4..740cb81f9 100644 --- a/metagpt/tools/libs/git.py +++ b/metagpt/tools/libs/git.py @@ -141,6 +141,8 @@ async def git_create_pull( Returns: PullRequest: The created pull request. """ + from metagpt.utils.git_repository import GitRepository + return await GitRepository.create_pull( base=base, head=head, @@ -187,4 +189,6 @@ async def git_create_issue( Returns: Issue: The created issue. """ + from metagpt.utils.git_repository import GitRepository + return await GitRepository.create_issue(repo_name=repo_name, title=title, body=body, access_token=access_token) From f0df3144d68ed288f5ccce0c34d3939f8462ba98 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Thu, 13 Jun 2024 15:14:34 +0800 Subject: [PATCH 09/41] fix ask human problem --- metagpt/roles/di/role_zero.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index dc28e53a6..6244a39a8 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -3,7 +3,7 @@ from __future__ import annotations import inspect import json import traceback -from typing import Literal, Tuple +from typing import Callable, Literal, Tuple from pydantic import model_validator @@ -42,7 +42,7 @@ class RoleZero(Role): # Tools tools: list[str] = [] # Use special symbol [""] to indicate use of all registered tools tool_recommender: ToolRecommender = None - tool_execution_map: dict[str, callable] = {} + tool_execution_map: dict[str, Callable] = {} special_tool_commands: list[str] = ["Plan.finish_current_task", "end"] # Equipped with three basic tools by default for optional use editor: Editor = Editor() @@ -247,7 +247,7 @@ class RoleZero(Role): if not isinstance(self.rc.env, MGXEnv): return "Not in MGXEnv, command will not be executed." - return await self.rc.env.get_human_input(question, sent_from=self) + return await self.rc.env.ask_human(question, sent_from=self) async def reply_to_human(self, content: str) -> str: """Reply to human user with the content provided. Use this when you have a clear answer or solution to the user's question.""" From 25ca3381101c94ed6dc249357aa03862b99169f3 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Thu, 13 Jun 2024 19:46:36 +0800 Subject: [PATCH 10/41] add back content display for Browser._view --- metagpt/tools/libs/browser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/tools/libs/browser.py b/metagpt/tools/libs/browser.py index 955058ea0..1b1b3d82d 100644 --- a/metagpt/tools/libs/browser.py +++ b/metagpt/tools/libs/browser.py @@ -189,8 +189,8 @@ class Browser: async def _view(self, keep_len: int = 5000) -> str: """simulate human viewing the current page, return the visible text with links""" - # visible_text_with_links = await self.current_page.evaluate(VIEW_CONTENT_JS) - # print("The visible text and their links (if any): ", visible_text_with_links[:keep_len]) + visible_text_with_links = await self.current_page.evaluate(VIEW_CONTENT_JS) + print("The visible text and their links (if any): ", visible_text_with_links[:keep_len]) # html_content = await self._view_page_html(keep_len=keep_len) # print("The html content: ", html_content) From 207f3f3b6ac271668d6f459857cb9576c5447bd1 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Thu, 13 Jun 2024 20:13:32 +0800 Subject: [PATCH 11/41] team leader common-sense qa, ask for clarification, increase max_react_loop, issue end command --- metagpt/prompts/di/team_leader.py | 4 +++- metagpt/roles/di/role_zero.py | 2 +- metagpt/roles/di/team_leader.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/metagpt/prompts/di/team_leader.py b/metagpt/prompts/di/team_leader.py index 2e4a17932..7af5a6e07 100644 --- a/metagpt/prompts/di/team_leader.py +++ b/metagpt/prompts/di/team_leader.py @@ -18,7 +18,9 @@ Note: 1. If the requirement is a pure DATA-RELATED requirement, such as bug fixes, issue reporting, environment setup, terminal operations, pip install, web browsing, web scraping, web searching, web imitation, data science, data analysis, machine learning, deep learning, text-to-image etc. DON'T decompose it, assign a single task with the original user requirement as instruction directly to Data Analyst. 2. If the requirement is developing a software, game, app, or website, excluding the above data-related tasks, you should decompose the requirement into multiple tasks and assign them to different team members based on their expertise, usually the sequence of Product Manager -> Architect -> Project Manager -> Engineer -> (optional: QaEngine if present) -> (optional: DataAnalyst if user requests deployment), each assigned ONE task. When publishing message to Product Manager, you should directly copy the full original user requirement. 3. If the requirement contains both DATA-RELATED part mentioned in 1 and software development part mentioned in 2, you should decompose the software development part and assign them to different team members based on their expertise, and assign the DATA-RELATED part to Data Analyst David directly. -4. It it helpful for Engineer to have both the system design and the project schedule for writing the code, so include paths of both files (if available) when publishing message to Engineer. +4. If the requirement is a common-sense, logical, or math problem, you should respond directly without assigning any task to team members. +5. If you think the requirement is not clear or ambiguous, you should ask the user for clarification immediately. Assign tasks only after all info is clear. +6. It is helpful for Engineer to have both the system design and the project schedule for writing the code, so include paths of both files (if available) when publishing message to Engineer. """ FINISH_CURRENT_TASK_CMD = """ diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index 6244a39a8..44e658ed4 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -175,7 +175,7 @@ class RoleZero(Role): actions_taken += 1 return rsp # return output from the last action - async def _run_commands(self, commands) -> list: + async def _run_commands(self, commands) -> str: outputs = [] for cmd in commands: # handle special command first diff --git a/metagpt/roles/di/team_leader.py b/metagpt/roles/di/team_leader.py index 92ddcab04..2932dd7f0 100644 --- a/metagpt/roles/di/team_leader.py +++ b/metagpt/roles/di/team_leader.py @@ -18,7 +18,8 @@ class TeamLeader(RoleZero): profile: str = "Team Leader" system_msg: list[str] = [SYSTEM_PROMPT] - max_react_loop: int = 1 # TeamLeader only reacts once each time + # TeamLeader only reacts once each time, but may encounter errors or need to ask human, thus allowing 2 more turns + max_react_loop: int = 3 tools: list[str] = ["Plan", "RoleZero", "TeamLeader"] From d57d4f3c01bb8550424c17490bec805e381beedf Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Thu, 13 Jun 2024 20:51:03 +0800 Subject: [PATCH 12/41] fix one line content (\\n issue) for editor --- metagpt/tools/libs/editor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index 23df02edd..eba1a1eac 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -1,4 +1,5 @@ import os +import re import shutil import subprocess @@ -26,6 +27,9 @@ class Editor: def write(self, path: str, content: str): """Write the whole content to a file. When used, make sure content arg contains the full content of the file.""" + if len(re.findall(r"\\n", content)) >= 5: + # A very raw rule to correct the content: Many \\n suggests all new line characters are mistaken as \\n whereas the correct one should be \n + content = content.replace("\\n", "\n") directory = os.path.dirname(path) if directory and not os.path.exists(directory): os.makedirs(directory) From aa0f8107262e73dbe41f17dd7956027901f4e2ae Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Fri, 14 Jun 2024 13:58:27 +0800 Subject: [PATCH 13/41] add end command to experience --- metagpt/strategy/experience_retriever.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index c122affb5..e268e9460 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -83,6 +83,9 @@ class SimpleExpRetriever(ExpRetriever): "args": { "content": "I have assigned the tasks to the team members. Alice will create the PRD, Bob will design the software architecture, Eve will break down the architecture into tasks, Alex will implement the core game logic, and Edward will write comprehensive tests. The team will work on the project accordingly", } + }, + { + "command_name": "end" } ] ``` @@ -113,6 +116,9 @@ class SimpleExpRetriever(ExpRetriever): "args": { "content": "I have assigned the task to David. He will break down the task further by himself and starts solving it.", } + }, + { + "command_name": "end" } ] ``` @@ -142,6 +148,9 @@ class SimpleExpRetriever(ExpRetriever): "args": { "content": "Alice has completed the PRD. I have marked her task as finished and sent the PRD to Bob. Bob will work on the software architecture.", } + }, + { + "command_name": "end" } ] ``` @@ -156,6 +165,9 @@ class SimpleExpRetriever(ExpRetriever): "args": { "content": "The team is currently working on ... We have completed ...", } + }, + { + "command_name": "end" } ] ``` @@ -213,6 +225,7 @@ Explanation: Launching a service requires Terminal tool with daemon mode, write "assignee": "David" } }, +] """ From 5d70c061e32759f8bb40e86656778b4c74ae41c8 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Mon, 17 Jun 2024 15:46:45 +0800 Subject: [PATCH 14/41] some formatting --- metagpt/prompts/di/role_zero.py | 2 +- metagpt/prompts/di/team_leader.py | 1 + metagpt/strategy/experience_retriever.py | 309 ++++++++++++----------- 3 files changed, 157 insertions(+), 155 deletions(-) diff --git a/metagpt/prompts/di/role_zero.py b/metagpt/prompts/di/role_zero.py index f098d2c4b..4d52476aa 100644 --- a/metagpt/prompts/di/role_zero.py +++ b/metagpt/prompts/di/role_zero.py @@ -20,7 +20,7 @@ class Task(BaseModel): # Available Commands {available_commands} -Special Command: Use {{"command_name": "pass"}} to do nothing and {{"command_name": "end"}} to indicate completion of all requirements and the end of actions. +Special Command: Use {{"command_name": "end"}} to do nothing or indicate completion of all requirements and the end of actions. # Current Plan {plan_status} diff --git a/metagpt/prompts/di/team_leader.py b/metagpt/prompts/di/team_leader.py index 7af5a6e07..8422cd97f 100644 --- a/metagpt/prompts/di/team_leader.py +++ b/metagpt/prompts/di/team_leader.py @@ -30,5 +30,6 @@ FINISH_CURRENT_TASK_CMD = """ "command_name": "Plan.finish_current_task", "args": {{}} } +] ``` """ diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index e268e9460..2bb734a8b 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -17,164 +17,165 @@ class DummyExpRetriever(ExpRetriever): return "" +TL_EXAMPLE = """ +## example 1 +User Requirement: Create a cli snake game using Python. +Explanation: The requirement is about software development. Assign each tasks to a different team member based on their expertise. When publishing message to Product Manager, we copy original user requirement directly to ensure no information loss. +```json +[ + { + "command_name": "Plan.append_task", + "args": { + "task_id": "1", + "dependent_task_ids": [], + "instruction": "Create a product requirement document (PRD) outlining the features, user interface, and user experience of the CLI python snake game.", + "assignee": "Alice" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "2", + "dependent_task_ids": ["1"], + "instruction": "Design the software architecture for the CLI snake game, including the choice of programming language, libraries, and data flow.", + "assignee": "Bob" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "3", + "dependent_task_ids": ["2"], + "instruction": "Break down the architecture into manageable tasks, identify task dependencies, and prepare a detailed task list for implementation.", + "assignee": "Eve" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "4", + "dependent_task_ids": ["3"], + "instruction": "Implement the core game logic for the CLI snake game, including snake movement, food generation, and score tracking.", + "assignee": "Alex" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "5", + "dependent_task_ids": ["4"], + "instruction": "Write comprehensive tests for the game logic and user interface to ensure functionality and reliability.", + "assignee": "Edward" + } + }, + { + "command_name": "TeamLeader.publish_message", + "args": { + "content": "Create a cli snake game using Python", + "send_to": "Alice" + } + }, + { + "command_name": "RoleZero.reply_to_human", + "args": { + "content": "I have assigned the tasks to the team members. Alice will create the PRD, Bob will design the software architecture, Eve will break down the architecture into tasks, Alex will implement the core game logic, and Edward will write comprehensive tests. The team will work on the project accordingly", + } + }, + { + "command_name": "end" + } +] +``` + +## example 2 +User Requirement: Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy. +Explanation: DON'T decompose requirement if it is a DATA-RELATED task, assign a single task directly to Data Analyst David. He will manage the decomposition and implementation. +```json +[ + { + "command_name": "Plan.append_task", + "args": { + "task_id": "1", + "dependent_task_ids": [], + "instruction": "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.", + "assignee": "David" + } + }, + { + "command_name": "TeamLeader.publish_message", + "args": { + "content": "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.", + "send_to": "David" + } + }, + { + "command_name": "RoleZero.reply_to_human", + "args": { + "content": "I have assigned the task to David. He will break down the task further by himself and starts solving it.", + } + }, + { + "command_name": "end" + } +] +``` + +## example 3 +Conversation History: +[ + ..., + {'role': 'assistant', 'content': 'from Alice(Product Manager) to {'Bob'}: {'docs': {'20240424153821.json': {'root_path': 'docs/prd', 'filename': '20240424153821.json', 'content': '{"Language":"en_us","Programming Language":"Python","Original Requirements":"create a cli snake game","Project Name":"snake_game","Product Goals":["Develop an intuitive and addictive snake game",...], ...}}}}}, +] +Explanation: You received a message from Alice, the Product Manager, that she has completed the PRD, use Plan.finish_current_task to mark her task as finished and moves the plan to the next task. Based on plan status, next task is for Bob (Architect), publish a message asking him to start. The message content should contain important path info. +```json +[ + { + "command_name": "Plan.finish_current_task", + "args": {} + }, + { + "command_name": "TeamLeader.publish_message", + "args": { + "content": "Please design the software architecture for the snake game based on the PRD created by Alice. The PRD is at 'docs/prd/20240424153821.json'. Include the choice of programming language, libraries, and data flow, etc.", + "send_to": "Bob" + } + }, + { + "command_name": "RoleZero.reply_to_human", + "args": { + "content": "Alice has completed the PRD. I have marked her task as finished and sent the PRD to Bob. Bob will work on the software architecture.", + } + }, + { + "command_name": "end" + } +] +``` + +## example 4 +User Question: how does the project go? +Explanation: The user is asking for a general update on the project status. Give a straight answer about the current task the team is working on and provide a summary of the completed tasks. +```json +[ + { + "command_name": "RoleZero.reply_to_human", + "args": { + "content": "The team is currently working on ... We have completed ...", + } + }, + { + "command_name": "end" + } +] +``` +""" + + class SimpleExpRetriever(ExpRetriever): """A simple experience retriever that returns manually crafted examples.""" - EXAMPLE: str = """ - ## example 1 - User Requirement: Create a cli snake game using Python. - Explanation: The requirement is about software development. Assign each tasks to a different team member based on their expertise. When publishing message to Product Manager, we copy original user requirement directly to ensure no information loss. - ```json - [ - { - "command_name": "Plan.append_task", - "args": { - "task_id": "1", - "dependent_task_ids": [], - "instruction": "Create a product requirement document (PRD) outlining the features, user interface, and user experience of the CLI python snake game.", - "assignee": "Alice" - } - }, - { - "command_name": "Plan.append_task", - "args": { - "task_id": "2", - "dependent_task_ids": ["1"], - "instruction": "Design the software architecture for the CLI snake game, including the choice of programming language, libraries, and data flow.", - "assignee": "Bob" - } - }, - { - "command_name": "Plan.append_task", - "args": { - "task_id": "3", - "dependent_task_ids": ["2"], - "instruction": "Break down the architecture into manageable tasks, identify task dependencies, and prepare a detailed task list for implementation.", - "assignee": "Eve" - } - }, - { - "command_name": "Plan.append_task", - "args": { - "task_id": "4", - "dependent_task_ids": ["3"], - "instruction": "Implement the core game logic for the CLI snake game, including snake movement, food generation, and score tracking.", - "assignee": "Alex" - } - }, - { - "command_name": "Plan.append_task", - "args": { - "task_id": "5", - "dependent_task_ids": ["4"], - "instruction": "Write comprehensive tests for the game logic and user interface to ensure functionality and reliability.", - "assignee": "Edward" - } - }, - { - "command_name": "TeamLeader.publish_message", - "args": { - "content": "Create a cli snake game using Python", - "send_to": "Alice" - } - }, - { - "command_name": "RoleZero.reply_to_human", - "args": { - "content": "I have assigned the tasks to the team members. Alice will create the PRD, Bob will design the software architecture, Eve will break down the architecture into tasks, Alex will implement the core game logic, and Edward will write comprehensive tests. The team will work on the project accordingly", - } - }, - { - "command_name": "end" - } - ] - ``` - - ## example 2 - User Requirement: Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy. - Explanation: DON'T decompose requirement if it is a DATA-RELATED task, assign a single task directly to Data Analyst David. He will manage the decomposition and implementation. - ```json - [ - { - "command_name": "Plan.append_task", - "args": { - "task_id": "1", - "dependent_task_ids": [], - "instruction": "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.", - "assignee": "David" - } - }, - { - "command_name": "TeamLeader.publish_message", - "args": { - "content": "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.", - "send_to": "David" - } - }, - { - "command_name": "RoleZero.reply_to_human", - "args": { - "content": "I have assigned the task to David. He will break down the task further by himself and starts solving it.", - } - }, - { - "command_name": "end" - } - ] - ``` - - ## example 3 - Conversation History: - [ - ..., - {'role': 'assistant', 'content': 'from Alice(Product Manager) to {'Bob'}: {'docs': {'20240424153821.json': {'root_path': 'docs/prd', 'filename': '20240424153821.json', 'content': '{"Language":"en_us","Programming Language":"Python","Original Requirements":"create a cli snake game","Project Name":"snake_game","Product Goals":["Develop an intuitive and addictive snake game",...], ...}}}}}, - ] - Explanation: You received a message from Alice, the Product Manager, that she has completed the PRD, use Plan.finish_current_task to mark her task as finished and moves the plan to the next task. Based on plan status, next task is for Bob (Architect), publish a message asking him to start. The message content should contain important path info. - ```json - [ - { - "command_name": "Plan.finish_current_task", - "args": {} - }, - { - "command_name": "TeamLeader.publish_message", - "args": { - "content": "Please design the software architecture for the snake game based on the PRD created by Alice. The PRD is at 'docs/prd/20240424153821.json'. Include the choice of programming language, libraries, and data flow, etc.", - "send_to": "Bob" - } - }, - { - "command_name": "RoleZero.reply_to_human", - "args": { - "content": "Alice has completed the PRD. I have marked her task as finished and sent the PRD to Bob. Bob will work on the software architecture.", - } - }, - { - "command_name": "end" - } - ] - ``` - - ## example 4 - User Question: how does the project go? - Explanation: The user is asking for a general update on the project status. Give a straight answer about the current task the team is working on and provide a summary of the completed tasks. - ```json - [ - { - "command_name": "RoleZero.reply_to_human", - "args": { - "content": "The team is currently working on ... We have completed ...", - } - }, - { - "command_name": "end" - } - ] - ``` - """ - def retrieve(self, context: str = "") -> str: - return self.EXAMPLE + return TL_EXAMPLE class KeywordExpRetriever(ExpRetriever): From e450a37daffa503b44272b17f1f84618ae839319 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Mon, 17 Jun 2024 15:49:48 +0800 Subject: [PATCH 15/41] a temp rule to fix TL's repetitive task assignment --- metagpt/environment/mgx/mgx_env.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/metagpt/environment/mgx/mgx_env.py b/metagpt/environment/mgx/mgx_env.py index 3bd41fd1e..97bca6b8c 100644 --- a/metagpt/environment/mgx/mgx_env.py +++ b/metagpt/environment/mgx/mgx_env.py @@ -17,8 +17,10 @@ from metagpt.utils.common import any_to_str, any_to_str_set class MGXEnv(Environment): """MGX Environment""" - # Before enabling TL to fully take over the routing, all software company roles need to be able to handle TL messages, which requires restructuring. - allow_bypass_team_leader: bool = True + # If True, fixed software sop bypassing TL is allowed, otherwise, TL will fully take over the routing + allow_bypass_team_leader: bool = False + + direct_chat_roles: set[str] = set() # record direct chat: @role_name def _publish_message(self, message: Message, peekable: bool = True) -> bool: return super().publish_message(message, peekable) @@ -28,9 +30,17 @@ class MGXEnv(Environment): tl = self.get_role("Team Leader") if user_defined_recipient: + # human user's direct chat message to a certain role self._publish_message(message) - # bypass team leader, team leader only needs to know but not to react - tl.rc.memory.add(self.move_message_info_to_content(message)) + self.direct_chat_roles.add(user_defined_recipient) + # # bypass team leader, team leader only needs to know but not to react (commented out because TL doesn't understand the message well in actual experiments) + # tl.rc.memory.add(self.move_message_info_to_content(message)) + + elif message.sent_from in self.direct_chat_roles: + # direct chat response from a certain role to human user, team leader and other roles in the env should not be involved, no need to publish + # NOTE: This is a temp rule to handle direct chat messages and has the following pitfalls: + # If human chats with a role directly when the role is undertaking a task assigned by TL, the rule prevents TL from processing the role's response, thus pausing global task progress. + self.direct_chat_roles.remove(message.sent_from) elif ( self.allow_bypass_team_leader @@ -106,3 +116,6 @@ class MGXEnv(Environment): sent_from = converted_msg.metadata[AGENT] if AGENT in converted_msg.metadata else converted_msg.sent_from converted_msg.content = f"from {sent_from} to {converted_msg.send_to}: {converted_msg.content}" return converted_msg + + def __repr__(self): + return "MGXEnv()" From 2083233d413be3ba905c2894d6ff7e4c9490eb60 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Mon, 17 Jun 2024 20:49:37 +0800 Subject: [PATCH 16/41] optimize direct chat logic --- metagpt/environment/base_env.py | 4 ++-- metagpt/environment/mgx/mgx_env.py | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/metagpt/environment/base_env.py b/metagpt/environment/base_env.py index 5d6d3a286..0776ae9ff 100644 --- a/metagpt/environment/base_env.py +++ b/metagpt/environment/base_env.py @@ -159,7 +159,7 @@ class Environment(ExtEnv): """增加一个在当前环境的角色 Add a role in the current environment """ - self.roles[role.profile] = role + self.roles[role.name] = role role.set_env(self) role.context = self.context @@ -168,7 +168,7 @@ class Environment(ExtEnv): Add a batch of characters in the current environment """ for role in roles: - self.roles[role.profile] = role + self.roles[role.name] = role for role in roles: # setup system message with roles role.context = self.context diff --git a/metagpt/environment/mgx/mgx_env.py b/metagpt/environment/mgx/mgx_env.py index 97bca6b8c..ab515e070 100644 --- a/metagpt/environment/mgx/mgx_env.py +++ b/metagpt/environment/mgx/mgx_env.py @@ -27,19 +27,22 @@ class MGXEnv(Environment): def publish_message(self, message: Message, user_defined_recipient: str = "", publicer: str = "") -> bool: """let the team leader take over message publishing""" - tl = self.get_role("Team Leader") + tl = self.get_role("Tim") # TeamLeader's name is Tim if user_defined_recipient: # human user's direct chat message to a certain role + + if self.get_role(user_defined_recipient).is_idle: + # User starts a new direct chat with a certain role, expecting a direct chat response from the role; Other roles including TL should not be involved. + # If the role is not idle, it means the user helps the role with its current work, in this case, we handle the role's response message as usual. + self.direct_chat_roles.add(user_defined_recipient) + self._publish_message(message) - self.direct_chat_roles.add(user_defined_recipient) # # bypass team leader, team leader only needs to know but not to react (commented out because TL doesn't understand the message well in actual experiments) # tl.rc.memory.add(self.move_message_info_to_content(message)) elif message.sent_from in self.direct_chat_roles: # direct chat response from a certain role to human user, team leader and other roles in the env should not be involved, no need to publish - # NOTE: This is a temp rule to handle direct chat messages and has the following pitfalls: - # If human chats with a role directly when the role is undertaking a task assigned by TL, the rule prevents TL from processing the role's response, thus pausing global task progress. self.direct_chat_roles.remove(message.sent_from) elif ( From 93e4c12610cdea066b2066376bb6115cbbddee2b Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Thu, 13 Jun 2024 20:24:14 +0800 Subject: [PATCH 17/41] add thought stream report --- metagpt/roles/di/data_analyst.py | 2 +- metagpt/roles/di/data_interpreter.py | 2 +- metagpt/roles/di/role_zero.py | 2 +- metagpt/tools/libs/editor.py | 2 +- metagpt/utils/report.py | 15 +++++++-------- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py index e489c9d98..ff985a9ef 100644 --- a/metagpt/roles/di/data_analyst.py +++ b/metagpt/roles/di/data_analyst.py @@ -82,7 +82,7 @@ class DataAnalyst(DataInterpreter): available_commands=prepare_command_prompt(self.available_commands), ) context = self.llm.format_msg(self.working_memory.get() + [Message(content=prompt, role="user")]) - async with ThoughtReporter(): + async with ThoughtReporter(enable_llm_stream=True): rsp = await self.llm.aask(context) self.commands = json.loads(CodeParser.parse_code(block=None, text=rsp)) self.rc.memory.add(Message(content=rsp, role="assistant")) diff --git a/metagpt/roles/di/data_interpreter.py b/metagpt/roles/di/data_interpreter.py index bdfc0e294..f90a928df 100644 --- a/metagpt/roles/di/data_interpreter.py +++ b/metagpt/roles/di/data_interpreter.py @@ -74,7 +74,7 @@ class DataInterpreter(Role): return True prompt = REACT_THINK_PROMPT.format(user_requirement=self.user_requirement, context=context) - async with ThoughtReporter(): + async with ThoughtReporter(enable_llm_stream=True): rsp = await self.llm.aask(prompt) rsp_dict = json.loads(CodeParser.parse_code(text=rsp)) self.working_memory.add(Message(content=rsp_dict["thoughts"], role="assistant")) diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index 44e658ed4..4f18060ee 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -129,7 +129,7 @@ class RoleZero(Role): ) context = self.llm.format_msg(self.rc.memory.get(self.memory_k) + [UserMessage(content=prompt)]) # print(*context, sep="\n" + "*" * 5 + "\n") - async with ThoughtReporter(): + async with ThoughtReporter(enable_llm_stream=True): self.command_rsp = await self.llm.aask(context, system_msgs=self.system_msg) self.rc.memory.add(AIMessage(content=self.command_rsp)) diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index eba1a1eac..d145fdabe 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -35,7 +35,7 @@ class Editor: os.makedirs(directory) with open(path, "w", encoding="utf-8") as f: f.write(content) - self.resource.report(path, "path") + # self.resource.report(path, "path") def read(self, path: str) -> FileBlock: """Read the whole content of a file.""" diff --git a/metagpt/utils/report.py b/metagpt/utils/report.py index 2d72af111..ed40e5dde 100644 --- a/metagpt/utils/report.py +++ b/metagpt/utils/report.py @@ -51,7 +51,6 @@ class ResourceReporter(BaseModel): block: BlockType = Field(description="The type of block that is reporting the resource") uuid: UUID = Field(default_factory=uuid4, description="The unique identifier for the resource") - is_chunk: bool = Field(False, description="Indicates whether the report is a chunk of a stream") enable_llm_stream: bool = Field(False, description="Indicates whether to connect to an LLM stream for reporting") callback_url: str = Field(METAGPT_REPORTER_DEFAULT_URL, description="The URL to which the report should be sent") _llm_task: Optional[asyncio.Task] = PrivateAttr(None) @@ -153,17 +152,14 @@ class ResourceReporter(BaseModel): def __enter__(self): """Enter the synchronous streaming callback context.""" - self.is_chunk = True return self def __exit__(self, *args, **kwargs): """Exit the synchronous streaming callback context.""" self.report(None, END_MARKER_NAME) - self.is_chunk = False async def __aenter__(self): """Enter the asynchronous streaming callback context.""" - self.is_chunk = True if self.enable_llm_stream: queue = create_llm_stream_queue() self._llm_task = asyncio.create_task(self._llm_stream_report(queue)) @@ -171,15 +167,18 @@ class ResourceReporter(BaseModel): async def __aexit__(self, *args, **kwargs): """Exit the asynchronous streaming callback context.""" - self.is_chunk = False if self.enable_llm_stream: - self._llm_task.cancel() + await get_llm_stream_queue().put(None) + await self._llm_task self._llm_task = None await self.async_report(None, END_MARKER_NAME) async def _llm_stream_report(self, queue: asyncio.Queue): - while self.is_chunk: - await self.async_report(await queue.get(), "content") + while True: + data = await queue.get() + if data is None: + return + await self.async_report(data, "content") async def wait_llm_stream_report(self): """Wait for the LLM stream report to complete.""" From 991bfd6f6837dd08c0aa5401eb5aba3ea4bb8a6d Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Tue, 18 Jun 2024 11:57:49 +0800 Subject: [PATCH 18/41] Fix: Correct regex pattern to accurately match code blocks in Markdown --- metagpt/utils/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index b58ac99da..72dc6ab94 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -275,7 +275,7 @@ class CodeParser: def parse_code(cls, text: str, lang: str = "", block: Optional[str] = None) -> str: if block: text = cls.parse_block(block, text) - pattern = rf"```{lang}.*?\s+(.*?)```" + pattern = rf"```{lang}.*?\s+(.*?)\n```" match = re.search(pattern, text, re.DOTALL) if match: code = match.group(1) From 351cf18c3a4915fd11e0856d9182e6b489c9e231 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Tue, 18 Jun 2024 12:01:53 +0800 Subject: [PATCH 19/41] Refactor: Update message publishing logic to handle multiple recipients --- metagpt/environment/mgx/mgx_env.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/environment/mgx/mgx_env.py b/metagpt/environment/mgx/mgx_env.py index ab515e070..33160cfd8 100644 --- a/metagpt/environment/mgx/mgx_env.py +++ b/metagpt/environment/mgx/mgx_env.py @@ -31,11 +31,11 @@ class MGXEnv(Environment): if user_defined_recipient: # human user's direct chat message to a certain role - - if self.get_role(user_defined_recipient).is_idle: - # User starts a new direct chat with a certain role, expecting a direct chat response from the role; Other roles including TL should not be involved. - # If the role is not idle, it means the user helps the role with its current work, in this case, we handle the role's response message as usual. - self.direct_chat_roles.add(user_defined_recipient) + for role_name in message.send_to: + if self.get_role(role_name).is_idle: + # User starts a new direct chat with a certain role, expecting a direct chat response from the role; Other roles including TL should not be involved. + # If the role is not idle, it means the user helps the role with its current work, in this case, we handle the role's response message as usual. + self.direct_chat_roles.add(role_name) self._publish_message(message) # # bypass team leader, team leader only needs to know but not to react (commented out because TL doesn't understand the message well in actual experiments) From 9d06f9f4f8a0214bff584289430e67a123f09c14 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Tue, 18 Jun 2024 14:38:40 +0800 Subject: [PATCH 20/41] Handle potential incorrect newline representations in FileWriter.write() --- metagpt/tools/libs/editor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index d145fdabe..c9f02c4f1 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -1,5 +1,4 @@ import os -import re import shutil import subprocess @@ -27,8 +26,9 @@ class Editor: def write(self, path: str, content: str): """Write the whole content to a file. When used, make sure content arg contains the full content of the file.""" - if len(re.findall(r"\\n", content)) >= 5: - # A very raw rule to correct the content: Many \\n suggests all new line characters are mistaken as \\n whereas the correct one should be \n + if "\n" not in content and "\\n" in content: + # A very raw rule to correct the content: If 'content' lacks actual newlines ('\n') but includes '\\n', consider + # replacing them with '\n' to potentially correct mistaken representations of newline characters. content = content.replace("\\n", "\n") directory = os.path.dirname(path) if directory and not os.path.exists(directory): From 63a41ba81ded62d306fe9b406dfaaefdc635a6d7 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Tue, 18 Jun 2024 21:35:46 +0800 Subject: [PATCH 21/41] add code_review tool to engineer2 to enhance code effectiveness --- metagpt/actions/di/rewrite_code.py | 80 +++++++++++++++++++++++++ metagpt/prompts/di/engineer2.py | 9 +++ metagpt/roles/di/engineer2.py | 13 +++- tests/metagpt/roles/di/run_engineer2.py | 8 +-- 4 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 metagpt/actions/di/rewrite_code.py diff --git a/metagpt/actions/di/rewrite_code.py b/metagpt/actions/di/rewrite_code.py new file mode 100644 index 000000000..a8586d1a6 --- /dev/null +++ b/metagpt/actions/di/rewrite_code.py @@ -0,0 +1,80 @@ +from tenacity import retry, stop_after_attempt, wait_random_exponential + +from metagpt.actions.action import Action +from metagpt.actions.write_code_review import ( + EXAMPLE_AND_INSTRUCTION, + FORMAT_EXAMPLE, + PROMPT_TEMPLATE, + REWRITE_CODE_TEMPLATE, +) +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.common import CodeParser, aread, awrite + + +@register_tool(tags=["RewriteCode"], include_functions=["run"]) +class RewriteCode(Action): + """Accordding design doc and task doc to review the code, to make the complete and correct code.""" + + name: str = "RewriteCode" + + async def run(self, code_path: str, design_doc: str = "", task_doc: str = "", code_review_k_times: int = 2) -> str: + """Reviews the provided code based on the accompanying design and task documentation, return the complete and correct code. + + Read the code from `code_path`, and write the final code to `code_path`. + + Args: + code_path (str): The file path of the code snippet to be reviewed. This should be a string containing the path to the source code file. + design_doc (str): The design document associated with the code. This should describe the system architecture, used in the code. It helps provide context for the review process. + task_doc (str): The task document describing what the code is intended to accomplish. This should outline the functional requirements or objectives of the code. + code_review_k_times (int, optional): The number of iterations for reviewing and potentially rewriting the code. Defaults to 2. + + Returns: + str: The potentially corrected or approved code after review. + + Example Usage: + # Example of how to call the run method with a code snippet and documentation + await WriteCodeReview().run( + code_path="/tmp/game.js", + design_doc='{"Implementation approach":"We will implement the 2048 game using plain JavaScript and HTML, ensuring no frameworks are used. The game logic will handle tile movements, merging, and game state updates. The UI will be simple and clean, with a responsive design to fit different screen sizes. We will use CSS for styling and ensure the game is playable with keyboard arrow keys. The game will display the current score, have a restart button, and show a game over message when no more moves are possible.","File list":["index.html","style.css","script.js"],"Data structures and interfaces":"\nclassDiagram\n class Game {\n -grid: int[][]\n -score: int\n +init(): void\n +move(direction: str): void\n +merge(direction: str): void\n +isGameOver(): bool\n +restart(): void\n }\n class UI {\n -game: Game\n +init(): void\n +update(): void\n +showGameOver(): void\n +bindEvents(): void\n }\n Game --> UI\n","Program call flow":"\nsequenceDiagram\n participant U as UI\n participant G as Game\n U->>G: init()\n G-->>U: return\n U->>U: bindEvents()\n U->>G: move(direction)\n G->>G: merge(direction)\n G->>U: update()\n U->>U: update()\n U->>G: isGameOver()\n G-->>U: return bool\n alt Game Over\n U->>U: showGameOver()\n end\n U->>G: restart()\n G-->>U: return\n","Anything UNCLEAR":"Clarify if there are any specific design preferences or additional features required beyond the basic 2048 game functionality."}', + task_doc='{"Required packages":["No third-party dependencies required"],"Required Other language third-party packages":["No third-party dependencies required"],"Logic Analysis":[["script.js","Contains Game and UI classes, and their methods: init, move, merge, isGameOver, restart, update, showGameOver, bindEvents"],["index.html","Contains the HTML structure for the game UI"],["style.css","Contains the CSS styles for the game UI"]],"Task list":["index.html","style.css","script.js"],"Full API spec":"","Shared Knowledge":"The `script.js` file will contain the core game logic and UI handling. The `index.html` file will provide the structure for the game, and `style.css` will handle the styling.","Anything UNCLEAR":"Clarify if there are any specific design preferences or additional features required beyond the basic 2048 game functionality."}' + ) + """ + code = await aread(code_path) + + context = "\n".join( + [ + "## System Design\n" + design_doc + "\n", + "## Task\n" + task_doc + "\n", + ] + ) + + for _ in range(code_review_k_times): + context_prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=code_path) + cr_prompt = EXAMPLE_AND_INSTRUCTION.format( + format_example=FORMAT_EXAMPLE.format(filename=code_path), + ) + result, rewrited_code = await self.write_code_review_and_rewrite( + context_prompt, cr_prompt, filename=code_path + ) + + if "LBTM" in result: + code = rewrited_code + elif "LGTM" in result: + break + + await awrite(filename=code_path, data=code) + + return code + + @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) + async def write_code_review_and_rewrite(self, context_prompt: str, cr_prompt: str, filename: str): + cr_rsp = await self._aask(context_prompt + cr_prompt) + result = CodeParser.parse_block("Code Review Result", cr_rsp) + if "LGTM" in result: + return result, None + + # if LBTM, rewrite code + rewrite_prompt = f"{context_prompt}\n{cr_rsp}\n{REWRITE_CODE_TEMPLATE.format(filename=filename)}" + code_rsp = await self._aask(rewrite_prompt) + code = CodeParser.parse_code(block="", text=code_rsp) + return result, code diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 4fd52e320..8e2722b49 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -5,5 +5,14 @@ EXTRA_INSTRUCTION = """ 5. Take on ONE task and write ONE code file in each response. DON'T attempt all tasks in one response. 6. When not specified, you should write files in a folder named "src". If you know the project path, then write in a "src" folder under the project path. 7. When provided system design or project schedule, read them first, then adhere to them in your implementation. +8. Write at most one file per task, do your best to implement THE ONLY ONE FILE. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. +9. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets. +10. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. +11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. +12. Do not use Editor to find start_line and end_line, just rewrite the file with the all complete code. +13. Revise task is to use RewriteCode.run to correct code. +14. At the end of the plan, add a Revise task for each file; for example, if there are three files, add three Revise tasks. """ + + ENGINEER2_INSTRUCTION = ROLE_INSTRUCTION + EXTRA_INSTRUCTION.strip() diff --git a/metagpt/roles/di/engineer2.py b/metagpt/roles/di/engineer2.py index e013ef09e..2720364ef 100644 --- a/metagpt/roles/di/engineer2.py +++ b/metagpt/roles/di/engineer2.py @@ -1,5 +1,6 @@ from __future__ import annotations +from metagpt.actions.di.rewrite_code import RewriteCode from metagpt.prompts.di.engineer2 import ENGINEER2_INSTRUCTION from metagpt.roles.di.role_zero import RoleZero @@ -10,4 +11,14 @@ class Engineer2(RoleZero): goal: str = "Take on game, app, and web development" instruction: str = ENGINEER2_INSTRUCTION - tools: str = ["Plan", "Editor:write,read,write_content", "RoleZero"] + tools: str = ["Plan", "Editor:write,read,write_content", "RoleZero", "RewriteCode"] + + def _update_tool_execution(self): + rewrite_code = RewriteCode() + + self.tool_execution_map.update( + { + "RewriteCode.run": rewrite_code.run, + "RewriteCode": rewrite_code.run, + } + ) diff --git a/tests/metagpt/roles/di/run_engineer2.py b/tests/metagpt/roles/di/run_engineer2.py index 4e948bad7..e5ae74485 100644 --- a/tests/metagpt/roles/di/run_engineer2.py +++ b/tests/metagpt/roles/di/run_engineer2.py @@ -67,18 +67,18 @@ Create a 2048 game, follow the design doc and task doc. Write your code under /U After writing all codes, write a code review for the codes, make improvement or adjustment based on the review. Notice: You MUST implement the full code, don't leave comment without implementation! Design doc: -{TASK_DOC_2048} -Task doc: {DESIGN_DOC_2048} +Task doc: +{TASK_DOC_2048} """ GAME_REQ_SNAKE = f""" Create a snake game, follow the design doc and task doc. Write your code under /Users/gary/Files/temp/workspace/snake_game/src. After writing all codes, write a code review for the codes, make improvement or adjustment based on the review. Notice: You MUST implement the full code, don't leave comment without implementation! Design doc: -{TASK_DOC_SNAKE} -Task doc: {DESIGN_DOC_SNAKE} +Task doc: +{TASK_DOC_SNAKE} """ GAME_REQ_2048_NO_DOC = """ Create a 2048 game with pygame. Write your code under /Users/gary/Files/temp/workspace/2048_game/src. From 47df4c09ba2e716e21c5f38047cb79f8af0cd67c Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Wed, 19 Jun 2024 15:42:07 +0800 Subject: [PATCH 22/41] add code_review tool to engineer2 to enhance code effectiveness --- metagpt/actions/di/rewrite_code.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/di/rewrite_code.py b/metagpt/actions/di/rewrite_code.py index a8586d1a6..57abbc56c 100644 --- a/metagpt/actions/di/rewrite_code.py +++ b/metagpt/actions/di/rewrite_code.py @@ -35,8 +35,8 @@ class RewriteCode(Action): # Example of how to call the run method with a code snippet and documentation await WriteCodeReview().run( code_path="/tmp/game.js", - design_doc='{"Implementation approach":"We will implement the 2048 game using plain JavaScript and HTML, ensuring no frameworks are used. The game logic will handle tile movements, merging, and game state updates. The UI will be simple and clean, with a responsive design to fit different screen sizes. We will use CSS for styling and ensure the game is playable with keyboard arrow keys. The game will display the current score, have a restart button, and show a game over message when no more moves are possible.","File list":["index.html","style.css","script.js"],"Data structures and interfaces":"\nclassDiagram\n class Game {\n -grid: int[][]\n -score: int\n +init(): void\n +move(direction: str): void\n +merge(direction: str): void\n +isGameOver(): bool\n +restart(): void\n }\n class UI {\n -game: Game\n +init(): void\n +update(): void\n +showGameOver(): void\n +bindEvents(): void\n }\n Game --> UI\n","Program call flow":"\nsequenceDiagram\n participant U as UI\n participant G as Game\n U->>G: init()\n G-->>U: return\n U->>U: bindEvents()\n U->>G: move(direction)\n G->>G: merge(direction)\n G->>U: update()\n U->>U: update()\n U->>G: isGameOver()\n G-->>U: return bool\n alt Game Over\n U->>U: showGameOver()\n end\n U->>G: restart()\n G-->>U: return\n","Anything UNCLEAR":"Clarify if there are any specific design preferences or additional features required beyond the basic 2048 game functionality."}', - task_doc='{"Required packages":["No third-party dependencies required"],"Required Other language third-party packages":["No third-party dependencies required"],"Logic Analysis":[["script.js","Contains Game and UI classes, and their methods: init, move, merge, isGameOver, restart, update, showGameOver, bindEvents"],["index.html","Contains the HTML structure for the game UI"],["style.css","Contains the CSS styles for the game UI"]],"Task list":["index.html","style.css","script.js"],"Full API spec":"","Shared Knowledge":"The `script.js` file will contain the core game logic and UI handling. The `index.html` file will provide the structure for the game, and `style.css` will handle the styling.","Anything UNCLEAR":"Clarify if there are any specific design preferences or additional features required beyond the basic 2048 game functionality."}' + design_doc='{"Implementation approach":"We will implement the 2048 game..."}', + task_doc='{"Required packages":["No third-party dependencies required"],"..."}' ) """ code = await aread(code_path) From 30286ce4b86fbf02a8178ff82ac9a73986cc4e1c Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Wed, 19 Jun 2024 16:04:55 +0800 Subject: [PATCH 23/41] fix da not responded to new req; rm user_req attribute from rolezero --- metagpt/roles/di/data_analyst.py | 11 +++++++---- metagpt/roles/di/role_zero.py | 4 +--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py index e489c9d98..1ccdd5e3c 100644 --- a/metagpt/roles/di/data_analyst.py +++ b/metagpt/roles/di/data_analyst.py @@ -41,6 +41,7 @@ class DataAnalyst(DataInterpreter): # Command.PASS, ] commands: list[dict] = [] # issued commands to be executed + user_requirement: str = "" @model_validator(mode="after") def set_plan_and_tool(self) -> "DataInterpreter": @@ -68,9 +69,6 @@ class DataAnalyst(DataInterpreter): self.user_requirement = self.get_memories()[-1].content self.planner.plan.goal = self.user_requirement example = KeywordExpRetriever().retrieve(self.user_requirement) - else: - self.working_memory.add_batch(self.rc.news) - # TODO: implement experience retrieval in multi-round setting plan_status = self.planner.plan.model_dump(include=["goal", "tasks"]) # for task in plan_status["tasks"]: @@ -82,10 +80,11 @@ class DataAnalyst(DataInterpreter): available_commands=prepare_command_prompt(self.available_commands), ) context = self.llm.format_msg(self.working_memory.get() + [Message(content=prompt, role="user")]) + # print(*context, sep="\n" + "*" * 5 + "\n") async with ThoughtReporter(): rsp = await self.llm.aask(context) self.commands = json.loads(CodeParser.parse_code(block=None, text=rsp)) - self.rc.memory.add(Message(content=rsp, role="assistant")) + self.rc.working_memory.add(Message(content=rsp, role="assistant")) await run_commands(self, self.commands, self.rc.working_memory) @@ -118,7 +117,11 @@ class DataAnalyst(DataInterpreter): rsp = Message(content="No actions taken yet", cause_by=Action) # will be overwritten after Role _act while actions_taken < self.rc.max_react_loop: # NOTE: Diff 2: Keep observing within _react, news will go into memory, allowing adapting to new info + # add news from self._observe, the one called in self.run, consider removing when switching from working_memory to memory + self.working_memory.add_batch(self.rc.news) await self._observe() + # add news from this self._observe, we need twice because _observe rewrites rc.news + self.working_memory.add_batch(self.rc.news) # think has_todo = await self._think() diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index 44e658ed4..3f6e53eb6 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -53,7 +53,6 @@ class RoleZero(Role): experience_retriever: ExpRetriever = DummyExpRetriever() # Others - user_requirement: str = "" command_rsp: str = "" # the raw string containing the commands commands: list[dict] = [] # commands to be executed memory_k: int = 20 # number of memories (messages) to use as historical context @@ -106,8 +105,7 @@ class RoleZero(Role): return False if not self.planner.plan.goal: - self.user_requirement = self.get_memories()[-1].content - self.planner.plan.goal = self.user_requirement + self.planner.plan.goal = self.get_memories()[-1].content ### 1. Experience ### example = self._retrieve_experience() From 942a6d61bb73eb9847c7c9b1f6a932964a03751a Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Wed, 19 Jun 2024 23:41:09 +0800 Subject: [PATCH 24/41] opt engineer2 --- metagpt/actions/di/rewrite_code.py | 8 +++++++- metagpt/prompts/di/engineer2.py | 6 +++--- metagpt/prompts/di/team_leader.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/di/rewrite_code.py b/metagpt/actions/di/rewrite_code.py index 57abbc56c..4ad612002 100644 --- a/metagpt/actions/di/rewrite_code.py +++ b/metagpt/actions/di/rewrite_code.py @@ -7,6 +7,7 @@ from metagpt.actions.write_code_review import ( PROMPT_TEMPLATE, REWRITE_CODE_TEMPLATE, ) +from metagpt.logs import logger from metagpt.tools.tool_registry import register_tool from metagpt.utils.common import CodeParser, aread, awrite @@ -21,6 +22,7 @@ class RewriteCode(Action): """Reviews the provided code based on the accompanying design and task documentation, return the complete and correct code. Read the code from `code_path`, and write the final code to `code_path`. + If there is no `design_doc` or `task_doc`, it will return and do nothing. Args: code_path (str): The file path of the code snippet to be reviewed. This should be a string containing the path to the source code file. @@ -39,6 +41,9 @@ class RewriteCode(Action): task_doc='{"Required packages":["No third-party dependencies required"],"..."}' ) """ + if not design_doc or not task_doc: + return + code = await aread(code_path) context = "\n".join( @@ -48,11 +53,12 @@ class RewriteCode(Action): ] ) - for _ in range(code_review_k_times): + for i in range(code_review_k_times): context_prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=code_path) cr_prompt = EXAMPLE_AND_INSTRUCTION.format( format_example=FORMAT_EXAMPLE.format(filename=code_path), ) + logger.info(f"The {i+1}th time to CodeReview: {code_path}.") result, rewrited_code = await self.write_code_review_and_rewrite( context_prompt, cr_prompt, filename=code_path ) diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 8e2722b49..55947f3b9 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -9,9 +9,9 @@ EXTRA_INSTRUCTION = """ 9. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets. 10. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. 11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. -12. Do not use Editor to find start_line and end_line, just rewrite the file with the all complete code. -13. Revise task is to use RewriteCode.run to correct code. -14. At the end of the plan, add a Revise task for each file; for example, if there are three files, add three Revise tasks. +12. To modify code in a file, read the entire file, make changes, and use Editor.write instead of Editor.write_content to update the file with the complete code. +13. Revise task is to use RewriteCode.run to correct code, must pass the content of system design and project schedule instead of just file path of them. +14. Only When provided system design, at the end of the plan, add a Revise task for each file; for example, if there are three files, add three Revise tasks. """ diff --git a/metagpt/prompts/di/team_leader.py b/metagpt/prompts/di/team_leader.py index 7af5a6e07..f166cee38 100644 --- a/metagpt/prompts/di/team_leader.py +++ b/metagpt/prompts/di/team_leader.py @@ -20,7 +20,7 @@ Note: 3. If the requirement contains both DATA-RELATED part mentioned in 1 and software development part mentioned in 2, you should decompose the software development part and assign them to different team members based on their expertise, and assign the DATA-RELATED part to Data Analyst David directly. 4. If the requirement is a common-sense, logical, or math problem, you should respond directly without assigning any task to team members. 5. If you think the requirement is not clear or ambiguous, you should ask the user for clarification immediately. Assign tasks only after all info is clear. -6. It is helpful for Engineer to have both the system design and the project schedule for writing the code, so include paths of both files (if available) when publishing message to Engineer. +6. It is helpful for Engineer to have both the system design and the project schedule for writing the code, so include paths of both files (if available) and remind Engineer to definitely read them when publishing message to Engineer. """ FINISH_CURRENT_TASK_CMD = """ From 599eecf1f460d2795a12d8945dd43de82c33dfa2 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Thu, 20 Jun 2024 10:34:20 +0800 Subject: [PATCH 25/41] opt engineer2 --- metagpt/actions/di/rewrite_code.py | 22 ++++++++++++++-------- metagpt/prompts/di/engineer2.py | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/metagpt/actions/di/rewrite_code.py b/metagpt/actions/di/rewrite_code.py index 4ad612002..772a9ee99 100644 --- a/metagpt/actions/di/rewrite_code.py +++ b/metagpt/actions/di/rewrite_code.py @@ -1,3 +1,5 @@ +import asyncio + from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action import Action @@ -18,16 +20,18 @@ class RewriteCode(Action): name: str = "RewriteCode" - async def run(self, code_path: str, design_doc: str = "", task_doc: str = "", code_review_k_times: int = 2) -> str: + async def run( + self, code_path: str, design_doc_path: str = "", task_doc_path: str = "", code_review_k_times: int = 2 + ) -> str: """Reviews the provided code based on the accompanying design and task documentation, return the complete and correct code. Read the code from `code_path`, and write the final code to `code_path`. - If there is no `design_doc` or `task_doc`, it will return and do nothing. + If there is no `design_doc_path` or `task_doc_path`, it will return and do nothing. Args: code_path (str): The file path of the code snippet to be reviewed. This should be a string containing the path to the source code file. - design_doc (str): The design document associated with the code. This should describe the system architecture, used in the code. It helps provide context for the review process. - task_doc (str): The task document describing what the code is intended to accomplish. This should outline the functional requirements or objectives of the code. + design_doc_path (str): The file path of the design document associated with the code. This should describe the system architecture, used in the code. It helps provide context for the review process. + task_doc_path (str): The file path of the task document describing what the code is intended to accomplish. This should outline the functional requirements or objectives of the code. code_review_k_times (int, optional): The number of iterations for reviewing and potentially rewriting the code. Defaults to 2. Returns: @@ -37,14 +41,16 @@ class RewriteCode(Action): # Example of how to call the run method with a code snippet and documentation await WriteCodeReview().run( code_path="/tmp/game.js", - design_doc='{"Implementation approach":"We will implement the 2048 game..."}', - task_doc='{"Required packages":["No third-party dependencies required"],"..."}' + design_doc="/tmp/design_doc.json", + task_doc="/tmp/task_doc.json" ) """ - if not design_doc or not task_doc: + if not design_doc_path or not design_doc_path: return - code = await aread(code_path) + code, design_doc, task_doc = await asyncio.gather( + aread(code_path), aread(design_doc_path), aread(task_doc_path) + ) context = "\n".join( [ diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 55947f3b9..eab8b44ff 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -10,7 +10,7 @@ EXTRA_INSTRUCTION = """ 10. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. 11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. 12. To modify code in a file, read the entire file, make changes, and use Editor.write instead of Editor.write_content to update the file with the complete code. -13. Revise task is to use RewriteCode.run to correct code, must pass the content of system design and project schedule instead of just file path of them. +13. Revise task is to use RewriteCode.run to correct code. 14. Only When provided system design, at the end of the plan, add a Revise task for each file; for example, if there are three files, add three Revise tasks. """ From 7538e715edd5af8577038cfbe39bfddb9314bbfb Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Thu, 20 Jun 2024 10:36:19 +0800 Subject: [PATCH 26/41] opt engineer2 --- metagpt/actions/di/rewrite_code.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/di/rewrite_code.py b/metagpt/actions/di/rewrite_code.py index 772a9ee99..0b00cd1b5 100644 --- a/metagpt/actions/di/rewrite_code.py +++ b/metagpt/actions/di/rewrite_code.py @@ -41,8 +41,8 @@ class RewriteCode(Action): # Example of how to call the run method with a code snippet and documentation await WriteCodeReview().run( code_path="/tmp/game.js", - design_doc="/tmp/design_doc.json", - task_doc="/tmp/task_doc.json" + design_doc_path="/tmp/design_doc.json", + task_doc_path="/tmp/task_doc.json" ) """ if not design_doc_path or not design_doc_path: From 0d3187e3089a64ea11db976d4815502ccf6d197d Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Thu, 20 Jun 2024 11:32:07 +0800 Subject: [PATCH 27/41] update --- metagpt/actions/di/rewrite_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/di/rewrite_code.py b/metagpt/actions/di/rewrite_code.py index 0b00cd1b5..03831f912 100644 --- a/metagpt/actions/di/rewrite_code.py +++ b/metagpt/actions/di/rewrite_code.py @@ -45,7 +45,7 @@ class RewriteCode(Action): task_doc_path="/tmp/task_doc.json" ) """ - if not design_doc_path or not design_doc_path: + if not design_doc_path or not task_doc_path: return code, design_doc, task_doc = await asyncio.gather( From 256e73e878b41ce261bfe8c71bc67f654c76cd48 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Thu, 20 Jun 2024 12:29:37 +0800 Subject: [PATCH 28/41] rewrite_code accept content or file_path of design doc --- metagpt/actions/di/rewrite_code.py | 33 ++++++++++++++++++------------ 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/metagpt/actions/di/rewrite_code.py b/metagpt/actions/di/rewrite_code.py index 03831f912..ea093b36c 100644 --- a/metagpt/actions/di/rewrite_code.py +++ b/metagpt/actions/di/rewrite_code.py @@ -1,4 +1,4 @@ -import asyncio +import os from tenacity import retry, stop_after_attempt, wait_random_exponential @@ -21,17 +21,17 @@ class RewriteCode(Action): name: str = "RewriteCode" async def run( - self, code_path: str, design_doc_path: str = "", task_doc_path: str = "", code_review_k_times: int = 2 + self, code_path: str, design_doc_input: str = "", task_doc_input: str = "", code_review_k_times: int = 2 ) -> str: """Reviews the provided code based on the accompanying design and task documentation, return the complete and correct code. Read the code from `code_path`, and write the final code to `code_path`. - If there is no `design_doc_path` or `task_doc_path`, it will return and do nothing. + If there is no `design_doc_input` or `task_doc_input`, it will return and do nothing. Args: code_path (str): The file path of the code snippet to be reviewed. This should be a string containing the path to the source code file. - design_doc_path (str): The file path of the design document associated with the code. This should describe the system architecture, used in the code. It helps provide context for the review process. - task_doc_path (str): The file path of the task document describing what the code is intended to accomplish. This should outline the functional requirements or objectives of the code. + design_doc_input (str): Content or file path of the design document associated with the code. This should describe the system architecture, used in the code. It helps provide context for the review process. + task_doc_input (str): Content or file path of the task document describing what the code is intended to accomplish. This should outline the functional requirements or objectives of the code. code_review_k_times (int, optional): The number of iterations for reviewing and potentially rewriting the code. Defaults to 2. Returns: @@ -41,21 +41,28 @@ class RewriteCode(Action): # Example of how to call the run method with a code snippet and documentation await WriteCodeReview().run( code_path="/tmp/game.js", - design_doc_path="/tmp/design_doc.json", - task_doc_path="/tmp/task_doc.json" + design_doc_input="/tmp/design_doc.json", + task_doc_input='{"Required packages":["No third-party dependencies required"], ...}' ) """ - if not design_doc_path or not task_doc_path: + if not design_doc_input or not task_doc_input: return - code, design_doc, task_doc = await asyncio.gather( - aread(code_path), aread(design_doc_path), aread(task_doc_path) - ) + code = await aread(code_path) + + # Check if design_doc_input and task_doc_input are paths or content, and read if they are paths + if os.path.exists(design_doc_input): + logger.info(f"read from {design_doc_input}") + design_doc_input = await aread(design_doc_input) + + if os.path.exists(task_doc_input): + logger.info(f"read from {task_doc_input}") + task_doc_input = await aread(task_doc_input) context = "\n".join( [ - "## System Design\n" + design_doc + "\n", - "## Task\n" + task_doc + "\n", + "## System Design\n" + design_doc_input + "\n", + "## Task\n" + task_doc_input + "\n", ] ) From 76f0d5aad825cd7ba2138150fdc78baa8bc9489f Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Thu, 20 Jun 2024 12:31:38 +0800 Subject: [PATCH 29/41] update comment --- metagpt/actions/di/rewrite_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/di/rewrite_code.py b/metagpt/actions/di/rewrite_code.py index ea093b36c..504ff72c8 100644 --- a/metagpt/actions/di/rewrite_code.py +++ b/metagpt/actions/di/rewrite_code.py @@ -39,7 +39,7 @@ class RewriteCode(Action): Example Usage: # Example of how to call the run method with a code snippet and documentation - await WriteCodeReview().run( + await RewriteCode().run( code_path="/tmp/game.js", design_doc_input="/tmp/design_doc.json", task_doc_input='{"Required packages":["No third-party dependencies required"], ...}' From 896c7d8d4cd774b644b0bbddc0c0bcc13bf6c818 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Thu, 20 Jun 2024 13:03:51 +0800 Subject: [PATCH 30/41] rewrite_code accept content or file_path of design doc --- metagpt/actions/di/rewrite_code.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/metagpt/actions/di/rewrite_code.py b/metagpt/actions/di/rewrite_code.py index 504ff72c8..0b03b534e 100644 --- a/metagpt/actions/di/rewrite_code.py +++ b/metagpt/actions/di/rewrite_code.py @@ -1,3 +1,4 @@ +import asyncio import os from tenacity import retry, stop_after_attempt, wait_random_exponential @@ -45,24 +46,18 @@ class RewriteCode(Action): task_doc_input='{"Required packages":["No third-party dependencies required"], ...}' ) """ + if not design_doc_input or not task_doc_input: return - code = await aread(code_path) - - # Check if design_doc_input and task_doc_input are paths or content, and read if they are paths - if os.path.exists(design_doc_input): - logger.info(f"read from {design_doc_input}") - design_doc_input = await aread(design_doc_input) - - if os.path.exists(task_doc_input): - logger.info(f"read from {task_doc_input}") - task_doc_input = await aread(task_doc_input) + code, design_doc, task_doc = await asyncio.gather( + aread(code_path), self._try_aread(design_doc_input), self._try_aread(task_doc_input) + ) context = "\n".join( [ - "## System Design\n" + design_doc_input + "\n", - "## Task\n" + task_doc_input + "\n", + "## System Design\n" + design_doc + "\n", + "## Task\n" + task_doc + "\n", ] ) @@ -97,3 +92,12 @@ class RewriteCode(Action): code_rsp = await self._aask(rewrite_prompt) code = CodeParser.parse_code(block="", text=code_rsp) return result, code + + @staticmethod + async def _try_aread(input: str) -> str: + """Try to read from the path if it's a file; return input directly if not.""" + + if os.path.exists(input): + return await aread(input) + + return input From 6c4cb765c78d1e58c69b6a9bc2a80aa3f515f470 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Thu, 20 Jun 2024 14:54:01 +0800 Subject: [PATCH 31/41] reuse write_code_review_and_rewrite --- metagpt/actions/di/rewrite_code.py | 35 ++++++++++++++---------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/metagpt/actions/di/rewrite_code.py b/metagpt/actions/di/rewrite_code.py index 0b03b534e..659635ad5 100644 --- a/metagpt/actions/di/rewrite_code.py +++ b/metagpt/actions/di/rewrite_code.py @@ -1,18 +1,18 @@ import asyncio import os - -from tenacity import retry, stop_after_attempt, wait_random_exponential +from pathlib import Path from metagpt.actions.action import Action from metagpt.actions.write_code_review import ( EXAMPLE_AND_INSTRUCTION, FORMAT_EXAMPLE, PROMPT_TEMPLATE, - REWRITE_CODE_TEMPLATE, + WriteCodeReview, ) from metagpt.logs import logger +from metagpt.schema import CodingContext, Document from metagpt.tools.tool_registry import register_tool -from metagpt.utils.common import CodeParser, aread, awrite +from metagpt.utils.common import aread, awrite @register_tool(tags=["RewriteCode"], include_functions=["run"]) @@ -53,6 +53,8 @@ class RewriteCode(Action): code, design_doc, task_doc = await asyncio.gather( aread(code_path), self._try_aread(design_doc_input), self._try_aread(task_doc_input) ) + code_doc = self._create_code_doc(code_path=code_path, code=code) + reviewer = WriteCodeReview(i_context=CodingContext(filename=code_doc.filename)) context = "\n".join( [ @@ -67,8 +69,8 @@ class RewriteCode(Action): format_example=FORMAT_EXAMPLE.format(filename=code_path), ) logger.info(f"The {i+1}th time to CodeReview: {code_path}.") - result, rewrited_code = await self.write_code_review_and_rewrite( - context_prompt, cr_prompt, filename=code_path + result, rewrited_code = await reviewer.write_code_review_and_rewrite( + context_prompt, cr_prompt, doc=code_doc ) if "LBTM" in result: @@ -80,19 +82,6 @@ class RewriteCode(Action): return code - @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) - async def write_code_review_and_rewrite(self, context_prompt: str, cr_prompt: str, filename: str): - cr_rsp = await self._aask(context_prompt + cr_prompt) - result = CodeParser.parse_block("Code Review Result", cr_rsp) - if "LGTM" in result: - return result, None - - # if LBTM, rewrite code - rewrite_prompt = f"{context_prompt}\n{cr_rsp}\n{REWRITE_CODE_TEMPLATE.format(filename=filename)}" - code_rsp = await self._aask(rewrite_prompt) - code = CodeParser.parse_code(block="", text=code_rsp) - return result, code - @staticmethod async def _try_aread(input: str) -> str: """Try to read from the path if it's a file; return input directly if not.""" @@ -101,3 +90,11 @@ class RewriteCode(Action): return await aread(input) return input + + @staticmethod + def _create_code_doc(code_path: str, code: str) -> Document: + """Create a Document to represent the code doc.""" + + path = Path(code_path) + + return Document(root_path=str(path.parent), filename=path.name, content=code) From 050bec2ccfcb23f21ef1170f2df5a6881a927f27 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Thu, 20 Jun 2024 15:40:56 +0800 Subject: [PATCH 32/41] update prompt --- metagpt/prompts/di/engineer2.py | 4 ++-- metagpt/roles/di/engineer2.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index eab8b44ff..102792001 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -4,12 +4,12 @@ EXTRA_INSTRUCTION = """ 4. Each time you write a code in your response, write with the Editor directly without preparing a repetitive code block beforehand. 5. Take on ONE task and write ONE code file in each response. DON'T attempt all tasks in one response. 6. When not specified, you should write files in a folder named "src". If you know the project path, then write in a "src" folder under the project path. -7. When provided system design or project schedule, read them first, then adhere to them in your implementation. +7. When provided system design or project schedule, read them first, then adhere to them in your implementation, especially in the programming language, package, or framework. You MUST implement all code files prescribed in the system design or project schedule. You can create a plan first with each task corresponding to implementing one code file. 8. Write at most one file per task, do your best to implement THE ONLY ONE FILE. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. 9. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets. 10. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. 11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. -12. To modify code in a file, read the entire file, make changes, and use Editor.write instead of Editor.write_content to update the file with the complete code. +12. To modify code in a file, read the entire file, make changes, and update the file with the complete code. 13. Revise task is to use RewriteCode.run to correct code. 14. Only When provided system design, at the end of the plan, add a Revise task for each file; for example, if there are three files, add three Revise tasks. """ diff --git a/metagpt/roles/di/engineer2.py b/metagpt/roles/di/engineer2.py index 2720364ef..152c62ca2 100644 --- a/metagpt/roles/di/engineer2.py +++ b/metagpt/roles/di/engineer2.py @@ -11,7 +11,7 @@ class Engineer2(RoleZero): goal: str = "Take on game, app, and web development" instruction: str = ENGINEER2_INSTRUCTION - tools: str = ["Plan", "Editor:write,read,write_content", "RoleZero", "RewriteCode"] + tools: str = ["Plan", "Editor:write,read", "RoleZero", "RewriteCode"] def _update_tool_execution(self): rewrite_code = RewriteCode() From 97e9409eadb6ef2356f34f57cb825e3058d09faf Mon Sep 17 00:00:00 2001 From: lidanyang Date: Thu, 20 Jun 2024 16:42:23 +0800 Subject: [PATCH 33/41] restrict to json lang --- metagpt/roles/di/data_analyst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py index 71e6bacd4..d4d67742b 100644 --- a/metagpt/roles/di/data_analyst.py +++ b/metagpt/roles/di/data_analyst.py @@ -83,7 +83,7 @@ class DataAnalyst(DataInterpreter): # print(*context, sep="\n" + "*" * 5 + "\n") async with ThoughtReporter(enable_llm_stream=True): rsp = await self.llm.aask(context) - self.commands = json.loads(CodeParser.parse_code(block=None, text=rsp)) + self.commands = json.loads(CodeParser.parse_code(block=None, lang='json', text=rsp)) self.rc.working_memory.add(Message(content=rsp, role="assistant")) await run_commands(self, self.commands, self.rc.working_memory) From a48ccfdcf9fa7b9ebfaeb69bea7d6675aead4dc0 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Thu, 20 Jun 2024 20:09:25 +0800 Subject: [PATCH 34/41] move RewriteCode to actions/write_code_review.py --- metagpt/actions/di/rewrite_code.py | 100 --------------------------- metagpt/actions/write_code_review.py | 91 +++++++++++++++++++++++- metagpt/prompts/di/engineer2.py | 3 +- metagpt/roles/di/engineer2.py | 2 +- 4 files changed, 92 insertions(+), 104 deletions(-) delete mode 100644 metagpt/actions/di/rewrite_code.py diff --git a/metagpt/actions/di/rewrite_code.py b/metagpt/actions/di/rewrite_code.py deleted file mode 100644 index 659635ad5..000000000 --- a/metagpt/actions/di/rewrite_code.py +++ /dev/null @@ -1,100 +0,0 @@ -import asyncio -import os -from pathlib import Path - -from metagpt.actions.action import Action -from metagpt.actions.write_code_review import ( - EXAMPLE_AND_INSTRUCTION, - FORMAT_EXAMPLE, - PROMPT_TEMPLATE, - WriteCodeReview, -) -from metagpt.logs import logger -from metagpt.schema import CodingContext, Document -from metagpt.tools.tool_registry import register_tool -from metagpt.utils.common import aread, awrite - - -@register_tool(tags=["RewriteCode"], include_functions=["run"]) -class RewriteCode(Action): - """Accordding design doc and task doc to review the code, to make the complete and correct code.""" - - name: str = "RewriteCode" - - async def run( - self, code_path: str, design_doc_input: str = "", task_doc_input: str = "", code_review_k_times: int = 2 - ) -> str: - """Reviews the provided code based on the accompanying design and task documentation, return the complete and correct code. - - Read the code from `code_path`, and write the final code to `code_path`. - If there is no `design_doc_input` or `task_doc_input`, it will return and do nothing. - - Args: - code_path (str): The file path of the code snippet to be reviewed. This should be a string containing the path to the source code file. - design_doc_input (str): Content or file path of the design document associated with the code. This should describe the system architecture, used in the code. It helps provide context for the review process. - task_doc_input (str): Content or file path of the task document describing what the code is intended to accomplish. This should outline the functional requirements or objectives of the code. - code_review_k_times (int, optional): The number of iterations for reviewing and potentially rewriting the code. Defaults to 2. - - Returns: - str: The potentially corrected or approved code after review. - - Example Usage: - # Example of how to call the run method with a code snippet and documentation - await RewriteCode().run( - code_path="/tmp/game.js", - design_doc_input="/tmp/design_doc.json", - task_doc_input='{"Required packages":["No third-party dependencies required"], ...}' - ) - """ - - if not design_doc_input or not task_doc_input: - return - - code, design_doc, task_doc = await asyncio.gather( - aread(code_path), self._try_aread(design_doc_input), self._try_aread(task_doc_input) - ) - code_doc = self._create_code_doc(code_path=code_path, code=code) - reviewer = WriteCodeReview(i_context=CodingContext(filename=code_doc.filename)) - - context = "\n".join( - [ - "## System Design\n" + design_doc + "\n", - "## Task\n" + task_doc + "\n", - ] - ) - - for i in range(code_review_k_times): - context_prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=code_path) - cr_prompt = EXAMPLE_AND_INSTRUCTION.format( - format_example=FORMAT_EXAMPLE.format(filename=code_path), - ) - logger.info(f"The {i+1}th time to CodeReview: {code_path}.") - result, rewrited_code = await reviewer.write_code_review_and_rewrite( - context_prompt, cr_prompt, doc=code_doc - ) - - if "LBTM" in result: - code = rewrited_code - elif "LGTM" in result: - break - - await awrite(filename=code_path, data=code) - - return code - - @staticmethod - async def _try_aread(input: str) -> str: - """Try to read from the path if it's a file; return input directly if not.""" - - if os.path.exists(input): - return await aread(input) - - return input - - @staticmethod - def _create_code_doc(code_path: str, code: str) -> Document: - """Create a Document to represent the code doc.""" - - path = Path(code_path) - - return Document(root_path=str(path.parent), filename=path.name, content=code) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index ad99de2dd..fce29c5bc 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -7,6 +7,9 @@ @Modified By: mashenquan, 2023/11/27. Following the think-act principle, solidify the task parameters when creating the WriteCode object, rather than passing them in when calling the run function. """ +import asyncio +import os +from pathlib import Path from typing import Optional from pydantic import BaseModel, Field @@ -16,7 +19,8 @@ from metagpt.actions import WriteCode from metagpt.actions.action import Action from metagpt.logs import logger from metagpt.schema import CodingContext, Document -from metagpt.utils.common import CodeParser +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.common import CodeParser, aread, awrite from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.report import EditorReporter @@ -205,3 +209,88 @@ class WriteCodeReview(Action): # 如果rewrited_code是None（原code perfect），那么直接返回code self.i_context.code_doc.content = iterative_code return self.i_context + + +@register_tool(tags=["CodeReview"], include_functions=["run"]) +class RewriteCode(Action): + """Accordding design doc and task doc to review the code, to make the complete and correct code.""" + + name: str = "RewriteCode" + + async def run( + self, code_path: str, design_doc_input: str = "", task_doc_input: str = "", code_review_k_times: int = 2 + ) -> str: + """Reviews the provided code based on the accompanying design and task documentation, return the complete and correct code. + + Read the code from `code_path`, and write the final code to `code_path`. + If there is no `design_doc_input` or `task_doc_input`, it will return and do nothing. + + Args: + code_path (str): The file path of the code snippet to be reviewed. This should be a string containing the path to the source code file. + design_doc_input (str): Content or file path of the design document associated with the code. This should describe the system architecture, used in the code. It helps provide context for the review process. + task_doc_input (str): Content or file path of the task document describing what the code is intended to accomplish. This should outline the functional requirements or objectives of the code. + code_review_k_times (int, optional): The number of iterations for reviewing and potentially rewriting the code. Defaults to 2. + + Returns: + str: The potentially corrected or approved code after review. + + Example Usage: + # Example of how to call the run method with a code snippet and documentation + await RewriteCode().run( + code_path="/tmp/game.js", + design_doc_input="/tmp/design_doc.json", + task_doc_input='{"Required packages":["No third-party dependencies required"], ...}' + ) + """ + + if not design_doc_input or not task_doc_input: + return + + code, design_doc, task_doc = await asyncio.gather( + aread(code_path), self._try_aread(design_doc_input), self._try_aread(task_doc_input) + ) + code_doc = self._create_code_doc(code_path=code_path, code=code) + reviewer = WriteCodeReview(i_context=CodingContext(filename=code_doc.filename)) + + context = "\n".join( + [ + "## System Design\n" + design_doc + "\n", + "## Task\n" + task_doc + "\n", + ] + ) + + for i in range(code_review_k_times): + context_prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=code_path) + cr_prompt = EXAMPLE_AND_INSTRUCTION.format( + format_example=FORMAT_EXAMPLE.format(filename=code_path), + ) + logger.info(f"The {i+1}th time to CodeReview: {code_path}.") + result, rewrited_code = await reviewer.write_code_review_and_rewrite( + context_prompt, cr_prompt, doc=code_doc + ) + + if "LBTM" in result: + code = rewrited_code + elif "LGTM" in result: + break + + await awrite(filename=code_path, data=code) + + return code + + @staticmethod + async def _try_aread(input: str) -> str: + """Try to read from the path if it's a file; return input directly if not.""" + + if os.path.exists(input): + return await aread(input) + + return input + + @staticmethod + def _create_code_doc(code_path: str, code: str) -> Document: + """Create a Document to represent the code doc.""" + + path = Path(code_path) + + return Document(root_path=str(path.parent), filename=path.name, content=code) diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 102792001..3e8c3723b 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -10,8 +10,7 @@ EXTRA_INSTRUCTION = """ 10. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. 11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. 12. To modify code in a file, read the entire file, make changes, and update the file with the complete code. -13. Revise task is to use RewriteCode.run to correct code. -14. Only When provided system design, at the end of the plan, add a Revise task for each file; for example, if there are three files, add three Revise tasks. +13. Only with a system design, at the end of the plan, add a CodeReview Task for each file; for example, if there are three files, add three CodeReview Tasks. """ diff --git a/metagpt/roles/di/engineer2.py b/metagpt/roles/di/engineer2.py index 152c62ca2..107101e82 100644 --- a/metagpt/roles/di/engineer2.py +++ b/metagpt/roles/di/engineer2.py @@ -1,6 +1,6 @@ from __future__ import annotations -from metagpt.actions.di.rewrite_code import RewriteCode +from metagpt.actions.write_code_review import RewriteCode from metagpt.prompts.di.engineer2 import ENGINEER2_INSTRUCTION from metagpt.roles.di.role_zero import RoleZero From bd3b4dc8c0744a22bdaf38a59b26752caae9f7e9 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Fri, 21 Jun 2024 15:25:00 +0800 Subject: [PATCH 35/41] update action name --- metagpt/actions/write_code_review.py | 23 ++++++++++++----------- metagpt/prompts/di/engineer2.py | 4 ++-- metagpt/roles/di/engineer2.py | 10 +++++----- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index fce29c5bc..62188786d 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -211,11 +211,11 @@ class WriteCodeReview(Action): return self.i_context -@register_tool(tags=["CodeReview"], include_functions=["run"]) -class RewriteCode(Action): - """Accordding design doc and task doc to review the code, to make the complete and correct code.""" +@register_tool(include_functions=["run"]) +class ReviewAndRewriteCode(Action): + """According to the design and task documents, review the code to ensure it is complete and correct.""" - name: str = "RewriteCode" + name: str = "ReviewAndRewriteCode" async def run( self, code_path: str, design_doc_input: str = "", task_doc_input: str = "", code_review_k_times: int = 2 @@ -223,7 +223,7 @@ class RewriteCode(Action): """Reviews the provided code based on the accompanying design and task documentation, return the complete and correct code. Read the code from `code_path`, and write the final code to `code_path`. - If there is no `design_doc_input` or `task_doc_input`, it will return and do nothing. + If both `design_doc_input` and `task_doc_input are absent`, it will return and do nothing. Args: code_path (str): The file path of the code snippet to be reviewed. This should be a string containing the path to the source code file. @@ -236,21 +236,22 @@ class RewriteCode(Action): Example Usage: # Example of how to call the run method with a code snippet and documentation - await RewriteCode().run( + await ReviewAndRewriteCode().run( code_path="/tmp/game.js", - design_doc_input="/tmp/design_doc.json", - task_doc_input='{"Required packages":["No third-party dependencies required"], ...}' + design_doc_input="/tmp/system_design.json", + task_doc_input="/tmp/project_task_list.json" ) """ - if not design_doc_input or not task_doc_input: + if not design_doc_input and not task_doc_input: + logger.info("Both design_doc_input and task_doc_input are absent, ReviewAndRewriteCode will do nothing.") return code, design_doc, task_doc = await asyncio.gather( aread(code_path), self._try_aread(design_doc_input), self._try_aread(task_doc_input) ) code_doc = self._create_code_doc(code_path=code_path, code=code) - reviewer = WriteCodeReview(i_context=CodingContext(filename=code_doc.filename)) + review_action = WriteCodeReview(i_context=CodingContext(filename=code_doc.filename)) context = "\n".join( [ @@ -265,7 +266,7 @@ class RewriteCode(Action): format_example=FORMAT_EXAMPLE.format(filename=code_path), ) logger.info(f"The {i+1}th time to CodeReview: {code_path}.") - result, rewrited_code = await reviewer.write_code_review_and_rewrite( + result, rewrited_code = await review_action.write_code_review_and_rewrite( context_prompt, cr_prompt, doc=code_doc ) diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 3e8c3723b..08259d380 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -4,13 +4,13 @@ EXTRA_INSTRUCTION = """ 4. Each time you write a code in your response, write with the Editor directly without preparing a repetitive code block beforehand. 5. Take on ONE task and write ONE code file in each response. DON'T attempt all tasks in one response. 6. When not specified, you should write files in a folder named "src". If you know the project path, then write in a "src" folder under the project path. -7. When provided system design or project schedule, read them first, then adhere to them in your implementation, especially in the programming language, package, or framework. You MUST implement all code files prescribed in the system design or project schedule. You can create a plan first with each task corresponding to implementing one code file. +7. When provided system design or project schedule, you MUST read them first before making a plan, then adhere to them in your implementation, especially in the programming language, package, or framework. You MUST implement all code files prescribed in the system design or project schedule. You can create a plan first with each task corresponding to implementing one code file. 8. Write at most one file per task, do your best to implement THE ONLY ONE FILE. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. 9. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets. 10. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. 11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. 12. To modify code in a file, read the entire file, make changes, and update the file with the complete code. -13. Only with a system design, at the end of the plan, add a CodeReview Task for each file; for example, if there are three files, add three CodeReview Tasks. +13. Only when a system design or a project schedule is provided, at the end of the plan, add a ReviewAndRewriteCode Task for each file; for example, if there are three files, add three ReviewAndRewriteCode Tasks. """ diff --git a/metagpt/roles/di/engineer2.py b/metagpt/roles/di/engineer2.py index 107101e82..845dd8960 100644 --- a/metagpt/roles/di/engineer2.py +++ b/metagpt/roles/di/engineer2.py @@ -1,6 +1,6 @@ from __future__ import annotations -from metagpt.actions.write_code_review import RewriteCode +from metagpt.actions.write_code_review import ReviewAndRewriteCode from metagpt.prompts.di.engineer2 import ENGINEER2_INSTRUCTION from metagpt.roles.di.role_zero import RoleZero @@ -11,14 +11,14 @@ class Engineer2(RoleZero): goal: str = "Take on game, app, and web development" instruction: str = ENGINEER2_INSTRUCTION - tools: str = ["Plan", "Editor:write,read", "RoleZero", "RewriteCode"] + tools: str = ["Plan", "Editor:write,read", "RoleZero", "ReviewAndRewriteCode"] def _update_tool_execution(self): - rewrite_code = RewriteCode() + review = ReviewAndRewriteCode() self.tool_execution_map.update( { - "RewriteCode.run": rewrite_code.run, - "RewriteCode": rewrite_code.run, + "ReviewAndRewriteCode.run": review.run, + "ReviewAndRewriteCode": review.run, } ) From 0f65ff5cb07d91a080298a3dafa8091055db4a4b Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Fri, 21 Jun 2024 16:31:10 +0800 Subject: [PATCH 36/41] change parameter names --- metagpt/actions/write_code_review.py | 24 +++++++++++++++--------- metagpt/prompts/di/engineer2.py | 2 +- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 62188786d..3740a80fe 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -218,17 +218,21 @@ class ReviewAndRewriteCode(Action): name: str = "ReviewAndRewriteCode" async def run( - self, code_path: str, design_doc_input: str = "", task_doc_input: str = "", code_review_k_times: int = 2 + self, + code_path: str, + system_design_input: str = "", + project_schedule_input: str = "", + code_review_k_times: int = 2, ) -> str: """Reviews the provided code based on the accompanying design and task documentation, return the complete and correct code. Read the code from `code_path`, and write the final code to `code_path`. - If both `design_doc_input` and `task_doc_input are absent`, it will return and do nothing. + If both `system_design_input` and `project_schedule_input are absent`, it will return and do nothing. Args: code_path (str): The file path of the code snippet to be reviewed. This should be a string containing the path to the source code file. - design_doc_input (str): Content or file path of the design document associated with the code. This should describe the system architecture, used in the code. It helps provide context for the review process. - task_doc_input (str): Content or file path of the task document describing what the code is intended to accomplish. This should outline the functional requirements or objectives of the code. + system_design_input (str): Content or file path of the design document associated with the code. This should describe the system architecture, used in the code. It helps provide context for the review process. + project_schedule_input (str): Content or file path of the task document describing what the code is intended to accomplish. This should outline the functional requirements or objectives of the code. code_review_k_times (int, optional): The number of iterations for reviewing and potentially rewriting the code. Defaults to 2. Returns: @@ -238,17 +242,19 @@ class ReviewAndRewriteCode(Action): # Example of how to call the run method with a code snippet and documentation await ReviewAndRewriteCode().run( code_path="/tmp/game.js", - design_doc_input="/tmp/system_design.json", - task_doc_input="/tmp/project_task_list.json" + system_design_input="/tmp/system_design.json", + project_schedule_input="/tmp/project_task_list.json" ) """ - if not design_doc_input and not task_doc_input: - logger.info("Both design_doc_input and task_doc_input are absent, ReviewAndRewriteCode will do nothing.") + if not system_design_input and not project_schedule_input: + logger.info( + "Both `system_design_input` and `project_schedule_input` are absent, ReviewAndRewriteCode will do nothing." + ) return code, design_doc, task_doc = await asyncio.gather( - aread(code_path), self._try_aread(design_doc_input), self._try_aread(task_doc_input) + aread(code_path), self._try_aread(system_design_input), self._try_aread(project_schedule_input) ) code_doc = self._create_code_doc(code_path=code_path, code=code) review_action = WriteCodeReview(i_context=CodingContext(filename=code_doc.filename)) diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 08259d380..54b68416a 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -10,7 +10,7 @@ EXTRA_INSTRUCTION = """ 10. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. 11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. 12. To modify code in a file, read the entire file, make changes, and update the file with the complete code. -13. Only when a system design or a project schedule is provided, at the end of the plan, add a ReviewAndRewriteCode Task for each file; for example, if there are three files, add three ReviewAndRewriteCode Tasks. +13. If a system design or project schedule is provided, at the end of the plan, add a ReviewAndRewriteCode Task for each file; for example, if there are three files, add three ReviewAndRewriteCode Tasks. """ From 17e606e1c313aa9aa56548c16075115c219b41f5 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Fri, 21 Jun 2024 18:13:47 +0800 Subject: [PATCH 37/41] update prompt --- metagpt/prompts/di/engineer2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 54b68416a..8a7a48cc2 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -10,7 +10,7 @@ EXTRA_INSTRUCTION = """ 10. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. 11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. 12. To modify code in a file, read the entire file, make changes, and update the file with the complete code. -13. If a system design or project schedule is provided, at the end of the plan, add a ReviewAndRewriteCode Task for each file; for example, if there are three files, add three ReviewAndRewriteCode Tasks. +13. If a system design or project schedule is provided, at the end of the plan, add a CodeReview Task for each file; for example, if there are three files, add three CodeReview Tasks; Each CodeReview Task should execute ReviewAndRewriteCode.run to review and update the code. """ From 7f732799698acc0f7727bde75a359df083d0ec6e Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Fri, 21 Jun 2024 19:56:16 +0800 Subject: [PATCH 38/41] update prompt --- metagpt/prompts/di/engineer2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 8a7a48cc2..5caa5e1a9 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -9,8 +9,8 @@ EXTRA_INSTRUCTION = """ 9. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets. 10. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. 11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. -12. To modify code in a file, read the entire file, make changes, and update the file with the complete code. -13. If a system design or project schedule is provided, at the end of the plan, add a CodeReview Task for each file; for example, if there are three files, add three CodeReview Tasks; Each CodeReview Task should execute ReviewAndRewriteCode.run to review and update the code. +12. To modify code in a file, read the entire file, make changes, and update the file with the complete code, ensuring that no line numbers are included in the final write. +13. Only if a system design or project schedule is provided, at the end of the plan, add a CodeReview Task for each file; for example, if there are three files, add three CodeReview Tasks; the CodeReview Task involves using ReviewAndRewriteCode to correct the code. """ From 9454c4dfd644507fa771cd83e266f9c1485f0864 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Sun, 23 Jun 2024 23:01:25 +0800 Subject: [PATCH 39/41] update prompt --- metagpt/actions/write_code_review.py | 2 +- metagpt/prompts/di/engineer2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 3740a80fe..e72fe5cd1 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -224,7 +224,7 @@ class ReviewAndRewriteCode(Action): project_schedule_input: str = "", code_review_k_times: int = 2, ) -> str: - """Reviews the provided code based on the accompanying design and task documentation, return the complete and correct code. + """Reviews the provided code based on the accompanying system design and project schedule documentation, return the complete and correct code. Read the code from `code_path`, and write the final code to `code_path`. If both `system_design_input` and `project_schedule_input are absent`, it will return and do nothing. diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 5caa5e1a9..4ac386b80 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -10,7 +10,7 @@ EXTRA_INSTRUCTION = """ 10. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. 11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. 12. To modify code in a file, read the entire file, make changes, and update the file with the complete code, ensuring that no line numbers are included in the final write. -13. Only if a system design or project schedule is provided, at the end of the plan, add a CodeReview Task for each file; for example, if there are three files, add three CodeReview Tasks; the CodeReview Task involves using ReviewAndRewriteCode to correct the code. +13. When a system design or project schedule is provided, at the end of the plan, add a Special Task for each file; for example, if there are three files, add three Special Tasks. For each Special Task, just call ReviewAndRewriteCode.run. """ From e44f4739f110f935cb04ffecabc120805a9314ad Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Mon, 24 Jun 2024 10:59:06 +0800 Subject: [PATCH 40/41] add alias to tolerate fault for architect --- metagpt/roles/architect.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index afa234a3c..8650f2640 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -50,5 +50,6 @@ class Architect(RoleZero): { "WriteDesign.run": wd.run, "WriteDesign": wd.run, # alias + "run": wd.run, # alias } ) From 5d0e99bb5dca191a35a5bf13602940f35d86f6c7 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Mon, 24 Jun 2024 21:22:43 +0800 Subject: [PATCH 41/41] add example to engineer --- metagpt/prompts/di/engineer2.py | 2 +- metagpt/roles/di/engineer2.py | 4 + metagpt/strategy/experience_retriever.py | 112 +++++++++++++++++++++++ 3 files changed, 117 insertions(+), 1 deletion(-) diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 4ac386b80..f3f2155b6 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -10,7 +10,7 @@ EXTRA_INSTRUCTION = """ 10. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. 11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. 12. To modify code in a file, read the entire file, make changes, and update the file with the complete code, ensuring that no line numbers are included in the final write. -13. When a system design or project schedule is provided, at the end of the plan, add a Special Task for each file; for example, if there are three files, add three Special Tasks. For each Special Task, just call ReviewAndRewriteCode.run. +13. When a system design or project schedule is provided, at the end of the plan, add a CodeRview Task for each file; for example, if there are three files, add three CodeRview Tasks. For each CodeRview Task, just call ReviewAndRewriteCode.run. """ diff --git a/metagpt/roles/di/engineer2.py b/metagpt/roles/di/engineer2.py index 845dd8960..8ea823c74 100644 --- a/metagpt/roles/di/engineer2.py +++ b/metagpt/roles/di/engineer2.py @@ -3,6 +3,7 @@ from __future__ import annotations from metagpt.actions.write_code_review import ReviewAndRewriteCode from metagpt.prompts.di.engineer2 import ENGINEER2_INSTRUCTION from metagpt.roles.di.role_zero import RoleZero +from metagpt.strategy.experience_retriever import ENGINEER_EXAMPLE class Engineer2(RoleZero): @@ -22,3 +23,6 @@ class Engineer2(RoleZero): "ReviewAndRewriteCode": review.run, } ) + + def _retrieve_experience(self) -> str: + return ENGINEER_EXAMPLE diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index 2bb734a8b..5e85b056a 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -336,3 +336,115 @@ editor.read(path="./main.py") - If no specific file is provided, search the symbol in the whole codebase to locate the issue. - If no specific symbol is provided, directly open and read the file to diagnose the problem. """ + +ENGINEER_EXAMPLE = """ +## example 1 +User Requirement: Please implement the core game logic for the 2048 game, including tile movements, merging logic, score tracking, and keyboard interaction. Refer to the project schedule located at '/tmp/project_schedule.json' and the system design document at '/tmp/system_design.json' for detailed information. +Explanation: I will first need to read the system design document and the project schedule to understand the specific requirements and architecture outlined for the game development. + +```json +[ + { + "command_name": "Editor.read", + "args": { + "path": "/tmp/docs/project_schedule.json" + } + }, + { + "command_name": "Editor.read", + "args": { + "path": "/tmp/docs/system_design.json" + } + } +] +``` + +## example 2 +To achieve the goal of writing a 2048 game using JavaScript and HTML without any frameworks, I will create a plan consisting of three tasks, each corresponding to the creation of one of the required files: `index.html`, `style.css`, and `script.js`. Following the completion of these tasks, I will add a code review task for each file to ensure the implementation aligns with the provided system design and project schedule documents. + +Here's the plan: + +1. **Task 1**: Create `index.html` - This file will contain the HTML structure necessary for the game's UI. +2. **Task 2**: Create `style.css` - This file will define the CSS styles to make the game visually appealing and responsive. +3. **Task 3**: Create `script.js` - This file will contain the JavaScript code for the game logic and UI interactions. +4. **Code Review Tasks**: Review each file to ensure they meet the project requirements and adhere to the system design. + +Let's start by appending the first task to the plan. + +```json +[ + { + "command_name": "Plan.append_task", + "args": { + "task_id": "1", + "dependent_task_ids": [], + "instruction": "Create the index.html file with the basic HTML structure for the 2048 game.", + "assignee": "Alex" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "2", + "dependent_task_ids": ["1"], + "instruction": "Create the style.css file with the necessary CSS to style the 2048 game.", + "assignee": "Alex" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "3", + "dependent_task_ids": ["1", "2"], + "instruction": "Create the script.js file containing the JavaScript logic for the 2048 game.", + "assignee": "Alex" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "4", + "dependent_task_ids": ["1"], + "instruction": "Use ReviewAndRewriteCode to review the code in index.html to ensure it meets the design specifications.", + "assignee": "Alex" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "5", + "dependent_task_ids": ["2"], + "instruction": "Use ReviewAndRewriteCode to review the code in style.css to ensure it meets the design specifications.", + "assignee": "Alex" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "6", + "dependent_task_ids": ["3"], + "instruction": "Use ReviewAndRewriteCode to review the code in script.js to ensure it meets the design specifications. ", + "assignee": "Alex" + } + } +] +``` + +## example 3 +I will now review the code in `script.js`. +Explanation: to review the code, call ReviewAndRewriteCode.run. + +```json +[ + { + "command_name": "ReviewAndRewriteCode.run", + "args": { + "code_path": "/tmp/src/script.js", + "system_design_input": "/tmp/docs/system_design.json", + "project_schedule_input": "/tmp/docs/project_schedule.json", + "code_review_k_times": 2 + } + } +] +``` +"""