diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 566e96efe..cc88171ff 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,7 +11,6 @@ @Modified By: mashenquan, 2024/5/31. Implement Chapter 3 of RFC 236. """ import json -import uuid from pathlib import Path from typing import List, Optional, Union @@ -87,84 +86,48 @@ class WriteDesign(Action): output_pathname (str, optional): The output path name of file that the system design should be saved to. Returns: - AIMessage: An AIMessage object containing the system design. + str: The file path of the generated system design. Example: - # Write a new system design. - >>> user_requirement = "Your user requirements" - >>> extra_info = "Your extra information" - >>> action = WriteDesign() - >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info) - >>> print(result) - System Design filename: "/path/to/design/filename" - - # Modify an exists system design. - >>> user_requirement = "Your user requirements" - >>> extra_info = "Your extra information" - >>> legacy_design_filename = "/path/to/exists/design/filename" - >>> action = WriteDesign() - >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename) - >>> print(result) - System Design filename: "/path/to/design/filename" - - # Write a new system design with the given PRD(Product Requirement Document). - >>> user_requirement = "Your user requirements" - >>> extra_info = "Your extra information" - >>> prd_filename = "/path/to/prd/filename" - >>> action = WriteDesign() - >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, prd_filename=prd_filename) - >>> print(result) - System Design filename: "/path/to/design/filename" - - # Modify an exists system design with the given PRD(Product Requirement Document). - >>> user_requirement = "Your user requirements" - >>> extra_info = "Your extra information" - >>> prd_filename = "/path/to/prd/filename" - >>> legacy_design_filename = "/path/to/exists/design/filename" - >>> action = WriteDesign() - >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename, prd_filename=prd_filename) - >>> print(result) - TSystem Design filename: "/path/to/design/filename" - # Write a new system design and save to the path name. - >>> user_requirement = "Your user requirements" + >>> user_requirement = "Write system design for a snake game" >>> extra_info = "Your extra information" - >>> output_pathname = "/path/to/design/filename" + >>> output_pathname = "snake_game/docs/system_design.json" >>> action = WriteDesign() >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, output_pathname=output_pathname) >>> print(result) - System Design filename: "/path/to/design/filename" + System Design filename: "/absolute/path/to/snake_game/docs/system_design.json" - # Modify an exists system design and save to the path name. - >>> user_requirement = "Your user requirements" + # Rewrite an existing system design and save to the path name. + >>> user_requirement = "Write system design for a snake game, include new features such as a web UI" >>> extra_info = "Your extra information" - >>> legacy_design_filename = "/path/to/exists/design/filename" - >>> output_pathname = "/path/to/design/filename" + >>> legacy_design_filename = "/absolute/path/to/snake_game/docs/system_design.json" + >>> output_pathname = "/absolute/path/to/snake_game/docs/system_design_new.json" >>> action = WriteDesign() >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename, output_pathname=output_pathname) >>> print(result) - System Design filename: "/path/to/design/filename" + System Design filename: "/absolute/path/to/snake_game/docs/system_design_new.json" # Write a new system design with the given PRD(Product Requirement Document) and save to the path name. - >>> user_requirement = "Your user requirements" + >>> user_requirement = "Write system design for a snake game based on the PRD at /absolute/path/to/snake_game/docs/prd.json" >>> extra_info = "Your extra information" - >>> prd_filename = "/path/to/prd/filename" - >>> output_pathname = "/path/to/design/filename" + >>> prd_filename = "/absolute/path/to/snake_game/docs/prd.json" + >>> output_pathname = "/absolute/path/to/snake_game/docs/sytem_design.json" >>> action = WriteDesign() >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, prd_filename=prd_filename, output_pathname=output_pathname) >>> print(result) - System Design filename: "/path/to/design/filename" + System Design filename: "/absolute/path/to/snake_game/docs/sytem_design.json" - # Modify an exists system design with the given PRD(Product Requirement Document) and save to the path name. - >>> user_requirement = "Your user requirements" + # Rewrite an existing system design with the given PRD(Product Requirement Document) and save to the path name. + >>> user_requirement = "Write system design for a snake game, include new features such as a web UI" >>> extra_info = "Your extra information" - >>> prd_filename = "/path/to/prd/filename" - >>> legacy_design_filename = "/path/to/exists/design/filename" - >>> output_pathname = "/path/to/design/filename" + >>> prd_filename = "/absolute/path/to/snake_game/docs/prd.json" + >>> legacy_design_filename = "/absolute/path/to/snake_game/docs/system_design.json" + >>> output_pathname = "/absolute/path/to/snake_game/docs/system_design_new.json" >>> action = WriteDesign() - >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename, prd_filename=prd_filename, output_pathname=output_pathname) + >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, prd_filename=prd_filename, legacy_design_filename=legacy_design_filename, output_pathname=output_pathname) >>> print(result) - System Design filename: "/path/to/design/filename" + System Design filename: "/absolute/path/to/snake_game/docs/system_design_new.json" """ if not with_messages: return await self._execute_api( @@ -301,9 +264,10 @@ class WriteDesign(Action): ) if not output_pathname: - output_path = DEFAULT_WORKSPACE_ROOT - output_path.mkdir(parents=True, exist_ok=True) - output_pathname = Path(output_path) / f"{uuid.uuid4().hex}.json" + output_pathname = Path(output_pathname) / "docs" / "sytem_design.json" + output_pathname.mkdir(parents=True, exist_ok=True) + elif not Path(output_pathname).is_absolute(): + output_pathname = DEFAULT_WORKSPACE_ROOT / output_pathname output_pathname = Path(output_pathname) await awrite(filename=output_pathname, data=design.content) output_filename = output_pathname.parent / f"{output_pathname.stem}-class-diagram" diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 38234cfb7..a39840bf1 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -19,11 +19,16 @@ from pydantic import BaseModel, Field from metagpt.actions.action import Action from metagpt.actions.project_management_an import PM_NODE, REFINED_PM_NODE -from metagpt.const import PACKAGE_REQUIREMENTS_FILENAME +from metagpt.const import DEFAULT_WORKSPACE_ROOT, PACKAGE_REQUIREMENTS_FILENAME from metagpt.logs import logger from metagpt.schema import AIMessage, Document, Documents, Message from metagpt.tools.tool_registry import register_tool -from metagpt.utils.common import aread, to_markdown_code_block +from metagpt.utils.common import ( + aread, + awrite, + save_json_to_markdown, + to_markdown_code_block, +) from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.report import DocsReporter @@ -44,7 +49,13 @@ class WriteTasks(Action): input_args: Optional[BaseModel] = Field(default=None, exclude=True) async def run( - self, with_messages: List[Message] = None, *, user_requirement: str = "", design_filename: str = "", **kwargs + self, + with_messages: List[Message] = None, + *, + user_requirement: str = "", + design_filename: str = "", + output_pathname: str = "", + **kwargs, ) -> Union[AIMessage, str]: """ Write a project schedule given a project system design file. @@ -52,29 +63,33 @@ class WriteTasks(Action): Args: user_requirement (str, optional): A string specifying the user's requirements. Defaults to an empty string. design_filename (str): The filename of the project system design file. Defaults to an empty string. + output_pathname (str, optional): The output path name of file that the project schedule should be saved to. **kwargs: Additional keyword arguments. Returns: - AIMessage: The generated project schedule. + str: Path to the generated project schedule. Example: - # Write a new project schedule. - >>> design_filename = "/path/to/design/filename" + # Write a project schedule with a given system design. + >>> design_filename = "/absolute/path/to/snake_game/docs/system_design.json" + >>> output_pathname = "/absolute/path/to/snake_game/docs/project_schedule.json" >>> action = WriteTasks() - >>> result = await action.run(design_filename=design_filename) + >>> result = await action.run(design_filename=design_filename, output_pathname=output_pathname) >>> print(result) - The project schedule is balabala... + The project schedule is at /absolute/path/to/snake_game/docs/project_schedule.json - # Write a new project schedule with the user requirement. - >>> design_filename = "/path/to/design/filename" - >>> user_requirement = "Your user requirements" + # Write a project schedule with a user requirement. + >>> user_requirement = "Write project schedule for a snake game following these requirements: ..." + >>> output_pathname = "/absolute/path/to/snake_game/docs/project_schedule.json" >>> action = WriteTasks() - >>> result = await action.run(design_filename=design_filename, user_requirement=user_requirement) + >>> result = await action.run(user_requirement=user_requirement, output_pathname=output_pathname) >>> print(result) - The project schedule is balabala... + The project schedule is at /absolute/path/to/snake_game/docs/project_schedule.json """ if not with_messages: - return await self._execute_api(user_requirement=user_requirement, design_filename=design_filename) + return await self._execute_api( + user_requirement=user_requirement, design_filename=design_filename, output_pathname=output_pathname + ) self.input_args = with_messages[-1].instruct_content self.repo = ProjectRepo(self.input_args.project_path) @@ -158,10 +173,23 @@ class WriteTasks(Action): packages.add(pkg) await self.repo.save(filename=PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages)) - async def _execute_api(self, user_requirement: str = "", design_filename: str = "") -> str: + async def _execute_api( + self, user_requirement: str = "", design_filename: str = "", output_pathname: str = "" + ) -> str: context = to_markdown_code_block(user_requirement) - if not design_filename: + if design_filename: content = await aread(filename=design_filename) context += to_markdown_code_block(content) node = await self._run_new_tasks(context) - return node.instruct_content.model_dump_json() + file_content = node.instruct_content.model_dump_json() + + if not output_pathname: + output_pathname = Path(output_pathname) / "docs" / "project_schedule.json" + output_pathname.mkdir(parents=True, exist_ok=True) + elif not Path(output_pathname).is_absolute(): + output_pathname = DEFAULT_WORKSPACE_ROOT / output_pathname + output_pathname = Path(output_pathname) + await awrite(filename=output_pathname, data=file_content) + await save_json_to_markdown(content=file_content, output_filename=output_pathname.with_suffix(".md")) + + return f'Project Schedule filename: "{str(output_pathname)}"' diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index ad99de2dd..e72fe5cd1 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -7,6 +7,9 @@ @Modified By: mashenquan, 2023/11/27. Following the think-act principle, solidify the task parameters when creating the WriteCode object, rather than passing them in when calling the run function. """ +import asyncio +import os +from pathlib import Path from typing import Optional from pydantic import BaseModel, Field @@ -16,7 +19,8 @@ from metagpt.actions import WriteCode from metagpt.actions.action import Action from metagpt.logs import logger from metagpt.schema import CodingContext, Document -from metagpt.utils.common import CodeParser +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.common import CodeParser, aread, awrite from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.report import EditorReporter @@ -205,3 +209,95 @@ class WriteCodeReview(Action): # 如果rewrited_code是None（原code perfect），那么直接返回code self.i_context.code_doc.content = iterative_code return self.i_context + + +@register_tool(include_functions=["run"]) +class ReviewAndRewriteCode(Action): + """According to the design and task documents, review the code to ensure it is complete and correct.""" + + name: str = "ReviewAndRewriteCode" + + async def run( + self, + code_path: str, + system_design_input: str = "", + project_schedule_input: str = "", + code_review_k_times: int = 2, + ) -> str: + """Reviews the provided code based on the accompanying system design and project schedule documentation, return the complete and correct code. + + Read the code from `code_path`, and write the final code to `code_path`. + If both `system_design_input` and `project_schedule_input are absent`, it will return and do nothing. + + Args: + code_path (str): The file path of the code snippet to be reviewed. This should be a string containing the path to the source code file. + system_design_input (str): Content or file path of the design document associated with the code. This should describe the system architecture, used in the code. It helps provide context for the review process. + project_schedule_input (str): Content or file path of the task document describing what the code is intended to accomplish. This should outline the functional requirements or objectives of the code. + code_review_k_times (int, optional): The number of iterations for reviewing and potentially rewriting the code. Defaults to 2. + + Returns: + str: The potentially corrected or approved code after review. + + Example Usage: + # Example of how to call the run method with a code snippet and documentation + await ReviewAndRewriteCode().run( + code_path="/tmp/game.js", + system_design_input="/tmp/system_design.json", + project_schedule_input="/tmp/project_task_list.json" + ) + """ + + if not system_design_input and not project_schedule_input: + logger.info( + "Both `system_design_input` and `project_schedule_input` are absent, ReviewAndRewriteCode will do nothing." + ) + return + + code, design_doc, task_doc = await asyncio.gather( + aread(code_path), self._try_aread(system_design_input), self._try_aread(project_schedule_input) + ) + code_doc = self._create_code_doc(code_path=code_path, code=code) + review_action = WriteCodeReview(i_context=CodingContext(filename=code_doc.filename)) + + context = "\n".join( + [ + "## System Design\n" + design_doc + "\n", + "## Task\n" + task_doc + "\n", + ] + ) + + for i in range(code_review_k_times): + context_prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=code_path) + cr_prompt = EXAMPLE_AND_INSTRUCTION.format( + format_example=FORMAT_EXAMPLE.format(filename=code_path), + ) + logger.info(f"The {i+1}th time to CodeReview: {code_path}.") + result, rewrited_code = await review_action.write_code_review_and_rewrite( + context_prompt, cr_prompt, doc=code_doc + ) + + if "LBTM" in result: + code = rewrited_code + elif "LGTM" in result: + break + + await awrite(filename=code_path, data=code) + + return code + + @staticmethod + async def _try_aread(input: str) -> str: + """Try to read from the path if it's a file; return input directly if not.""" + + if os.path.exists(input): + return await aread(input) + + return input + + @staticmethod + def _create_code_doc(code_path: str, code: str) -> Document: + """Create a Document to represent the code doc.""" + + path = Path(code_path) + + return Document(root_path=str(path.parent), filename=path.name, content=code) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index b6062c36c..7199ec415 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -15,7 +15,6 @@ from __future__ import annotations import json -import uuid from pathlib import Path from typing import List, Optional, Union @@ -105,45 +104,27 @@ class WritePRD(Action): **kwargs: Additional keyword arguments. Returns: - str: The resulting message after generating the Product Requirement Document. + str: The file path of the generated Product Requirement Document. Example: - # Write a new PRD(Product Requirement Document) - >>> user_requirement = "YOUR REQUIREMENTS" - >>> extra_info = "YOUR EXTRA INFO" + # Write a new PRD (Product Requirement Document) + >>> user_requirement = "Write PRD for a snake game" + >>> output_pathname = "snake_game/docs/prd.json" + >>> extra_info = "YOUR EXTRA INFO, if any" >>> write_prd = WritePRD() - >>> result = await write_prd.run(user_requirement=user_requirement, extra_info=extra_info) + >>> result = await write_prd.run(user_requirement=user_requirement, output_pathname=output_pathname, extra_info=extra_info) >>> print(result) - PRD filename: "/path/to/prd/directory/213434ad.json" + PRD filename: "/absolute/path/to/snake_game/docs/prd.json" - # Modify a exists PRD(Product Requirement Document) - >>> user_requirement = "YOUR REQUIREMENTS" - >>> extra_info = "YOUR EXTRA INFO" - >>> legacy_prd_filename = "/path/to/exists/prd_filename" + # Rewrite an existing PRD (Product Requirement Document) and save to a new path. + >>> user_requirement = "Write PRD for a snake game, include new features such as a web UI" + >>> legacy_prd_filename = "/absolute/path/to/snake_game/docs/prd.json" + >>> output_pathname = "/absolute/path/to/snake_game/docs/prd_new.json" + >>> extra_info = "YOUR EXTRA INFO, if any" >>> write_prd = WritePRD() - >>> result = await write_prd.run(user_requirement=user_requirement, extra_info=extra_info, legacy_prd_filename=legacy_prd_filename) + >>> result = await write_prd.run(user_requirement=user_requirement, legacy_prd_filename=legacy_prd_filename, extra_info=extra_info) >>> print(result) - PRD filename: "/path/to/prd/directory/213434ad.json" - - # Write and save a new PRD(Product Requirement Document) to the path name. - >>> user_requirement = "YOUR REQUIREMENTS" - >>> extra_info = "YOUR EXTRA INFO" - >>> output_pathname = "/path/to/prd/directory/213434ad.json" - >>> write_prd = WritePRD() - >>> result = await write_prd.run(user_requirement=user_requirement, extra_info=extra_info, output_pathname=output_pathname) - >>> print(result) - PRD filename: "/path/to/prd/directory/213434ad.json" - - # Modify a exists PRD(Product Requirement Document) and save to the path name. - >>> user_requirement = "YOUR REQUIREMENTS" - >>> extra_info = "YOUR EXTRA INFO" - >>> legacy_prd_filename = "/path/to/exists/prd_filename" - >>> output_pathname = "/path/to/prd/directory/213434ad.json" - >>> write_prd = WritePRD() - >>> result = await write_prd.run(user_requirement=user_requirement, extra_info=extra_info, legacy_prd_filename=legacy_prd_filename, output_pathname=output_pathname) - >>> print(result) - PRD filename: "/path/to/prd/directory/213434ad.json" - + PRD filename: "/absolute/path/to/snake_game/docs/prd_new.json" """ if not with_messages: return await self._execute_api( @@ -329,9 +310,10 @@ class WritePRD(Action): new_prd = await self._merge(req=req, related_doc=old_prd) if not output_pathname: - output_path = DEFAULT_WORKSPACE_ROOT - output_path.mkdir(parents=True, exist_ok=True) - output_pathname = Path(output_path) / f"{uuid.uuid4().hex}.json" + output_pathname = DEFAULT_WORKSPACE_ROOT / "docs" / "prd.json" + output_pathname.mkdir(parents=True, exist_ok=True) + elif not Path(output_pathname).is_absolute(): + output_pathname = DEFAULT_WORKSPACE_ROOT / output_pathname output_pathname = Path(output_pathname) await awrite(filename=output_pathname, data=new_prd.content) competitive_analysis_filename = output_pathname.parent / f"{output_pathname.stem}-competitive-analysis" diff --git a/metagpt/environment/base_env.py b/metagpt/environment/base_env.py index 5d6d3a286..0776ae9ff 100644 --- a/metagpt/environment/base_env.py +++ b/metagpt/environment/base_env.py @@ -159,7 +159,7 @@ class Environment(ExtEnv): """增加一个在当前环境的角色 Add a role in the current environment """ - self.roles[role.profile] = role + self.roles[role.name] = role role.set_env(self) role.context = self.context @@ -168,7 +168,7 @@ class Environment(ExtEnv): Add a batch of characters in the current environment """ for role in roles: - self.roles[role.profile] = role + self.roles[role.name] = role for role in roles: # setup system message with roles role.context = self.context diff --git a/metagpt/environment/mgx/mgx_env.py b/metagpt/environment/mgx/mgx_env.py index 3bd41fd1e..33160cfd8 100644 --- a/metagpt/environment/mgx/mgx_env.py +++ b/metagpt/environment/mgx/mgx_env.py @@ -17,20 +17,33 @@ from metagpt.utils.common import any_to_str, any_to_str_set class MGXEnv(Environment): """MGX Environment""" - # Before enabling TL to fully take over the routing, all software company roles need to be able to handle TL messages, which requires restructuring. - allow_bypass_team_leader: bool = True + # If True, fixed software sop bypassing TL is allowed, otherwise, TL will fully take over the routing + allow_bypass_team_leader: bool = False + + direct_chat_roles: set[str] = set() # record direct chat: @role_name def _publish_message(self, message: Message, peekable: bool = True) -> bool: return super().publish_message(message, peekable) def publish_message(self, message: Message, user_defined_recipient: str = "", publicer: str = "") -> bool: """let the team leader take over message publishing""" - tl = self.get_role("Team Leader") + tl = self.get_role("Tim") # TeamLeader's name is Tim if user_defined_recipient: + # human user's direct chat message to a certain role + for role_name in message.send_to: + if self.get_role(role_name).is_idle: + # User starts a new direct chat with a certain role, expecting a direct chat response from the role; Other roles including TL should not be involved. + # If the role is not idle, it means the user helps the role with its current work, in this case, we handle the role's response message as usual. + self.direct_chat_roles.add(role_name) + self._publish_message(message) - # bypass team leader, team leader only needs to know but not to react - tl.rc.memory.add(self.move_message_info_to_content(message)) + # # bypass team leader, team leader only needs to know but not to react (commented out because TL doesn't understand the message well in actual experiments) + # tl.rc.memory.add(self.move_message_info_to_content(message)) + + elif message.sent_from in self.direct_chat_roles: + # direct chat response from a certain role to human user, team leader and other roles in the env should not be involved, no need to publish + self.direct_chat_roles.remove(message.sent_from) elif ( self.allow_bypass_team_leader @@ -106,3 +119,6 @@ class MGXEnv(Environment): sent_from = converted_msg.metadata[AGENT] if AGENT in converted_msg.metadata else converted_msg.sent_from converted_msg.content = f"from {sent_from} to {converted_msg.send_to}: {converted_msg.content}" return converted_msg + + def __repr__(self): + return "MGXEnv()" diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 346f2fc5a..f3f2155b6 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -1,6 +1,17 @@ from metagpt.prompts.di.role_zero import ROLE_INSTRUCTION -ENGINEER2_INSTRUCTION = ( - ROLE_INSTRUCTION - + "4. Each time you write a code in your response, write with the Editor directly without preparing a repetitive code block beforehand." -) +EXTRA_INSTRUCTION = """ +4. Each time you write a code in your response, write with the Editor directly without preparing a repetitive code block beforehand. +5. Take on ONE task and write ONE code file in each response. DON'T attempt all tasks in one response. +6. When not specified, you should write files in a folder named "src". If you know the project path, then write in a "src" folder under the project path. +7. When provided system design or project schedule, you MUST read them first before making a plan, then adhere to them in your implementation, especially in the programming language, package, or framework. You MUST implement all code files prescribed in the system design or project schedule. You can create a plan first with each task corresponding to implementing one code file. +8. Write at most one file per task, do your best to implement THE ONLY ONE FILE. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. +9. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets. +10. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. +11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. +12. To modify code in a file, read the entire file, make changes, and update the file with the complete code, ensuring that no line numbers are included in the final write. +13. When a system design or project schedule is provided, at the end of the plan, add a CodeRview Task for each file; for example, if there are three files, add three CodeRview Tasks. For each CodeRview Task, just call ReviewAndRewriteCode.run. +""" + + +ENGINEER2_INSTRUCTION = ROLE_INSTRUCTION + EXTRA_INSTRUCTION.strip() diff --git a/metagpt/prompts/di/role_zero.py b/metagpt/prompts/di/role_zero.py index f098d2c4b..4d52476aa 100644 --- a/metagpt/prompts/di/role_zero.py +++ b/metagpt/prompts/di/role_zero.py @@ -20,7 +20,7 @@ class Task(BaseModel): # Available Commands {available_commands} -Special Command: Use {{"command_name": "pass"}} to do nothing and {{"command_name": "end"}} to indicate completion of all requirements and the end of actions. +Special Command: Use {{"command_name": "end"}} to do nothing or indicate completion of all requirements and the end of actions. # Current Plan {plan_status} diff --git a/metagpt/prompts/di/team_leader.py b/metagpt/prompts/di/team_leader.py index 14297b026..9304fd24d 100644 --- a/metagpt/prompts/di/team_leader.py +++ b/metagpt/prompts/di/team_leader.py @@ -11,13 +11,16 @@ Your team member: You should NOT assign consecutive tasks to the same team member, instead, assign an aggregated task (or the complete requirement) and let the team member to decompose it. When creating a new plan involving multiple members, create all tasks at once. If plan is created, you should track the progress based on team member feedback message, and update plan accordingly, such as Plan.finish_current_task, Plan.reset_task, Plan.replace_task, etc. -You should use TeamLeader.publish_team_message to team members, asking them to start their task. +You should use TeamLeader.publish_team_message to team members, asking them to start their task. DONT omit any necessary info such as path, link, environment, programming language, framework, requirement, constraint from original content to team members because you are their sole info source. Pay close attention to new user message, review the conversation history, use RoleZero.reply_to_human to respond to the user directly, DON'T ask your team members. Note: 1. If the requirement is a pure DATA-RELATED requirement, such as bug fixes, issue reporting, environment setup, terminal operations, pip install, web browsing, web scraping, web searching, web imitation, data science, data analysis, machine learning, deep learning, text-to-image etc. DON'T decompose it, assign a single task with the original user requirement as instruction directly to Data Analyst. 2. If the requirement is developing a software, game, app, or website, excluding the above data-related tasks, you should decompose the requirement into multiple tasks and assign them to different team members based on their expertise, usually the sequence of Product Manager -> Architect -> Project Manager -> Engineer -> (optional: QaEngine if present) -> (optional: DataAnalyst if user requests deployment), each assigned ONE task. When publishing message to Product Manager, you should directly copy the full original user requirement. 3. If the requirement contains both DATA-RELATED part mentioned in 1 and software development part mentioned in 2, you should decompose the software development part and assign them to different team members based on their expertise, and assign the DATA-RELATED part to Data Analyst David directly. +4. If the requirement is a common-sense, logical, or math problem, you should respond directly without assigning any task to team members. +5. If you think the requirement is not clear or ambiguous, you should ask the user for clarification immediately. Assign tasks only after all info is clear. +6. It is helpful for Engineer to have both the system design and the project schedule for writing the code, so include paths of both files (if available) and remind Engineer to definitely read them when publishing message to Engineer. """ FINISH_CURRENT_TASK_CMD = """ @@ -27,5 +30,6 @@ FINISH_CURRENT_TASK_CMD = """ "command_name": "Plan.finish_current_task", "args": {{}} } +] ``` """ diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index afa234a3c..8650f2640 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -50,5 +50,6 @@ class Architect(RoleZero): { "WriteDesign.run": wd.run, "WriteDesign": wd.run, # alias + "run": wd.run, # alias } ) diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py index fc298ea4c..d4d67742b 100644 --- a/metagpt/roles/di/data_analyst.py +++ b/metagpt/roles/di/data_analyst.py @@ -41,6 +41,7 @@ class DataAnalyst(DataInterpreter): # Command.PASS, ] commands: list[dict] = [] # issued commands to be executed + user_requirement: str = "" @model_validator(mode="after") def set_plan_and_tool(self) -> "DataInterpreter": @@ -54,7 +55,6 @@ class DataAnalyst(DataInterpreter): if self.tools and not self.tool_recommender: self.tool_recommender = BM25ToolRecommender(tools=self.tools) self.set_actions([WriteAnalysisCode]) - self._set_state(0) # HACK: Init Planner, control it through dynamic thinking; Consider formalizing as a react mode self.planner = Planner(goal="", working_memory=self.rc.working_memory, auto_run=True) @@ -69,9 +69,6 @@ class DataAnalyst(DataInterpreter): self.user_requirement = self.get_memories()[-1].content self.planner.plan.goal = self.user_requirement example = KeywordExpRetriever().retrieve(self.user_requirement) - else: - self.working_memory.add_batch(self.rc.news) - # TODO: implement experience retrieval in multi-round setting plan_status = self.planner.plan.model_dump(include=["goal", "tasks"]) # for task in plan_status["tasks"]: @@ -83,10 +80,11 @@ class DataAnalyst(DataInterpreter): available_commands=prepare_command_prompt(self.available_commands), ) context = self.llm.format_msg(self.working_memory.get() + [Message(content=prompt, role="user")]) - async with ThoughtReporter(): + # print(*context, sep="\n" + "*" * 5 + "\n") + async with ThoughtReporter(enable_llm_stream=True): rsp = await self.llm.aask(context) - self.commands = json.loads(CodeParser.parse_code(block=None, text=rsp)) - self.rc.memory.add(Message(content=rsp, role="assistant")) + self.commands = json.loads(CodeParser.parse_code(block=None, lang='json', text=rsp)) + self.rc.working_memory.add(Message(content=rsp, role="assistant")) await run_commands(self, self.commands, self.rc.working_memory) @@ -112,11 +110,19 @@ class DataAnalyst(DataInterpreter): return Message(content="Task completed", role="assistant", sent_from=self._setting, cause_by=WriteAnalysisCode) async def _react(self) -> Message: + # NOTE: Diff 1: Each time landing here means observing news, set todo to allow news processing in _think + self._set_state(0) + actions_taken = 0 rsp = Message(content="No actions taken yet", cause_by=Action) # will be overwritten after Role _act while actions_taken < self.rc.max_react_loop: - # NOTE: difference here, keep observing within react + # NOTE: Diff 2: Keep observing within _react, news will go into memory, allowing adapting to new info + # add news from self._observe, the one called in self.run, consider removing when switching from working_memory to memory + self.working_memory.add_batch(self.rc.news) await self._observe() + # add news from this self._observe, we need twice because _observe rewrites rc.news + self.working_memory.add_batch(self.rc.news) + # think has_todo = await self._think() if not has_todo: diff --git a/metagpt/roles/di/data_interpreter.py b/metagpt/roles/di/data_interpreter.py index bdfc0e294..f90a928df 100644 --- a/metagpt/roles/di/data_interpreter.py +++ b/metagpt/roles/di/data_interpreter.py @@ -74,7 +74,7 @@ class DataInterpreter(Role): return True prompt = REACT_THINK_PROMPT.format(user_requirement=self.user_requirement, context=context) - async with ThoughtReporter(): + async with ThoughtReporter(enable_llm_stream=True): rsp = await self.llm.aask(prompt) rsp_dict = json.loads(CodeParser.parse_code(text=rsp)) self.working_memory.add(Message(content=rsp_dict["thoughts"], role="assistant")) diff --git a/metagpt/roles/di/engineer2.py b/metagpt/roles/di/engineer2.py index e013ef09e..8ea823c74 100644 --- a/metagpt/roles/di/engineer2.py +++ b/metagpt/roles/di/engineer2.py @@ -1,7 +1,9 @@ from __future__ import annotations +from metagpt.actions.write_code_review import ReviewAndRewriteCode from metagpt.prompts.di.engineer2 import ENGINEER2_INSTRUCTION from metagpt.roles.di.role_zero import RoleZero +from metagpt.strategy.experience_retriever import ENGINEER_EXAMPLE class Engineer2(RoleZero): @@ -10,4 +12,17 @@ class Engineer2(RoleZero): goal: str = "Take on game, app, and web development" instruction: str = ENGINEER2_INSTRUCTION - tools: str = ["Plan", "Editor:write,read,write_content", "RoleZero"] + tools: str = ["Plan", "Editor:write,read", "RoleZero", "ReviewAndRewriteCode"] + + def _update_tool_execution(self): + review = ReviewAndRewriteCode() + + self.tool_execution_map.update( + { + "ReviewAndRewriteCode.run": review.run, + "ReviewAndRewriteCode": review.run, + } + ) + + def _retrieve_experience(self) -> str: + return ENGINEER_EXAMPLE diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index 9c678c600..b5342409f 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -3,7 +3,7 @@ from __future__ import annotations import inspect import json import traceback -from typing import Literal, Tuple +from typing import Callable, Literal, Tuple from pydantic import model_validator @@ -42,7 +42,7 @@ class RoleZero(Role): # Tools tools: list[str] = [] # Use special symbol [""] to indicate use of all registered tools tool_recommender: ToolRecommender = None - tool_execution_map: dict[str, callable] = {} + tool_execution_map: dict[str, Callable] = {} special_tool_commands: list[str] = ["Plan.finish_current_task", "end"] # Equipped with three basic tools by default for optional use editor: Editor = Editor() @@ -53,7 +53,6 @@ class RoleZero(Role): experience_retriever: ExpRetriever = DummyExpRetriever() # Others - user_requirement: str = "" command_rsp: str = "" # the raw string containing the commands commands: list[dict] = [] # commands to be executed memory_k: int = 20 # number of memories (messages) to use as historical context @@ -106,8 +105,7 @@ class RoleZero(Role): return False if not self.planner.plan.goal: - self.user_requirement = self.get_memories()[-1].content - self.planner.plan.goal = self.user_requirement + self.planner.plan.goal = self.get_memories()[-1].content ### 1. Experience ### example = self._retrieve_experience() @@ -129,7 +127,7 @@ class RoleZero(Role): ) context = self.llm.format_msg(self.rc.memory.get(self.memory_k) + [UserMessage(content=prompt)]) # print(*context, sep="\n" + "*" * 5 + "\n") - async with ThoughtReporter(): + async with ThoughtReporter(enable_llm_stream=True): self.command_rsp = await self.llm.aask(context, system_msgs=self.system_msg) self.rc.memory.add(AIMessage(content=self.command_rsp)) @@ -175,7 +173,7 @@ class RoleZero(Role): actions_taken += 1 return rsp # return output from the last action - async def _run_commands(self, commands) -> list: + async def _run_commands(self, commands) -> str: outputs = [] for cmd in commands: # handle special command first @@ -195,7 +193,7 @@ class RoleZero(Role): outputs.append(output) except Exception as e: tb = traceback.format_exc() - logger.exception(e + tb) + logger.exception(str(e) + tb) outputs.append(output + f": {tb}") break # Stop executing if any command fails else: @@ -247,7 +245,7 @@ class RoleZero(Role): if not isinstance(self.rc.env, MGXEnv): return "Not in MGXEnv, command will not be executed." - return await self.rc.env.get_human_input(question, sent_from=self) + return await self.rc.env.ask_human(question, sent_from=self) async def reply_to_human(self, content: str) -> str: """Reply to human user with the content provided. Use this when you have a clear answer or solution to the user's question.""" diff --git a/metagpt/roles/di/team_leader.py b/metagpt/roles/di/team_leader.py index 92ddcab04..2932dd7f0 100644 --- a/metagpt/roles/di/team_leader.py +++ b/metagpt/roles/di/team_leader.py @@ -18,7 +18,8 @@ class TeamLeader(RoleZero): profile: str = "Team Leader" system_msg: list[str] = [SYSTEM_PROMPT] - max_react_loop: int = 1 # TeamLeader only reacts once each time + # TeamLeader only reacts once each time, but may encounter errors or need to ask human, thus allowing 2 more turns + max_react_loop: int = 3 tools: list[str] = ["Plan", "RoleZero", "TeamLeader"] diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index c122affb5..5e85b056a 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -17,152 +17,165 @@ class DummyExpRetriever(ExpRetriever): return "" +TL_EXAMPLE = """ +## example 1 +User Requirement: Create a cli snake game using Python. +Explanation: The requirement is about software development. Assign each tasks to a different team member based on their expertise. When publishing message to Product Manager, we copy original user requirement directly to ensure no information loss. +```json +[ + { + "command_name": "Plan.append_task", + "args": { + "task_id": "1", + "dependent_task_ids": [], + "instruction": "Create a product requirement document (PRD) outlining the features, user interface, and user experience of the CLI python snake game.", + "assignee": "Alice" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "2", + "dependent_task_ids": ["1"], + "instruction": "Design the software architecture for the CLI snake game, including the choice of programming language, libraries, and data flow.", + "assignee": "Bob" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "3", + "dependent_task_ids": ["2"], + "instruction": "Break down the architecture into manageable tasks, identify task dependencies, and prepare a detailed task list for implementation.", + "assignee": "Eve" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "4", + "dependent_task_ids": ["3"], + "instruction": "Implement the core game logic for the CLI snake game, including snake movement, food generation, and score tracking.", + "assignee": "Alex" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "5", + "dependent_task_ids": ["4"], + "instruction": "Write comprehensive tests for the game logic and user interface to ensure functionality and reliability.", + "assignee": "Edward" + } + }, + { + "command_name": "TeamLeader.publish_message", + "args": { + "content": "Create a cli snake game using Python", + "send_to": "Alice" + } + }, + { + "command_name": "RoleZero.reply_to_human", + "args": { + "content": "I have assigned the tasks to the team members. Alice will create the PRD, Bob will design the software architecture, Eve will break down the architecture into tasks, Alex will implement the core game logic, and Edward will write comprehensive tests. The team will work on the project accordingly", + } + }, + { + "command_name": "end" + } +] +``` + +## example 2 +User Requirement: Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy. +Explanation: DON'T decompose requirement if it is a DATA-RELATED task, assign a single task directly to Data Analyst David. He will manage the decomposition and implementation. +```json +[ + { + "command_name": "Plan.append_task", + "args": { + "task_id": "1", + "dependent_task_ids": [], + "instruction": "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.", + "assignee": "David" + } + }, + { + "command_name": "TeamLeader.publish_message", + "args": { + "content": "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.", + "send_to": "David" + } + }, + { + "command_name": "RoleZero.reply_to_human", + "args": { + "content": "I have assigned the task to David. He will break down the task further by himself and starts solving it.", + } + }, + { + "command_name": "end" + } +] +``` + +## example 3 +Conversation History: +[ + ..., + {'role': 'assistant', 'content': 'from Alice(Product Manager) to {'Bob'}: {'docs': {'20240424153821.json': {'root_path': 'docs/prd', 'filename': '20240424153821.json', 'content': '{"Language":"en_us","Programming Language":"Python","Original Requirements":"create a cli snake game","Project Name":"snake_game","Product Goals":["Develop an intuitive and addictive snake game",...], ...}}}}}, +] +Explanation: You received a message from Alice, the Product Manager, that she has completed the PRD, use Plan.finish_current_task to mark her task as finished and moves the plan to the next task. Based on plan status, next task is for Bob (Architect), publish a message asking him to start. The message content should contain important path info. +```json +[ + { + "command_name": "Plan.finish_current_task", + "args": {} + }, + { + "command_name": "TeamLeader.publish_message", + "args": { + "content": "Please design the software architecture for the snake game based on the PRD created by Alice. The PRD is at 'docs/prd/20240424153821.json'. Include the choice of programming language, libraries, and data flow, etc.", + "send_to": "Bob" + } + }, + { + "command_name": "RoleZero.reply_to_human", + "args": { + "content": "Alice has completed the PRD. I have marked her task as finished and sent the PRD to Bob. Bob will work on the software architecture.", + } + }, + { + "command_name": "end" + } +] +``` + +## example 4 +User Question: how does the project go? +Explanation: The user is asking for a general update on the project status. Give a straight answer about the current task the team is working on and provide a summary of the completed tasks. +```json +[ + { + "command_name": "RoleZero.reply_to_human", + "args": { + "content": "The team is currently working on ... We have completed ...", + } + }, + { + "command_name": "end" + } +] +``` +""" + + class SimpleExpRetriever(ExpRetriever): """A simple experience retriever that returns manually crafted examples.""" - EXAMPLE: str = """ - ## example 1 - User Requirement: Create a cli snake game using Python. - Explanation: The requirement is about software development. Assign each tasks to a different team member based on their expertise. When publishing message to Product Manager, we copy original user requirement directly to ensure no information loss. - ```json - [ - { - "command_name": "Plan.append_task", - "args": { - "task_id": "1", - "dependent_task_ids": [], - "instruction": "Create a product requirement document (PRD) outlining the features, user interface, and user experience of the CLI python snake game.", - "assignee": "Alice" - } - }, - { - "command_name": "Plan.append_task", - "args": { - "task_id": "2", - "dependent_task_ids": ["1"], - "instruction": "Design the software architecture for the CLI snake game, including the choice of programming language, libraries, and data flow.", - "assignee": "Bob" - } - }, - { - "command_name": "Plan.append_task", - "args": { - "task_id": "3", - "dependent_task_ids": ["2"], - "instruction": "Break down the architecture into manageable tasks, identify task dependencies, and prepare a detailed task list for implementation.", - "assignee": "Eve" - } - }, - { - "command_name": "Plan.append_task", - "args": { - "task_id": "4", - "dependent_task_ids": ["3"], - "instruction": "Implement the core game logic for the CLI snake game, including snake movement, food generation, and score tracking.", - "assignee": "Alex" - } - }, - { - "command_name": "Plan.append_task", - "args": { - "task_id": "5", - "dependent_task_ids": ["4"], - "instruction": "Write comprehensive tests for the game logic and user interface to ensure functionality and reliability.", - "assignee": "Edward" - } - }, - { - "command_name": "TeamLeader.publish_message", - "args": { - "content": "Create a cli snake game using Python", - "send_to": "Alice" - } - }, - { - "command_name": "RoleZero.reply_to_human", - "args": { - "content": "I have assigned the tasks to the team members. Alice will create the PRD, Bob will design the software architecture, Eve will break down the architecture into tasks, Alex will implement the core game logic, and Edward will write comprehensive tests. The team will work on the project accordingly", - } - } - ] - ``` - - ## example 2 - User Requirement: Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy. - Explanation: DON'T decompose requirement if it is a DATA-RELATED task, assign a single task directly to Data Analyst David. He will manage the decomposition and implementation. - ```json - [ - { - "command_name": "Plan.append_task", - "args": { - "task_id": "1", - "dependent_task_ids": [], - "instruction": "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.", - "assignee": "David" - } - }, - { - "command_name": "TeamLeader.publish_message", - "args": { - "content": "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.", - "send_to": "David" - } - }, - { - "command_name": "RoleZero.reply_to_human", - "args": { - "content": "I have assigned the task to David. He will break down the task further by himself and starts solving it.", - } - } - ] - ``` - - ## example 3 - Conversation History: - [ - ..., - {'role': 'assistant', 'content': 'from Alice(Product Manager) to {'Bob'}: {'docs': {'20240424153821.json': {'root_path': 'docs/prd', 'filename': '20240424153821.json', 'content': '{"Language":"en_us","Programming Language":"Python","Original Requirements":"create a cli snake game","Project Name":"snake_game","Product Goals":["Develop an intuitive and addictive snake game",...], ...}}}}}, - ] - Explanation: You received a message from Alice, the Product Manager, that she has completed the PRD, use Plan.finish_current_task to mark her task as finished and moves the plan to the next task. Based on plan status, next task is for Bob (Architect), publish a message asking him to start. The message content should contain important path info. - ```json - [ - { - "command_name": "Plan.finish_current_task", - "args": {} - }, - { - "command_name": "TeamLeader.publish_message", - "args": { - "content": "Please design the software architecture for the snake game based on the PRD created by Alice. The PRD is at 'docs/prd/20240424153821.json'. Include the choice of programming language, libraries, and data flow, etc.", - "send_to": "Bob" - } - }, - { - "command_name": "RoleZero.reply_to_human", - "args": { - "content": "Alice has completed the PRD. I have marked her task as finished and sent the PRD to Bob. Bob will work on the software architecture.", - } - } - ] - ``` - - ## example 4 - User Question: how does the project go? - Explanation: The user is asking for a general update on the project status. Give a straight answer about the current task the team is working on and provide a summary of the completed tasks. - ```json - [ - { - "command_name": "RoleZero.reply_to_human", - "args": { - "content": "The team is currently working on ... We have completed ...", - } - } - ] - ``` - """ - def retrieve(self, context: str = "") -> str: - return self.EXAMPLE + return TL_EXAMPLE class KeywordExpRetriever(ExpRetriever): @@ -213,6 +226,7 @@ Explanation: Launching a service requires Terminal tool with daemon mode, write "assignee": "David" } }, +] """ @@ -322,3 +336,115 @@ editor.read(path="./main.py") - If no specific file is provided, search the symbol in the whole codebase to locate the issue. - If no specific symbol is provided, directly open and read the file to diagnose the problem. """ + +ENGINEER_EXAMPLE = """ +## example 1 +User Requirement: Please implement the core game logic for the 2048 game, including tile movements, merging logic, score tracking, and keyboard interaction. Refer to the project schedule located at '/tmp/project_schedule.json' and the system design document at '/tmp/system_design.json' for detailed information. +Explanation: I will first need to read the system design document and the project schedule to understand the specific requirements and architecture outlined for the game development. + +```json +[ + { + "command_name": "Editor.read", + "args": { + "path": "/tmp/docs/project_schedule.json" + } + }, + { + "command_name": "Editor.read", + "args": { + "path": "/tmp/docs/system_design.json" + } + } +] +``` + +## example 2 +To achieve the goal of writing a 2048 game using JavaScript and HTML without any frameworks, I will create a plan consisting of three tasks, each corresponding to the creation of one of the required files: `index.html`, `style.css`, and `script.js`. Following the completion of these tasks, I will add a code review task for each file to ensure the implementation aligns with the provided system design and project schedule documents. + +Here's the plan: + +1. **Task 1**: Create `index.html` - This file will contain the HTML structure necessary for the game's UI. +2. **Task 2**: Create `style.css` - This file will define the CSS styles to make the game visually appealing and responsive. +3. **Task 3**: Create `script.js` - This file will contain the JavaScript code for the game logic and UI interactions. +4. **Code Review Tasks**: Review each file to ensure they meet the project requirements and adhere to the system design. + +Let's start by appending the first task to the plan. + +```json +[ + { + "command_name": "Plan.append_task", + "args": { + "task_id": "1", + "dependent_task_ids": [], + "instruction": "Create the index.html file with the basic HTML structure for the 2048 game.", + "assignee": "Alex" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "2", + "dependent_task_ids": ["1"], + "instruction": "Create the style.css file with the necessary CSS to style the 2048 game.", + "assignee": "Alex" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "3", + "dependent_task_ids": ["1", "2"], + "instruction": "Create the script.js file containing the JavaScript logic for the 2048 game.", + "assignee": "Alex" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "4", + "dependent_task_ids": ["1"], + "instruction": "Use ReviewAndRewriteCode to review the code in index.html to ensure it meets the design specifications.", + "assignee": "Alex" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "5", + "dependent_task_ids": ["2"], + "instruction": "Use ReviewAndRewriteCode to review the code in style.css to ensure it meets the design specifications.", + "assignee": "Alex" + } + }, + { + "command_name": "Plan.append_task", + "args": { + "task_id": "6", + "dependent_task_ids": ["3"], + "instruction": "Use ReviewAndRewriteCode to review the code in script.js to ensure it meets the design specifications. ", + "assignee": "Alex" + } + } +] +``` + +## example 3 +I will now review the code in `script.js`. +Explanation: to review the code, call ReviewAndRewriteCode.run. + +```json +[ + { + "command_name": "ReviewAndRewriteCode.run", + "args": { + "code_path": "/tmp/src/script.js", + "system_design_input": "/tmp/docs/system_design.json", + "project_schedule_input": "/tmp/docs/project_schedule.json", + "code_review_k_times": 2 + } + } +] +``` +""" diff --git a/metagpt/tools/libs/browser.py b/metagpt/tools/libs/browser.py index 955058ea0..1b1b3d82d 100644 --- a/metagpt/tools/libs/browser.py +++ b/metagpt/tools/libs/browser.py @@ -189,8 +189,8 @@ class Browser: async def _view(self, keep_len: int = 5000) -> str: """simulate human viewing the current page, return the visible text with links""" - # visible_text_with_links = await self.current_page.evaluate(VIEW_CONTENT_JS) - # print("The visible text and their links (if any): ", visible_text_with_links[:keep_len]) + visible_text_with_links = await self.current_page.evaluate(VIEW_CONTENT_JS) + print("The visible text and their links (if any): ", visible_text_with_links[:keep_len]) # html_content = await self._view_page_html(keep_len=keep_len) # print("The html content: ", html_content) diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index 23df02edd..c9f02c4f1 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -26,12 +26,16 @@ class Editor: def write(self, path: str, content: str): """Write the whole content to a file. When used, make sure content arg contains the full content of the file.""" + if "\n" not in content and "\\n" in content: + # A very raw rule to correct the content: If 'content' lacks actual newlines ('\n') but includes '\\n', consider + # replacing them with '\n' to potentially correct mistaken representations of newline characters. + content = content.replace("\\n", "\n") directory = os.path.dirname(path) if directory and not os.path.exists(directory): os.makedirs(directory) with open(path, "w", encoding="utf-8") as f: f.write(content) - self.resource.report(path, "path") + # self.resource.report(path, "path") def read(self, path: str) -> FileBlock: """Read the whole content of a file.""" diff --git a/metagpt/tools/libs/git.py b/metagpt/tools/libs/git.py index b4d759bf4..740cb81f9 100644 --- a/metagpt/tools/libs/git.py +++ b/metagpt/tools/libs/git.py @@ -141,6 +141,8 @@ async def git_create_pull( Returns: PullRequest: The created pull request. """ + from metagpt.utils.git_repository import GitRepository + return await GitRepository.create_pull( base=base, head=head, @@ -187,4 +189,6 @@ async def git_create_issue( Returns: Issue: The created issue. """ + from metagpt.utils.git_repository import GitRepository + return await GitRepository.create_issue(repo_name=repo_name, title=title, body=body, access_token=access_token) diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index e6d059636..faf2893a7 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -26,7 +26,7 @@ class Terminal: stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - executable="/bin/bash", + executable="/bin/bash" ) self.stdout_queue = Queue() self.observer = TerminalReporter() diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index b58ac99da..72dc6ab94 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -275,7 +275,7 @@ class CodeParser: def parse_code(cls, text: str, lang: str = "", block: Optional[str] = None) -> str: if block: text = cls.parse_block(block, text) - pattern = rf"```{lang}.*?\s+(.*?)```" + pattern = rf"```{lang}.*?\s+(.*?)\n```" match = re.search(pattern, text, re.DOTALL) if match: code = match.group(1) diff --git a/metagpt/utils/report.py b/metagpt/utils/report.py index 2d72af111..ed40e5dde 100644 --- a/metagpt/utils/report.py +++ b/metagpt/utils/report.py @@ -51,7 +51,6 @@ class ResourceReporter(BaseModel): block: BlockType = Field(description="The type of block that is reporting the resource") uuid: UUID = Field(default_factory=uuid4, description="The unique identifier for the resource") - is_chunk: bool = Field(False, description="Indicates whether the report is a chunk of a stream") enable_llm_stream: bool = Field(False, description="Indicates whether to connect to an LLM stream for reporting") callback_url: str = Field(METAGPT_REPORTER_DEFAULT_URL, description="The URL to which the report should be sent") _llm_task: Optional[asyncio.Task] = PrivateAttr(None) @@ -153,17 +152,14 @@ class ResourceReporter(BaseModel): def __enter__(self): """Enter the synchronous streaming callback context.""" - self.is_chunk = True return self def __exit__(self, *args, **kwargs): """Exit the synchronous streaming callback context.""" self.report(None, END_MARKER_NAME) - self.is_chunk = False async def __aenter__(self): """Enter the asynchronous streaming callback context.""" - self.is_chunk = True if self.enable_llm_stream: queue = create_llm_stream_queue() self._llm_task = asyncio.create_task(self._llm_stream_report(queue)) @@ -171,15 +167,18 @@ class ResourceReporter(BaseModel): async def __aexit__(self, *args, **kwargs): """Exit the asynchronous streaming callback context.""" - self.is_chunk = False if self.enable_llm_stream: - self._llm_task.cancel() + await get_llm_stream_queue().put(None) + await self._llm_task self._llm_task = None await self.async_report(None, END_MARKER_NAME) async def _llm_stream_report(self, queue: asyncio.Queue): - while self.is_chunk: - await self.async_report(await queue.get(), "content") + while True: + data = await queue.get() + if data is None: + return + await self.async_report(data, "content") async def wait_llm_stream_report(self): """Wait for the LLM stream report to complete.""" diff --git a/tests/metagpt/roles/di/run_architect.py b/tests/metagpt/roles/di/run_architect.py index e615af4eb..455b60d92 100644 --- a/tests/metagpt/roles/di/run_architect.py +++ b/tests/metagpt/roles/di/run_architect.py @@ -2,6 +2,7 @@ import asyncio import os from metagpt.roles.architect import Architect +from metagpt.schema import Message DESIGN_DOC_SNAKE = """ { @@ -30,9 +31,9 @@ async def main(requirement): with open("temp_design.json", "w") as f: f.write(DESIGN_DOC_SNAKE) architect = Architect() - await architect.run(requirement) + await architect.run(Message(content=requirement, send_to="Bob")) os.remove("temp_design.json") if __name__ == "__main__": - asyncio.run(main(WRITE_SNAKE)) + asyncio.run(main(REWRITE_SNAKE)) diff --git a/tests/metagpt/roles/di/run_engineer2.py b/tests/metagpt/roles/di/run_engineer2.py index 4e948bad7..e5ae74485 100644 --- a/tests/metagpt/roles/di/run_engineer2.py +++ b/tests/metagpt/roles/di/run_engineer2.py @@ -67,18 +67,18 @@ Create a 2048 game, follow the design doc and task doc. Write your code under /U After writing all codes, write a code review for the codes, make improvement or adjustment based on the review. Notice: You MUST implement the full code, don't leave comment without implementation! Design doc: -{TASK_DOC_2048} -Task doc: {DESIGN_DOC_2048} +Task doc: +{TASK_DOC_2048} """ GAME_REQ_SNAKE = f""" Create a snake game, follow the design doc and task doc. Write your code under /Users/gary/Files/temp/workspace/snake_game/src. After writing all codes, write a code review for the codes, make improvement or adjustment based on the review. Notice: You MUST implement the full code, don't leave comment without implementation! Design doc: -{TASK_DOC_SNAKE} -Task doc: {DESIGN_DOC_SNAKE} +Task doc: +{TASK_DOC_SNAKE} """ GAME_REQ_2048_NO_DOC = """ Create a 2048 game with pygame. Write your code under /Users/gary/Files/temp/workspace/2048_game/src. diff --git a/tests/metagpt/roles/di/run_product_manager.py b/tests/metagpt/roles/di/run_product_manager.py new file mode 100644 index 000000000..3ab1e9bab --- /dev/null +++ b/tests/metagpt/roles/di/run_product_manager.py @@ -0,0 +1,18 @@ +import asyncio + +from metagpt.roles.product_manager import ProductManager + +WRITE_2048 = """Write a PRD for a cli 2048 game""" + +REWRITE_2048 = """Rewrite the prd at /Users/gary/Files/temp/workspace/2048_game/docs/prd.json, add a web UI""" + +CASUAL_CHAT = """What's your name?""" + + +async def main(requirement): + product_manager = ProductManager() + await product_manager.run(requirement) + + +if __name__ == "__main__": + asyncio.run(main(WRITE_2048)) diff --git a/tests/metagpt/roles/di/run_project_manager.py b/tests/metagpt/roles/di/run_project_manager.py new file mode 100644 index 000000000..30889c59c --- /dev/null +++ b/tests/metagpt/roles/di/run_project_manager.py @@ -0,0 +1,36 @@ +import asyncio +import os + +from metagpt.roles.project_manager import ProjectManager +from metagpt.schema import Message + +DESIGN_DOC_2048 = '{"Implementation approach":"We will use the Pygame library to implement the 2048 game logic and user interface. Pygame is a set of Python modules designed for writing video games, which will help us create a responsive and visually appealing UI. For the mobile responsiveness, we will ensure that the game scales appropriately on different screen sizes. We will also use the Pygame GUI library to create buttons for restarting the game and choosing difficulty levels.","File list":["main.py","game.py","ui.py"],"Data structures and interfaces":"\\nclassDiagram\\n class Game {\\n -grid: list[list[int]]\\n -score: int\\n +__init__()\\n +move(direction: str) bool\\n +merge() bool\\n +spawn_tile() None\\n +is_game_over() bool\\n +reset() None\\n }\\n class UI {\\n -game: Game\\n +__init__(game: Game)\\n +draw_grid() None\\n +draw_score() None\\n +draw_buttons() None\\n +handle_input() None\\n }\\n class Main {\\n -ui: UI\\n +main() None\\n }\\n Main --> UI\\n UI --> Game\\n","Program call flow":"\\nsequenceDiagram\\n participant M as Main\\n participant U as UI\\n participant G as Game\\n M->>U: __init__(game)\\n U->>G: __init__()\\n M->>U: draw_grid()\\n U->>G: move(direction)\\n G-->>U: return bool\\n U->>G: merge()\\n G-->>U: return bool\\n U->>G: spawn_tile()\\n G-->>U: return None\\n U->>G: is_game_over()\\n G-->>U: return bool\\n U->>G: reset()\\n G-->>U: return None\\n M->>U: draw_score()\\n M->>U: draw_buttons()\\n M->>U: handle_input()\\n","Anything UNCLEAR":"Clarification needed on the specific design elements for the UI to ensure it meets the \'beautiful\' requirement. Additionally, we need to confirm the exact difficulty levels and how they should affect the game mechanics."}' +DESIGN_DOC_SNAKE = """ +{ + "Implementation approach": "We will use the Pygame library to create the CLI-based snake game. Pygame is a set of Python modules designed for writing video games, which will help us handle graphics, sound, and input. The game will be structured into different modules to handle the main game loop, snake movement, food generation, collision detection, and user interface. We will ensure the game is engaging and responsive by optimizing the game loop and input handling. The score display and different speed levels will be implemented to enhance the user experience.", + "File list": [ + "main.py", + "game.py", + "snake.py", + "food.py", + "ui.py" + ], + "Data structures and interfaces": "\nclassDiagram\n class Main {\n +main() void\n }\n class Game {\n -Snake snake\n -Food food\n -int score\n -int speed\n +__init__(speed: int)\n +run() void\n +restart() void\n +update_score() void\n }\n class Snake {\n -list body\n -str direction\n +__init__()\n +move() void\n +change_direction(new_direction: str) void\n +check_collision() bool\n +grow() void\n }\n class Food {\n -tuple position\n +__init__()\n +generate_new_position() void\n }\n class UI {\n +display_score(score: int) void\n +display_game_over() void\n +display_game(snake: Snake, food: Food) void\n }\n Main --> Game\n Game --> Snake\n Game --> Food\n Game --> UI\n", + "Program call flow": "\nsequenceDiagram\n participant M as Main\n participant G as Game\n participant S as Snake\n participant F as Food\n participant U as UI\n M->>G: __init__(speed)\n M->>G: run()\n G->>S: __init__()\n G->>F: __init__()\n loop Game Loop\n G->>S: move()\n G->>S: check_collision()\n alt Collision Detected\n G->>G: restart()\n G->>U: display_game_over()\n else No Collision\n G->>F: generate_new_position()\n G->>S: grow()\n G->>G: update_score()\n G->>U: display_score(score)\n end\n G->>U: display_game(snake, food)\n end\n", + "Anything UNCLEAR": "Currently, all aspects of the project are clear." +} +""" +REQ = """Write a project schedule based on the design at temp_design.json""" +CASUAL_CHAT = """what's your name?""" + + +async def main(requirement): + with open("temp_design.json", "w") as f: + f.write(DESIGN_DOC_2048) + project_manager = ProjectManager() + await project_manager.run(Message(content=requirement, send_to="Eve")) + os.remove("temp_design.json") + + +if __name__ == "__main__": + asyncio.run(main(REQ))