Merge branch 'opt_speed_quality' into 'mgx_ops'

Faster & Better Engineer2, & some small updates

See merge request pub/MetaGPT!359

metagpt/prompts/di/engineer2.py

@@ -1,6 +1,6 @@
 from metagpt.prompts.di.role_zero import ROLE_INSTRUCTION
 
-EXTRA_INSTRUCTION = """
+EXTRA_INSTRUCTION_DEPRECATED = """
 4. Each time you write a code in your response, write with the Editor directly without preparing a repetitive code block beforehand.
 5. Take on ONE task and write ONE code file in each response. DON'T attempt all tasks in one response.
 6. When not specified, you should write files in a folder named "src". If you know the project path, then write in a "src" folder under the project path.
@@ -18,5 +18,39 @@ EXTRA_INSTRUCTION = """
 18. If you plan to read a file, do not include other plans in the same response.
 """
 
+EXTRA_INSTRUCTION = """
+6. When not specified, you should write files in a folder named "src". If you know the project path, then write in a "src" folder under the project path.
+7. When provided system design or project schedule, you MUST read them first before making a plan, then adhere to them in your implementation, especially in the programming language, package, or framework. You MUST implement all code files prescribed in the system design or project schedule. You can create a plan first with each task corresponding to implementing one code file.
+8. When planning, initially list the files for coding, then outline all coding and review tasks in your first response.
+9. If you plan to read a file, do not include other plans in the same response.
+10. Use Engineer2.write_new_code to create or modify a file. Write only one code file each time.
+11. When the requirement is simple, you don't need to create a plan, just do it right away.
+"""
 
 ENGINEER2_INSTRUCTION = ROLE_INSTRUCTION + EXTRA_INSTRUCTION.strip()
+
+WRITE_CODE_SYSTEM_PROMPT = """
+You are a world-class engineer, your goal is to write google-style, elegant, modular, readable, maintainable, fully functional, and ready-for-production code.
+
+Pay attention to the conversation history and the following constraints:
+1. When provided system design, YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design.
+2. When modifying a code, rewrite the full code instead of updating or inserting a snippet.
+3. Write out EVERY CODE DETAIL, DON'T LEAVE TODO.
+"""
+
+WRITE_CODE_PROMPT = """
+# User Requirement
+{user_requirement}
+
+# Plan Status
+{plan_status}
+
+# Further Instruction
+{instruction}
+
+# Output
+While some concise thoughts are helpful, code is absolutely required. Always output one and only one code block in your response. Output code in the following format:
+```
+your code
+```
+"""

metagpt/roles/di/engineer2.py

@@ -1,14 +1,25 @@
 from __future__ import annotations
 
+from pathlib import Path
+
 from pydantic import Field
 
-from metagpt.actions.write_code_review import ValidateAndRewriteCode
-from metagpt.prompts.di.engineer2 import ENGINEER2_INSTRUCTION
+# from metagpt.actions.write_code_review import ValidateAndRewriteCode
+from metagpt.prompts.di.engineer2 import (
+    ENGINEER2_INSTRUCTION,
+    WRITE_CODE_PROMPT,
+    WRITE_CODE_SYSTEM_PROMPT,
+)
 from metagpt.roles.di.role_zero import RoleZero
+from metagpt.schema import UserMessage
 from metagpt.strategy.experience_retriever import ENGINEER_EXAMPLE
 from metagpt.tools.libs.terminal import Terminal
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import CodeParser, awrite
+from metagpt.utils.report import EditorReporter
 
 
+@register_tool(include_functions=["write_new_code"])
 class Engineer2(RoleZero):
     name: str = "Alex"
     profile: str = "Engineer"
@@ -17,16 +28,16 @@ class Engineer2(RoleZero):
 
     terminal: Terminal = Field(default_factory=Terminal, exclude=True)
 
-    tools: list[str] = ["Plan", "Editor:write,read", "RoleZero", "Terminal:run_command", "ValidateAndRewriteCode"]
+    tools: list[str] = ["Plan", "Editor:read", "RoleZero", "Terminal:run_command", "Engineer2"]
 
     def _update_tool_execution(self):
-        validate = ValidateAndRewriteCode()
-
+        # validate = ValidateAndRewriteCode()
         self.tool_execution_map.update(
             {
                 "Terminal.run_command": self.terminal.run_command,
-                "ValidateAndRewriteCode.run": validate.run,
-                "ValidateAndRewriteCode": validate.run,
+                "Engineer2.write_new_code": self.write_new_code,
+                # "ValidateAndRewriteCode.run": validate.run,
+                # "ValidateAndRewriteCode": validate.run,
             }
         )
 
@@ -42,3 +53,28 @@ class Engineer2(RoleZero):
             command_output += "All tasks are finished.\n"
         command_output += await super()._run_special_command(cmd)
         return command_output
+
+    async def write_new_code(self, path: str, instruction: str = "") -> str:
+        """Write a new code file.
+
+        Args:
+            path (str): The absolute path of the file to be created.
+            instruction (optional, str): Further hints or notice other than the current task instruction, must be very concise and can be empty. Defaults to "".
+        """
+        plan_status, _ = self._get_plan_status()
+        prompt = WRITE_CODE_PROMPT.format(
+            user_requirement=self.planner.plan.goal,
+            plan_status=plan_status,
+            instruction=instruction,
+        )
+        context = self.llm.format_msg(self.rc.memory.get(self.memory_k) + [UserMessage(content=prompt)])
+
+        async with EditorReporter(enable_llm_stream=True) as reporter:
+            await reporter.async_report({"type": "code", "filename": Path(path).name, "src_path": path}, "meta")
+            rsp = await self.llm.aask(context, system_msgs=[WRITE_CODE_SYSTEM_PROMPT])
+            code = CodeParser.parse_code(text=rsp)
+            await awrite(path, code)
+            await reporter.async_report(path, "path")
+
+        # TODO: Consider adding line no to be ready for editing.
+        return f"The file {path} has been successfully created, with content:\n{code}"

metagpt/roles/di/role_zero.py

@@ -4,6 +4,7 @@ import inspect
 import json
 import re
 import traceback
+from datetime import datetime
 from typing import Annotated, Callable, Dict, List, Literal, Optional, Tuple
 
 from pydantic import Field, model_validator
@@ -67,7 +68,7 @@ class RoleZero(Role):
 
     # React Mode
     react_mode: Literal["react"] = "react"
-    max_react_loop: int = 20  # used for react mode
+    max_react_loop: int = 50  # used for react mode
 
     # Tools
     tools: list[str] = []  # Use special symbol ["<all>"] to indicate use of all registered tools
@@ -233,6 +234,10 @@ class RoleZero(Role):
                 msg.add_metadata(IMAGES, images)
         return memory
 
+    def _get_prefix(self) -> str:
+        time_info = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        return super()._get_prefix() + f" The current time is {time_info}."
+
     async def _act(self) -> Message:
         if self.use_fixed_sop:
             return await super()._act()
@@ -276,6 +281,14 @@ class RoleZero(Role):
             logger.debug(f"{self._setting}: {self.rc.state=}, will do {self.rc.todo}")
             rsp = await self._act()
             actions_taken += 1
+
+            # post-check
+            if self.rc.max_react_loop >= 10 and actions_taken >= self.rc.max_react_loop:
+                # If max_react_loop is a small value (e.g. < 10), it is intended to be reached and make the agent stop
+                logger.warning(f"reached max_react_loop: {actions_taken}")
+                rsp = await self.ask_human("I have reached my max action rounds, do you want me to continue? Yes or no")
+                if "yes" in rsp.lower():
+                    actions_taken = 0
         return rsp  # return output from the last action
 
     def format_quick_system_prompt(self) -> str:

metagpt/strategy/experience_retriever.py

@@ -849,7 +849,6 @@ Here's the plan:
 1. **Task 1**: Create `index.html` - This file will contain the HTML structure necessary for the game's UI.
 2. **Task 2**: Create `style.css` - This file will define the CSS styles to make the game visually appealing and responsive.
 3. **Task 3**: Create `script.js` - This file will contain the JavaScript code for the game logic and UI interactions.
-4. **Code Review Tasks**: Review each file to ensure they meet the project requirements and adhere to the system design.
 
 Let's start by appending the first task to the plan.
 
@@ -881,33 +880,6 @@ Let's start by appending the first task to the plan.
             "instruction": "Create the script.js file containing the JavaScript logic for the 2048 game.",
             "assignee": "Alex"
         }
-    },
-    {
-        "command_name": "Plan.append_task",
-        "args": {
-            "task_id": "4",
-            "dependent_task_ids": ["1"],
-            "instruction": "Use ValidateAndRewriteCode to review the code in index.html to ensure it meets the design specifications.",
-            "assignee": "Alex"
-        }
-    },
-    {
-        "command_name": "Plan.append_task",
-        "args": {
-            "task_id": "5",
-            "dependent_task_ids": ["2"],
-            "instruction": "Use ValidateAndRewriteCode to review the code in style.css to ensure it meets the design specifications.",
-            "assignee": "Alex"
-        }
-    },
-    {
-        "command_name": "Plan.append_task",
-        "args": {
-            "task_id": "6",
-            "dependent_task_ids": ["3"],
-            "instruction": "Use ValidateAndRewriteCode to review the code in script.js to ensure it meets the design specifications. ",
-            "assignee": "Alex"
-        }
     }
 ]
 ```
@@ -918,32 +890,9 @@ Explanation: Take on one task, such as writing a file. Upon completion, finish c
 ```json
 [
     {
-        "command_name": "Editor.write",
+        "command_name": "Engineer2.write_new_code",
         "args": {
-            "path": "/Users/gary/Files/temp/workspace/snake_game/src/index.html",
-            "content": "the code ..."
-        }
-    },
-    {
-        "command_name": "Plan.finish_current_task",
-        "args": {{}}
-    }
-]
-```
-
-## example 4
-I will now review the code in `script.js`.
-Explanation: to review the code, call ValidateAndRewriteCode.run.
-
-```json
-[
-    {
-        "command_name": "ValidateAndRewriteCode.run",
-        "args": {
-            "code_path": "/tmp/src/script.js",
-            "system_design_input": "/tmp/docs/system_design.json",
-            "project_schedule_input": "/tmp/docs/project_schedule.json",
-            "code_validate_k_times": 2
+            "path": "src/index.html"
         }
     },
     {