From 47bc87d76ac647946d205ee1d5864bd0fc41f2bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Fri, 13 Sep 2024 14:48:09 +0800 Subject: [PATCH 01/21] engineer2 must use browser before git clone. --- metagpt/actions/write_prd_an.py | 4 ++-- metagpt/prompts/di/engineer2.py | 8 +++++--- metagpt/strategy/experience_retriever.py | 8 ++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py index f64997bd2..4c0403c36 100644 --- a/metagpt/actions/write_prd_an.py +++ b/metagpt/actions/write_prd_an.py @@ -19,8 +19,8 @@ LANGUAGE = ActionNode( PROGRAMMING_LANGUAGE = ActionNode( key="Programming Language", expected_type=str, - instruction="Mainstream programming language. If not specified in the requirements, use HTML, CSS, and Pure JavaScript.", - example="HTML, CSS, and Pure JavaScript", + instruction="Mainstream programming language. If not specified in the requirements, native web technologies", + example="native web technologies", ) ORIGINAL_REQUIREMENTS = ActionNode( diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index 73127a2be..a743f6530 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -10,9 +10,11 @@ You can use terminal commands (e.g., cat, ls, cd) by calling Terminal.run_comman You should carefully observe the behavior and results of the previous action, and avoid triggering repeated errors. -In addition to the terminal, I also provide additional tools. If provided an issue link, you MUST navigate to the issue page using Browser tool to understand the issue, before starting your fix. +In addition to the terminal, I also provide additional tools. -Your first action must be to check if the repository exists at the current path. If it exists, navigate to the repository path. If the repository doesn't exist, please download it and then navigate to it. +If provided an issue link, you first action must be navigate to the issue page using Browser tool to understand the issue. + +Your must check if the repository exists at the current path. If it exists, navigate to the repository path. If the repository doesn't exist, please download it and then navigate to it. All subsequent actions must be performed within this repository path. Do not leave this directory to execute any actions at any time. Note: @@ -76,7 +78,7 @@ Note: 19. When the requirement is simple, you don't need to create a plan, just do it right away. 20. If the code exists, use the Editor tool's open and edit commands to modify it. Since it is not a new code, do not use write_new_code. 21. When using the editor, pay attention to the editor's current directory. When you use editor tools, the paths must be either absolute or relative to the editor's current directory. -22. The default programming languages are HTML (.html), CSS (.css), and Pure JavaScript (.js). +22. The default programming languages are Native HTML. """ CURRENT_STATE = """ The current editor state is: diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index 0f51f60db..7229982d4 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -479,7 +479,7 @@ Explanation: The requirement is about software development. Assign each tasks to "args": { "task_id": "1", "dependent_task_ids": [], - "instruction": "Use HTML, CSS, Pure JavaScrip as the programming language. And create a product requirement document (PRD) outlining the features, user interface. ", + "instruction": "Use native web technologies for the program. And create a product requirement document (PRD) outlining the features, user interface. ", "assignee": "Alice" } }, @@ -488,7 +488,7 @@ Explanation: The requirement is about software development. Assign each tasks to "args": { "task_id": "2", "dependent_task_ids": ["1"], - "instruction": "Use HTML, CSS, Pure JavaScrip as the programming language. Design the software architecture for the CLI snake game, including the data flow.", + "instruction": "Use native web technologies for the program. Design the software architecture for the CLI snake game, including the data flow.", "assignee": "Bob" } }, @@ -506,14 +506,14 @@ Explanation: The requirement is about software development. Assign each tasks to "args": { "task_id": "4", "dependent_task_ids": ["3"], - "instruction": "Use HTML, CSS, Pure JavaScrip as the programming language. Implement the core game logic for the CLI snake game, including snake movement, food generation, and score tracking.", + "instruction": "Use native web technologies for the program. Implement the core game logic for the CLI snake game, including snake movement, food generation, and score tracking.", "assignee": "Alex" } }, { "command_name": "TeamLeader.publish_message", "args": { - "content": "Use HTML, CSS, Pure JavaScrip as the programming language. Create a cli snake game.", + "content": "Use native web technologies for the program. Create a cli snake game.", "send_to": "Alice" } }, From be2868dfcfeb0999a396897b02276475a47851a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Fri, 13 Sep 2024 14:54:18 +0800 Subject: [PATCH 02/21] replace default web technologies to native HTML --- metagpt/actions/write_prd_an.py | 4 ++-- metagpt/strategy/experience_retriever.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py index 4c0403c36..574dcef89 100644 --- a/metagpt/actions/write_prd_an.py +++ b/metagpt/actions/write_prd_an.py @@ -19,8 +19,8 @@ LANGUAGE = ActionNode( PROGRAMMING_LANGUAGE = ActionNode( key="Programming Language", expected_type=str, - instruction="Mainstream programming language. If not specified in the requirements, native web technologies", - example="native web technologies", + instruction="Mainstream programming language. If not specified in the requirements, use native HTML", + example="native HTML", ) ORIGINAL_REQUIREMENTS = ActionNode( diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index 7229982d4..a4bb5a0cf 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -479,7 +479,7 @@ Explanation: The requirement is about software development. Assign each tasks to "args": { "task_id": "1", "dependent_task_ids": [], - "instruction": "Use native web technologies for the program. And create a product requirement document (PRD) outlining the features, user interface. ", + "instruction": "Use native HTML for the program. And create a product requirement document (PRD) outlining the features, user interface. ", "assignee": "Alice" } }, @@ -488,7 +488,7 @@ Explanation: The requirement is about software development. Assign each tasks to "args": { "task_id": "2", "dependent_task_ids": ["1"], - "instruction": "Use native web technologies for the program. Design the software architecture for the CLI snake game, including the data flow.", + "instruction": "Use native HTML for the program. Design the software architecture for the CLI snake game, including the data flow.", "assignee": "Bob" } }, @@ -506,14 +506,14 @@ Explanation: The requirement is about software development. Assign each tasks to "args": { "task_id": "4", "dependent_task_ids": ["3"], - "instruction": "Use native web technologies for the program. Implement the core game logic for the CLI snake game, including snake movement, food generation, and score tracking.", + "instruction": "Use native HTML for the program. Implement the core game logic for the CLI snake game, including snake movement, food generation, and score tracking.", "assignee": "Alex" } }, { "command_name": "TeamLeader.publish_message", "args": { - "content": "Use native web technologies for the program. Create a cli snake game.", + "content": "Use native HTML for the program. Create a cli snake game.", "send_to": "Alice" } }, From fa68d071a58e5e3c21def10b2e7f135823005530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Fri, 13 Sep 2024 17:54:38 +0800 Subject: [PATCH 03/21] use start_line_number and end_line_number to guide editor.replace --- metagpt/roles/di/role_zero.py | 8 +- metagpt/strategy/experience_retriever.py | 10 ++- metagpt/tools/libs/editor.py | 106 ++++++++++++++++++++--- 3 files changed, 107 insertions(+), 17 deletions(-) diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index ab8179618..6ea2bf7b4 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -310,8 +310,10 @@ class RoleZero(Role): if self.rc.max_react_loop >= 10 and actions_taken >= self.rc.max_react_loop: # If max_react_loop is a small value (e.g. < 10), it is intended to be reached and make the agent stop logger.warning(f"reached max_react_loop: {actions_taken}") - rsp = await self.ask_human("I have reached my max action rounds, do you want me to continue? Yes or no") - if "yes" in rsp.lower(): + human_rsp = await self.ask_human( + "I have reached my max action rounds, do you want me to continue? Yes or no" + ) + if "yes" in human_rsp.lower(): actions_taken = 0 return rsp # return output from the last action @@ -541,7 +543,7 @@ class RoleZero(Role): from metagpt.environment.mgx.mgx_env import MGXEnv # avoid circular import if not isinstance(self.rc.env, MGXEnv): - return "Not in MGXEnv, command will not be executed." + return "Not in MGXEnv, command will not be executed. If you no longer need to take action, use the command ‘end’ to stop." return await self.rc.env.reply_to_human(content, sent_from=self) async def _end(self, **kwarg): diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index a4bb5a0cf..d9645ea84 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -957,6 +957,13 @@ I have opened the openai_api.py file. However, the range of lines shown is from ## example 9 I've found the bug and will start fixing it. I'll pay close attention to the indentation. +the previous file look like: + +739| can_fold_brackets = self._settings['fold_func_brackets'] and \ +740| len(args) == 1 and \ +741| not self._needs_function_brackets(expr.args[0]) +743| inv_trig_table = ["asin", "acos", "atan", "acsc", "asec", "acot"] +745| # If the function is an inverse trig function, handle the style Since I only need to modify a few lines in this file, I will use Editor.edit_file_by_replace. The original content will be replaced by the new code. Editor tool is exclusive. If I use this tool, I cannot use any other commands in the current response. ```json @@ -965,7 +972,8 @@ Editor tool is exclusive. If I use this tool, I cannot use any other commands in "command_name": "Editor.edit_file_by_replace", "args": { "file_name":"/workspace/MetaGPT/provider/openai_api.py", - "to_replace": " inv_trig_table = ["asin", "acos", "atan", "acot"]" + "start_line": 741, + "end_line": 741, "new_content": " inv_trig_table = ["asin", "acos", "atan", "acsc", "asec", "acot"]" } } diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index 63d7f9fd9..1d3f4c38b 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -57,8 +57,9 @@ Your changes have NOT been applied. Please fix your edit command and try again SUCCESS_EDIT_INFO = """ [File: {file_name} ({n_total_lines} lines total after edit)] {window_after_applied} -[File updated (edited at line {line_number}). Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.] +[File updated (edited at line {line_number}). """ +# Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.] class FileBlock(BaseModel): @@ -207,7 +208,7 @@ class Editor(BaseModel): else: output += "(this is the beginning of the file)\n" for i in range(start, end + 1): - _new_line = f"{i}|{lines[i - 1]}" + _new_line = f"{i:03d}|{lines[i - 1]}" if not _new_line.endswith("\n"): _new_line += "\n" output += _new_line @@ -665,7 +666,86 @@ class Editor(BaseModel): ).strip() return success_edit_info - def edit_file_by_replace(self, file_name: str, to_replace: str, new_content: str) -> str: + def edit_file_by_replace( + self, file_name: str, start_line: Optional[int], end_line: Optional[int], new_content: str + ) -> str: + """ + Line numbers start from 1. Replaces lines start_line through end_line (inclusive) with the given text in the open file. + All of the new_content will be entered, so makesure your indentation is formatted properly. + + Example 1: + Given a file "/workspace/example.txt" with the following content: + ``` + 001|line 1 + 002|line 2 + 003|line 3 + 004|line 4 + ``` + + EDITING: If you want to replace line 2 and line 3 + + edit_file_by_replace( + '/workspace/example.txt', + start_line=2, + end_line=3, + new_content='new line', + ) + This will replace only the second line 2 and line 3 with "new line". + + The resulting file will be: + ``` + 001|line 1 + 002|new line + 003|line 4 + ``` + Example 2: + Given a file "/workspace/example.txt" with the following content: + ``` + 001|line 1 + 002|line 2 + 003|line 3 + 004|line 4 + ``` + EDITING: If you want to remove the line 2 and line 3 + edit_file_by_replace( + '/workspace/example.txt', + start_line=2, + end_line=3, + new_content='new line', + ) + This will remove line 2 and line 3 + The resulting file will be: + ``` + 001|line 1 + 002| + 003|line 4 + ``` + Args: + file_name str:The name of the file to edit. + start_line int: The line number to start the edit at, starting from 1. + end_line int: The line number to end the edit at (inclusive), starting from 1. + new_content str: The text to replace the current selection with, must conform to PEP8 standards. + + """ + # FIXME: support replacing *all* occurrences + + # search for `to_replace` in the file + # if found, replace it with `new_content` + # if not found, perform a fuzzy search to find the closest match and replace it with `new_content` + file_name = self._try_fix_path(file_name) + + ret_str = self._edit_file_impl( + file_name, + start=start_line, + end=end_line, + content=new_content, + ) + # lint_error = bool(LINTER_ERROR_MSG in ret_str) + # TODO: automatically tries to fix linter error (maybe involve some static analysis tools on the location near the edit to figure out indentation) + self.resource.report(file_name, "path") + return ret_str + + def _edit_file_by_replace(self, file_name: str, to_replace: str, new_content: str) -> str: """Edit a file. This will search for `to_replace` in the given file and replace it with `new_content`. Every *to_replace* must *EXACTLY MATCH* the existing source code, character for character, including all comments, docstrings, etc. @@ -710,7 +790,6 @@ class Editor(BaseModel): file_name: str: The name of the file to edit. to_replace: str: The content to search for and replace. new_content: str: The new content to replace the old content with. - NOTE: This tool is exclusive. If you use this tool, you cannot use any other commands in the current response. If you need to use it multiple times, wait for the next turn. @@ -736,7 +815,6 @@ class Editor(BaseModel): raise ValueError( "`to_replace` appears more than once, please include enough lines to make code in `to_replace` unique." ) - start = file_content.find(to_replace) if start != -1: # Convert start from index to line number @@ -772,21 +850,23 @@ class Editor(BaseModel): return ret_str def insert_content_at_line(self, file_name: str, line_number: int, content: str) -> str: - """Insert content at the given line number in a file. + """Insert content at the given line number in a file. That is, the new content will start at line_number after the insertion. This will NOT modify the content of the lines before OR after the given line number. For example, if the file has the following content: ``` - line 1 - line 2 - line 3 + 001|line 1 + 002|line 2 + 003|line 3 + 004|line 4 ``` and you call `insert_content_at_line('file.txt', 2, 'new line')`, the file will be updated to: ``` - line 1 - new line - line 2 - line 3 + 001|line 1 + 002|new line + 003|line 2 + 004|line 3 + 005|line 4 ``` Args: From 99d8cbc28b12c240364473de6eb4ba62ee0d112c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Fri, 13 Sep 2024 18:02:28 +0800 Subject: [PATCH 04/21] fix issues --- metagpt/roles/di/role_zero.py | 2 +- metagpt/tools/libs/editor.py | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index 6ea2bf7b4..e308f4bc9 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -543,7 +543,7 @@ class RoleZero(Role): from metagpt.environment.mgx.mgx_env import MGXEnv # avoid circular import if not isinstance(self.rc.env, MGXEnv): - return "Not in MGXEnv, command will not be executed. If you no longer need to take action, use the command ‘end’ to stop." + return "Not in MGXEnv, command will not be executed." return await self.rc.env.reply_to_human(content, sent_from=self) async def _end(self, **kwarg): diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index 1d3f4c38b..2ba1d4341 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -57,9 +57,9 @@ Your changes have NOT been applied. Please fix your edit command and try again SUCCESS_EDIT_INFO = """ [File: {file_name} ({n_total_lines} lines total after edit)] {window_after_applied} -[File updated (edited at line {line_number}). +[File updated (edited at line {line_number})]. """ -# Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.] +# Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary. class FileBlock(BaseModel): @@ -727,11 +727,7 @@ class Editor(BaseModel): new_content str: The text to replace the current selection with, must conform to PEP8 standards. """ - # FIXME: support replacing *all* occurrences - # search for `to_replace` in the file - # if found, replace it with `new_content` - # if not found, perform a fuzzy search to find the closest match and replace it with `new_content` file_name = self._try_fix_path(file_name) ret_str = self._edit_file_impl( @@ -740,7 +736,6 @@ class Editor(BaseModel): end=end_line, content=new_content, ) - # lint_error = bool(LINTER_ERROR_MSG in ret_str) # TODO: automatically tries to fix linter error (maybe involve some static analysis tools on the location near the edit to figure out indentation) self.resource.report(file_name, "path") return ret_str From e4f9af904ec7f097cec5a571ca6c90ce12ec929b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Fri, 13 Sep 2024 19:27:11 +0800 Subject: [PATCH 05/21] fix issues --- metagpt/roles/di/engineer2.py | 4 +--- metagpt/strategy/experience_retriever.py | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/metagpt/roles/di/engineer2.py b/metagpt/roles/di/engineer2.py index b4fcd34c8..5933e9eff 100644 --- a/metagpt/roles/di/engineer2.py +++ b/metagpt/roles/di/engineer2.py @@ -61,9 +61,7 @@ class Engineer2(RoleZero): This information will be dynamically added to the command prompt. """ current_directory = (await self.terminal.run_command("pwd")).strip() - # Synchronize Terminal and Editor Working Directories - if str(self.editor.working_dir.absolute()).strip() != current_directory: - self.editor._set_workdir(current_directory) + self.editor._set_workdir(current_directory) state = { "editor_open_file": self.editor.current_file, "current_directory": current_directory, diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index a45733102..faeb5faec 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -962,8 +962,8 @@ the previous file look like: 739| can_fold_brackets = self._settings['fold_func_brackets'] and \ 740| len(args) == 1 and \ 741| not self._needs_function_brackets(expr.args[0]) -743| inv_trig_table = ["asin", "acos", "atan", "acsc", "asec", "acot"] -745| # If the function is an inverse trig function, handle the style +742| inv_trig_table = ["asin", "acos", "atan", "acsc", "asec", "acot"] +743| # If the function is an inverse trig function, handle the style Since I only need to modify a few lines in this file, I will use Editor.edit_file_by_replace. The original content will be replaced by the new code. Editor tool is exclusive. If I use this tool, I cannot use any other commands in the current response. ```json @@ -972,8 +972,8 @@ Editor tool is exclusive. If I use this tool, I cannot use any other commands in "command_name": "Editor.edit_file_by_replace", "args": { "file_name":"/workspace/MetaGPT/provider/openai_api.py", - "start_line": 741, - "end_line": 741, + "start_line": 742, + "end_line": 742, "new_content": " inv_trig_table = ["asin", "acos", "atan", "acsc", "asec", "acot"]" } } From 72f2aaa02bf6347c67b72fbe533076b1fd2f463e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Fri, 13 Sep 2024 19:39:56 +0800 Subject: [PATCH 06/21] enlarge print_window --- metagpt/tools/libs/editor.py | 10 ++++------ metagpt/tools/libs/linter.py | 1 + 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index c4b2e9a08..a315d7370 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -84,7 +84,7 @@ class Editor(BaseModel): resource: EditorReporter = EditorReporter() current_file: Optional[Path] = None current_line: int = 1 - window: int = 100 + window: int = 200 enable_auto_lint: bool = False working_dir: Path = DEFAULT_WORKSPACE_ROOT @@ -633,8 +633,8 @@ class Editor(BaseModel): ) error_info = ERROR_GUIDANCE.format( linter_error_msg=LINTER_ERROR_MSG + str(e), - window_after_applied=self._print_window(file_name, start or len(lines), 40), - window_before_applied=self._print_window(Path(temp_backup_file.name), start or len(lines), 40), + window_after_applied=self._print_window(file_name, start or len(lines), 100), + window_before_applied=self._print_window(Path(temp_backup_file.name), start or len(lines), 100), guidance_message=guidance_message, ).strip() # Clean up the temporary file if an error occurs @@ -662,9 +662,7 @@ class Editor(BaseModel): ).strip() return success_edit_info - def edit_file_by_replace( - self, file_name: str, start_line: Optional[int], end_line: Optional[int], new_content: str - ) -> str: + def edit_file_by_replace(self, file_name: str, start_line: int, end_line: int, new_content: str) -> str: """ Line numbers start from 1. Replaces lines start_line through end_line (inclusive) with the given text in the open file. All of the new_content will be entered, so makesure your indentation is formatted properly. diff --git a/metagpt/tools/libs/linter.py b/metagpt/tools/libs/linter.py index 61de7acce..8a0024f83 100644 --- a/metagpt/tools/libs/linter.py +++ b/metagpt/tools/libs/linter.py @@ -34,6 +34,7 @@ class Linter: python=self.py_lint, sql=self.fake_lint, # base_lint lacks support for full SQL syntax. Use fake_lint to bypass the validation. css=self.fake_lint, # base_lint lacks support for css syntax. Use fake_lint to bypass the validation. + js=self.fake_lint, # base_lint lacks support for javascipt syntax. Use fake_lint to bypass the validation. ) self.all_lint_cmd = None From ed0dd9b7fabb777896b18c3c8c2a873e70a52e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Fri, 13 Sep 2024 20:15:26 +0800 Subject: [PATCH 07/21] update test editor --- tests/metagpt/tools/libs/test_editor.py | 72 ++++++++++++------------- 1 file changed, 33 insertions(+), 39 deletions(-) diff --git a/tests/metagpt/tools/libs/test_editor.py b/tests/metagpt/tools/libs/test_editor.py index a716b3de4..07efd8132 100644 --- a/tests/metagpt/tools/libs/test_editor.py +++ b/tests/metagpt/tools/libs/test_editor.py @@ -25,7 +25,7 @@ def test_function_for_fm(): # this is the 7th line """.strip() -WINDOW = 100 +WINDOW = 200 @pytest.fixture @@ -196,11 +196,11 @@ def test_open_file(temp_file_path): expected = ( f"[File: {temp_file_path} (5 lines total)]\n" "(this is the beginning of the file)\n" - "1|Line 1\n" - "2|Line 2\n" - "3|Line 3\n" - "4|Line 4\n" - "5|Line 5\n" + "001|Line 1\n" + "002|Line 2\n" + "003|Line 3\n" + "004|Line 4\n" + "005|Line 5\n" "(this is the end of the file)" ) assert result.split("\n") == expected.split("\n") @@ -215,11 +215,11 @@ def test_open_file_with_indentation(temp_file_path): expected = ( f"[File: {temp_file_path} (5 lines total)]\n" "(this is the beginning of the file)\n" - "1|Line 1\n" - "2| Line 2\n" - "3|Line 3\n" - "4|Line 4\n" - "5|Line 5\n" + "001|Line 1\n" + "002| Line 2\n" + "003|Line 3\n" + "004|Line 4\n" + "005|Line 5\n" "(this is the end of the file)" ) assert result.split("\n") == expected.split("\n") @@ -235,7 +235,7 @@ def test_open_file_long(temp_file_path): expected = f"[File: {temp_file_path} (1000 lines total)]\n" expected += "(this is the beginning of the file)\n" for i in range(1, 51): - expected += f"{i}|Line {i}\n" + expected += f"{i:03d}|Line {i}\n" expected += "(950 more lines below)" assert result.split("\n") == expected.split("\n") @@ -245,7 +245,7 @@ def test_open_file_long_with_lineno(temp_file_path): content = "\n".join([f"Line {i}" for i in range(1, 1001)]) temp_file_path.write_text(content) - cur_line = 100 + cur_line = 300 result = editor.open_file(str(temp_file_path), cur_line) assert result is not None @@ -256,7 +256,7 @@ def test_open_file_long_with_lineno(temp_file_path): else: expected += f"({start - 1} more lines above)\n" for i in range(start, end + 1): - expected += f"{i}|Line {i}\n" + expected += f"{i:03d}|Line {i}\n" if end == 1000: expected += "(this is the end of the file)\n" else: @@ -290,7 +290,7 @@ def test_goto_line(temp_file_path): expected = f"[File: {temp_file_path} ({total_lines} lines total)]\n" expected += "(this is the beginning of the file)\n" for i in range(1, WINDOW + 1): - expected += f"{i}|Line {i}\n" + expected += f"{i:03d}|Line {i}\n" expected += f"({total_lines - WINDOW} more lines below)" assert result.split("\n") == expected.split("\n") @@ -306,7 +306,7 @@ def test_goto_line(temp_file_path): else: expected += f"({start - 1} more lines above)\n" for i in range(start, end + 1): - expected += f"{i}|Line {i}\n" + expected += f"{i:03d}|Line {i}\n" if end == total_lines: expected += "(this is the end of the file)\n" else: @@ -349,7 +349,7 @@ def test_scroll_down(temp_file_path): else: expected += f"({start - 1} more lines above)\n" for i in range(start, end + 1): - expected += f"{i}|Line {i}\n" + expected += f"{i:03d}|Line {i}\n" if end == total_lines: expected += "(this is the end of the file)" else: @@ -367,7 +367,7 @@ def test_scroll_down(temp_file_path): else: expected += f"({start - 1} more lines above)\n" for i in range(start, end + 1): - expected += f"{i}|Line {i}\n" + expected += f"{i:03d}|Line {i}\n" if end == total_lines: expected += "(this is the end of the file)\n" else: @@ -381,7 +381,7 @@ def test_scroll_up(temp_file_path): content = "\n".join([f"Line {i}" for i in range(1, total_lines + 1)]) temp_file_path.write_text(content) - cur_line = 300 + cur_line = 500 result = editor.open_file(str(temp_file_path), cur_line) assert result is not None @@ -393,11 +393,12 @@ def test_scroll_up(temp_file_path): else: expected += f"({start - 1} more lines above)\n" for i in range(start, end + 1): - expected += f"{i}|Line {i}\n" + expected += f"{i:03d}|Line {i}\n" if end == total_lines: expected += "(this is the end of the file)\n" else: expected += f"({total_lines - end} more lines below)" + assert result.split("\n") == expected.split("\n") result = editor.scroll_up() assert result is not None @@ -411,11 +412,13 @@ def test_scroll_up(temp_file_path): else: expected += f"({start - 1} more lines above)\n" for i in range(start, end + 1): - expected += f"{i}|Line {i}\n" + expected += f"{i:03d}|Line {i}\n" if end == total_lines: expected += "(this is the end of the file)\n" else: expected += f"({total_lines - end} more lines below)" + print(result) + print(expected) assert result.split("\n") == expected.split("\n") @@ -430,7 +433,7 @@ def test_scroll_down_edge(temp_file_path): expected = f"[File: {temp_file_path} (9 lines total)]\n" expected += "(this is the beginning of the file)\n" for i in range(1, 10): - expected += f"{i}|Line {i}\n" + expected += f"{i:03d}|Line {i}\n" expected += "(this is the end of the file)" result = editor.scroll_down() @@ -450,7 +453,7 @@ def test_print_window_internal(temp_file_path): window = 2 result = editor._print_window(temp_file_path, current_line, window) - expected = "(48 more lines above)\n" "49|Line `49`\n" "50|Line `50`\n" "51|Line `51`\n" "(49 more lines below)" + expected = "(48 more lines above)\n" "049|Line `49`\n" "050|Line `50`\n" "051|Line `51`\n" "(49 more lines below)" assert result == expected @@ -533,21 +536,12 @@ def test_function_for_fm(): def test_edit_file_by_replace(temp_py_file): editor = Editor() - editor.edit_file_by_replace(file_name=str(temp_py_file), to_replace=" b = 2", new_content=" b = 9") + editor.edit_file_by_replace(file_name=str(temp_py_file), start_line=5, end_line=5, new_content=" b = 9") with open(temp_py_file, "r") as f: new_content = f.read() assert new_content.strip() == EXPECTED_CONTENT_AFTER_REPLACE_TEXT.strip() -def test_edit_file_by_replace_to_palce_empty(empty_file): - editor = Editor() - with pytest.raises(ValueError) as exc_info: - editor.edit_file_by_replace( - file_name=str(empty_file), to_replace="", new_content=EXPECTED_CONTENT_AFTER_REPLACE_TEXT.strip() - ) - assert "is empty. Use the append method to add content." in str(exc_info.value) - - def test_append_file(temp_file_path): editor = Editor() # 写入初始内容 @@ -571,13 +565,13 @@ def test_append_file(temp_file_path): expected_output = ( f"[File: {temp_file_path.resolve()} (5 lines total after edit)]\n" "(this is the beginning of the file)\n" - "1|Line 1\n" - "2|Line 2\n" - "3|Line 3\n" - "4|Line 4\n" - "5|Line 5\n" + "001|Line 1\n" + "002|Line 2\n" + "003|Line 3\n" + "004|Line 4\n" + "005|Line 5\n" "(this is the end of the file)\n" - "[File updated (edited at line 3). Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.]" + "[File updated (edited at line 3)]." ) assert result.split("\n") == expected_output.split("\n") From 2cae99fa881dfb725f81075285b23fcf02906b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Sat, 14 Sep 2024 09:21:47 +0800 Subject: [PATCH 08/21] delete an empty line --- metagpt/strategy/experience_retriever.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index faeb5faec..eb535fb94 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -958,7 +958,6 @@ I have opened the openai_api.py file. However, the range of lines shown is from ## example 9 I've found the bug and will start fixing it. I'll pay close attention to the indentation. the previous file look like: - 739| can_fold_brackets = self._settings['fold_func_brackets'] and \ 740| len(args) == 1 and \ 741| not self._needs_function_brackets(expr.args[0]) From a80b41db5d6cec165887a795f99ab7f73576140d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Sat, 14 Sep 2024 12:09:49 +0800 Subject: [PATCH 09/21] ignore javascirip validation and add tool_selector in editr class --- metagpt/tools/libs/editor.py | 19 ++++++++++++++++++- metagpt/tools/libs/linter.py | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index a315d7370..834615ab9 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -73,7 +73,24 @@ class LineNumberError(Exception): pass -@register_tool() +@register_tool( + include_functions=[ + "write", + "read", + "open_file", + "goto_line", + "scroll_down", + "scroll_up", + "create_file", + "edit_file_by_replace", + "insert_content_at_line", + "append_file", + "search_dir", + "search_file", + "find_file", + "search_index_repo", + ] +) class Editor(BaseModel): """ A tool for reading, understanding, writing, and editing files. diff --git a/metagpt/tools/libs/linter.py b/metagpt/tools/libs/linter.py index 8a0024f83..0497e49c0 100644 --- a/metagpt/tools/libs/linter.py +++ b/metagpt/tools/libs/linter.py @@ -35,6 +35,7 @@ class Linter: sql=self.fake_lint, # base_lint lacks support for full SQL syntax. Use fake_lint to bypass the validation. css=self.fake_lint, # base_lint lacks support for css syntax. Use fake_lint to bypass the validation. js=self.fake_lint, # base_lint lacks support for javascipt syntax. Use fake_lint to bypass the validation. + javascript=self.fake_lint, ) self.all_lint_cmd = None From 384dc78e70adefdfdd99753a2839f15893694f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Sat, 14 Sep 2024 15:35:26 +0800 Subject: [PATCH 10/21] add engineer2 test script --- metagpt/roles/di/engineer2.py | 3 +- tests/metagpt/roles/di/run_engineer2.py | 67 ++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/metagpt/roles/di/engineer2.py b/metagpt/roles/di/engineer2.py index 5933e9eff..6313c5b33 100644 --- a/metagpt/roles/di/engineer2.py +++ b/metagpt/roles/di/engineer2.py @@ -71,6 +71,7 @@ class Engineer2(RoleZero): def _update_tool_execution(self): # validate = ValidateAndRewriteCode() cr = CodeReview() + image_getter = ImageGetter() self.exclusive_tool_commands.append("Engineer2.write_new_code") if self.run_eval is True: # Evalute tool map @@ -78,6 +79,7 @@ class Engineer2(RoleZero): { "git_create_pull": git_create_pull, "Engineer2.write_new_code": self.write_new_code, + "ImageGetter.get_image": image_getter.get_image, "CodeReview.review": cr.review, "CodeReview.fix": cr.fix, "Terminal.run_command": self._eval_terminal_run, @@ -87,7 +89,6 @@ class Engineer2(RoleZero): ) else: # Default tool map - image_getter = ImageGetter() self.tool_execution_map.update( { "git_create_pull": git_create_pull, diff --git a/tests/metagpt/roles/di/run_engineer2.py b/tests/metagpt/roles/di/run_engineer2.py index e5ae74485..a5ceec93e 100644 --- a/tests/metagpt/roles/di/run_engineer2.py +++ b/tests/metagpt/roles/di/run_engineer2.py @@ -1,5 +1,9 @@ import asyncio +import sys +import uuid +from pathlib import Path +from metagpt.logs import logger from metagpt.roles.di.engineer2 import Engineer2 DESIGN_DOC_2048 = '{"Implementation approach":"We will use the Pygame library to implement the 2048 game logic and user interface. Pygame is a set of Python modules designed for writing video games, which will help us create a responsive and visually appealing UI. For the mobile responsiveness, we will ensure that the game scales appropriately on different screen sizes. We will also use the Pygame GUI library to create buttons for restarting the game and choosing difficulty levels.","File list":["main.py","game.py","ui.py"],"Data structures and interfaces":"\\nclassDiagram\\n class Game {\\n -grid: list[list[int]]\\n -score: int\\n +__init__()\\n +move(direction: str) bool\\n +merge() bool\\n +spawn_tile() None\\n +is_game_over() bool\\n +reset() None\\n }\\n class UI {\\n -game: Game\\n +__init__(game: Game)\\n +draw_grid() None\\n +draw_score() None\\n +draw_buttons() None\\n +handle_input() None\\n }\\n class Main {\\n -ui: UI\\n +main() None\\n }\\n Main --> UI\\n UI --> Game\\n","Program call flow":"\\nsequenceDiagram\\n participant M as Main\\n participant U as UI\\n participant G as Game\\n M->>U: __init__(game)\\n U->>G: __init__()\\n M->>U: draw_grid()\\n U->>G: move(direction)\\n G-->>U: return bool\\n U->>G: merge()\\n G-->>U: return bool\\n U->>G: spawn_tile()\\n G-->>U: return None\\n U->>G: is_game_over()\\n G-->>U: return bool\\n U->>G: reset()\\n G-->>U: return None\\n M->>U: draw_score()\\n M->>U: draw_buttons()\\n M->>U: handle_input()\\n","Anything UNCLEAR":"Clarification needed on the specific design elements for the UI to ensure it meets the \'beautiful\' requirement. Additionally, we need to confirm the exact difficulty levels and how they should affect the game mechanics."}' @@ -99,6 +103,65 @@ Then correct any issues you find. You can review all code in one time, and solve CASUAL_CHAT = """what's your name?""" +# increment development +INC_DEVELOPMENT_CASE1 = [ + "Complete the Snake game with the root directory at '/home/mgx/mgx/MetaGPT/workspace/snake_game'", + "Use the up button to control the snake to move down, the left button to move right, and so on", + "Place the restart/start button at the top", + "Add a pause button", + "Display the score and leaderboard in real-time on the page", +] + +INC_DEVELOPMENT_CASE2 = [ + "Develop a Snake game using Python in the '/home/mgx/mgx/MetaGPT/workspace/snake_game_py' folder", + "Change the title to 'Special Snake'", + "Use the up button to control the snake to move down, the left button to move right, and so on", + "Add a pause button", + "Display the score and leaderboard in real-time on the page", + "Design a more attractive style for the leaderboard", +] + +INC_DEVELOPMENT_CASE3 = [ + "Complete the 2048 game with the root directory at '/home/mgx/mgx/MetaGPT/workspace/2048_game'", + "Place the start button at the top", + "Display the score and leaderboard in real-time on the page", + "Design a more attractive style for the leaderboard", + "Add a restart button", +] + +INC_DEVELOPMENT_CASE4 = [ + "Develop a 2048 game using Python in the '/home/mgx/mgx/MetaGPT/workspace/2048_game_py' folder", + "Display the score and leaderboard in real-time on the page", + "Add a restart button", +] +INC_DEVELOPMENT_CASE5 = [ + "Root path is '/home/mgx/mgx/MetaGPT/workspace/to_list' Create a website widget for TODO list management. Users should be able to add, mark as complete, and delete tasks. Include features like prioritization, due dates, and categories. Make it visually appealing, responsive, and user-friendly. Use HTML, CSS, and JavaScript. Consider additional features like notifications or task export. Keep it simple and enjoyable for users.dont use vue or react.dont use third party library, use localstorage to save data.", + "Add a `clean all` buttonn", +] +INC_DEVELOPMENT_CASE6 = [ + '使用原生HTML开发一个塔罗牌角色介绍网站\n1. 主题是塔罗牌占卜的网站\n2. 超前的网页布局\n3. 页面需要时响应式的\n4. 页面需要美观大气 root path "”/home/mgx/mgx/MetaGPT/workspace/taro"', + "扩充更多的角色，添加3个自己想出来的角色", + "让每一个角色的描述更加清楚", + "将中文内容全部替换为英文包括js里面的内容", +] + + +async def increment_development(): + engineer2 = Engineer2(run_eval=True) + example = INC_DEVELOPMENT_CASE6 + logger.remove() + logger.add(sys.stderr, level="INFO") + logger.add(Path("logs") / f"{str(uuid.uuid4())[-12:]}.log", level="DEBUG") + logger.info("user requirement:\n" + "\n".join(example)) + try: + for user_requirement in example: + logger.info(f"input:{user_requirement}") + await engineer2.run(user_requirement) + except Exception as e: + print(e) + + if __name__ == "__main__": - engineer2 = Engineer2() - asyncio.run(engineer2.run(GAME_REQ_2048_NO_DOC)) + asyncio.run(increment_development()) + # engineer2 = Engineer2() + # asyncio.run(engineer2.run(GAME_REQ_2048_NO_DOC)) From 120e92821075ad1586f1a74e2c72c6cb8103a0e6 Mon Sep 17 00:00:00 2001 From: zhanglei Date: Sat, 14 Sep 2024 16:44:11 +0800 Subject: [PATCH 11/21] update: pwd not report --- metagpt/tools/libs/terminal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index 24daa184d..10972dbe9 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -108,6 +108,8 @@ class Terminal: return "\n".join(output_lines) async def _read_and_process_output(self, cmd, daemon=False) -> str: + if "pwd" in cmd: + return async with self.observer as observer: cmd_output = [] await observer.async_report(cmd + self.command_terminator, "cmd") From 9d5e871e3087ced9d1eed923dd83d544f304d37a Mon Sep 17 00:00:00 2001 From: zhanglei Date: Sat, 14 Sep 2024 16:52:08 +0800 Subject: [PATCH 12/21] update: pwd not report --- metagpt/tools/libs/terminal.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index 10972dbe9..6d6934ac7 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -108,11 +108,10 @@ class Terminal: return "\n".join(output_lines) async def _read_and_process_output(self, cmd, daemon=False) -> str: - if "pwd" in cmd: - return async with self.observer as observer: cmd_output = [] - await observer.async_report(cmd + self.command_terminator, "cmd") + if cmd != "pwd": + await observer.async_report(cmd + self.command_terminator, "cmd") # report the command # Read the output until the unique marker is found. # We read bytes directly from stdout instead of text because when reading text, @@ -129,12 +128,14 @@ class Terminal: if ix >= 0: line = line[0:ix] if line: - await observer.async_report(line, "output") + if cmd != "pwd": + await observer.async_report(line, "output") # report stdout in real-time cmd_output.append(line) return "".join(cmd_output) # log stdout in real-time - await observer.async_report(line, "output") + if cmd != "pwd": + await observer.async_report(line, "output") cmd_output.append(line) if daemon: await self.stdout_queue.put(line) From 726dc65067a953a3e5d6e2fd0e58dbc56ca5ee3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Sat, 14 Sep 2024 20:57:03 +0800 Subject: [PATCH 13/21] Update the editor parameters and enhance the engineer experience. --- metagpt/prompts/di/role_zero.py | 1 + metagpt/strategy/experience_retriever.py | 60 ++++++------- metagpt/tools/libs/editor.py | 103 ++++++++++++++--------- tests/metagpt/tools/libs/test_editor.py | 11 ++- 4 files changed, 103 insertions(+), 72 deletions(-) diff --git a/metagpt/prompts/di/role_zero.py b/metagpt/prompts/di/role_zero.py index 3ed9389a6..820466b6f 100644 --- a/metagpt/prompts/di/role_zero.py +++ b/metagpt/prompts/di/role_zero.py @@ -69,6 +69,7 @@ CMD_PROMPT = ( you must respond in {respond_language}. Pay close attention to the Example provided, you can reuse the example for your current situation if it fits. +If you open a file, the line number is displayed at the front of each line. You may use any of the available commands to create a plan or update the plan. You may output mutiple commands, they will be executed sequentially. If you finish current task, you will automatically take the next task in the existing plan, use Plan.finish_task, DON'T append a new task. Review the latest plan's outcome, focusing on achievements. If your completed task matches the current, consider it finished. diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index eb535fb94..266461f25 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -956,14 +956,15 @@ I have opened the openai_api.py file. However, the range of lines shown is from ``` ## example 9 -I've found the bug and will start fixing it. I'll pay close attention to the indentation. +I want to change the key bindings from (w/s) to the arrow keys (up, down). And add the space bar to pause. the previous file look like: -739| can_fold_brackets = self._settings['fold_func_brackets'] and \ -740| len(args) == 1 and \ -741| not self._needs_function_brackets(expr.args[0]) -742| inv_trig_table = ["asin", "acos", "atan", "acsc", "asec", "acot"] -743| # If the function is an inverse trig function, handle the style -Since I only need to modify a few lines in this file, I will use Editor.edit_file_by_replace. The original content will be replaced by the new code. +142| while not self.is_game_over(): +143| if event.key == pygame.K_w: +144| self.move_up() +145| elif event.key == pygame.K_s: +146| self.move_down() +147| self.add_random_tile() +Since I only need to modify the lines 143 to 146, I will use Editor.edit_file_by_replace. The original content will be replaced by the new code. Editor tool is exclusive. If I use this tool, I cannot use any other commands in the current response. ```json [ @@ -971,46 +972,47 @@ Editor tool is exclusive. If I use this tool, I cannot use any other commands in "command_name": "Editor.edit_file_by_replace", "args": { "file_name":"/workspace/MetaGPT/provider/openai_api.py", - "start_line": 742, - "end_line": 742, - "new_content": " inv_trig_table = ["asin", "acos", "atan", "acsc", "asec", "acot"]" - } + "start_line_number": 143, + "start_line_content":" if event.key == pygame.K_w:", + "new_content": " if event.key == pygame.K_UP:\\n self.move_up()\\n elif event.key == pygame.K_DOWN:\\n self.move_down()\\n elif event.key == pygame.K_SPACE:\\n self.stop()" + "end_line_number": 146, + "end_line_content": " self.move_down()", + } } ] ``` ## example 10 +I want to add a score variable in the initialization of the game. +the previous file look like: +028| if restart: +029| self.snake = Snake() +030| self.food = Food(self.board_size) +031| self.start_game() +032| self.location = (0,0) I only need to add a few lines to the file, so I will use Editor.insert_content_at_line. The new code will not cover the original code. Note that the Editor command must be executed in a single response, so this step will only involve using the Editor command. + ```json [ { "command_name": "Editor.insert_content_at_line", "args": { "file_name":"/workspace/MetaGPT/provider/openai_api.py" - "line_number":727, - "content": "if hasattr(self, '_print_' + func) and not isinstance(expr.func, UndefinedFunction):\\n return getattr(self, '_print_' + func)(expr, exp)" - } - } -] -``` + "line_number":31, + "insert_content": " self.score = Score()" -## example 10.1 -To enhance the functionality of the 2048 game, including game end detection and score tracking, we need to add these features to the existing game_2048.py file. First, we will add a score tracking feature, and then we will insert game end detection logic into the game loop. -We will use the Editor.insert_content_at_line command to insert new code into the file for adding score tracking and game end detection. -Since Editor.insert_content_at_line can only be used once per response, this time I will use it to create the variable self.score -```json -[ - { - "command_name": "Editor.insert_content_at_line", - "args": { - "file_name": "/home/mgx/mgx/MetaGPT/workspace/2048_game_py/game_2048.py", - "line_number": 4, - "content": " self.score = 0\n" } } ] ``` +After executing the command, the file will be: +028| if restart: +029| self.snake = Snake() +030| self.food = Food(self.board_size) +031| self.score = Score() +032| self.start_game() +033| self.location = (0,0) In the next turn, I will try to add another code snippet ## example 11 diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index 834615ab9..75f6f2760 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -679,63 +679,78 @@ class Editor(BaseModel): ).strip() return success_edit_info - def edit_file_by_replace(self, file_name: str, start_line: int, end_line: int, new_content: str) -> str: + def edit_file_by_replace( + self, + file_name: str, + start_line_number: int, + start_line_content: str, + end_line_number: int, + end_line_content: str, + new_content: str, + ) -> str: """ Line numbers start from 1. Replaces lines start_line through end_line (inclusive) with the given text in the open file. All of the new_content will be entered, so makesure your indentation is formatted properly. + The new_content must be a complete block of code . Example 1: Given a file "/workspace/example.txt" with the following content: ``` - 001|line 1 - 002|line 2 - 003|line 3 - 004|line 4 + 001|contain f + 002|contain g + 003|contain h + 004|contain i ``` EDITING: If you want to replace line 2 and line 3 edit_file_by_replace( '/workspace/example.txt', - start_line=2, - end_line=3, - new_content='new line', + start_line_number=2, + start_line_content="contain g", + end_line_number=3, + end_line_content="contain h", + new_content='new content', ) - This will replace only the second line 2 and line 3 with "new line". + This will replace only the second line 2 and line 3 with "new content". The resulting file will be: ``` - 001|line 1 - 002|new line - 003|line 4 + 001|contain f + 002|new content + 003|contain i ``` Example 2: Given a file "/workspace/example.txt" with the following content: ``` - 001|line 1 - 002|line 2 - 003|line 3 - 004|line 4 + 001|contain f + 002|contain g + 003|contain h + 004|contain i ``` EDITING: If you want to remove the line 2 and line 3 edit_file_by_replace( '/workspace/example.txt', - start_line=2, - end_line=3, + start_line_number=2, + start_line_content="contain g", + end_line_number=3, + end_line_content="contain h", new_content='new line', ) This will remove line 2 and line 3 The resulting file will be: ``` - 001|line 1 + 001|contain f 002| - 003|line 4 + 003|contain i ``` Args: file_name str:The name of the file to edit. - start_line int: The line number to start the edit at, starting from 1. - end_line int: The line number to end the edit at (inclusive), starting from 1. - new_content str: The text to replace the current selection with, must conform to PEP8 standards. + start_line_number int:The line number to start the edit at, starting from 1. + start_line_content str:The content of the start replace line, according to the start_line_number. + end_line_number int:The line number to end the edit at (inclusive), starting from 1. + end_line_content str:The content of the end replace line, according to the end_line_number. + new_content str: The text to replace the current selection with, must conform to PEP8 standards.The content in the start line and end line will also be replaced. """ @@ -743,8 +758,8 @@ class Editor(BaseModel): ret_str = self._edit_file_impl( file_name, - start=start_line, - end=end_line, + start=start_line_number, + end=end_line_number, content=new_content, ) # TODO: automatically tries to fix linter error (maybe involve some static analysis tools on the location near the edit to figure out indentation) @@ -855,30 +870,36 @@ class Editor(BaseModel): self.resource.report(file_name, "path") return ret_str - def insert_content_at_line(self, file_name: str, line_number: int, content: str) -> str: - """Insert content at the given line number in a file. That is, the new content will start at line_number after the insertion. - This will NOT modify the content of the lines before OR after the given line number. - + def insert_content_at_line(self, file_name: str, line_number: int, insert_content: str) -> str: + """Insert a complete block of code before the given line number in a file That is, the new content will start at the beginning of the specified line, and the existing content of that line will be moved down. + This operation will NOT modify the content of the lines before or after the given line number. + This function can not insert content the end of the file. Please use append_file instead, For example, if the file has the following content: ``` - 001|line 1 - 002|line 2 - 003|line 3 - 004|line 4 + 001|contain g + 002|contain h + 003|contain i + 004|contain j ``` - and you call `insert_content_at_line('file.txt', 2, 'new line')`, the file will be updated to: + and you call + insert_content_at_line( + file_name='file.txt', + line_number=2, + insert_content='new line')` + the file will be updated to: ``` - 001|line 1 + 001|contain g 002|new line - 003|line 2 - 004|line 3 - 005|line 4 + 003|contain h + 004|contain i + 005|contain j ``` Args: file_name: str: The name of the file to edit. - line_number: int: The line number (starting from 1) to insert the content after. - content: str: The content to insert. + line_number int: The line number (starting from 1) to insert the content after.the insert content will be add between the line of line_number-1 and line_number + insert_content (str): The content to insert betweed the previous_line_content and current_line_content.The insert_content must be a complete block of code at. + NOTE: This tool is exclusive. If you use this tool, you cannot use any other commands in the current response. If you need to use it multiple times, wait for the next turn. @@ -888,7 +909,7 @@ class Editor(BaseModel): file_name, start=line_number, end=line_number, - content=content, + content=insert_content, is_insert=True, is_append=False, ) diff --git a/tests/metagpt/tools/libs/test_editor.py b/tests/metagpt/tools/libs/test_editor.py index 07efd8132..de773f04d 100644 --- a/tests/metagpt/tools/libs/test_editor.py +++ b/tests/metagpt/tools/libs/test_editor.py @@ -120,7 +120,7 @@ def test_insert_content(temp_py_file): editor.insert_content_at_line( file_name=temp_py_file, line_number=3, - content=" # This is the new line to be inserted, at line 3", + insert_content=" # This is the new line to be inserted, at line 3", ) with open(temp_py_file, "r") as f: new_content = f.read() @@ -536,7 +536,14 @@ def test_function_for_fm(): def test_edit_file_by_replace(temp_py_file): editor = Editor() - editor.edit_file_by_replace(file_name=str(temp_py_file), start_line=5, end_line=5, new_content=" b = 9") + editor.edit_file_by_replace( + file_name=str(temp_py_file), + start_line_number=5, + start_line_content="", + new_content=" b = 9", + end_line_number=5, + end_line_content="", + ) with open(temp_py_file, "r") as f: new_content = f.read() assert new_content.strip() == EXPECTED_CONTENT_AFTER_REPLACE_TEXT.strip() From 3ee4bf94a8edf5315cde5a895e2aecee261a00b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Wed, 18 Sep 2024 09:54:16 +0800 Subject: [PATCH 14/21] fix typo --- metagpt/strategy/experience_retriever.py | 1 - metagpt/tools/libs/editor.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index 266461f25..7cefe863c 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -992,7 +992,6 @@ the previous file look like: 032| self.location = (0,0) I only need to add a few lines to the file, so I will use Editor.insert_content_at_line. The new code will not cover the original code. Note that the Editor command must be executed in a single response, so this step will only involve using the Editor command. - ```json [ { diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index 75f6f2760..fd4784f65 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -871,7 +871,7 @@ class Editor(BaseModel): return ret_str def insert_content_at_line(self, file_name: str, line_number: int, insert_content: str) -> str: - """Insert a complete block of code before the given line number in a file That is, the new content will start at the beginning of the specified line, and the existing content of that line will be moved down. + """Insert a complete block of code before the given line number in a file. That is, the new content will start at the beginning of the specified line, and the existing content of that line will be moved down. This operation will NOT modify the content of the lines before or after the given line number. This function can not insert content the end of the file. Please use append_file instead, For example, if the file has the following content: From b1120740304d6251002307bfc26d90bcb883db21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Wed, 18 Sep 2024 16:11:32 +0800 Subject: [PATCH 15/21] fix:line_number and line_content parameters mismatch --- metagpt/tools/libs/editor.py | 33 ++++++++++++++++++ tests/metagpt/tools/libs/test_editor.py | 45 +++++++++++++++++++++++-- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index fd4784f65..46a3bf970 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -54,6 +54,13 @@ Your changes have NOT been applied. Please fix your edit command and try again """ +LINE_NUMBER_AND_CONTENT_MISMATCH = """ +Error: The `{position}_line_number` does not match the `{position}_line_content`. Please correct the parameters. +The `{position}_line_number` is {line_number} and the corresponding content is "{true_content}". +But the `{position}_line_content` is "{fake_content}". +The content around the specified line is: +{context} +""".strip() SUCCESS_EDIT_INFO = """ [File: {file_name} ({n_total_lines} lines total after edit)] {window_after_applied} @@ -756,6 +763,32 @@ class Editor(BaseModel): file_name = self._try_fix_path(file_name) + # Check if the start_line_number and end_line_number correspond to the appropriate content. + mismatch_error = "" + with file_name.open() as file: + content = file.read() + # Ensure the content ends with a newline character + if not content.endswith("\n"): + content += "\n" + lines = content.splitlines(True) + total_lines = len(lines) + check_list = [("start", start_line_number, start_line_content), ("end", end_line_number, end_line_content)] + for position, line_number, line_content in check_list: + if lines[line_number - 1].rstrip() != line_content: + start = max(1, line_number - 3) + end = min(total_lines, line_number + 3) + context = "".join( + [f"{line_number:03d}|{lines[line_number-1]}" for line_number in range(start, end + 1)] + ) + mismatch_error += LINE_NUMBER_AND_CONTENT_MISMATCH.format( + position=position, + line_number=line_number, + true_content=lines[line_number - 1].rstrip(), + fake_content=line_content, + context=context, + ) + if mismatch_error: + return mismatch_error ret_str = self._edit_file_impl( file_name, start=start_line_number, diff --git a/tests/metagpt/tools/libs/test_editor.py b/tests/metagpt/tools/libs/test_editor.py index de773f04d..91c74085e 100644 --- a/tests/metagpt/tools/libs/test_editor.py +++ b/tests/metagpt/tools/libs/test_editor.py @@ -537,6 +537,45 @@ def test_function_for_fm(): def test_edit_file_by_replace(temp_py_file): editor = Editor() editor.edit_file_by_replace( + file_name=str(temp_py_file), + start_line_number=5, + start_line_content=" b = 2", + new_content=" b = 9", + end_line_number=5, + end_line_content=" b = 2", + ) + with open(temp_py_file, "r") as f: + new_content = f.read() + assert new_content.strip() == EXPECTED_CONTENT_AFTER_REPLACE_TEXT.strip() + + +MISMATCH_ERROR = """ +Error: The `start_line_number` does not match the `start_line_content`. Please correct the parameters. +The `start_line_number` is 5 and the corresponding content is " b = 2". +But the `start_line_content` is "". +The content around the specified line is: +002|def test_function_for_fm(): +003| "some docstring" +004| a = 1 +005| b = 2 +006| c = 3 +007| # this is the 7th line +Error: The `end_line_number` does not match the `end_line_content`. Please correct the parameters. +The `end_line_number` is 5 and the corresponding content is " b = 2". +But the `end_line_content` is "". +The content around the specified line is: +002|def test_function_for_fm(): +003| "some docstring" +004| a = 1 +005| b = 2 +006| c = 3 +007| # this is the 7th line +""".strip() + + +def test_edit_file_by_replace_mismatch(temp_py_file): + editor = Editor() + output = editor.edit_file_by_replace( file_name=str(temp_py_file), start_line_number=5, start_line_content="", @@ -544,9 +583,9 @@ def test_edit_file_by_replace(temp_py_file): end_line_number=5, end_line_content="", ) - with open(temp_py_file, "r") as f: - new_content = f.read() - assert new_content.strip() == EXPECTED_CONTENT_AFTER_REPLACE_TEXT.strip() + with open("tmp.txt", "w", encoding="utf-8") as f: + f.write(output) + assert output.strip() == MISMATCH_ERROR.strip() def test_append_file(temp_file_path): From a8aaece8fb09ab1afbac6450fddfeb4e6562a6d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Wed, 18 Sep 2024 19:34:39 +0800 Subject: [PATCH 16/21] remove the outdated editor results. --- metagpt/roles/di/role_zero.py | 19 ++++++++ metagpt/strategy/experience_retriever.py | 8 ++-- metagpt/tools/libs/editor.py | 55 +++++++++++++----------- tests/metagpt/tools/libs/test_editor.py | 32 +++++++------- 4 files changed, 68 insertions(+), 46 deletions(-) diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index 5e07f4895..f93ad9058 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -213,6 +213,7 @@ class RoleZero(Role): ### Recent Observation ### memory = self.rc.memory.get(self.memory_k) memory = await self.parse_browser_actions(memory) + memory = await self.parse_editor_result(memory) memory = self.parse_images(memory) req = self.llm.format_msg(memory + [UserMessage(content=prompt)]) @@ -246,6 +247,24 @@ class RoleZero(Role): break return memory + async def parse_editor_result(self, memory: list[Message]) -> list[Message]: + """Retain the latest result for each editor command and remove outdated editor results.""" + # Set to keep track of unique editor commands + record = set() + pattern = re.compile(r"Command Editor\.(\w+) executed") + # Iterate over the memory in reverse order + for msg in reversed(memory): + matches = pattern.findall(msg.content) + if not matches: + continue + # If all matches are already in the record, remove the editor content + if all(match in record for match in matches): + msg.content = msg.content[: msg.content.find("Command Editor")] + else: + # Add new matches to the record + record.update(matches) + return memory + def parse_images(self, memory: list[Message]) -> list[Message]: if not self.llm.support_image_input(): return memory diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index 7cefe863c..aec8a3337 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -972,11 +972,11 @@ Editor tool is exclusive. If I use this tool, I cannot use any other commands in "command_name": "Editor.edit_file_by_replace", "args": { "file_name":"/workspace/MetaGPT/provider/openai_api.py", - "start_line_number": 143, - "start_line_content":" if event.key == pygame.K_w:", + "first_replaced_line_number": 143, + "first_replaced_line_content":" if event.key == pygame.K_w:", "new_content": " if event.key == pygame.K_UP:\\n self.move_up()\\n elif event.key == pygame.K_DOWN:\\n self.move_down()\\n elif event.key == pygame.K_SPACE:\\n self.stop()" - "end_line_number": 146, - "end_line_content": " self.move_down()", + "last_replaced_line_number": 146, + "last_replaced_line_content": " self.move_down()", } } ] diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index 46a3bf970..376c86db5 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -54,13 +54,13 @@ Your changes have NOT been applied. Please fix your edit command and try again """ -LINE_NUMBER_AND_CONTENT_MISMATCH = """ -Error: The `{position}_line_number` does not match the `{position}_line_content`. Please correct the parameters. -The `{position}_line_number` is {line_number} and the corresponding content is "{true_content}". -But the `{position}_line_content` is "{fake_content}". +LINE_NUMBER_AND_CONTENT_MISMATCH = """Error: The `{position}_replaced_line_number` does not match the `{position}_replaced_line_content`. Please correct the parameters. +The `{position}_replaced_line_number` is {line_number} and the corresponding content is "{true_content}". +But the `{position}_replaced_line_content ` is "{fake_content}". The content around the specified line is: {context} -""".strip() +Pay attention to the new content. Ensure that it aligns with the new parameters. +""" SUCCESS_EDIT_INFO = """ [File: {file_name} ({n_total_lines} lines total after edit)] {window_after_applied} @@ -689,10 +689,10 @@ class Editor(BaseModel): def edit_file_by_replace( self, file_name: str, - start_line_number: int, - start_line_content: str, - end_line_number: int, - end_line_content: str, + first_replaced_line_number: int, + first_replaced_line_content: str, + last_replaced_line_number: int, + last_replaced_line_content: str, new_content: str, ) -> str: """ @@ -713,10 +713,10 @@ class Editor(BaseModel): edit_file_by_replace( '/workspace/example.txt', - start_line_number=2, - start_line_content="contain g", - end_line_number=3, - end_line_content="contain h", + first_replaced_line_number =2, + first_replaced_line_content="contain g", + last_replaced_line_number =3, + last_replaced_line_content="contain h", new_content='new content', ) This will replace only the second line 2 and line 3 with "new content". @@ -738,10 +738,10 @@ class Editor(BaseModel): EDITING: If you want to remove the line 2 and line 3 edit_file_by_replace( '/workspace/example.txt', - start_line_number=2, - start_line_content="contain g", - end_line_number=3, - end_line_content="contain h", + first_replaced_line_number =2, + first_replaced_line_content="contain g", + last_replaced_line_number =3, + last_replaced_line_content="contain h", new_content='new line', ) This will remove line 2 and line 3 @@ -753,17 +753,17 @@ class Editor(BaseModel): ``` Args: file_name str:The name of the file to edit. - start_line_number int:The line number to start the edit at, starting from 1. - start_line_content str:The content of the start replace line, according to the start_line_number. - end_line_number int:The line number to end the edit at (inclusive), starting from 1. - end_line_content str:The content of the end replace line, according to the end_line_number. + first_replaced_line_number int:The line number to start the edit at, starting from 1. + first_replaced_line_content str:The content of the start replace line, according to the first_replaced_line_number . + last_replaced_line_number int:The line number to end the edit at (inclusive), starting from 1. + last_replaced_line_content str:The content of the end replace line, according to the last_replaced_line_number . new_content str: The text to replace the current selection with, must conform to PEP8 standards.The content in the start line and end line will also be replaced. """ file_name = self._try_fix_path(file_name) - # Check if the start_line_number and end_line_number correspond to the appropriate content. + # Check if the first_replaced_line_number and last_replaced_line_number correspond to the appropriate content. mismatch_error = "" with file_name.open() as file: content = file.read() @@ -772,7 +772,10 @@ class Editor(BaseModel): content += "\n" lines = content.splitlines(True) total_lines = len(lines) - check_list = [("start", start_line_number, start_line_content), ("end", end_line_number, end_line_content)] + check_list = [ + ("first_replaced", first_replaced_line_number, first_replaced_line_content), + ("last_replaced", last_replaced_line_number, last_replaced_line_content), + ] for position, line_number, line_content in check_list: if lines[line_number - 1].rstrip() != line_content: start = max(1, line_number - 3) @@ -785,14 +788,14 @@ class Editor(BaseModel): line_number=line_number, true_content=lines[line_number - 1].rstrip(), fake_content=line_content, - context=context, + context=context.strip(), ) if mismatch_error: return mismatch_error ret_str = self._edit_file_impl( file_name, - start=start_line_number, - end=end_line_number, + start=first_replaced_line_number, + end=last_replaced_line_number, content=new_content, ) # TODO: automatically tries to fix linter error (maybe involve some static analysis tools on the location near the edit to figure out indentation) diff --git a/tests/metagpt/tools/libs/test_editor.py b/tests/metagpt/tools/libs/test_editor.py index 91c74085e..40b905e13 100644 --- a/tests/metagpt/tools/libs/test_editor.py +++ b/tests/metagpt/tools/libs/test_editor.py @@ -538,11 +538,11 @@ def test_edit_file_by_replace(temp_py_file): editor = Editor() editor.edit_file_by_replace( file_name=str(temp_py_file), - start_line_number=5, - start_line_content=" b = 2", + first_replaced_line_number=5, + first_replaced_line_content=" b = 2", new_content=" b = 9", - end_line_number=5, - end_line_content=" b = 2", + last_replaced_line_number=5, + last_replaced_line_content=" b = 2", ) with open(temp_py_file, "r") as f: new_content = f.read() @@ -550,9 +550,9 @@ def test_edit_file_by_replace(temp_py_file): MISMATCH_ERROR = """ -Error: The `start_line_number` does not match the `start_line_content`. Please correct the parameters. -The `start_line_number` is 5 and the corresponding content is " b = 2". -But the `start_line_content` is "". +Error: The `first_replaced_replaced_line_number` does not match the `first_replaced_replaced_line_content`. Please correct the parameters. +The `first_replaced_replaced_line_number` is 5 and the corresponding content is " b = 2". +But the `first_replaced_replaced_line_content ` is "". The content around the specified line is: 002|def test_function_for_fm(): 003| "some docstring" @@ -560,9 +560,10 @@ The content around the specified line is: 005| b = 2 006| c = 3 007| # this is the 7th line -Error: The `end_line_number` does not match the `end_line_content`. Please correct the parameters. -The `end_line_number` is 5 and the corresponding content is " b = 2". -But the `end_line_content` is "". +Pay attention to the new content. Ensure that it aligns with the new parameters. +Error: The `last_replaced_replaced_line_number` does not match the `last_replaced_replaced_line_content`. Please correct the parameters. +The `last_replaced_replaced_line_number` is 5 and the corresponding content is " b = 2". +But the `last_replaced_replaced_line_content ` is "". The content around the specified line is: 002|def test_function_for_fm(): 003| "some docstring" @@ -570,6 +571,7 @@ The content around the specified line is: 005| b = 2 006| c = 3 007| # this is the 7th line +Pay attention to the new content. Ensure that it aligns with the new parameters. """.strip() @@ -577,14 +579,12 @@ def test_edit_file_by_replace_mismatch(temp_py_file): editor = Editor() output = editor.edit_file_by_replace( file_name=str(temp_py_file), - start_line_number=5, - start_line_content="", + first_replaced_line_number=5, + first_replaced_line_content="", new_content=" b = 9", - end_line_number=5, - end_line_content="", + last_replaced_line_number=5, + last_replaced_line_content="", ) - with open("tmp.txt", "w", encoding="utf-8") as f: - f.write(output) assert output.strip() == MISMATCH_ERROR.strip() From e495ea90d06188e4c3f886807bb21ba3b798330a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Thu, 19 Sep 2024 15:12:19 +0800 Subject: [PATCH 17/21] fix:line numbers and line contents do not match. --- metagpt/roles/di/role_zero.py | 32 +++++++++++-------------- metagpt/tools/libs/editor.py | 9 ++++--- tests/metagpt/tools/libs/test_editor.py | 24 +++++++++---------- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index f93ad9058..eaba43277 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -248,22 +248,22 @@ class RoleZero(Role): return memory async def parse_editor_result(self, memory: list[Message]) -> list[Message]: - """Retain the latest result for each editor command and remove outdated editor results.""" - # Set to keep track of unique editor commands - record = set() + """Retain the latest result and remove outdated editor results.""" + keep_count = 5 pattern = re.compile(r"Command Editor\.(\w+) executed") - # Iterate over the memory in reverse order + new_memory = [] for msg in reversed(memory): matches = pattern.findall(msg.content) - if not matches: - continue - # If all matches are already in the record, remove the editor content - if all(match in record for match in matches): - msg.content = msg.content[: msg.content.find("Command Editor")] - else: - # Add new matches to the record - record.update(matches) - return memory + if matches: + if keep_count < 0: + new_content = msg.content[: msg.content.find("Command Editor")] + new_content += "\n".join([f"Command Editor\.{match} executed." for match in matches]) + msg = UserMessage(content=new_content) + keep_count -= 1 + new_memory.append(msg) + # Reverse the new memory list so the latest message is at the end + new_memory.reverse() + return new_memory def parse_images(self, memory: list[Message]) -> list[Message]: if not self.llm.support_image_input(): @@ -446,11 +446,7 @@ class RoleZero(Role): if command_flag.count(False) > 1: # Keep only the first exclusive command index_of_first_exclusive = command_flag.index(False) - commands = [ - cmd - for index, cmd in enumerate(commands) - if index == index_of_first_exclusive or cmd["command_name"] not in self.exclusive_tool_commands - ] + commands = commands[: index_of_first_exclusive + 1] command_rsp = "```json\n" + json.dumps(commands, indent=4, ensure_ascii=False) + "\n```" logger.info( "exclusive command more than one in current command list. change the command list.\n" + command_rsp diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index 376c86db5..1fe75f732 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -780,14 +780,17 @@ class Editor(BaseModel): if lines[line_number - 1].rstrip() != line_content: start = max(1, line_number - 3) end = min(total_lines, line_number + 3) - context = "".join( - [f"{line_number:03d}|{lines[line_number-1]}" for line_number in range(start, end + 1)] + context = "\n".join( + [ + f'The {line_number:03d} line is "{lines[line_number-1].rstrip()}"' + for line_number in range(start, end + 1) + ] ) mismatch_error += LINE_NUMBER_AND_CONTENT_MISMATCH.format( position=position, line_number=line_number, true_content=lines[line_number - 1].rstrip(), - fake_content=line_content, + fake_content=line_content.replace("\n", "\\n"), context=context.strip(), ) if mismatch_error: diff --git a/tests/metagpt/tools/libs/test_editor.py b/tests/metagpt/tools/libs/test_editor.py index 40b905e13..b56f7bf0e 100644 --- a/tests/metagpt/tools/libs/test_editor.py +++ b/tests/metagpt/tools/libs/test_editor.py @@ -554,23 +554,23 @@ Error: The `first_replaced_replaced_line_number` does not match the `first_repla The `first_replaced_replaced_line_number` is 5 and the corresponding content is " b = 2". But the `first_replaced_replaced_line_content ` is "". The content around the specified line is: -002|def test_function_for_fm(): -003| "some docstring" -004| a = 1 -005| b = 2 -006| c = 3 -007| # this is the 7th line +The 002 line is "def test_function_for_fm():" +The 003 line is " "some docstring"" +The 004 line is " a = 1" +The 005 line is " b = 2" +The 006 line is " c = 3" +The 007 line is " # this is the 7th line" Pay attention to the new content. Ensure that it aligns with the new parameters. Error: The `last_replaced_replaced_line_number` does not match the `last_replaced_replaced_line_content`. Please correct the parameters. The `last_replaced_replaced_line_number` is 5 and the corresponding content is " b = 2". But the `last_replaced_replaced_line_content ` is "". The content around the specified line is: -002|def test_function_for_fm(): -003| "some docstring" -004| a = 1 -005| b = 2 -006| c = 3 -007| # this is the 7th line +The 002 line is "def test_function_for_fm():" +The 003 line is " "some docstring"" +The 004 line is " a = 1" +The 005 line is " b = 2" +The 006 line is " c = 3" +The 007 line is " # this is the 7th line" Pay attention to the new content. Ensure that it aligns with the new parameters. """.strip() From deb8bb6b2910332fc017923eeb1712db84dd16d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Thu, 19 Sep 2024 19:25:27 +0800 Subject: [PATCH 18/21] fix:command exists in the code file --- metagpt/roles/di/engineer2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/di/engineer2.py b/metagpt/roles/di/engineer2.py index 6313c5b33..3d80e6a26 100644 --- a/metagpt/roles/di/engineer2.py +++ b/metagpt/roles/di/engineer2.py @@ -113,7 +113,7 @@ class Engineer2(RoleZero): command_output += await super()._run_special_command(cmd) return command_output - async def write_new_code(self, path: str, instruction: str = "") -> str: + async def write_new_code(self, path: str, instruction: str = "Write code for the current file.") -> str: """Write a new code file. Args: From 1fa20757c01938edce2cca0b4768fef76f695d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Thu, 19 Sep 2024 20:29:54 +0800 Subject: [PATCH 19/21] fix typo --- metagpt/roles/di/role_zero.py | 12 ++++++------ metagpt/tools/libs/editor.py | 29 +++++++++++++++-------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index eaba43277..b4c7553ea 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -247,19 +247,19 @@ class RoleZero(Role): break return memory - async def parse_editor_result(self, memory: list[Message]) -> list[Message]: + async def parse_editor_result(self, memory: list[Message], keep_latest_count=5) -> list[Message]: """Retain the latest result and remove outdated editor results.""" - keep_count = 5 - pattern = re.compile(r"Command Editor\.(\w+) executed") + pattern = re.compile(r"Command Editor\.(\w+?) executed") new_memory = [] + i = 0 for msg in reversed(memory): matches = pattern.findall(msg.content) if matches: - if keep_count < 0: + i += 1 + if i > keep_latest_count: new_content = msg.content[: msg.content.find("Command Editor")] - new_content += "\n".join([f"Command Editor\.{match} executed." for match in matches]) + new_content += "\n".join([f"Command Editor.{match} executed." for match in matches]) msg = UserMessage(content=new_content) - keep_count -= 1 new_memory.append(msg) # Reverse the new memory list so the latest message is at the end new_memory.reverse() diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index 1fe75f732..9d1dc9730 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -698,7 +698,7 @@ class Editor(BaseModel): """ Line numbers start from 1. Replaces lines start_line through end_line (inclusive) with the given text in the open file. All of the new_content will be entered, so makesure your indentation is formatted properly. - The new_content must be a complete block of code . + The new_content must be a complete block of code. Example 1: Given a file "/workspace/example.txt" with the following content: @@ -742,7 +742,7 @@ class Editor(BaseModel): first_replaced_line_content="contain g", last_replaced_line_number =3, last_replaced_line_content="contain h", - new_content='new line', + new_content='', ) This will remove line 2 and line 3 The resulting file will be: @@ -752,12 +752,12 @@ class Editor(BaseModel): 003|contain i ``` Args: - file_name str:The name of the file to edit. - first_replaced_line_number int:The line number to start the edit at, starting from 1. - first_replaced_line_content str:The content of the start replace line, according to the first_replaced_line_number . - last_replaced_line_number int:The line number to end the edit at (inclusive), starting from 1. - last_replaced_line_content str:The content of the end replace line, according to the last_replaced_line_number . - new_content str: The text to replace the current selection with, must conform to PEP8 standards.The content in the start line and end line will also be replaced. + file_name (str):The name of the file to edit. + first_replaced_line_number (int):The line number to start the edit at, starting from 1. + first_replaced_line_content (str):The content of the start replace line, according to the first_replaced_line_number . + last_replaced_line_number (int):The line number to end the edit at (inclusive), starting from 1. + last_replaced_line_content (str):The content of the end replace line, according to the last_replaced_line_number . + new_content (str): The text to replace the current selection with, must conform to PEP8 standards.The content in the start line and end line will also be replaced. """ @@ -847,9 +847,9 @@ class Editor(BaseModel): ) Args: - file_name: str: The name of the file to edit. - to_replace: str: The content to search for and replace. - new_content: str: The new content to replace the old content with. + file_name: (str): The name of the file to edit. + to_replace: (str): The content to search for and replace. + new_content: (str): The new content to replace the old content with. NOTE: This tool is exclusive. If you use this tool, you cannot use any other commands in the current response. If you need to use it multiple times, wait for the next turn. @@ -924,7 +924,8 @@ class Editor(BaseModel): insert_content_at_line( file_name='file.txt', line_number=2, - insert_content='new line')` + insert_content='new line' + ) the file will be updated to: ``` 001|contain g @@ -935,8 +936,8 @@ class Editor(BaseModel): ``` Args: - file_name: str: The name of the file to edit. - line_number int: The line number (starting from 1) to insert the content after.the insert content will be add between the line of line_number-1 and line_number + file_name: (str): The name of the file to edit. + line_number (int): The line number (starting from 1) to insert the content after.the insert content will be add between the line of line_number-1 and line_number insert_content (str): The content to insert betweed the previous_line_content and current_line_content.The insert_content must be a complete block of code at. NOTE: From 45b20f9ebc32af06d1a17d86bb96368a81059113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=A5=9A=E5=9F=8E?= Date: Thu, 19 Sep 2024 12:33:07 +0000 Subject: [PATCH 20/21] Update terminal.py --- metagpt/tools/libs/terminal.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index 6d6934ac7..24daa184d 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -110,8 +110,7 @@ class Terminal: async def _read_and_process_output(self, cmd, daemon=False) -> str: async with self.observer as observer: cmd_output = [] - if cmd != "pwd": - await observer.async_report(cmd + self.command_terminator, "cmd") + await observer.async_report(cmd + self.command_terminator, "cmd") # report the command # Read the output until the unique marker is found. # We read bytes directly from stdout instead of text because when reading text, @@ -128,14 +127,12 @@ class Terminal: if ix >= 0: line = line[0:ix] if line: - if cmd != "pwd": - await observer.async_report(line, "output") + await observer.async_report(line, "output") # report stdout in real-time cmd_output.append(line) return "".join(cmd_output) # log stdout in real-time - if cmd != "pwd": - await observer.async_report(line, "output") + await observer.async_report(line, "output") cmd_output.append(line) if daemon: await self.stdout_queue.put(line) From 8eab27e7854b2db931d39ee0364c951af0cc6aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E4=BC=9F=E9=9F=AC?= Date: Fri, 20 Sep 2024 12:57:19 +0800 Subject: [PATCH 21/21] add rule to forbiden repeat finish current task --- metagpt/prompts/di/engineer2.py | 1 + metagpt/prompts/di/role_zero.py | 11 ++++++++++ metagpt/roles/di/role_zero.py | 9 ++++++-- metagpt/tools/libs/editor.py | 38 ++++++++++++++++----------------- 4 files changed, 38 insertions(+), 21 deletions(-) diff --git a/metagpt/prompts/di/engineer2.py b/metagpt/prompts/di/engineer2.py index cef90ec3e..016884d2b 100644 --- a/metagpt/prompts/di/engineer2.py +++ b/metagpt/prompts/di/engineer2.py @@ -80,6 +80,7 @@ Note: 21. When using the editor, pay attention to the editor's current directory. When you use editor tools, the paths must be either absolute or relative to the editor's current directory. 22. The default programming languages are Native HTML. 23. When planning, consider whether images are needed. If you are developing a showcase website, start by using ImageGetter.get_image to obtain the necessary images. +24. If you finish all the tasks, use the command "end" to end. """ CURRENT_STATE = """ The current editor state is: diff --git a/metagpt/prompts/di/role_zero.py b/metagpt/prompts/di/role_zero.py index 820466b6f..2697ecef4 100644 --- a/metagpt/prompts/di/role_zero.py +++ b/metagpt/prompts/di/role_zero.py @@ -108,6 +108,17 @@ Describe if you should terminate using **end** command, or use **RoleZero.ask_hu You should use "end" to stop when all tasks have been completed and the requirements are satisfied. Your reflection, then the commands in a json array: """ +END_COMMAND = """ +```json +[ + { + "command_name": "end", + "args": {} + } +] +``` +""" + ASK_HUMAN_COMMAND = """ ```json [ diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index b4c7553ea..0e7e04969 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -21,6 +21,7 @@ from metagpt.prompts.di.role_zero import ( ASK_HUMAN_COMMAND, CMD_PROMPT, DETECT_LANGUAGE_PROMPT, + END_COMMAND, JSON_REPAIR_PROMPT, QUICK_RESPONSE_SYSTEM_PROMPT, QUICK_THINK_EXAMPLES, @@ -383,8 +384,8 @@ class RoleZero(Role): return rsp_msg, intent_result - async def _check_duplicates(self, req: list[dict], command_rsp: str): - past_rsp = [mem.content for mem in self.rc.memory.get(self.memory_k)] + async def _check_duplicates(self, req: list[dict], command_rsp: str, check_window: int = 10): + past_rsp = [mem.content for mem in self.rc.memory.get(check_window)] if command_rsp in past_rsp: # Normal response with thought contents are highly unlikely to reproduce # If an identical response is detected, it is a bad response, mostly due to LLM repeating generated content @@ -393,6 +394,10 @@ class RoleZero(Role): # Hard rule to ask human for help if past_rsp.count(command_rsp) >= 3: + if '"command_name": "Plan.finish_current_task",' in command_rsp: + # Detect the deplicate of "Plan.finish_current_task" command, use command "end" to finish the task + logger.warning(f"Duplicate response detected: {command_rsp}") + return END_COMMAND return ASK_HUMAN_COMMAND # Try correction by self logger.warning(f"Duplicate response detected: {command_rsp}") diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index 9d1dc9730..ca8cd9ccf 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -696,7 +696,7 @@ class Editor(BaseModel): new_content: str, ) -> str: """ - Line numbers start from 1. Replaces lines start_line through end_line (inclusive) with the given text in the open file. + Line numbers start from 1. Replace lines from start_line to end_line (inclusive) with the new_content in the open file. All of the new_content will be entered, so makesure your indentation is formatted properly. The new_content must be a complete block of code. @@ -712,14 +712,14 @@ class Editor(BaseModel): EDITING: If you want to replace line 2 and line 3 edit_file_by_replace( - '/workspace/example.txt', - first_replaced_line_number =2, + "/workspace/example.txt", + first_replaced_line_number=2, first_replaced_line_content="contain g", - last_replaced_line_number =3, + last_replaced_line_number=3, last_replaced_line_content="contain h", - new_content='new content', + new_content="new content", ) - This will replace only the second line 2 and line 3 with "new content". + This will replace the second line 2 and line 3 with "new content". The resulting file will be: ``` @@ -735,16 +735,16 @@ class Editor(BaseModel): 003|contain h 004|contain i ``` - EDITING: If you want to remove the line 2 and line 3 + EDITING: If you want to remove the line 2 and line 3. edit_file_by_replace( - '/workspace/example.txt', - first_replaced_line_number =2, + "/workspace/example.txt", + first_replaced_line_number=2, first_replaced_line_content="contain g", - last_replaced_line_number =3, + last_replaced_line_number=3, last_replaced_line_content="contain h", - new_content='', + new_content="", ) - This will remove line 2 and line 3 + This will remove line 2 and line 3. The resulting file will be: ``` 001|contain f @@ -752,12 +752,12 @@ class Editor(BaseModel): 003|contain i ``` Args: - file_name (str):The name of the file to edit. - first_replaced_line_number (int):The line number to start the edit at, starting from 1. - first_replaced_line_content (str):The content of the start replace line, according to the first_replaced_line_number . - last_replaced_line_number (int):The line number to end the edit at (inclusive), starting from 1. - last_replaced_line_content (str):The content of the end replace line, according to the last_replaced_line_number . - new_content (str): The text to replace the current selection with, must conform to PEP8 standards.The content in the start line and end line will also be replaced. + file_name (str): The name of the file to edit. + first_replaced_line_number (int): The line number to start the edit at, starting from 1. + first_replaced_line_content (str): The content of the start replace line, according to the first_replaced_line_number. + last_replaced_line_number (int): The line number to end the edit at (inclusive), starting from 1. + last_replaced_line_content (str): The content of the end replace line, according to the last_replaced_line_number. + new_content (str): The text to replace the current selection with, must conform to PEP8 standards. The content in the start line and end line will also be replaced. """ @@ -938,7 +938,7 @@ class Editor(BaseModel): Args: file_name: (str): The name of the file to edit. line_number (int): The line number (starting from 1) to insert the content after.the insert content will be add between the line of line_number-1 and line_number - insert_content (str): The content to insert betweed the previous_line_content and current_line_content.The insert_content must be a complete block of code at. + insert_content (str): The content to insert betweed the previous_line_content and current_line_content.The insert_content must be a complete block of code at. NOTE: This tool is exclusive. If you use this tool, you cannot use any other commands in the current response.