From 27157e33a05c24b5aebba417cad623bea13640d4 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Wed, 15 May 2024 21:26:13 +0800 Subject: [PATCH 1/5] include more plan status in thinking, add row num for editor --- metagpt/prompts/di/data_analyst.py | 3 +- metagpt/roles/di/data_analyst.py | 6 +- metagpt/tools/libs/editor.py | 98 ++++++++++++++++-------------- 3 files changed, 58 insertions(+), 49 deletions(-) diff --git a/metagpt/prompts/di/data_analyst.py b/metagpt/prompts/di/data_analyst.py index a0624f047..8450b2fe1 100644 --- a/metagpt/prompts/di/data_analyst.py +++ b/metagpt/prompts/di/data_analyst.py @@ -22,7 +22,8 @@ If plan is created, you should track the progress and update the plan accordingl Pay close attention to new user message, review the conversation history, use reply_to_human to respond to new user requirement. Note: 1. If you keeping encountering errors, unexpected situation, or you are not sure of proceeding, use ask_human to ask for help. -2. Each time you finish a task, use reply_to_human to report your progress. +2. Carefully review your progress at the current task, if your actions so far has not fulfilled the task instruction, you should continue with current task. Otherwise, finish current task. +3. Each time you finish a task, use reply_to_human to report your progress. Pay close attention to the Example provided, you can reuse the example for your current situation if it fits. You may use any of the available commands to create a plan or update the plan. You may output mutiple commands, they will be executed sequentially. diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py index df0b8d288..0fc95b9d6 100644 --- a/metagpt/roles/di/data_analyst.py +++ b/metagpt/roles/di/data_analyst.py @@ -73,9 +73,9 @@ class DataAnalyst(DataInterpreter): # TODO: implement experience retrieval in multi-round setting plan_status = self.planner.plan.model_dump(include=["goal", "tasks"]) - for task in plan_status["tasks"]: - task.pop("code") - task.pop("result") + # for task in plan_status["tasks"]: + # task.pop("code") + # task.pop("result") prompt = CMD_PROMPT.format( plan_status=plan_status, example=example, diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index f973bad91..fcf892fdb 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -2,7 +2,7 @@ import os import shutil import subprocess -from pydantic import BaseModel, Field +from pydantic import BaseModel from metagpt.tools.tool_registry import register_tool from metagpt.utils.report import EditorReporter @@ -13,12 +13,6 @@ class FileBlock(BaseModel): file_path: str block_content: str - block_start_line: int - block_end_line: int - symbol: str = Field(default="", description="The symbol of interest in the block, empty if not applicable.") - symbol_start_line: int = Field( - default=-1, description="The line number of the symbol in the file, -1 if not applicable" - ) @register_tool() @@ -34,13 +28,19 @@ class Editor: f.write(content) self.resource.report(path, "path") - def read(self, path: str) -> str: + def read(self, path: str) -> FileBlock: """Read the whole content of a file.""" with open(path, "r") as f: self.resource.report(path, "path") - return f.read() + lines = f.readlines() + lines_with_num = [f"{i + 1:03}|{line}" for i, line in enumerate(lines)] + result = FileBlock( + file_path=path, + block_content="".join(lines_with_num), + ) + return result - def search_content(self, symbol: str, root_path: str = ".", window: int = 20) -> FileBlock: + def search_content(self, symbol: str, root_path: str = ".", window: int = 50) -> FileBlock: """ Search symbol in all files under root_path, return the context of symbol with window size Useful for locating class or function in a large codebase. Example symbol can be "def some_function", "class SomeClass", etc. @@ -48,7 +48,7 @@ class Editor: Args: symbol (str): The symbol to search. - root_path (str, optional): The root path to search in. If not provided, search in the current directory. Defaults to ".". + root_path (str, optional): The root path to search in, the path can be a folder or a file. If not provided, search in the current directory. Defaults to ".". window (int, optional): The window size to return. Defaults to 20. Returns: @@ -56,42 +56,50 @@ class Editor: class FileBlock(BaseModel): file_path: str block_content: str - block_start_line: int - block_end_line: int - symbol: str = Field(default="", description="The symbol of interest in the block, empty if not applicable.") - symbol_start_line: int = Field(default=-1, description="The line number of the symbol in the file, -1 if not applicable") """ if not os.path.exists(root_path): print(f"Currently at {os.getcwd()}. Path {root_path} does not exist.") return None + not_found_msg = ( + "symbol not found, you may try searching another one, or break down your search term to search a part of it" + ) + if os.path.isfile(root_path): + result = self._search_content_in_file(symbol, root_path, window) + if not result: + print(not_found_msg) + return result for root, _, files in os.walk(root_path or "."): for file in files: file_path = os.path.join(root, file) - if not file.endswith(".py"): - continue - with open(file_path, "r", encoding="utf-8") as f: - try: - lines = f.readlines() - except Exception: - continue - for i, line in enumerate(lines): - if symbol in line: - start = max(i - window, 0) - end = min(i + window, len(lines) - 1) - block_content = "".join(lines[start : end + 1]) - result = FileBlock( - file_path=file_path, - block_content=block_content, - block_start_line=start + 1, - block_end_line=end + 1, - symbol=symbol, - symbol_start_line=i + 1, - ) - self.resource.report(result.file_path, "path") - return result - print( - "symbol not found, you may try searching another one, or break down your search term to search a part of it" - ) + result = self._search_content_in_file(symbol, file_path, window) + if result: + # FIXME: This returns the first found result, not all results. + return result + print(not_found_msg) + return None + + def _search_content_in_file(self, symbol: str, file_path: str, window: int = 50) -> FileBlock: + print("search in", file_path) + if not file_path.endswith(".py"): + return None + with open(file_path, "r", encoding="utf-8") as f: + try: + lines = f.readlines() + except Exception: + return None + for i, line in enumerate(lines): + if symbol in line: + start = max(i - window, 0) + end = min(i + window, len(lines) - 1) + for row_num in range(start, end + 1): + lines[row_num] = f"{(row_num + 1):03}|{lines[row_num]}" + block_content = "".join(lines[start : end + 1]) + result = FileBlock( + file_path=file_path, + block_content=block_content, + ) + self.resource.report(result.file_path, "path") + return result return None def write_content(self, file_path: str, start_line: int, end_line: int, new_block_content: str = "") -> str: @@ -100,12 +108,13 @@ class Editor: 1. If the new block content is empty, the original block will be deleted. 2. If the new block content is not empty and end_line < start_line (e.g. set end_line = -1) the new block content will be inserted at start_line. 3. If the new block content is not empty and end_line >= start_line, the original block from start_line to end_line (both inclusively) will be replaced by the new block content. - This function can sometimes be used given a FileBlock upstream. Think carefully if you want to use block_start_line or symbol_start_line in the FileBlock as your start_line input. Your new_block_content will be placed at the start_line. + This function can sometimes be used given a FileBlock upstream. You should carefully review its row number. Determine the start_line and end_line based on the row number of the FileBlock. + The file content from start_line to end_line will be replaced by your new_block_content. DON'T replace more than you intend to. Args: file_path (str): The file path to write the new block content. - start_line (int): start line of the original block to be updated. - end_line (int): end line of the original block to be updated. + start_line (int): start line of the original block to be updated (inclusive). + end_line (int): end line of the original block to be updated (inclusive). new_block_content (str): The new block content to write. Returns: @@ -130,8 +139,6 @@ class Editor: new_file_block = FileBlock( file_path=file_path, block_content=new_block_content, - block_start_line=start_line, - block_end_line=-1 if end_line < start_line else start_line + new_block_content.count("\n"), ) self.resource.report(new_file_block.file_path, "path") @@ -143,6 +150,7 @@ class Editor: os.remove(temp_file_path) def _write_content(self, file_path: str, start_line: int, end_line: int, new_block_content: str = ""): + """start_line and end_line are both 1-based indices and inclusive.""" with open(file_path, "r") as file: lines = file.readlines() From df2eff04de3ebb3ecd29044edcc7b52d7a8f6f76 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Wed, 15 May 2024 21:32:12 +0800 Subject: [PATCH 2/5] add issue fixing experience --- metagpt/strategy/experience_retriever.py | 51 +++++++++++++++++++----- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index a57abe72f..d141e61d2 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -211,8 +211,8 @@ Explanation: Launching a service requires Terminal tool with daemon mode, write FIX_ISSUE_EXAMPLE = """ ## example 1 -User Requirement: Write a fix for this issue: https://github.com/xxx/xxx/issues/xxx, and commit and push your changes. -Explanation: The requirement is for software development, focusing on fixing an issue in an existing repository. The process is broken down into several steps, each demanding specific actions and tools. +User Requirement: Write a fix for this issue: https://github.com/xxx/xxx/issues/xxx, and commit, push your changes, and create a PR to the target repo. +Explanation: The requirement is to fix an issue in an existing repository. The process is broken down into several steps, each demanding specific actions and tools. ```json [ { @@ -256,7 +256,7 @@ Explanation: The requirement is for software development, focusing on fixing an "args": { "task_id": "5", "dependent_task_ids": ["4"], - "instruction": "Commit, push the changes to the repository.", + "instruction": "Commit, push the changes to the repository, and create a pull request to the target repository.", "assignee": "David" } }, @@ -266,21 +266,52 @@ Explanation: The requirement is for software development, focusing on fixing an SEARCH_SYMBOL_EXAMPLE = """ -## Past Experience -Issue: PaiEasChatEndpoint._call_eas should return bytes type instead of str type -Explanation: To understand the issue, we first need to know the content of the method in the codebase. Therefore, we search the symbol `def _call_eas` in the cloned repo langchain. - +## Past Experience 1 +Issue: _achat_completion_stream in metagpt/provider/openai_api.py produced TypeError: openai.types.completion_usage.CompletionUsage() argument after ** must be a mapping, not NoneType +Explanation: To understand the issue, we first need to know the content of the method. Since the issue provides the specific file, we should directly search the symbol `def _achat_completion_stream` in that file. Notice in previous steps, we already cd to the cloned project folder, the source code is at `metagpt` under the project folder. ```python from metagpt.tools.libs.editor import Editor # Initialize the Editor tool editor = Editor() -# Search for the PaiEasChatEndpoint._call_eas method in the codebase to understand the issue -symbol_to_search = "def _call_eas" -file_block = editor.search_content(symbol=symbol_to_search, root_path="langchain") +# Search for the '_achat_completion_stream' function in the 'openai_api.py' file +symbol_to_search = "def _achat_completion_stream" +file_block = editor.search_content(symbol=symbol_to_search, root_path="metagpt/provider/openai_api.py") # Output the file block containing the method to diagnose the problem file_block ``` + +## Past Experience 2 +Issue: PaiEasChatEndpoint._call_eas should return bytes type instead of str type +Explanation: To understand the issue, we first need to know the content of the method. Since no specific file is provided, we can search it in the whole codebase. Therefore, we search the symbol `def _call_eas` in the cloned repo. Notice in previous steps, we already cd to the cloned project folder, the source code is at langchain under the project folder. +```python +from metagpt.tools.libs.editor import Editor + +editor = Editor() + +# Search for the PaiEasChatEndpoint._call_eas method in the codebase to understand the issue +symbol_to_search = "def _call_eas" +file_block = editor.search_content(symbol=symbol_to_search, root_path="langchain") + +file_block +``` + +## Past Experience 3 +Issue: When I run main.py, a message 'Running on http://127.0.0.1:5000' will appear on the console, but when I click on this link, the front-end web page cannot be displayed correctly. +Explanation: To understand the issue, we first need to know the content of the file. Since the issue doesn't provide a specific method or class, we will not search_content, but directly open and read the file main.py to diagnose the problem. Notice in previous steps, we already cd to the cloned project folder. +```python +from metagpt.tools.libs.editor import Editor + +editor = Editor() + +# Open the file to understand the issue +editor.read(path="./main.py") +``` + +## Experience Summary +- When diagnosing an issue, search for the specific symbol in the corresponding file to understand the problem. +- If no specific file is provided, search the symbol in the whole codebase to locate the issue. +- If no specific symbol is provided, directly open and read the file to diagnose the problem. """ From 4d05bba39ca22ea853cea11f2e80dccad4e7c084 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Wed, 15 May 2024 21:43:31 +0800 Subject: [PATCH 3/5] fix redundant new line --- metagpt/tools/libs/editor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index fcf892fdb..94836a9fe 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -165,7 +165,7 @@ class Editor: # irrespective of the length difference between the original and new content. lines[start_line_index:end_line_index] = new_content_lines else: - lines.insert(start_line_index, "\n".join(new_content_lines)) + lines.insert(start_line_index, "".join(new_content_lines)) else: del lines[start_line_index:end_line_index] From 653809223538fd1220dacbb63a72d9152d54667e Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Wed, 15 May 2024 23:07:34 +0800 Subject: [PATCH 4/5] small improvement --- metagpt/strategy/experience_retriever.py | 2 +- metagpt/tools/libs/editor.py | 4 +++- metagpt/tools/libs/git.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index d141e61d2..4b209717c 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -238,7 +238,7 @@ Explanation: The requirement is to fix an issue in an existing repository. The p "args": { "task_id": "3", "dependent_task_ids": ["2"], - "instruction": "Use Editor to search the relevant function(s), then diagnose and identify the source of the problem.", + "instruction": "Use Editor to search relevant function(s) or open relevant files, then diagnose and identify the source of the problem.", "assignee": "David" } }, diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index 94836a9fe..e032dcef5 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -4,6 +4,7 @@ import subprocess from pydantic import BaseModel +from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.tools.tool_registry import register_tool from metagpt.utils.report import EditorReporter @@ -20,6 +21,7 @@ class Editor: """A tool for reading, understanding, writing, and editing files""" def __init__(self) -> None: + print(f"Editor initialized with root path at: {DEFAULT_WORKSPACE_ROOT}") self.resource = EditorReporter() def write(self, path: str, content: str): @@ -58,7 +60,7 @@ class Editor: block_content: str """ if not os.path.exists(root_path): - print(f"Currently at {os.getcwd()}. Path {root_path} does not exist.") + print(f"Currently at {os.getcwd()} containing: {os.listdir()}. Path {root_path} does not exist.") return None not_found_msg = ( "symbol not found, you may try searching another one, or break down your search term to search a part of it" diff --git a/metagpt/tools/libs/git.py b/metagpt/tools/libs/git.py index ae10e969e..eb3fd6822 100644 --- a/metagpt/tools/libs/git.py +++ b/metagpt/tools/libs/git.py @@ -69,7 +69,7 @@ async def git_create_pull( issue: Optional[Issue] = None, ) -> PullRequest: """ - Creates a pull request on a Git repository. + Creates a pull request on a Git repository. Use this tool in priority over Browser to create a pull request. Args: base (str): The base branch of the pull request. From e54cc424e8ba604d4c4a667de580272251ec9fed Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Thu, 16 May 2024 10:20:20 +0800 Subject: [PATCH 5/5] add example issue --- tests/metagpt/environment/mgx_env/run_mgx_env.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/metagpt/environment/mgx_env/run_mgx_env.py b/tests/metagpt/environment/mgx_env/run_mgx_env.py index f0fdbc244..42f550ab6 100644 --- a/tests/metagpt/environment/mgx_env/run_mgx_env.py +++ b/tests/metagpt/environment/mgx_env/run_mgx_env.py @@ -72,10 +72,20 @@ Write a fix for this issue: https://github.com/langchain-ai/langchain/issues/204 you can fix it on this repo https://github.com/garylin2099/langchain, checkout a branch named test-fix, commit your changes, push, and create a PR to the master branch of https://github.com/iorisa/langchain """ +FIX_ISSUE2 = """ +Write a fix for this issue https://github.com/geekan/MetaGPT/issues/1275. +You can fix it on the v0.8-release branch of this repo https://github.com/garylin2099/MetaGPT, +during fixing, checkout a branch named test-fix-1275, commit your changes, push, and create a PR to the v0.8-release branch of https://github.com/garylin2099/MetaGPT +""" +FIX_ISSUE3 = """ +Write a fix for this issue https://github.com/geekan/MetaGPT/issues/1262. +You can fix it on this repo https://github.com/garylin2099/MetaGPT, +during fixing, checkout a branch named test-fix-1262, commit your changes, push, and create a PR to https://github.com/garylin2099/MetaGPT +""" FIX_ISSUE_SIMPLE = """ Write a fix for this issue: https://github.com/mannaandpoem/simple_calculator/issues/1, you can fix it on this repo https://github.com/garylin2099/simple_calculator, -checkout a branch named test, commit your changes, push, and create a PR to the original repo. +checkout a branch named test, commit your changes, push, and create a PR to the master branch of original repo. """ PUSH_PR_REQ = """ clone https://github.com/garylin2099/simple_calculator, checkout a new branch named test-branch, add an empty file test_file.py to the repo.