From fafe2ce68feae2c6855bc3704bb64486bc00b308 Mon Sep 17 00:00:00 2001 From: seeker Date: Wed, 3 Jul 2024 20:11:32 +0800 Subject: [PATCH 1/9] update: SWE Agent --- metagpt/const.py | 2 +- metagpt/prompts/di/swe.py | 87 +++++++++++++------ metagpt/roles/di/swe.py | 8 +- metagpt/tools/libs/browser.py | 2 + metagpt/tools/libs/terminal.py | 7 +- metagpt/tools/swe_agent_commands/defaults.sh | 9 +- .../swe_agent_commands/swe_agent_utils.py | 8 +- tests/metagpt/roles/di/run_swe.py | 71 +++++---------- 8 files changed, 103 insertions(+), 91 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index c78a22641..94d22bc70 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -149,6 +149,6 @@ METAGPT_REPORTER_DEFAULT_URL = os.environ.get("METAGPT_REPORTER_URL", "") # Metadata defines AGENT = "agent" - +SWE_WORKSPACE_ROOT = Path("/tmp/swe_workspace") # SWE agent SWE_SETUP_PATH = METAGPT_ROOT / "metagpt/tools/swe_agent_commands/setup_default.sh" diff --git a/metagpt/prompts/di/swe.py b/metagpt/prompts/di/swe.py index 64c67b09b..ed1f8a011 100644 --- a/metagpt/prompts/di/swe.py +++ b/metagpt/prompts/di/swe.py @@ -4,19 +4,17 @@ You can find the original examples from the SWE-agent project here: https://github.com/princeton-nlp/SWE-agent/tree/main/config/configs """ - SWE_AGENT_SYSTEM_TEMPLATE = """ -SETTING: You are an autonomous programmer, and you're working directly in the command line with a special interface. +SETTING: You are an autonomous programmer, and you're working directly in the environment line with a special interface. The special interface consists of a file editor that shows you {WINDOW} lines of a file at a time. Please note that THE EDIT COMMAND REQUIRES PROPER INDENTATION. Pay attention to the original indentation when replacing the function. If you'd like to add the line ' print(x)' you must fully write that out, with all those spaces before the code! Indentation is important and code that is not indented correctly will fail and require fixing before it can be run. - Always review your changes post-edit to ensure they accurately reflect your intentions. If the changes are not as desired, don't hesitate to issue another command to correct them. Your output should always contain a section of reasoning and a command described in JSON format. -The command must always contain command_name and args fields. The command_name field should always be Bash.run, and the args field should always include a cmd field containing the bash command. + Use \\n to represent line breaks, ensuring the command conforms to the JSON format and is displayed on a single line. Except for the `edit` command, each parameter of the command needs to be enclosed in single quotes. As shown in the example below: @@ -31,42 +29,75 @@ First I'll start by using ls to see what files are in the current directory. The }} ``` - You should only include a *SINGLE* command in the command section and then wait for a response from the shell before continuing with more discussion and commands. Everything you include in the DISCUSSION section will be saved for future reference. If you'd like to issue two commands at once, PLEASE DO NOT DO THAT! Please instead first submit just the first command, and then after receiving a response you'll be able to issue the second command. -You're free to use any other bash commands you want (e.g. find, grep, cat, ls, cd) in addition to the special commands listed above. +Remember, YOU CAN ONLY ENTER ONE COMMAND AT A TIME. You should always wait for feedback after every command. + +You can use any bash commands you want (e.g., find, grep, cat, ls, cd) or any custom special tools (including `edit`) by calling Bash.run. Edit all the files you need. You should carefully observe the behavior and results of the previous action, and avoid triggering repeated errors. -However, the environment does NOT support interactive session commands (e.g. python, vim), so please do not invoke them. +However, the Bash.run does NOT support interactive session commands (e.g. python, vim), so please do not invoke them. + +In addition to the terminal, I also provide additional tools. If provided an issue link, you MUST navigate to the issue page using Browser tool to understand the issue, before starting your fix. + +# INSTRUCTIONS: +Your first action must be to check if the repository exists at the current path. If it exists, navigate to the repository path. If the repository doesn't exist, please download it and then navigate to it. +All subsequent actions must be performed within this repository path. Do not leave this directory to execute any actions at any time. +Your terminal session has started, and you can use any bash commands or the special interface to help you. Edit all the files you need. """ MINIMAL_EXAMPLE = """ ## Example of a actions trajectory User Requirement and Issue: Fix the bug in the repo. Because the environment is not available, you DO NOT need to run and modify any existing test case files or add new test case files to ensure that the bug is fixed. -### Locate issue(Require): Locate the issue in the code by searching for the relevant file, function, or class and open the file to view the code. -cd /workspace/django__django_3.0 +### Read and understand issue(Require): +{{ + "command_name": "Browser.goto", + "args": {{ + "url": "https://github.com/geekan/MetaGPT/issues/1275" + }} +}} -> -search_dir_and_preview ASCIIUsernameValidator --> -open /workspace/django__django_3.0/django/contrib/auth/validators.py --> -### Fix the Bug(Require): Fix the bug in the code by editing the relevant function, class or code snippet. -edit 10:20 < + +Bash.run(cmd='search_dir_and_preview ASCIIUsernameValidator') +{{ + "command_name": "Bash.run", + "args": {{ + "cmd": "open /workspace/django__django_3.0/django/contrib/auth/validators.py" + }} +}} +-> + +### Fix the Bug(Require): Fix the bug in the code by editing the relevant function, class or code snippet. +{{ + "command_name": "Bash.run", + "args": {{ + "cmd": "edit 10:20 < + ### Submit the Changes(Require): Submit the changes to the repository. -submit +{{ + "command_name": "Bash.run", + "args": {{ + "cmd": "submit" + }} +}} +Bash.run(cmd='submit') +-> +{{ + "command_name": "end", +}} """ @@ -132,6 +163,10 @@ IMPORTANT_TIPS = """ - Based on feedback of observation or bash command in trajectory to guide adjustments in your search strategy. 13. If the task results in succeed, fail, or NO PROGRESS, output `submit`. + +14. If provided an issue link, you MUST go to the issue page using Browser tool to understand the issue before starting your fix. + +15. When the edit fails, try to enlarge the starting line. """ NEXT_STEP_TEMPLATE = f""" diff --git a/metagpt/roles/di/swe.py b/metagpt/roles/di/swe.py index 6d357c02b..915d186b4 100644 --- a/metagpt/roles/di/swe.py +++ b/metagpt/roles/di/swe.py @@ -22,8 +22,7 @@ class SWE(RoleZero): _system_msg: str = SWE_AGENT_SYSTEM_TEMPLATE system_msg: list[str] = [_system_msg.format(WINDOW=_bash_window_size)] _instruction: str = NEXT_STEP_TEMPLATE - # tools: list[str] = ["Bash", "Browser"] - tools: list[str] = ["Bash"] + tools: list[str] = ["Bash", "Browser:goto,scroll"] terminal: Bash = Field(default_factory=Bash, exclude=True) output_diff: str = "" max_react_loop: int = 30 @@ -75,11 +74,10 @@ class SWE(RoleZero): if not ok: return for cmd in commands: - if "submit" not in cmd.get("args", {}).get("cmd", ""): + if "end" != cmd.get("command_name", ""): return try: - # Generate patch by git diff - diff_output = self.terminal.run("git diff") + diff_output = self.terminal.run("git diff --cached") clear_diff = extract_patch(diff_output) logger.info(f"Diff output: \n{clear_diff}") if clear_diff: diff --git a/metagpt/tools/libs/browser.py b/metagpt/tools/libs/browser.py index c6ea71bd5..864996e8c 100644 --- a/metagpt/tools/libs/browser.py +++ b/metagpt/tools/libs/browser.py @@ -122,6 +122,8 @@ class Browser: async def goto(self, url: str, timeout: float = 30000): """Navigate to a specific URL.""" + if self.page is None: + await self.start() async with self.reporter as reporter: await reporter.async_report(url, "url") await self.page.goto(url, timeout=timeout) diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index 938eadff4..a04acb8e9 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -2,7 +2,7 @@ import subprocess import threading from queue import Queue -from metagpt.const import SWE_SETUP_PATH +from metagpt.const import SWE_SETUP_PATH, SWE_WORKSPACE_ROOT from metagpt.tools.tool_registry import register_tool from metagpt.utils.report import END_MARKER_VALUE, TerminalReporter @@ -136,13 +136,14 @@ class Terminal: class Bash(Terminal): """ A class to run bash commands directly and provides custom shell functions. + All custom functions in this class can ONLY be called via the `Bash.run` method. """ def __init__(self): """init""" super().__init__() setup_cmd = f"source {SWE_SETUP_PATH}" - self.run_command(setup_cmd) + self.run_command(f"cd {SWE_WORKSPACE_ROOT} && {setup_cmd}") def run(self, cmd) -> str: """ @@ -184,7 +185,7 @@ class Bash(Terminal): filename (str): The name of the file to create. - submit - Submits your current code and terminates the session. + Submits your current code. it can only be executed once, the last action before the `end`. - search_dir_and_preview [] Searches for search_term in all files in dir and gives their code preview diff --git a/metagpt/tools/swe_agent_commands/defaults.sh b/metagpt/tools/swe_agent_commands/defaults.sh index f0898aabc..d416dcbf5 100644 --- a/metagpt/tools/swe_agent_commands/defaults.sh +++ b/metagpt/tools/swe_agent_commands/defaults.sh @@ -177,7 +177,7 @@ create() { # @yaml # signature: submit -# docstring: submits your current code and terminates the session. this is the only submit action needed; no need to run git add or git commit before this. +# docstring: submits your current code. the last action before the `end`, it can only be executed once. submit() { # Check if the patch file exists and is non-empty if [ -s "$SWE_CMD_WORK_DIR/test.patch" ]; then @@ -186,8 +186,7 @@ submit() { fi git add -A - git diff --cached > model.patch - echo "<>" + echo "<>" } diff --git a/metagpt/tools/swe_agent_commands/swe_agent_utils.py b/metagpt/tools/swe_agent_commands/swe_agent_utils.py index 8c01dc9c9..9e293f4d2 100644 --- a/metagpt/tools/swe_agent_commands/swe_agent_utils.py +++ b/metagpt/tools/swe_agent_commands/swe_agent_utils.py @@ -16,10 +16,12 @@ def extract_patch(command_output): def load_hf_dataset(dataset_name_or_path: str, cache_dir, split: str = "test", existing_ids: list = []): - if Path(dataset_name_or_path).exists(): - dataset = load_from_disk(dataset_name_or_path) + data_dir = cache_dir / dataset_name_or_path + if Path(data_dir).exists(): + dataset = load_from_disk(data_dir) else: - dataset = load_dataset(dataset_name_or_path, cache_dir=cache_dir) + dataset = load_dataset(dataset_name_or_path) + dataset.save_to_disk(data_dir) print(dataset) if split not in dataset: raise ValueError(f"Invalid split {split} for dataset {dataset_name_or_path}") diff --git a/tests/metagpt/roles/di/run_swe.py b/tests/metagpt/roles/di/run_swe.py index c6cc56fd1..f9d19be74 100644 --- a/tests/metagpt/roles/di/run_swe.py +++ b/tests/metagpt/roles/di/run_swe.py @@ -12,7 +12,7 @@ from metagpt.tools.swe_agent_commands.swe_agent_utils import load_hf_dataset # Specify by yourself TEST_REPO_DIR = Path("/Users/seeker/Projects/sdfz/mg/mg-swe-agent") / "benchmark" / "swe_bench" / "data" / "test_repo" -DATA_DIR = METAGPT_ROOT / "benchmark" / "swe_bench" / "data" +DATA_DIR = METAGPT_ROOT / "data/hugging_face" INSTANCE_TEMPLATE = """ ## User Requirement @@ -27,8 +27,9 @@ hints text is the comment under issue: {hints_text} The repository may already exist at the path `{repo_path}`. If it doesn't, please download the repository to this path. -All your subsequent actions should use the project path as your root directory, and you should never leave that directory to execute any actions. +Your first action must be to navigate to the repository path `{repo_path}`. This issue occurred in version {version}, with the corresponding base commit being {base_commit}. You need to switch to the code version associated with this commit. +All subsequent actions must be performed within this repository path. Do not leave this directory to execute any actions at any time. # INSTRUCTIONS: Now, you're going to solve this issue on your own from the perspective of a programmer. Your terminal session has started and you're in the repository's root directory. You can use any bash commands or the special interface to help you. Edit all the files you need. @@ -36,38 +37,6 @@ Remember, YOU CAN ONLY ENTER ONE COMMAND AT A TIME. You should always wait for f """ -def split_dataset_equally(dataset): - # 计算索引 - # fixme: 设置django - - part1 = dataset.filter( - lambda x: x["repo"] - not in [ - "django/django", - "sympy/sympy", - "pytest-dev/pytest", - ], - desc="Filtering out existing ids", - load_from_cache_file=True, - ) - - part2 = dataset.filter( - lambda x: x["repo"] in ["sympy/sympy", "pytest-dev/pytest"], - desc="Filtering out existing ids", - load_from_cache_file=True, - ) - - part3 = dataset.filter( - lambda x: x["repo"] in ["django/django"], - desc="Filtering out existing ids", - load_from_cache_file=False, - ) - - print(len(part1), len(part2), len(part3)) - - return [part1, part2, part3] - - def check_instance_status(instance, swe_result_dir): output_file = swe_result_dir / "all_preds.jsonl" res = True @@ -87,12 +56,20 @@ async def run(instance, swe_result_dir): logger.info(f"Instance {instance['instance_id']} already exists, skipping execution.") return - repo_path = TEST_REPO_DIR / (instance["repo"].replace("-", "_").replace("/", "__") + "_" + instance["version"]) - """ - All your subsequent actions should use the project path as your root directory, and you should never leave that directory to execute any actions. - """ + repo_path = Path("/Users/seeker/Projects/other/test_repo") / ( + instance["repo"].replace("-", "_").replace("/", "__") + "_" + instance["version"] + ) + # repo_path = Path("/Users/seeker/Projects/other/test_repo") / instance["repo"].split("/")[-1] + + # 前处理 terminal = Terminal() - terminal.run_command(f"cd {repo_path} && git checkout . && git clean -n -d && git clean -f -d") + terminal.run_command(f"cd {repo_path} && git reset --hard && git clean -n -d && git clean -f -d") + terminal.run_command("BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}')") + logger.info(terminal.run_command("echo $BRANCH")) + # logger.info(terminal.run_command(f'Branch name: $BRANCH')) + logger.info(terminal.run_command('git checkout "$BRANCH"')) + logger.info(terminal.run_command("git branch")) + user_requirement_and_issue = INSTANCE_TEMPLATE.format( issue=instance["problem_statement"], hints_text=instance["hints_text"], @@ -126,16 +103,14 @@ async def async_main(): dataset_path = "manna-ai/SWE-bench_Nano" # "princeton-nlp/SWE-bench_Lite" #"manna-ai/SWE-bench_Nano" dataset = load_hf_dataset(dataset_name_or_path=dataset_path, cache_dir=DATA_DIR, split="test") - sample_datasets = split_dataset_equally(dataset) date_time = datetime.now().strftime("%m-%d") - round_ = "third" - - for idx, sub_dataset in enumerate(sample_datasets): - exp_name = f"nano_mgx_{date_time}_{round_}_part_{idx}" - swe_result_dir = DEFAULT_WORKSPACE_ROOT / f"result_{config.llm.model}" / exp_name - swe_result_dir.mkdir(parents=True, exist_ok=True) - for instance in sub_dataset: - await run(instance, swe_result_dir) + # _round = "first" + _round = "second" + exp_name = f"nano_mgx_{date_time}_{_round}" + swe_result_dir = DEFAULT_WORKSPACE_ROOT / f"result_{config.llm.model.replace('/', '_')}" / exp_name + swe_result_dir.mkdir(parents=True, exist_ok=True) + for instance in dataset: + await run(instance, swe_result_dir) if __name__ == "__main__": From d571fe0f90a00877921141264ea95a815fdafe19 Mon Sep 17 00:00:00 2001 From: seeker Date: Wed, 3 Jul 2024 20:26:55 +0800 Subject: [PATCH 2/9] update: SWE Agent --- tests/metagpt/roles/di/run_swe_new.py | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/metagpt/roles/di/run_swe_new.py diff --git a/tests/metagpt/roles/di/run_swe_new.py b/tests/metagpt/roles/di/run_swe_new.py new file mode 100644 index 000000000..94ef1e644 --- /dev/null +++ b/tests/metagpt/roles/di/run_swe_new.py @@ -0,0 +1,38 @@ +import asyncio + +from metagpt.logs import logger +from metagpt.roles.di.swe import SWE + +FIX_ISSUE1 = """ +Write a fix for this issue: https://github.com/langchain-ai/langchain/issues/20453, +you can fix it on this repo https://github.com/garylin2099/langchain, +""" + +"checkout a branch named test-fix, commit your changes, push, and create a PR to the master branch of https://github.com/iorisa/langchain" + +FIX_ISSUE2 = """ +Write a fix for this issue https://github.com/geekan/MetaGPT/issues/1275. +You can fix it on the v0.8-release branch of this repo https://github.com/garylin2099/MetaGPT, +during fixing, checkout a branch named test-fix-1275, commit your changes, push, and create a PR to the v0.8-release branch of https://github.com/garylin2099/MetaGPT +""" +FIX_ISSUE3 = """ +Write a fix for this issue https://github.com/geekan/MetaGPT/issues/1262. +You can fix it on this repo https://github.com/garylin2099/MetaGPT, +during fixing, checkout a branch named test-fix-1262, commit your changes, push, and create a PR to https://github.com/garylin2099/MetaGPT +""" +FIX_ISSUE_SIMPLE = """ +Write a fix for this issue: https://github.com/mannaandpoem/simple_calculator/issues/1, +you can fix it on this repo https://github.com/garylin2099/simple_calculator, +checkout a branch named test, commit your changes, push, and create a PR to the master branch of original repo. +""" + +if __name__ == "__main__": + swe_agent = SWE() + logger.info("**** Starting run ****") + user_requirement_and_issue = ( + FIX_ISSUE1 + + """Because the environment is not available, you DO NOT need to run and modify any existing test case files or add new test case files to ensure that the bug is fixed.""" + ) + asyncio.run(swe_agent.run(user_requirement_and_issue)) + logger.info("**** Finished running ****") + logger.info(f"Patch: {swe_agent.output_diff}") From ca0cb7639f7d764044bd3df380f68fe90b03eaca Mon Sep 17 00:00:00 2001 From: seeker Date: Wed, 3 Jul 2024 22:37:17 +0800 Subject: [PATCH 3/9] update: SWE Agent --- metagpt/const.py | 2 ++ metagpt/roles/di/swe.py | 2 +- tests/metagpt/roles/di/run_swe_new.py | 11 +++++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index 94d22bc70..6a0e2c4bb 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -150,5 +150,7 @@ METAGPT_REPORTER_DEFAULT_URL = os.environ.get("METAGPT_REPORTER_URL", "") # Metadata defines AGENT = "agent" SWE_WORKSPACE_ROOT = Path("/tmp/swe_workspace") +if not SWE_WORKSPACE_ROOT.exists(): + SWE_WORKSPACE_ROOT.mkdir(parents=True) # SWE agent SWE_SETUP_PATH = METAGPT_ROOT / "metagpt/tools/swe_agent_commands/setup_default.sh" diff --git a/metagpt/roles/di/swe.py b/metagpt/roles/di/swe.py index 915d186b4..dde686e6a 100644 --- a/metagpt/roles/di/swe.py +++ b/metagpt/roles/di/swe.py @@ -25,7 +25,7 @@ class SWE(RoleZero): tools: list[str] = ["Bash", "Browser:goto,scroll"] terminal: Bash = Field(default_factory=Bash, exclude=True) output_diff: str = "" - max_react_loop: int = 30 + max_react_loop: int = 40 async def _think(self) -> bool: self._set_system_msg() diff --git a/tests/metagpt/roles/di/run_swe_new.py b/tests/metagpt/roles/di/run_swe_new.py index 94ef1e644..8ed1e9107 100644 --- a/tests/metagpt/roles/di/run_swe_new.py +++ b/tests/metagpt/roles/di/run_swe_new.py @@ -7,14 +7,14 @@ FIX_ISSUE1 = """ Write a fix for this issue: https://github.com/langchain-ai/langchain/issues/20453, you can fix it on this repo https://github.com/garylin2099/langchain, """ - -"checkout a branch named test-fix, commit your changes, push, and create a PR to the master branch of https://github.com/iorisa/langchain" +# + "checkout a branch named test-fix, commit your changes, push, and create a PR to the master branch of https://github.com/iorisa/langchain" FIX_ISSUE2 = """ Write a fix for this issue https://github.com/geekan/MetaGPT/issues/1275. You can fix it on the v0.8-release branch of this repo https://github.com/garylin2099/MetaGPT, -during fixing, checkout a branch named test-fix-1275, commit your changes, push, and create a PR to the v0.8-release branch of https://github.com/garylin2099/MetaGPT """ +# + "during fixing, checkout a branch named test-fix-1275, commit your changes, push, and create a PR to the v0.8-release branch of https://github.com/garylin2099/MetaGPT" + FIX_ISSUE3 = """ Write a fix for this issue https://github.com/geekan/MetaGPT/issues/1262. You can fix it on this repo https://github.com/garylin2099/MetaGPT, @@ -30,7 +30,10 @@ if __name__ == "__main__": swe_agent = SWE() logger.info("**** Starting run ****") user_requirement_and_issue = ( - FIX_ISSUE1 + # FIX_ISSUE1 + # FIX_ISSUE2 + # FIX_ISSUE3 + FIX_ISSUE_SIMPLE + """Because the environment is not available, you DO NOT need to run and modify any existing test case files or add new test case files to ensure that the bug is fixed.""" ) asyncio.run(swe_agent.run(user_requirement_and_issue)) From 5c0f490c049387dc334ea623f9efc2afb505ee7d Mon Sep 17 00:00:00 2001 From: seeker Date: Wed, 3 Jul 2024 22:46:21 +0800 Subject: [PATCH 4/9] update: SWE Agent --- metagpt/tools/libs/terminal.py | 4 ++-- tests/metagpt/roles/di/run_swe.py | 15 +++++---------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index a04acb8e9..3270b0623 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -142,8 +142,8 @@ class Bash(Terminal): def __init__(self): """init""" super().__init__() - setup_cmd = f"source {SWE_SETUP_PATH}" - self.run_command(f"cd {SWE_WORKSPACE_ROOT} && {setup_cmd}") + self.run_command(f"cd {SWE_WORKSPACE_ROOT}") + self.run_command(f"source {SWE_SETUP_PATH}") def run(self, cmd) -> str: """ diff --git a/tests/metagpt/roles/di/run_swe.py b/tests/metagpt/roles/di/run_swe.py index f9d19be74..9965107db 100644 --- a/tests/metagpt/roles/di/run_swe.py +++ b/tests/metagpt/roles/di/run_swe.py @@ -1,7 +1,6 @@ import asyncio import json from datetime import datetime -from pathlib import Path from metagpt.config2 import config from metagpt.const import DEFAULT_WORKSPACE_ROOT, METAGPT_ROOT @@ -11,7 +10,7 @@ from metagpt.tools.libs.terminal import Terminal from metagpt.tools.swe_agent_commands.swe_agent_utils import load_hf_dataset # Specify by yourself -TEST_REPO_DIR = Path("/Users/seeker/Projects/sdfz/mg/mg-swe-agent") / "benchmark" / "swe_bench" / "data" / "test_repo" +TEST_REPO_DIR = METAGPT_ROOT / "data" / "test_repo" DATA_DIR = METAGPT_ROOT / "data/hugging_face" INSTANCE_TEMPLATE = """ @@ -56,17 +55,13 @@ async def run(instance, swe_result_dir): logger.info(f"Instance {instance['instance_id']} already exists, skipping execution.") return - repo_path = Path("/Users/seeker/Projects/other/test_repo") / ( - instance["repo"].replace("-", "_").replace("/", "__") + "_" + instance["version"] - ) - # repo_path = Path("/Users/seeker/Projects/other/test_repo") / instance["repo"].split("/")[-1] + repo_path = TEST_REPO_DIR / instance["repo"].replace("-", "_").replace("/", "__") + "_" + instance["version"] # 前处理 terminal = Terminal() terminal.run_command(f"cd {repo_path} && git reset --hard && git clean -n -d && git clean -f -d") terminal.run_command("BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}')") logger.info(terminal.run_command("echo $BRANCH")) - # logger.info(terminal.run_command(f'Branch name: $BRANCH')) logger.info(terminal.run_command('git checkout "$BRANCH"')) logger.info(terminal.run_command("git branch")) @@ -103,9 +98,9 @@ async def async_main(): dataset_path = "manna-ai/SWE-bench_Nano" # "princeton-nlp/SWE-bench_Lite" #"manna-ai/SWE-bench_Nano" dataset = load_hf_dataset(dataset_name_or_path=dataset_path, cache_dir=DATA_DIR, split="test") - date_time = datetime.now().strftime("%m-%d") - # _round = "first" - _round = "second" + date_time = datetime.now().strftime("%m%d") + _round = "first" + # _round = "second" exp_name = f"nano_mgx_{date_time}_{_round}" swe_result_dir = DEFAULT_WORKSPACE_ROOT / f"result_{config.llm.model.replace('/', '_')}" / exp_name swe_result_dir.mkdir(parents=True, exist_ok=True) From 816a6eadde46c0ea72ffcbae060a5bda20f3c29d Mon Sep 17 00:00:00 2001 From: seeker Date: Thu, 4 Jul 2024 16:07:22 +0800 Subject: [PATCH 5/9] update: SWE Agent --- metagpt/roles/di/{swe.py => swe_agent.py} | 12 +++--- metagpt/tools/libs/terminal.py | 4 +- metagpt/tools/swe_agent_commands/__init__.py | 2 +- ...n_swe.py => run_sweagent_for_benchmark.py} | 7 ++-- ...w.py => run_sweagent_open_source_issue.py} | 41 ++++++++++--------- 5 files changed, 36 insertions(+), 30 deletions(-) rename metagpt/roles/di/{swe.py => swe_agent.py} (92%) rename tests/metagpt/roles/di/{run_swe.py => run_sweagent_for_benchmark.py} (96%) rename tests/metagpt/roles/di/{run_swe_new.py => run_sweagent_open_source_issue.py} (53%) diff --git a/metagpt/roles/di/swe.py b/metagpt/roles/di/swe_agent.py similarity index 92% rename from metagpt/roles/di/swe.py rename to metagpt/roles/di/swe_agent.py index dde686e6a..e86b50a9d 100644 --- a/metagpt/roles/di/swe.py +++ b/metagpt/roles/di/swe_agent.py @@ -14,9 +14,9 @@ from metagpt.tools.libs.terminal import Bash from metagpt.tools.swe_agent_commands.swe_agent_utils import extract_patch -class SWE(RoleZero): - name: str = "SweAgent" - profile: str = "Software Engineer" +class SWEAgent(RoleZero): + name: str = "Swen" + profile: str = "Issue Solver" goal: str = "Resolve GitHub issue" _bash_window_size: int = 100 _system_msg: str = SWE_AGENT_SYSTEM_TEMPLATE @@ -26,12 +26,14 @@ class SWE(RoleZero): terminal: Bash = Field(default_factory=Bash, exclude=True) output_diff: str = "" max_react_loop: int = 40 + run_eval: bool = False async def _think(self) -> bool: self._set_system_msg() self._format_instruction() res = await super()._think() - await self._handle_action() + if self.run_eval: + await self._parse_commands_for_eval() return res def _set_system_msg(self): @@ -61,7 +63,7 @@ class SWE(RoleZero): return self.instruction - async def _handle_action(self): + async def _parse_commands_for_eval(self): """ Handles actions based on parsed commands. diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index 3270b0623..bcf039a5e 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -2,7 +2,7 @@ import subprocess import threading from queue import Queue -from metagpt.const import SWE_SETUP_PATH, SWE_WORKSPACE_ROOT +from metagpt.const import DEFAULT_WORKSPACE_ROOT, SWE_SETUP_PATH from metagpt.tools.tool_registry import register_tool from metagpt.utils.report import END_MARKER_VALUE, TerminalReporter @@ -142,7 +142,7 @@ class Bash(Terminal): def __init__(self): """init""" super().__init__() - self.run_command(f"cd {SWE_WORKSPACE_ROOT}") + self.run_command(f"cd {DEFAULT_WORKSPACE_ROOT}") self.run_command(f"source {SWE_SETUP_PATH}") def run(self, cmd) -> str: diff --git a/metagpt/tools/swe_agent_commands/__init__.py b/metagpt/tools/swe_agent_commands/__init__.py index 42e92a12d..c0d3e2a60 100644 --- a/metagpt/tools/swe_agent_commands/__init__.py +++ b/metagpt/tools/swe_agent_commands/__init__.py @@ -1,5 +1,5 @@ """ -This tool is originally developed by the team behind the princeton-nlp/SWE-agent repository. +This folder is borrowed from princeton-nlp/SWE-agent You can find the original repository here: https://github.com/princeton-nlp/SWE-agent/tree/main/config/commands We are using a modified version from OpenDevin: diff --git a/tests/metagpt/roles/di/run_swe.py b/tests/metagpt/roles/di/run_sweagent_for_benchmark.py similarity index 96% rename from tests/metagpt/roles/di/run_swe.py rename to tests/metagpt/roles/di/run_sweagent_for_benchmark.py index 9965107db..cdd8df5d7 100644 --- a/tests/metagpt/roles/di/run_swe.py +++ b/tests/metagpt/roles/di/run_sweagent_for_benchmark.py @@ -5,7 +5,7 @@ from datetime import datetime from metagpt.config2 import config from metagpt.const import DEFAULT_WORKSPACE_ROOT, METAGPT_ROOT from metagpt.logs import logger -from metagpt.roles.di.swe import SWE +from metagpt.roles.di.swe_agent import SWEAgent from metagpt.tools.libs.terminal import Terminal from metagpt.tools.swe_agent_commands.swe_agent_utils import load_hf_dataset @@ -74,13 +74,14 @@ async def run(instance, swe_result_dir): ) logger.info(f"**** Starting to run {instance['instance_id']}****") - swe_agent = SWE() + swe_agent = SWEAgent() + swe_agent.run_eval = True await swe_agent.run(user_requirement_and_issue) save_predictions(swe_agent, instance, swe_result_dir) logger.info(f"**** Finished running {instance['instance_id']}****") -def save_predictions(swe_agent: SWE, instance, swe_result_dir): +def save_predictions(swe_agent: SWEAgent, instance, swe_result_dir): output_file = swe_result_dir / "all_preds.jsonl" instance["model_name_or_path"] = swe_agent.config.llm.model instance["model_patch"] = swe_agent.output_diff diff --git a/tests/metagpt/roles/di/run_swe_new.py b/tests/metagpt/roles/di/run_sweagent_open_source_issue.py similarity index 53% rename from tests/metagpt/roles/di/run_swe_new.py rename to tests/metagpt/roles/di/run_sweagent_open_source_issue.py index 8ed1e9107..ec87dd7e2 100644 --- a/tests/metagpt/roles/di/run_swe_new.py +++ b/tests/metagpt/roles/di/run_sweagent_open_source_issue.py @@ -1,41 +1,44 @@ import asyncio from metagpt.logs import logger -from metagpt.roles.di.swe import SWE +from metagpt.roles.di.swe_agent import SWEAgent FIX_ISSUE1 = """ Write a fix for this issue: https://github.com/langchain-ai/langchain/issues/20453, -you can fix it on this repo https://github.com/garylin2099/langchain, +you can fix it on this repo https://github.com/garylin2099/langchain """ -# + "checkout a branch named test-fix, commit your changes, push, and create a PR to the master branch of https://github.com/iorisa/langchain" - +# + "checkout a branch named test-fix, commit your changes, push, +# and create a PR to the master branch of https://github.com/iorisa/langchain" +# """ FIX_ISSUE2 = """ Write a fix for this issue https://github.com/geekan/MetaGPT/issues/1275. -You can fix it on the v0.8-release branch of this repo https://github.com/garylin2099/MetaGPT, +You can fix it on the v0.8-release branch of this repo https://github.com/garylin2099/MetaGPT """ -# + "during fixing, checkout a branch named test-fix-1275, commit your changes, push, and create a PR to the v0.8-release branch of https://github.com/garylin2099/MetaGPT" +# + "during fixing, checkout a branch named test-fix-1275, commit your changes, push, +# and create a PR to the v0.8-release branch of https://github.com/garylin2099/MetaGPT" FIX_ISSUE3 = """ Write a fix for this issue https://github.com/geekan/MetaGPT/issues/1262. -You can fix it on this repo https://github.com/garylin2099/MetaGPT, -during fixing, checkout a branch named test-fix-1262, commit your changes, push, and create a PR to https://github.com/garylin2099/MetaGPT +You can fix it on this repo https://github.com/garylin2099/MetaGPT """ +# during fixing, checkout a branch named test-fix-1262, commit your changes, push, +# and create a PR to https://github.com/garylin2099/MetaGPT +# """ FIX_ISSUE_SIMPLE = """ Write a fix for this issue: https://github.com/mannaandpoem/simple_calculator/issues/1, -you can fix it on this repo https://github.com/garylin2099/simple_calculator, -checkout a branch named test, commit your changes, push, and create a PR to the master branch of original repo. +you can fix it on this repo https://github.com/garylin2099/simple_calculator """ +# checkout a branch named test, commit your changes, push, and create a PR to the master branch of original repo. +# """ + +NO_ENV_TIP = """ +Because the environment is not available, you DO NOT need to run and modify any existing test case files or +add new test case files to ensure that the bug is fixed. +""" if __name__ == "__main__": - swe_agent = SWE() + swe_agent = SWEAgent() logger.info("**** Starting run ****") - user_requirement_and_issue = ( - # FIX_ISSUE1 - # FIX_ISSUE2 - # FIX_ISSUE3 - FIX_ISSUE_SIMPLE - + """Because the environment is not available, you DO NOT need to run and modify any existing test case files or add new test case files to ensure that the bug is fixed.""" - ) + user_requirement_and_issue = FIX_ISSUE1 + NO_ENV_TIP asyncio.run(swe_agent.run(user_requirement_and_issue)) logger.info("**** Finished running ****") - logger.info(f"Patch: {swe_agent.output_diff}") From 51bba81415a3096a4ebb974b24a4fee10bbccd84 Mon Sep 17 00:00:00 2001 From: seeker Date: Thu, 4 Jul 2024 16:12:49 +0800 Subject: [PATCH 6/9] update: SWE Agent --- metagpt/roles/di/swe_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/di/swe_agent.py b/metagpt/roles/di/swe_agent.py index e86b50a9d..166de3918 100644 --- a/metagpt/roles/di/swe_agent.py +++ b/metagpt/roles/di/swe_agent.py @@ -29,14 +29,14 @@ class SWEAgent(RoleZero): run_eval: bool = False async def _think(self) -> bool: - self._set_system_msg() + self._update_system_msg() self._format_instruction() res = await super()._think() if self.run_eval: await self._parse_commands_for_eval() return res - def _set_system_msg(self): + def _update_system_msg(self): """ Sets the system message for the SWE agent. From 729c3914635751869da9fd94d1253e47c43af802 Mon Sep 17 00:00:00 2001 From: seeker Date: Thu, 4 Jul 2024 16:18:36 +0800 Subject: [PATCH 7/9] update: SWE Agent --- metagpt/prompts/di/{swe.py => swe_agent.py} | 0 ...n_sweagent_for_benchmark.py => run_swe_agent_for_benchmark.py} | 0 ...nt_open_source_issue.py => run_swe_agent_open_source_issue.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename metagpt/prompts/di/{swe.py => swe_agent.py} (100%) rename tests/metagpt/roles/di/{run_sweagent_for_benchmark.py => run_swe_agent_for_benchmark.py} (100%) rename tests/metagpt/roles/di/{run_sweagent_open_source_issue.py => run_swe_agent_open_source_issue.py} (100%) diff --git a/metagpt/prompts/di/swe.py b/metagpt/prompts/di/swe_agent.py similarity index 100% rename from metagpt/prompts/di/swe.py rename to metagpt/prompts/di/swe_agent.py diff --git a/tests/metagpt/roles/di/run_sweagent_for_benchmark.py b/tests/metagpt/roles/di/run_swe_agent_for_benchmark.py similarity index 100% rename from tests/metagpt/roles/di/run_sweagent_for_benchmark.py rename to tests/metagpt/roles/di/run_swe_agent_for_benchmark.py diff --git a/tests/metagpt/roles/di/run_sweagent_open_source_issue.py b/tests/metagpt/roles/di/run_swe_agent_open_source_issue.py similarity index 100% rename from tests/metagpt/roles/di/run_sweagent_open_source_issue.py rename to tests/metagpt/roles/di/run_swe_agent_open_source_issue.py From cdd222c4c50e6f6f1169c84cc4193e249023e7a8 Mon Sep 17 00:00:00 2001 From: seeker Date: Thu, 4 Jul 2024 16:25:13 +0800 Subject: [PATCH 8/9] update: SWE Agent --- metagpt/const.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index 6a0e2c4bb..c78a22641 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -149,8 +149,6 @@ METAGPT_REPORTER_DEFAULT_URL = os.environ.get("METAGPT_REPORTER_URL", "") # Metadata defines AGENT = "agent" -SWE_WORKSPACE_ROOT = Path("/tmp/swe_workspace") -if not SWE_WORKSPACE_ROOT.exists(): - SWE_WORKSPACE_ROOT.mkdir(parents=True) + # SWE agent SWE_SETUP_PATH = METAGPT_ROOT / "metagpt/tools/swe_agent_commands/setup_default.sh" From c5a39fcf408eb4ed2efd977d396319207f905e7e Mon Sep 17 00:00:00 2001 From: seeker Date: Thu, 4 Jul 2024 17:40:36 +0800 Subject: [PATCH 9/9] fix: bug --- metagpt/roles/di/swe_agent.py | 2 +- tests/metagpt/roles/di/run_swe_agent_for_benchmark.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/di/swe_agent.py b/metagpt/roles/di/swe_agent.py index 166de3918..5acce5f9d 100644 --- a/metagpt/roles/di/swe_agent.py +++ b/metagpt/roles/di/swe_agent.py @@ -4,7 +4,7 @@ import os from pydantic import Field from metagpt.logs import logger -from metagpt.prompts.di.swe import ( +from metagpt.prompts.di.swe_agent import ( MINIMAL_EXAMPLE, NEXT_STEP_TEMPLATE, SWE_AGENT_SYSTEM_TEMPLATE, diff --git a/tests/metagpt/roles/di/run_swe_agent_for_benchmark.py b/tests/metagpt/roles/di/run_swe_agent_for_benchmark.py index cdd8df5d7..54b3623a4 100644 --- a/tests/metagpt/roles/di/run_swe_agent_for_benchmark.py +++ b/tests/metagpt/roles/di/run_swe_agent_for_benchmark.py @@ -55,7 +55,7 @@ async def run(instance, swe_result_dir): logger.info(f"Instance {instance['instance_id']} already exists, skipping execution.") return - repo_path = TEST_REPO_DIR / instance["repo"].replace("-", "_").replace("/", "__") + "_" + instance["version"] + repo_path = TEST_REPO_DIR / (instance["repo"].replace("-", "_").replace("/", "__") + "_" + instance["version"]) # 前处理 terminal = Terminal()