From 631a2642e81e5746c11a933ff301ddca02b8bc4b Mon Sep 17 00:00:00 2001 From: stellahsr Date: Tue, 19 Mar 2024 23:19:58 +0800 Subject: [PATCH 1/7] 1. add testbed path 2. update repo parse and git ops --- data/inference/const.py | 3 ++- data/inference/run_api.py | 22 +++++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/data/inference/const.py b/data/inference/const.py index 42e63ec0e..62d96fe69 100644 --- a/data/inference/const.py +++ b/data/inference/const.py @@ -3,10 +3,11 @@ # @Desc : import pandas as pd -from metagpt.const import METAGPT_ROOT +from metagpt.const import DATA_PATH, METAGPT_ROOT SUBSET_DATASET = METAGPT_ROOT / "sub_swebench_dataset" / "sub_swebench.csv" SUBSET_DATASET_SKLERARN = METAGPT_ROOT / "sub_swebench_dataset" / "scikit-learn-68.csv" +TESTBED = DATA_PATH / "repos" # SCIKIT_LEARN_IDS: A list of instance identifiers from 'sub_swebench.csv' within SUBSET_DATASET. # This collection represents a subset specifically related to scikit-learn content. diff --git a/data/inference/run_api.py b/data/inference/run_api.py index 7882f13e7..00dbdd0e0 100644 --- a/data/inference/run_api.py +++ b/data/inference/run_api.py @@ -10,7 +10,9 @@ from make_datasets.utils import extract_diff from tenacity import retry, stop_after_attempt, wait_random_exponential from tqdm.auto import tqdm -from data.inference.const import SCIKIT_LEARN_IDS +from data.inference.const import SCIKIT_LEARN_IDS, TESTBED +from data.inference.make_datasets.parse_diff import extract_scripts_from_codetext +from data.inference.make_datasets.repo_utils import EnvManager from metagpt.config2 import config from metagpt.logs import logger from metagpt.roles.di.data_interpreter import DataInterpreter @@ -33,8 +35,12 @@ async def call_chat(inputs, interpreter): requirement = "Please rewrite the code and generate test case to address the issues existing in the repository. If the test code passes, it is considered that the execution code has passed and use the `git diff` command to output the patch based on the correct code." system_messages = inputs.split("\n", 1)[0] user_message = inputs.split("\n", 1)[1] + cleaned_user_message = user_message.split( + "I need you to solve this issue by generating a single patch file that I can apply directly to this repository using git apply. Please respond with a single patch file in the following format." + )[0] + try: - await interpreter.run([requirement, system_messages, user_message]) + await interpreter.run([requirement, system_messages, cleaned_user_message]) return interpreter.get_last_cell_source() except Exception as e: logger.error(f"Error: {e}\nInputs: {inputs}") @@ -70,8 +76,18 @@ async def openai_inference( with open(output_file, "a+") as f: for datum in tqdm(test_dataset, desc=f"Inference for {model_name_or_path}"): di = DataInterpreter(use_reflection=use_reflection) - instance_id = datum["instance_id"] + env_manager = EnvManager(testbed=TESTBED) + instance_id = datum["instance_id"] + script_names = extract_scripts_from_codetext(datum["text"]) + logger.info(script_names) + repo = datum["repo"] + repo_prefix = repo.replace("/", "__") + repo_path = os.path.join(env_manager.testbed, repo_prefix) + if not os.path.exists(repo_path): + continue + os.chdir(repo_path) + env_manager.reset_task_env(instance=datum) if instance_id in existing_ids: continue output_dict = {"instance_id": instance_id} From 740c963c8374510faa9feb8883f2fdcc4d82351a Mon Sep 17 00:00:00 2001 From: stellahsr Date: Tue, 19 Mar 2024 23:21:33 +0800 Subject: [PATCH 2/7] parse oracle file name from xx --- data/inference/make_datasets/parse_utils.py | 27 +++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 data/inference/make_datasets/parse_utils.py diff --git a/data/inference/make_datasets/parse_utils.py b/data/inference/make_datasets/parse_utils.py new file mode 100644 index 000000000..ace137a44 --- /dev/null +++ b/data/inference/make_datasets/parse_utils.py @@ -0,0 +1,27 @@ +import re + + +def extract_scripts_from_codetext(codetext: str): + script_names = [] + # 提供的文本内容，可能包含多个 [start of ... .py] + """ + [end of README.rst] + [start of sklearn/compose/_target.py] + ... 文件内容 ... + [end of sklearn/compose/_target.py] + [start of another_module/example.py] + ... 文件内容 ... + [end of another_module/example.py] + """ + + # 使用正则表达式匹配所有 “[start of 任意字符.py]” + matches = re.findall(r"\[start of ([^\]]+\.py)\]", codetext) + + if matches: + # 遍历所有匹配的文件名并打印 + for script_name in matches: + print("Extracted script name:", script_name) + script_names.append(script_name) + else: + print("No script names found in the text.") + return script_names From 63fbbe25b40e9c4ef29c98f5904ffadce678ec0b Mon Sep 17 00:00:00 2001 From: stellahsr Date: Tue, 19 Mar 2024 23:28:00 +0800 Subject: [PATCH 3/7] add git basic ops --- data/inference/make_datasets/repo_utils.py | 86 ++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 data/inference/make_datasets/repo_utils.py diff --git a/data/inference/make_datasets/repo_utils.py b/data/inference/make_datasets/repo_utils.py new file mode 100644 index 000000000..ce5f2d418 --- /dev/null +++ b/data/inference/make_datasets/repo_utils.py @@ -0,0 +1,86 @@ +import os +import subprocess +from pathlib import Path +from traceback import format_exc +from typing import Dict + +import git +from git.exc import GitError + +from metagpt.logs import logger + +KEY_INSTANCE_ID = "instance_id" +RESET_FAILED = ">>>>> Reset Failed" + + +class ExecWrapper: + def __init__(self, subprocess_args: Dict = None): + self.subprocess_args = subprocess_args or {} + + def __call__(self, cmd, raise_error=True, **kwargs): + try: + combined_args = {**self.subprocess_args, **kwargs} + output = subprocess.run(cmd, **combined_args) + return output + except subprocess.CalledProcessError as e: + if raise_error: + error_message = ( + f"Error: {e}\nError stdout: {e.stdout}\nError stderr: {e.stderr}\nError traceback: {format_exc()}" + ) + logger.error(error_message) + raise e + + +class EnvManager: + def __init__(self, testbed): + shellenv = os.environ.copy() + self.testbed = testbed + + self.exec = ExecWrapper( + subprocess_args={ + "check": True, + "shell": False, + "capture_output": True, + "text": True, + "env": shellenv, + } + ) + + def clone_repo(self, repo_name: str, path: str, token: str = None): + if token is None: + token = os.environ.get("GITHUB_TOKEN", "git") + if not token: + raise ValueError("GitHub token is required for cloning repositories.") + + repo_url = f"https://{token}@github.com/swe-bench/{repo_name.replace('/', '__')}.git" + + try: + # Ensure the destination directory exists + os.makedirs(path, exist_ok=True) + + # Clone the repository + git.Repo.clone_from(repo_url, path) + print(f"Repository '{repo_name}' cloned successfully.") + except GitError as e: + print(f"Failed to clone repository '{repo_name}': {e}") + + def reset_task_env(self, instance: Dict): + """ + Reset task environment + testbed and checkout base commit of given task instance + """ + try: + gitignore_path = Path(".gitignore") + if gitignore_path.exists(): + self.exec(["git", "ls-files", "--ignored", "--exclude-standard", "-o", "-z"], raise_error=False) + # self.exec(["xargs", "-0", "-r", "rm", "-rf"], input=gitignore_path.read_text()) + + self.exec(["git", "restore", "."]) + self.exec(["git", "reset", "HEAD", "."]) + self.exec(["git", "clean", "-fdx"]) + self.exec(["git", "-c", "advice.detachedHead=false", "checkout", instance["base_commit"]]) + logger.info(f"[{instance['instance_id']}] Reset task environment to {instance['base_commit']}") + return True + except Exception as e: + err_msg = f"{RESET_FAILED}; Failed to reset task environment to {instance['base_commit']}: {e}" + logger.error(err_msg) + return False From 6e28eafcd7161ee81c6c3fcfd7332ced4acf3758 Mon Sep 17 00:00:00 2001 From: stellahsr Date: Tue, 19 Mar 2024 23:33:42 +0800 Subject: [PATCH 4/7] add fixme comments --- data/inference/make_datasets/repo_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data/inference/make_datasets/repo_utils.py b/data/inference/make_datasets/repo_utils.py index ce5f2d418..e9483e6d0 100644 --- a/data/inference/make_datasets/repo_utils.py +++ b/data/inference/make_datasets/repo_utils.py @@ -72,6 +72,7 @@ class EnvManager: gitignore_path = Path(".gitignore") if gitignore_path.exists(): self.exec(["git", "ls-files", "--ignored", "--exclude-standard", "-o", "-z"], raise_error=False) + # fixme: need detect platform and change this cmd # self.exec(["xargs", "-0", "-r", "rm", "-rf"], input=gitignore_path.read_text()) self.exec(["git", "restore", "."]) From 3fac156d66f8d631ae55278a79381d13331b1c58 Mon Sep 17 00:00:00 2001 From: stellahsr Date: Tue, 19 Mar 2024 23:51:57 +0800 Subject: [PATCH 5/7] rm Chinese comments --- data/inference/make_datasets/parse_utils.py | 23 +++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/data/inference/make_datasets/parse_utils.py b/data/inference/make_datasets/parse_utils.py index ace137a44..79b6189e1 100644 --- a/data/inference/make_datasets/parse_utils.py +++ b/data/inference/make_datasets/parse_utils.py @@ -2,26 +2,37 @@ import re def extract_scripts_from_codetext(codetext: str): - script_names = [] - # 提供的文本内容，可能包含多个 [start of ... .py] """ + Extracts Python script file names from a given text that contains multiple sections. + Each section starts with '[start of .py]' and ends with '[end of .py]'. + + Parameters: + - codetext (str): A string that may contain multiple sections, each indicating the start of a Python script file. + + Returns: + - list: A list of extracted Python script file names. + + Example of codetext: + ''' [end of README.rst] [start of sklearn/compose/_target.py] - ... 文件内容 ... + ... file content ... [end of sklearn/compose/_target.py] [start of another_module/example.py] - ... 文件内容 ... + ... file content ... [end of another_module/example.py] + ''' """ + script_names = [] - # 使用正则表达式匹配所有 “[start of 任意字符.py]” + # Match all occurrences of '[start of .py]' matches = re.findall(r"\[start of ([^\]]+\.py)\]", codetext) if matches: - # 遍历所有匹配的文件名并打印 for script_name in matches: print("Extracted script name:", script_name) script_names.append(script_name) else: print("No script names found in the text.") + return script_names From 99d2678d2007016972b4d041d8bf07f081e664da Mon Sep 17 00:00:00 2001 From: stellahsr Date: Thu, 21 Mar 2024 01:59:46 +0800 Subject: [PATCH 6/7] use handle_exception add cp option --- data/inference/make_datasets/repo_utils.py | 91 ++++++++++++---------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/data/inference/make_datasets/repo_utils.py b/data/inference/make_datasets/repo_utils.py index e9483e6d0..fdee07a75 100644 --- a/data/inference/make_datasets/repo_utils.py +++ b/data/inference/make_datasets/repo_utils.py @@ -1,13 +1,14 @@ import os import subprocess from pathlib import Path -from traceback import format_exc +import shutil from typing import Dict import git from git.exc import GitError from metagpt.logs import logger +from metagpt.utils.exceptions import handle_exception KEY_INSTANCE_ID = "instance_id" RESET_FAILED = ">>>>> Reset Failed" @@ -16,26 +17,19 @@ RESET_FAILED = ">>>>> Reset Failed" class ExecWrapper: def __init__(self, subprocess_args: Dict = None): self.subprocess_args = subprocess_args or {} - + + @handle_exception(exception_type=subprocess.CalledProcessError) def __call__(self, cmd, raise_error=True, **kwargs): - try: - combined_args = {**self.subprocess_args, **kwargs} - output = subprocess.run(cmd, **combined_args) - return output - except subprocess.CalledProcessError as e: - if raise_error: - error_message = ( - f"Error: {e}\nError stdout: {e.stdout}\nError stderr: {e.stderr}\nError traceback: {format_exc()}" - ) - logger.error(error_message) - raise e + combined_args = {**self.subprocess_args, **kwargs} + output = subprocess.run(cmd, **combined_args) + return output class EnvManager: def __init__(self, testbed): shellenv = os.environ.copy() self.testbed = testbed - + self.exec = ExecWrapper( subprocess_args={ "check": True, @@ -45,43 +39,54 @@ class EnvManager: "env": shellenv, } ) - + + @handle_exception(exception_type=GitError) def clone_repo(self, repo_name: str, path: str, token: str = None): if token is None: token = os.environ.get("GITHUB_TOKEN", "git") if not token: raise ValueError("GitHub token is required for cloning repositories.") - + repo_url = f"https://{token}@github.com/swe-bench/{repo_name.replace('/', '__')}.git" - + os.makedirs(path, exist_ok=True) + + # Clone the repository + git.Repo.clone_from(repo_url, path) + logger.info(f"Repository '{repo_name}' cloned successfully.") + + @handle_exception(exception_type=Exception) # Using a broad exception type for the example + def copy_repo(self, source_path: str, destination_path: str): + if not os.path.isdir(source_path): + raise ValueError("Source path does not exist or is not a directory.") + + os.makedirs(destination_path, exist_ok=True) + + # Copy the repository try: - # Ensure the destination directory exists - os.makedirs(path, exist_ok=True) - - # Clone the repository - git.Repo.clone_from(repo_url, path) - print(f"Repository '{repo_name}' cloned successfully.") - except GitError as e: - print(f"Failed to clone repository '{repo_name}': {e}") - + shutil.copytree(source_path, destination_path, + dirs_exist_ok=True) # For Python 3.8+, dirs_exist_ok handles existing directories + except TypeError: + # Fallback for Python < 3.8, where dirs_exist_ok is not available + if os.listdir(destination_path): # If destination is not empty + raise ValueError("Destination directory is not empty and dirs_exist_ok is not supported.") + shutil.copytree(source_path, destination_path) + + logger.info(f"Repository contents from '{source_path}' copied successfully to '{destination_path}'.") + + @handle_exception(exception_type=Exception, default_return=False) def reset_task_env(self, instance: Dict): """ Reset task environment + testbed and checkout base commit of given task instance """ - try: - gitignore_path = Path(".gitignore") - if gitignore_path.exists(): - self.exec(["git", "ls-files", "--ignored", "--exclude-standard", "-o", "-z"], raise_error=False) - # fixme: need detect platform and change this cmd - # self.exec(["xargs", "-0", "-r", "rm", "-rf"], input=gitignore_path.read_text()) - - self.exec(["git", "restore", "."]) - self.exec(["git", "reset", "HEAD", "."]) - self.exec(["git", "clean", "-fdx"]) - self.exec(["git", "-c", "advice.detachedHead=false", "checkout", instance["base_commit"]]) - logger.info(f"[{instance['instance_id']}] Reset task environment to {instance['base_commit']}") - return True - except Exception as e: - err_msg = f"{RESET_FAILED}; Failed to reset task environment to {instance['base_commit']}: {e}" - logger.error(err_msg) - return False + gitignore_path = Path(".gitignore") + if gitignore_path.exists(): + self.exec(["git", "ls-files", "--ignored", "--exclude-standard", "-o", "-z"], raise_error=False) + # fixme: need detect platform and change this cmd + # self.exec(["xargs", "-0", "-r", "rm", "-rf"], input=gitignore_path.read_text()) + + self.exec(["git", "restore", "."]) + self.exec(["git", "reset", "HEAD", "."]) + self.exec(["git", "clean", "-fdx"]) + self.exec(["git", "-c", "advice.detachedHead=false", "checkout", instance["base_commit"]]) + logger.info(f"[{instance['instance_id']}] Reset task environment to {instance['base_commit']}") + return True From c8a511037c91dc824428817130b1a48e73b8bf5c Mon Sep 17 00:00:00 2001 From: stellahsr Date: Thu, 21 Mar 2024 02:00:52 +0800 Subject: [PATCH 7/7] format --- data/inference/make_datasets/repo_utils.py | 29 +++++++++++----------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/data/inference/make_datasets/repo_utils.py b/data/inference/make_datasets/repo_utils.py index fdee07a75..8bdf77605 100644 --- a/data/inference/make_datasets/repo_utils.py +++ b/data/inference/make_datasets/repo_utils.py @@ -1,7 +1,7 @@ import os +import shutil import subprocess from pathlib import Path -import shutil from typing import Dict import git @@ -17,7 +17,7 @@ RESET_FAILED = ">>>>> Reset Failed" class ExecWrapper: def __init__(self, subprocess_args: Dict = None): self.subprocess_args = subprocess_args or {} - + @handle_exception(exception_type=subprocess.CalledProcessError) def __call__(self, cmd, raise_error=True, **kwargs): combined_args = {**self.subprocess_args, **kwargs} @@ -29,7 +29,7 @@ class EnvManager: def __init__(self, testbed): shellenv = os.environ.copy() self.testbed = testbed - + self.exec = ExecWrapper( subprocess_args={ "check": True, @@ -39,40 +39,41 @@ class EnvManager: "env": shellenv, } ) - + @handle_exception(exception_type=GitError) def clone_repo(self, repo_name: str, path: str, token: str = None): if token is None: token = os.environ.get("GITHUB_TOKEN", "git") if not token: raise ValueError("GitHub token is required for cloning repositories.") - + repo_url = f"https://{token}@github.com/swe-bench/{repo_name.replace('/', '__')}.git" os.makedirs(path, exist_ok=True) - + # Clone the repository git.Repo.clone_from(repo_url, path) logger.info(f"Repository '{repo_name}' cloned successfully.") - + @handle_exception(exception_type=Exception) # Using a broad exception type for the example def copy_repo(self, source_path: str, destination_path: str): if not os.path.isdir(source_path): raise ValueError("Source path does not exist or is not a directory.") - + os.makedirs(destination_path, exist_ok=True) - + # Copy the repository try: - shutil.copytree(source_path, destination_path, - dirs_exist_ok=True) # For Python 3.8+, dirs_exist_ok handles existing directories + shutil.copytree( + source_path, destination_path, dirs_exist_ok=True + ) # For Python 3.8+, dirs_exist_ok handles existing directories except TypeError: # Fallback for Python < 3.8, where dirs_exist_ok is not available if os.listdir(destination_path): # If destination is not empty raise ValueError("Destination directory is not empty and dirs_exist_ok is not supported.") shutil.copytree(source_path, destination_path) - + logger.info(f"Repository contents from '{source_path}' copied successfully to '{destination_path}'.") - + @handle_exception(exception_type=Exception, default_return=False) def reset_task_env(self, instance: Dict): """ @@ -83,7 +84,7 @@ class EnvManager: self.exec(["git", "ls-files", "--ignored", "--exclude-standard", "-o", "-z"], raise_error=False) # fixme: need detect platform and change this cmd # self.exec(["xargs", "-0", "-r", "rm", "-rf"], input=gitignore_path.read_text()) - + self.exec(["git", "restore", "."]) self.exec(["git", "reset", "HEAD", "."]) self.exec(["git", "clean", "-fdx"])