diff --git a/metagpt/ext/sela/README.md b/metagpt/ext/sela/README.md index 3fa03ee08..829306e36 100644 --- a/metagpt/ext/sela/README.md +++ b/metagpt/ext/sela/README.md @@ -213,7 +213,7 @@ #### Run The `log` folder will contain the experimental configuration and the generated scheme, and the `workspace` folder will save the final results generated by aide ``` -python experimenter/aide.py +python runner/aide.py ``` ### Autogluon diff --git a/metagpt/ext/sela/data/custom_task.py b/metagpt/ext/sela/data/custom_task.py index 3371d5b1c..08a7cbabb 100644 --- a/metagpt/ext/sela/data/custom_task.py +++ b/metagpt/ext/sela/data/custom_task.py @@ -1,7 +1,7 @@ import os from metagpt.ext.sela.data.dataset import SPECIAL_INSTRUCTIONS -from metagpt.ext.sela.experimenter.mle_bench.instructions import ( +from metagpt.ext.sela.runner.mle_bench.instructions import ( ADDITIONAL_NOTES, INSTRUCTIONS, INSTRUCTIONS_OBFUSCATED, diff --git a/metagpt/ext/sela/research_assistant.py b/metagpt/ext/sela/experimenter.py similarity index 100% rename from metagpt/ext/sela/research_assistant.py rename to metagpt/ext/sela/experimenter.py diff --git a/metagpt/ext/sela/run_experiment.py b/metagpt/ext/sela/run_experiment.py index 4cced19c3..32130a6fb 100644 --- a/metagpt/ext/sela/run_experiment.py +++ b/metagpt/ext/sela/run_experiment.py @@ -2,12 +2,12 @@ import argparse import asyncio from metagpt.ext.sela.data.custom_task import get_mle_is_lower_better, get_mle_task_id -from metagpt.ext.sela.experimenter.autogluon import GluonExperimenter -from metagpt.ext.sela.experimenter.autosklearn import AutoSklearnExperimenter -from metagpt.ext.sela.experimenter.custom import CustomExperimenter -from metagpt.ext.sela.experimenter.experimenter import Experimenter -from metagpt.ext.sela.experimenter.mcts import MCTSExperimenter -from metagpt.ext.sela.experimenter.random_search import RandomSearchExperimenter +from metagpt.ext.sela.runner.autogluon import GluonRunner +from metagpt.ext.sela.runner.autosklearn import AutoSklearnRunner +from metagpt.ext.sela.runner.custom import CustomRunner +from metagpt.ext.sela.runner.mcts import MCTSRunner +from metagpt.ext.sela.runner.random_search import RandomSearchRunner +from metagpt.ext.sela.runner.runner import Runner def get_args(cmd=True): @@ -74,24 +74,24 @@ def get_di_args(parser): async def main(args): if args.exp_mode == "mcts": - experimenter = MCTSExperimenter(args) + runner = MCTSRunner(args) elif args.exp_mode == "greedy": - experimenter = MCTSExperimenter(args, tree_mode="greedy") + runner = MCTSRunner(args, tree_mode="greedy") elif args.exp_mode == "random": - experimenter = MCTSExperimenter(args, tree_mode="random") + runner = MCTSRunner(args, tree_mode="random") elif args.exp_mode == "rs": - experimenter = RandomSearchExperimenter(args) + runner = RandomSearchRunner(args) elif args.exp_mode == "base": - experimenter = Experimenter(args) + runner = Runner(args) elif args.exp_mode == "autogluon": - experimenter = GluonExperimenter(args) + runner = GluonRunner(args) elif args.exp_mode == "custom": - experimenter = CustomExperimenter(args) + runner = CustomRunner(args) elif args.exp_mode == "autosklearn": - experimenter = AutoSklearnExperimenter(args) + runner = AutoSklearnRunner(args) else: raise ValueError(f"Invalid exp_mode: {args.exp_mode}") - await experimenter.run_experiment() + await runner.run_experiment() if __name__ == "__main__": diff --git a/metagpt/ext/sela/experimenter/__init__.py b/metagpt/ext/sela/runner/__init__.py similarity index 100% rename from metagpt/ext/sela/experimenter/__init__.py rename to metagpt/ext/sela/runner/__init__.py diff --git a/metagpt/ext/sela/experimenter/aide.py b/metagpt/ext/sela/runner/aide.py similarity index 100% rename from metagpt/ext/sela/experimenter/aide.py rename to metagpt/ext/sela/runner/aide.py diff --git a/metagpt/ext/sela/experimenter/autogluon.py b/metagpt/ext/sela/runner/autogluon.py similarity index 98% rename from metagpt/ext/sela/experimenter/autogluon.py rename to metagpt/ext/sela/runner/autogluon.py index f547ce4ba..48737da04 100644 --- a/metagpt/ext/sela/experimenter/autogluon.py +++ b/metagpt/ext/sela/runner/autogluon.py @@ -3,7 +3,7 @@ from datetime import datetime import pandas as pd -from metagpt.ext.sela.experimenter.custom import CustomExperimenter +from metagpt.ext.sela.runner.custom import CustomRunner class AGRunner: @@ -102,7 +102,7 @@ class AGRunner: return train_data, dev_data, dev_wo_target_data, test_data -class GluonExperimenter(CustomExperimenter): +class GluonRunner(CustomRunner): result_path: str = "results/autogluon" def __init__(self, args, **kwargs): diff --git a/metagpt/ext/sela/experimenter/autosklearn.py b/metagpt/ext/sela/runner/autosklearn.py similarity index 96% rename from metagpt/ext/sela/experimenter/autosklearn.py rename to metagpt/ext/sela/runner/autosklearn.py index f6ff267e7..7d0eb364e 100644 --- a/metagpt/ext/sela/experimenter/autosklearn.py +++ b/metagpt/ext/sela/runner/autosklearn.py @@ -4,7 +4,7 @@ from functools import partial import pandas as pd from metagpt.ext.sela.evaluation.evaluation import evaluate_score -from metagpt.ext.sela.experimenter.custom import CustomExperimenter +from metagpt.ext.sela.runner.custom import CustomRunner def custom_scorer(y_true, y_pred, metric_name): @@ -69,7 +69,7 @@ class ASRunner: return {"test_preds": test_preds, "dev_preds": dev_preds} -class AutoSklearnExperimenter(CustomExperimenter): +class AutoSklearnRunner(CustomRunner): result_path: str = "results/autosklearn" def __init__(self, args, **kwargs): diff --git a/metagpt/ext/sela/experimenter/custom.py b/metagpt/ext/sela/runner/custom.py similarity index 95% rename from metagpt/ext/sela/experimenter/custom.py rename to metagpt/ext/sela/runner/custom.py index 70df1a78e..e9a8ee276 100644 --- a/metagpt/ext/sela/experimenter/custom.py +++ b/metagpt/ext/sela/runner/custom.py @@ -3,11 +3,11 @@ import os import pandas as pd from metagpt.ext.sela.evaluation.evaluation import evaluate_score -from metagpt.ext.sela.experimenter.experimenter import Experimenter +from metagpt.ext.sela.runner.runner import Runner from metagpt.ext.sela.search.tree_search import create_initial_state -class CustomExperimenter(Experimenter): +class CustomRunner(Runner): result_path: str = "results/custom" def __init__(self, args, **kwargs): diff --git a/metagpt/ext/sela/experimenter/mcts.py b/metagpt/ext/sela/runner/mcts.py similarity index 96% rename from metagpt/ext/sela/experimenter/mcts.py rename to metagpt/ext/sela/runner/mcts.py index 9fd66121d..8b6c14100 100644 --- a/metagpt/ext/sela/experimenter/mcts.py +++ b/metagpt/ext/sela/runner/mcts.py @@ -5,11 +5,11 @@ from metagpt.ext.sela.evaluation.evaluation import ( node_evaluate_score_sela, ) from metagpt.ext.sela.evaluation.visualize_mcts import get_tree_text -from metagpt.ext.sela.experimenter.experimenter import Experimenter +from metagpt.ext.sela.runner.runner import Runner from metagpt.ext.sela.search.search_algorithm import MCTS, Greedy, Random -class MCTSExperimenter(Experimenter): +class MCTSRunner(Runner): result_path: str = "results/mcts" def __init__(self, args, tree_mode=None, **kwargs): diff --git a/metagpt/ext/sela/experimenter/mle_bench/instructions.py b/metagpt/ext/sela/runner/mle_bench/instructions.py similarity index 98% rename from metagpt/ext/sela/experimenter/mle_bench/instructions.py rename to metagpt/ext/sela/runner/mle_bench/instructions.py index 5c9c7b70f..136726bcf 100644 --- a/metagpt/ext/sela/experimenter/mle_bench/instructions.py +++ b/metagpt/ext/sela/runner/mle_bench/instructions.py @@ -8,7 +8,7 @@ You will be given a machine learning task. You must solve the task by training a BENCHMARK INSTRUCTIONS ------ -First, please read the instructions below which are specific to our benchmark setup. Note, these instructions can be found again in `experimenter/mle_bench/instructions.py`. +First, please read the instructions below which are specific to our benchmark setup. Note, these instructions can be found again in `runner/mle_bench/instructions.py`. - **Competition-specific instructions**: The instructions for the competition are available in `{dataset_dir}/description.md`. Read this file to understand the problem setup for the competition you are currently solving. - **Dataset**: The dataset files for the competition are available in `{dataset_dir}`. @@ -28,7 +28,7 @@ You will be given a machine learning task. You must solve the task by training a BENCHMARK INSTRUCTIONS ------ -First, please read the instructions below which are specific to our benchmark setup. Note, these instructions can be found again in `experimenter/mle_bench/instructions.py`. +First, please read the instructions below which are specific to our benchmark setup. Note, these instructions can be found again in `runner/mle_bench/instructions.py`. - **Task-specific instructions**: The instructions for the task are available in `{dataset_dir}/description.md`. Read this file to understand the problem setup for the task you are currently solving. - **Dataset**: The dataset files for the task are available in `{dataset_dir}/`. diff --git a/metagpt/ext/sela/experimenter/random_search.py b/metagpt/ext/sela/runner/random_search.py similarity index 92% rename from metagpt/ext/sela/experimenter/random_search.py rename to metagpt/ext/sela/runner/random_search.py index 5617ee601..8ce42f0ff 100644 --- a/metagpt/ext/sela/experimenter/random_search.py +++ b/metagpt/ext/sela/runner/random_search.py @@ -1,6 +1,6 @@ -from metagpt.ext.sela.experimenter.experimenter import Experimenter +from metagpt.ext.sela.experimenter import ResearchAssistant from metagpt.ext.sela.insights.instruction_generator import InstructionGenerator -from metagpt.ext.sela.research_assistant import ResearchAssistant +from metagpt.ext.sela.runner.runner import Runner from metagpt.ext.sela.utils import get_exp_pool_path EXPS_PROMPT = """ @@ -10,7 +10,7 @@ When doing the tasks, you can refer to the insights below: """ -class RandomSearchExperimenter(Experimenter): +class RandomSearchRunner(Runner): result_path: str = "results/random_search" async def run_experiment(self): diff --git a/metagpt/ext/sela/experimenter/experimenter.py b/metagpt/ext/sela/runner/runner.py similarity index 98% rename from metagpt/ext/sela/experimenter/experimenter.py rename to metagpt/ext/sela/runner/runner.py index 3df46b74b..7ab83c6c3 100644 --- a/metagpt/ext/sela/experimenter/experimenter.py +++ b/metagpt/ext/sela/runner/runner.py @@ -6,12 +6,12 @@ import numpy as np import pandas as pd from metagpt.ext.sela.evaluation.evaluation import evaluate_score -from metagpt.ext.sela.research_assistant import ResearchAssistant +from metagpt.ext.sela.experimenter import ResearchAssistant from metagpt.ext.sela.search.tree_search import create_initial_state from metagpt.ext.sela.utils import DATA_CONFIG, save_notebook -class Experimenter: +class Runner: result_path: str = "results/base" data_config = DATA_CONFIG start_task_id = 1 diff --git a/metagpt/ext/sela/search/tree_search.py b/metagpt/ext/sela/search/tree_search.py index cde8dc82a..684426fe6 100644 --- a/metagpt/ext/sela/search/tree_search.py +++ b/metagpt/ext/sela/search/tree_search.py @@ -15,8 +15,8 @@ from metagpt.ext.sela.data.dataset import ( get_split_dataset_path, ) from metagpt.ext.sela.evaluation.evaluation import evaluate_score +from metagpt.ext.sela.experimenter import ResearchAssistant, TimeoutException from metagpt.ext.sela.insights.instruction_generator import InstructionGenerator -from metagpt.ext.sela.research_assistant import ResearchAssistant, TimeoutException from metagpt.ext.sela.utils import get_exp_pool_path, load_execute_notebook, mcts_logger from metagpt.tools.tool_recommend import ToolRecommender from metagpt.utils.common import read_json_file