From 9ba0d217fc25ae4732394173a1c3626de5679128 Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Mon, 9 Sep 2024 14:52:25 +0800 Subject: [PATCH 1/9] update mcts logic --- expo/MCTS.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/expo/MCTS.py b/expo/MCTS.py index 4090331cd..360baac8d 100644 --- a/expo/MCTS.py +++ b/expo/MCTS.py @@ -279,7 +279,8 @@ class MCTS: def best_path(self, root: Node): best_child = root - best_score = 0 + global_best_score = root.normalized_reward["test_score"] + dev_best_score = root.normalized_reward["dev_score"] def bfs(node: Node, best_score, best_child: Node, split): assert split in ["test_score", "dev_score"] @@ -294,10 +295,10 @@ class MCTS: best_score, best_child = bfs(child, best_score, best_child, split) return best_score, best_child - _, best_child = bfs(root, best_score, best_child, "test_score") - _, dev_best_child = bfs(root, best_score, best_child, "dev_score") + _, global_best_child = bfs(root, global_best_score, best_child, "test_score") + _, dev_best_child = bfs(root, dev_best_score, best_child, "dev_score") - return {"dev_best": dev_best_child, "global_best": best_child} + return {"dev_best": dev_best_child, "global_best": global_best_child} def get_num_simulations(self): return self.root_node.visited From 294d0fe70968ae66ff45200da3569e90c127951a Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Mon, 9 Sep 2024 16:59:39 +0800 Subject: [PATCH 2/9] fix nbclient --- expo/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expo/utils.py b/expo/utils.py index 9c6295fa9..270842e41 100644 --- a/expo/utils.py +++ b/expo/utils.py @@ -110,7 +110,7 @@ async def load_execute_notebook(role): codes = [task.code for task in tasks if task.code] executor = role.execute_code executor.nb = nbformat.v4.new_notebook() - executor.nb.client = NotebookClient(executor.nb) + executor.nb_client = NotebookClient(executor.nb) # await executor.build() for code in codes: outputs, success = await executor.run(code) From 60e8e3eab8f41b52a03137bfb8f8c99be9bf926b Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Tue, 10 Sep 2024 13:51:54 +0800 Subject: [PATCH 3/9] fix hfdataset; make dirs when save notebook --- expo/data/hf_data.py | 9 ++++++--- expo/utils.py | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/expo/data/hf_data.py b/expo/data/hf_data.py index a7e2a1afe..9ed2b2c48 100644 --- a/expo/data/hf_data.py +++ b/expo/data/hf_data.py @@ -38,18 +38,21 @@ class HFExpDataset(ExpDataset): df = pd.read_csv(Path(raw_dir, "train.csv")) else: df = self.dataset["train"].to_pandas() - df.to_csv(Path(raw_dir, "train.csv")) + df.to_csv(Path(raw_dir, "train.csv"), index=False) if os.path.exists(Path(raw_dir, "test.csv")): - test_df = pd.read_csv(Path(raw_dir, "test.csv")) + test_df = pd.read_csv(Path(raw_dir, "test.csv"), index=False) else: if "test" in self.dataset: test_df = self.dataset["test"].to_pandas() - test_df.to_csv(Path(raw_dir, "test.csv")) + test_df.to_csv(Path(raw_dir, "test.csv"), index=False) else: test_df = None return df, test_df + # def get_df_head(self, raw_df): + # return raw_df.head() + if __name__ == "__main__": dataset_dir = "D:/work/automl/datasets" diff --git a/expo/utils.py b/expo/utils.py index 270842e41..44de8cf9b 100644 --- a/expo/utils.py +++ b/expo/utils.py @@ -99,6 +99,7 @@ def save_notebook(role: Role, save_dir: str = "", name: str = ""): for code in codes: clean_nb.cells.append(nbformat.v4.new_code_cell(code)) nb = process_cells(role.execute_code.nb) + os.makedirs(save_dir, exist_ok=True) file_path = save_dir / f"{name}.ipynb" clean_file_path = save_dir / f"{name}_clean.ipynb" nbformat.write(nb, file_path) From d34a482faf0ec5478f9dad0291f65e6a85931006 Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Tue, 10 Sep 2024 14:11:47 +0800 Subject: [PATCH 4/9] give dev label --- expo/data/dataset.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/expo/data/dataset.py b/expo/data/dataset.py index 8bcce0b1a..88528eb5c 100644 --- a/expo/data/dataset.py +++ b/expo/data/dataset.py @@ -20,23 +20,19 @@ Report {metric} on the eval data. Do not plot or make any visualizations. DI_INSTRUCTION = """\ **Attention** 1. Please do not leak the target label in any form during training. -2. Dev and Test sets do not have the target column. +2. Test set does not have the target column. 3. You should perform transformations on train, dev, and test sets at the same time (it's a good idea to define functions for this and avoid code repetition). 4. If labels are transformed during training, they should be transformed back to the original format before saving the predictions. +5. You could split the training set further to make cross-validation and hyperparameter tuning. ## Saving Dev and Test Predictions 1. Save the prediction results of BOTH the dev set and test set in `dev_predictions.csv` and `test_predictions.csv` respectively in the output directory. - Both files should contain a single column named `target` with the predicted values. 2. Make sure the prediction results are in the same format as the target column in the training set. -- The labels should be transformed back to the original format if any transformation was applied during training. -## Output Training Set Performance +## Output Performance Make sure the performance of the model is printed in python in the last step even if it has been printed in the previous steps. The value should be a float number. -Print the training set performance in the last step. Write in this format: -```python -... -print("Train score:", train_score) -``` +Print the training set performance in the last step. # Output dir {output_dir} @@ -48,7 +44,7 @@ TASK_PROMPT = """\ {additional_instruction} # Data dir training (with labels): {train_path} -dev (without labels): {dev_path} +dev (with labels): {dev_path} testing (without labels): {test_path} dataset description: {data_info_path} (You can use this file to get additional information about the dataset) """ @@ -147,7 +143,7 @@ def generate_task_requirement(task_name, data_config, is_di=True): user_requirement = get_user_requirement(task_name, data_config) split_dataset_path = get_split_dataset_path(task_name, data_config) train_path = split_dataset_path["train"] - dev_path = split_dataset_path["dev_wo_target"] + dev_path = split_dataset_path["dev"] test_path = split_dataset_path["test_wo_target"] work_dir = data_config["work_dir"] output_dir = f"{work_dir}/{task_name}" @@ -225,7 +221,7 @@ class ExpDataset: "NumberOfSymbolicFeatures": raw_df.select_dtypes(include=["object"]).shape[1], } - df_head_text = raw_df.head().to_string(index=False) + df_head_text = self.get_df_head(raw_df) dataset_info = { "name": self.name, @@ -236,6 +232,9 @@ class ExpDataset: } return dataset_info + def get_df_head(self, raw_df): + return raw_df.head().to_string(index=False) + def get_metric(self): dataset_info = self.get_dataset_info() num_classes = dataset_info["metadata"]["NumberOfClasses"] From b776c7309bc64c0ec62c8efa4acbe93fdbd1a75f Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Tue, 10 Sep 2024 15:30:23 +0800 Subject: [PATCH 5/9] add random tree search --- expo/Greedy.py | 10 ++++++++++ expo/data/dataset.py | 2 +- expo/experimenter/mcts.py | 10 ++++++---- expo/run_experiment.py | 8 ++++++-- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/expo/Greedy.py b/expo/Greedy.py index f6f60db01..8c8d865cd 100644 --- a/expo/Greedy.py +++ b/expo/Greedy.py @@ -1,3 +1,5 @@ +import random + from expo.MCTS import MCTS @@ -7,3 +9,11 @@ class Greedy(MCTS): return self.root_node all_children = [child for children in self.children.values() for child in children] return max(all_children, key=lambda x: x.normalized_reward.get("dev_score", 0)) + + +class Random(MCTS): + def best_child(self): + if len(self.children) == 0: + return self.root_node + all_children = [child for children in self.children.values() for child in children] + return random.choice(all_children) diff --git a/expo/data/dataset.py b/expo/data/dataset.py index 88528eb5c..43ac8ee0d 100644 --- a/expo/data/dataset.py +++ b/expo/data/dataset.py @@ -23,7 +23,7 @@ DI_INSTRUCTION = """\ 2. Test set does not have the target column. 3. You should perform transformations on train, dev, and test sets at the same time (it's a good idea to define functions for this and avoid code repetition). 4. If labels are transformed during training, they should be transformed back to the original format before saving the predictions. -5. You could split the training set further to make cross-validation and hyperparameter tuning. +5. You could utilize dev set to improve the model. ## Saving Dev and Test Predictions 1. Save the prediction results of BOTH the dev set and test set in `dev_predictions.csv` and `test_predictions.csv` respectively in the output directory. diff --git a/expo/experimenter/mcts.py b/expo/experimenter/mcts.py index 9bf7306c4..fbe2f35f1 100644 --- a/expo/experimenter/mcts.py +++ b/expo/experimenter/mcts.py @@ -1,19 +1,21 @@ from expo.evaluation.visualize_mcts import get_tree_text from expo.experimenter.experimenter import Experimenter -from expo.Greedy import Greedy +from expo.Greedy import Greedy, Random from expo.MCTS import MCTS class MCTSExperimenter(Experimenter): result_path: str = "results/mcts" - def __init__(self, args, greedy=False, **kwargs): + def __init__(self, args, tree_mode=None, **kwargs): super().__init__(args, **kwargs) - self.greedy = greedy + self.tree_mode = tree_mode async def run_experiment(self): - if self.greedy: + if self.tree_mode == "greedy": mcts = Greedy(root_node=None, max_depth=5) + elif self.tree_mode == "random": + mcts = Random(root_node=None, max_depth=5) else: mcts = MCTS(root_node=None, max_depth=5) best_nodes = await mcts.search( diff --git a/expo/run_experiment.py b/expo/run_experiment.py index 83237741a..b68607d79 100644 --- a/expo/run_experiment.py +++ b/expo/run_experiment.py @@ -10,7 +10,9 @@ from expo.experimenter.mcts import MCTSExperimenter def get_args(): parser = argparse.ArgumentParser() parser.add_argument("--name", type=str, default="") - parser.add_argument("--exp_mode", type=str, default="mcts", choices=["mcts", "aug", "base", "custom", "greedy"]) + parser.add_argument( + "--exp_mode", type=str, default="mcts", choices=["mcts", "aug", "base", "custom", "greedy", "random"] + ) get_di_args(parser) get_mcts_args(parser) get_aug_exp_args(parser) @@ -42,7 +44,9 @@ async def main(args): if args.exp_mode == "mcts": experimenter = MCTSExperimenter(args) elif args.exp_mode == "greedy": - experimenter = MCTSExperimenter(args, greedy=True) + experimenter = MCTSExperimenter(args, tree_mode="greedy") + elif args.exp_mode == "random": + experimenter = MCTSExperimenter(args, tree_mode="random") elif args.exp_mode == "aug": experimenter = AugExperimenter(args) elif args.exp_mode == "base": From a373e684aec59a70270c8be99c721dd71a0c4de3 Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Tue, 10 Sep 2024 16:05:22 +0800 Subject: [PATCH 6/9] update di instruction --- expo/data/dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/expo/data/dataset.py b/expo/data/dataset.py index 43ac8ee0d..510c39fce 100644 --- a/expo/data/dataset.py +++ b/expo/data/dataset.py @@ -23,7 +23,7 @@ DI_INSTRUCTION = """\ 2. Test set does not have the target column. 3. You should perform transformations on train, dev, and test sets at the same time (it's a good idea to define functions for this and avoid code repetition). 4. If labels are transformed during training, they should be transformed back to the original format before saving the predictions. -5. You could utilize dev set to improve the model. +5. You could utilize dev set to improve model training. ## Saving Dev and Test Predictions 1. Save the prediction results of BOTH the dev set and test set in `dev_predictions.csv` and `test_predictions.csv` respectively in the output directory. @@ -46,7 +46,7 @@ TASK_PROMPT = """\ training (with labels): {train_path} dev (with labels): {dev_path} testing (without labels): {test_path} -dataset description: {data_info_path} (You can use this file to get additional information about the dataset) +dataset description: {data_info_path} (During EDA, you can use this file to get additional information about the dataset) """ From 35b9ea097e26784c16c638e5f839c754739ebca1 Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Wed, 11 Sep 2024 11:59:52 +0800 Subject: [PATCH 7/9] update di instruction --- expo/data/dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/expo/data/dataset.py b/expo/data/dataset.py index 510c39fce..c83f7b926 100644 --- a/expo/data/dataset.py +++ b/expo/data/dataset.py @@ -24,6 +24,7 @@ DI_INSTRUCTION = """\ 3. You should perform transformations on train, dev, and test sets at the same time (it's a good idea to define functions for this and avoid code repetition). 4. If labels are transformed during training, they should be transformed back to the original format before saving the predictions. 5. You could utilize dev set to improve model training. +6. Use techniques to avoid overfitting. ## Saving Dev and Test Predictions 1. Save the prediction results of BOTH the dev set and test set in `dev_predictions.csv` and `test_predictions.csv` respectively in the output directory. From c91b2ada88036f8d2575e023e53fa895467478ad Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Wed, 11 Sep 2024 13:54:41 +0800 Subject: [PATCH 8/9] remove unnecessary prompt instruction --- expo/data/dataset.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/expo/data/dataset.py b/expo/data/dataset.py index c83f7b926..1494eb267 100644 --- a/expo/data/dataset.py +++ b/expo/data/dataset.py @@ -23,7 +23,7 @@ DI_INSTRUCTION = """\ 2. Test set does not have the target column. 3. You should perform transformations on train, dev, and test sets at the same time (it's a good idea to define functions for this and avoid code repetition). 4. If labels are transformed during training, they should be transformed back to the original format before saving the predictions. -5. You could utilize dev set to improve model training. +5. You could utilize dev set to validate and improve model training. 6. Use techniques to avoid overfitting. ## Saving Dev and Test Predictions @@ -32,8 +32,7 @@ DI_INSTRUCTION = """\ 2. Make sure the prediction results are in the same format as the target column in the training set. ## Output Performance -Make sure the performance of the model is printed in python in the last step even if it has been printed in the previous steps. The value should be a float number. -Print the training set performance in the last step. +Print the train and dev set performance in the last step. # Output dir {output_dir} @@ -44,9 +43,9 @@ TASK_PROMPT = """\ {user_requirement} {additional_instruction} # Data dir -training (with labels): {train_path} -dev (with labels): {dev_path} -testing (without labels): {test_path} +train set (with labels): {train_path} +dev set (with labels): {dev_path} +test set (without labels): {test_path} dataset description: {data_info_path} (During EDA, you can use this file to get additional information about the dataset) """ From eaf1b62343f1aade858ddc82255b7d9380ee51ef Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Wed, 11 Sep 2024 16:14:50 +0800 Subject: [PATCH 9/9] add timeout to client --- expo/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expo/utils.py b/expo/utils.py index 44de8cf9b..f3c0c392d 100644 --- a/expo/utils.py +++ b/expo/utils.py @@ -111,7 +111,7 @@ async def load_execute_notebook(role): codes = [task.code for task in tasks if task.code] executor = role.execute_code executor.nb = nbformat.v4.new_notebook() - executor.nb_client = NotebookClient(executor.nb) + executor.nb_client = NotebookClient(executor.nb, timeout=executor.timeout) # await executor.build() for code in codes: outputs, success = await executor.run(code)