diff --git a/expo/Greedy.py b/expo/Greedy.py index f6f60db01..8c8d865cd 100644 --- a/expo/Greedy.py +++ b/expo/Greedy.py @@ -1,3 +1,5 @@ +import random + from expo.MCTS import MCTS @@ -7,3 +9,11 @@ class Greedy(MCTS): return self.root_node all_children = [child for children in self.children.values() for child in children] return max(all_children, key=lambda x: x.normalized_reward.get("dev_score", 0)) + + +class Random(MCTS): + def best_child(self): + if len(self.children) == 0: + return self.root_node + all_children = [child for children in self.children.values() for child in children] + return random.choice(all_children) diff --git a/expo/MCTS.py b/expo/MCTS.py index 4090331cd..360baac8d 100644 --- a/expo/MCTS.py +++ b/expo/MCTS.py @@ -279,7 +279,8 @@ class MCTS: def best_path(self, root: Node): best_child = root - best_score = 0 + global_best_score = root.normalized_reward["test_score"] + dev_best_score = root.normalized_reward["dev_score"] def bfs(node: Node, best_score, best_child: Node, split): assert split in ["test_score", "dev_score"] @@ -294,10 +295,10 @@ class MCTS: best_score, best_child = bfs(child, best_score, best_child, split) return best_score, best_child - _, best_child = bfs(root, best_score, best_child, "test_score") - _, dev_best_child = bfs(root, best_score, best_child, "dev_score") + _, global_best_child = bfs(root, global_best_score, best_child, "test_score") + _, dev_best_child = bfs(root, dev_best_score, best_child, "dev_score") - return {"dev_best": dev_best_child, "global_best": best_child} + return {"dev_best": dev_best_child, "global_best": global_best_child} def get_num_simulations(self): return self.root_node.visited diff --git a/expo/data/dataset.py b/expo/data/dataset.py index 8bcce0b1a..1494eb267 100644 --- a/expo/data/dataset.py +++ b/expo/data/dataset.py @@ -20,23 +20,19 @@ Report {metric} on the eval data. Do not plot or make any visualizations. DI_INSTRUCTION = """\ **Attention** 1. Please do not leak the target label in any form during training. -2. Dev and Test sets do not have the target column. +2. Test set does not have the target column. 3. You should perform transformations on train, dev, and test sets at the same time (it's a good idea to define functions for this and avoid code repetition). 4. If labels are transformed during training, they should be transformed back to the original format before saving the predictions. +5. You could utilize dev set to validate and improve model training. +6. Use techniques to avoid overfitting. ## Saving Dev and Test Predictions 1. Save the prediction results of BOTH the dev set and test set in `dev_predictions.csv` and `test_predictions.csv` respectively in the output directory. - Both files should contain a single column named `target` with the predicted values. 2. Make sure the prediction results are in the same format as the target column in the training set. -- The labels should be transformed back to the original format if any transformation was applied during training. -## Output Training Set Performance -Make sure the performance of the model is printed in python in the last step even if it has been printed in the previous steps. The value should be a float number. -Print the training set performance in the last step. Write in this format: -```python -... -print("Train score:", train_score) -``` +## Output Performance +Print the train and dev set performance in the last step. # Output dir {output_dir} @@ -47,10 +43,10 @@ TASK_PROMPT = """\ {user_requirement} {additional_instruction} # Data dir -training (with labels): {train_path} -dev (without labels): {dev_path} -testing (without labels): {test_path} -dataset description: {data_info_path} (You can use this file to get additional information about the dataset) +train set (with labels): {train_path} +dev set (with labels): {dev_path} +test set (without labels): {test_path} +dataset description: {data_info_path} (During EDA, you can use this file to get additional information about the dataset) """ @@ -147,7 +143,7 @@ def generate_task_requirement(task_name, data_config, is_di=True): user_requirement = get_user_requirement(task_name, data_config) split_dataset_path = get_split_dataset_path(task_name, data_config) train_path = split_dataset_path["train"] - dev_path = split_dataset_path["dev_wo_target"] + dev_path = split_dataset_path["dev"] test_path = split_dataset_path["test_wo_target"] work_dir = data_config["work_dir"] output_dir = f"{work_dir}/{task_name}" @@ -225,7 +221,7 @@ class ExpDataset: "NumberOfSymbolicFeatures": raw_df.select_dtypes(include=["object"]).shape[1], } - df_head_text = raw_df.head().to_string(index=False) + df_head_text = self.get_df_head(raw_df) dataset_info = { "name": self.name, @@ -236,6 +232,9 @@ class ExpDataset: } return dataset_info + def get_df_head(self, raw_df): + return raw_df.head().to_string(index=False) + def get_metric(self): dataset_info = self.get_dataset_info() num_classes = dataset_info["metadata"]["NumberOfClasses"] diff --git a/expo/data/hf_data.py b/expo/data/hf_data.py index a7e2a1afe..9ed2b2c48 100644 --- a/expo/data/hf_data.py +++ b/expo/data/hf_data.py @@ -38,18 +38,21 @@ class HFExpDataset(ExpDataset): df = pd.read_csv(Path(raw_dir, "train.csv")) else: df = self.dataset["train"].to_pandas() - df.to_csv(Path(raw_dir, "train.csv")) + df.to_csv(Path(raw_dir, "train.csv"), index=False) if os.path.exists(Path(raw_dir, "test.csv")): - test_df = pd.read_csv(Path(raw_dir, "test.csv")) + test_df = pd.read_csv(Path(raw_dir, "test.csv"), index=False) else: if "test" in self.dataset: test_df = self.dataset["test"].to_pandas() - test_df.to_csv(Path(raw_dir, "test.csv")) + test_df.to_csv(Path(raw_dir, "test.csv"), index=False) else: test_df = None return df, test_df + # def get_df_head(self, raw_df): + # return raw_df.head() + if __name__ == "__main__": dataset_dir = "D:/work/automl/datasets" diff --git a/expo/experimenter/mcts.py b/expo/experimenter/mcts.py index 9bf7306c4..fbe2f35f1 100644 --- a/expo/experimenter/mcts.py +++ b/expo/experimenter/mcts.py @@ -1,19 +1,21 @@ from expo.evaluation.visualize_mcts import get_tree_text from expo.experimenter.experimenter import Experimenter -from expo.Greedy import Greedy +from expo.Greedy import Greedy, Random from expo.MCTS import MCTS class MCTSExperimenter(Experimenter): result_path: str = "results/mcts" - def __init__(self, args, greedy=False, **kwargs): + def __init__(self, args, tree_mode=None, **kwargs): super().__init__(args, **kwargs) - self.greedy = greedy + self.tree_mode = tree_mode async def run_experiment(self): - if self.greedy: + if self.tree_mode == "greedy": mcts = Greedy(root_node=None, max_depth=5) + elif self.tree_mode == "random": + mcts = Random(root_node=None, max_depth=5) else: mcts = MCTS(root_node=None, max_depth=5) best_nodes = await mcts.search( diff --git a/expo/insights/solution_designer.py b/expo/insights/solution_designer.py index fc05afeea..b1fcf4188 100644 --- a/expo/insights/solution_designer.py +++ b/expo/insights/solution_designer.py @@ -19,7 +19,8 @@ DATASET_INSIGHT_PROMPT = """ Propose insights to help improve the performance of the model on this dataset. The insights should be proposed based on the dataset description with different task types. Each task type should have at least 5 insights. -Make sure each method is independent and can be implemented separately. +Make sure each method is diverse enough and can be implemented separately. +Be specific about models' choices, ensemble and tuning techniques, and preprocessing & feature engineering techniques. # Format ```json diff --git a/expo/run_experiment.py b/expo/run_experiment.py index cfdd295b2..2123fade3 100644 --- a/expo/run_experiment.py +++ b/expo/run_experiment.py @@ -2,16 +2,21 @@ import argparse import asyncio from expo.experimenter.aug import AugExperimenter +from expo.experimenter.autogluon import GluonExperimenter from expo.experimenter.custom import CustomExperimenter from expo.experimenter.experimenter import Experimenter from expo.experimenter.mcts import MCTSExperimenter -from expo.experimenter.autogluon import GluonExperimenter def get_args(): parser = argparse.ArgumentParser() parser.add_argument("--name", type=str, default="") - parser.add_argument("--exp_mode", type=str, default="mcts", choices=["mcts", "aug", "base", "custom", "greedy", "autogluon"]) + parser.add_argument( + "--exp_mode", + type=str, + default="mcts", + choices=["mcts", "aug", "base", "custom", "greedy", "autogluon", "random"], + ) get_di_args(parser) get_mcts_args(parser) get_aug_exp_args(parser) @@ -43,7 +48,9 @@ async def main(args): if args.exp_mode == "mcts": experimenter = MCTSExperimenter(args) elif args.exp_mode == "greedy": - experimenter = MCTSExperimenter(args, greedy=True) + experimenter = MCTSExperimenter(args, tree_mode="greedy") + elif args.exp_mode == "random": + experimenter = MCTSExperimenter(args, tree_mode="random") elif args.exp_mode == "aug": experimenter = AugExperimenter(args) elif args.exp_mode == "base": diff --git a/expo/utils.py b/expo/utils.py index 9c6295fa9..f3c0c392d 100644 --- a/expo/utils.py +++ b/expo/utils.py @@ -99,6 +99,7 @@ def save_notebook(role: Role, save_dir: str = "", name: str = ""): for code in codes: clean_nb.cells.append(nbformat.v4.new_code_cell(code)) nb = process_cells(role.execute_code.nb) + os.makedirs(save_dir, exist_ok=True) file_path = save_dir / f"{name}.ipynb" clean_file_path = save_dir / f"{name}_clean.ipynb" nbformat.write(nb, file_path) @@ -110,7 +111,7 @@ async def load_execute_notebook(role): codes = [task.code for task in tasks if task.code] executor = role.execute_code executor.nb = nbformat.v4.new_notebook() - executor.nb.client = NotebookClient(executor.nb) + executor.nb_client = NotebookClient(executor.nb, timeout=executor.timeout) # await executor.build() for code in codes: outputs, success = await executor.run(code)