Merge branch 'improve_mcts' into 'expo'

更新实验配置

See merge request agents/exp_optimizer!8

expo/Greedy.py

@@ -1,3 +1,5 @@
+import random
+
 from expo.MCTS import MCTS
 
 
@@ -7,3 +9,11 @@ class Greedy(MCTS):
             return self.root_node
         all_children = [child for children in self.children.values() for child in children]
         return max(all_children, key=lambda x: x.normalized_reward.get("dev_score", 0))
+
+
+class Random(MCTS):
+    def best_child(self):
+        if len(self.children) == 0:
+            return self.root_node
+        all_children = [child for children in self.children.values() for child in children]
+        return random.choice(all_children)

expo/MCTS.py

@@ -279,7 +279,8 @@ class MCTS:
 
     def best_path(self, root: Node):
         best_child = root
-        best_score = 0
+        global_best_score = root.normalized_reward["test_score"]
+        dev_best_score = root.normalized_reward["dev_score"]
 
         def bfs(node: Node, best_score, best_child: Node, split):
             assert split in ["test_score", "dev_score"]
@@ -294,10 +295,10 @@ class MCTS:
                 best_score, best_child = bfs(child, best_score, best_child, split)
             return best_score, best_child
 
-        _, best_child = bfs(root, best_score, best_child, "test_score")
-        _, dev_best_child = bfs(root, best_score, best_child, "dev_score")
+        _, global_best_child = bfs(root, global_best_score, best_child, "test_score")
+        _, dev_best_child = bfs(root, dev_best_score, best_child, "dev_score")
 
-        return {"dev_best": dev_best_child, "global_best": best_child}
+        return {"dev_best": dev_best_child, "global_best": global_best_child}
 
     def get_num_simulations(self):
         return self.root_node.visited

expo/data/dataset.py

@@ -20,23 +20,19 @@ Report {metric} on the eval data. Do not plot or make any visualizations.
 DI_INSTRUCTION = """\
 **Attention** 
 1. Please do not leak the target label in any form during training.
-2. Dev and Test sets do not have the target column.
+2. Test set does not have the target column.
 3. You should perform transformations on train, dev, and test sets at the same time (it's a good idea to define functions for this and avoid code repetition).
 4. If labels are transformed during training, they should be transformed back to the original format before saving the predictions.
+5. You could utilize dev set to validate and improve model training.
+6. Use techniques to avoid overfitting.
 
 ## Saving Dev and Test Predictions
 1. Save the prediction results of BOTH the dev set and test set in `dev_predictions.csv` and `test_predictions.csv` respectively in the output directory. 
 - Both files should contain a single column named `target` with the predicted values.
 2. Make sure the prediction results are in the same format as the target column in the training set. 
-- The labels should be transformed back to the original format if any transformation was applied during training.
 
-## Output Training Set Performance
-Make sure the performance of the model is printed in python in the last step even if it has been printed in the previous steps. The value should be a float number.
-Print the training set performance in the last step. Write in this format:
-```python
-...
-print("Train score:", train_score)
-```
+## Output Performance
+Print the train and dev set performance in the last step.
 
 # Output dir
 {output_dir}
@@ -47,10 +43,10 @@ TASK_PROMPT = """\
 {user_requirement}
 {additional_instruction}
 # Data dir
-training (with labels): {train_path}
-dev (without labels): {dev_path}
-testing (without labels): {test_path}
-dataset description: {data_info_path} (You can use this file to get additional information about the dataset)
+train set (with labels): {train_path}
+dev set (with labels): {dev_path}
+test set (without labels): {test_path}
+dataset description: {data_info_path} (During EDA, you can use this file to get additional information about the dataset)
 """
 
 
@@ -147,7 +143,7 @@ def generate_task_requirement(task_name, data_config, is_di=True):
     user_requirement = get_user_requirement(task_name, data_config)
     split_dataset_path = get_split_dataset_path(task_name, data_config)
     train_path = split_dataset_path["train"]
-    dev_path = split_dataset_path["dev_wo_target"]
+    dev_path = split_dataset_path["dev"]
     test_path = split_dataset_path["test_wo_target"]
     work_dir = data_config["work_dir"]
     output_dir = f"{work_dir}/{task_name}"
@@ -225,7 +221,7 @@ class ExpDataset:
             "NumberOfSymbolicFeatures": raw_df.select_dtypes(include=["object"]).shape[1],
         }
 
-        df_head_text = raw_df.head().to_string(index=False)
+        df_head_text = self.get_df_head(raw_df)
 
         dataset_info = {
             "name": self.name,
@@ -236,6 +232,9 @@ class ExpDataset:
         }
         return dataset_info
 
+    def get_df_head(self, raw_df):
+        return raw_df.head().to_string(index=False)
+
     def get_metric(self):
         dataset_info = self.get_dataset_info()
         num_classes = dataset_info["metadata"]["NumberOfClasses"]

expo/data/hf_data.py

@@ -38,18 +38,21 @@ class HFExpDataset(ExpDataset):
             df = pd.read_csv(Path(raw_dir, "train.csv"))
         else:
             df = self.dataset["train"].to_pandas()
-            df.to_csv(Path(raw_dir, "train.csv"))
+            df.to_csv(Path(raw_dir, "train.csv"), index=False)
 
         if os.path.exists(Path(raw_dir, "test.csv")):
-            test_df = pd.read_csv(Path(raw_dir, "test.csv"))
+            test_df = pd.read_csv(Path(raw_dir, "test.csv"), index=False)
         else:
             if "test" in self.dataset:
                 test_df = self.dataset["test"].to_pandas()
-                test_df.to_csv(Path(raw_dir, "test.csv"))
+                test_df.to_csv(Path(raw_dir, "test.csv"), index=False)
             else:
                 test_df = None
         return df, test_df
 
+    # def get_df_head(self, raw_df):
+    #     return raw_df.head()
+
 
 if __name__ == "__main__":
     dataset_dir = "D:/work/automl/datasets"

expo/experimenter/mcts.py

@@ -1,19 +1,21 @@
 from expo.evaluation.visualize_mcts import get_tree_text
 from expo.experimenter.experimenter import Experimenter
-from expo.Greedy import Greedy
+from expo.Greedy import Greedy, Random
 from expo.MCTS import MCTS
 
 
 class MCTSExperimenter(Experimenter):
     result_path: str = "results/mcts"
 
-    def __init__(self, args, greedy=False, **kwargs):
+    def __init__(self, args, tree_mode=None, **kwargs):
         super().__init__(args, **kwargs)
-        self.greedy = greedy
+        self.tree_mode = tree_mode
 
     async def run_experiment(self):
-        if self.greedy:
+        if self.tree_mode == "greedy":
             mcts = Greedy(root_node=None, max_depth=5)
+        elif self.tree_mode == "random":
+            mcts = Random(root_node=None, max_depth=5)
         else:
             mcts = MCTS(root_node=None, max_depth=5)
         best_nodes = await mcts.search(

expo/insights/solution_designer.py

@@ -19,7 +19,8 @@ DATASET_INSIGHT_PROMPT = """
 Propose insights to help improve the performance of the model on this dataset.
 The insights should be proposed based on the dataset description with different task types.
 Each task type should have at least 5 insights.
-Make sure each method is independent and can be implemented separately.
+Make sure each method is diverse enough and can be implemented separately.
+Be specific about models' choices, ensemble and tuning techniques, and preprocessing & feature engineering techniques.
 
 # Format
 ```json

expo/run_experiment.py

@@ -2,16 +2,21 @@ import argparse
 import asyncio
 
 from expo.experimenter.aug import AugExperimenter
+from expo.experimenter.autogluon import GluonExperimenter
 from expo.experimenter.custom import CustomExperimenter
 from expo.experimenter.experimenter import Experimenter
 from expo.experimenter.mcts import MCTSExperimenter
-from expo.experimenter.autogluon import GluonExperimenter
 
 
 def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("--name", type=str, default="")
-    parser.add_argument("--exp_mode", type=str, default="mcts", choices=["mcts", "aug", "base", "custom", "greedy", "autogluon"])
+    parser.add_argument(
+        "--exp_mode",
+        type=str,
+        default="mcts",
+        choices=["mcts", "aug", "base", "custom", "greedy", "autogluon", "random"],
+    )
     get_di_args(parser)
     get_mcts_args(parser)
     get_aug_exp_args(parser)
@@ -43,7 +48,9 @@ async def main(args):
     if args.exp_mode == "mcts":
         experimenter = MCTSExperimenter(args)
     elif args.exp_mode == "greedy":
-        experimenter = MCTSExperimenter(args, greedy=True)
+        experimenter = MCTSExperimenter(args, tree_mode="greedy")
+    elif args.exp_mode == "random":
+        experimenter = MCTSExperimenter(args, tree_mode="random")
     elif args.exp_mode == "aug":
         experimenter = AugExperimenter(args)
     elif args.exp_mode == "base":

expo/utils.py

@@ -99,6 +99,7 @@ def save_notebook(role: Role, save_dir: str = "", name: str = ""):
     for code in codes:
         clean_nb.cells.append(nbformat.v4.new_code_cell(code))
     nb = process_cells(role.execute_code.nb)
+    os.makedirs(save_dir, exist_ok=True)
     file_path = save_dir / f"{name}.ipynb"
     clean_file_path = save_dir / f"{name}_clean.ipynb"
     nbformat.write(nb, file_path)
@@ -110,7 +111,7 @@ async def load_execute_notebook(role):
     codes = [task.code for task in tasks if task.code]
     executor = role.execute_code
     executor.nb = nbformat.v4.new_notebook()
-    executor.nb.client = NotebookClient(executor.nb)
+    executor.nb_client = NotebookClient(executor.nb, timeout=executor.timeout)
     # await executor.build()
     for code in codes:
         outputs, success = await executor.run(code)