From 6344046c31d0d4c673ddfcd6c9b041f0dbfc3e6f Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Tue, 24 Sep 2024 20:48:44 +0800 Subject: [PATCH 01/11] update aug result summarization --- expo/experimenter/aug.py | 6 +----- expo/experimenter/experimenter.py | 29 +++++++++++++++++------------ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/expo/experimenter/aug.py b/expo/experimenter/aug.py index ffe0d04c5..97b819802 100644 --- a/expo/experimenter/aug.py +++ b/expo/experimenter/aug.py @@ -49,9 +49,5 @@ class AugExperimenter(Experimenter): "args": vars(self.args), } ) - scores = [result["score_dict"]["test_score"] for result in results] - avg_score = sum(scores) / len(scores) - best_score = max(scores) if not self.args.low_is_better else min(scores) - best_score_idx = scores.index(best_score) - results.insert(0, {"avg_score": avg_score, "best_score": best_score, "best_score_idx": best_score_idx}) + results = self.summarize_results(results) self.save_result(results) diff --git a/expo/experimenter/experimenter.py b/expo/experimenter/experimenter.py index 77cb5fa45..c6ead281b 100644 --- a/expo/experimenter/experimenter.py +++ b/expo/experimenter/experimenter.py @@ -47,18 +47,7 @@ class Experimenter: score_dict = {"train_score": -1, "dev_score": -1, "test_score": -1, "score": -1} return score_dict - async def run_experiment(self): - state = self.state - user_requirement = state["requirement"] - results = [] - - for i in range(self.args.num_experiments): - di = ResearchAssistant(node_id="0", use_reflection=self.args.reflection) - score_dict = await self.run_di(di, user_requirement, run_idx=i) - results.append( - {"idx": i, "score_dict": score_dict, "user_requirement": user_requirement, "args": vars(self.args)} - ) - self.save_result(results) # save intermediate results + def summarize_results(self, results): dev_scores = [result["score_dict"]["dev_score"] for result in results] best_dev_score = ( max(dev_scores) @@ -85,6 +74,22 @@ class Experimenter: "global_best_test_score": global_best_score, }, ) + return results + + async def run_experiment(self): + state = self.state + user_requirement = state["requirement"] + results = [] + + for i in range(self.args.num_experiments): + di = ResearchAssistant(node_id="0", use_reflection=self.args.reflection) + score_dict = await self.run_di(di, user_requirement, run_idx=i) + results.append( + {"idx": i, "score_dict": score_dict, "user_requirement": user_requirement, "args": vars(self.args)} + ) + self.save_result(results) # save intermediate results + results = self.summarize_results(results) + self.save_result(results) def evaluate_prediction(self, split, state): From 31adaee23f919da61e8b949a1651d36a734de105 Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Tue, 24 Sep 2024 20:58:41 +0800 Subject: [PATCH 02/11] add special instruction for img/text dataset --- expo/data/dataset.py | 13 ++++++++++++- expo/run_experiment.py | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/expo/data/dataset.py b/expo/data/dataset.py index 28bd26d2e..8ad6f2854 100644 --- a/expo/data/dataset.py +++ b/expo/data/dataset.py @@ -20,12 +20,23 @@ USE_AG = """ 7. Please use autogluon for model training with presets='medium_quality', time_limit=None, give dev dataset to tuning_data, and use right eval_metric. """ +TEXT_MODALITY = """ +7. You could use models from transformers library for this text dataset. +8. Use gpu if available for faster training. +""" + +IMAGE_MODALITY = """ +7. You could use models from torchvision library for this image dataset. +8. Use gpu if available for faster training. +""" + STACKING = """ 7. To avoid overfitting, train a weighted ensemble model such as StackingClassifier or StackingRegressor. 8. You could do some quick model prototyping to see which models work best and then use them in the ensemble. """ -SPECIAL_INSTRUCTIONS = {"ag": USE_AG, "stacking": STACKING} + +SPECIAL_INSTRUCTIONS = {"ag": USE_AG, "stacking": STACKING, "text": TEXT_MODALITY, "image": IMAGE_MODALITY} DI_INSTRUCTION = """ ## Attention diff --git a/expo/run_experiment.py b/expo/run_experiment.py index be028c47e..8dd66577c 100644 --- a/expo/run_experiment.py +++ b/expo/run_experiment.py @@ -3,10 +3,10 @@ import asyncio from expo.experimenter.aug import AugExperimenter from expo.experimenter.autogluon import GluonExperimenter +from expo.experimenter.autosklearn import AutoSklearnExperimenter from expo.experimenter.custom import CustomExperimenter from expo.experimenter.experimenter import Experimenter from expo.experimenter.mcts import MCTSExperimenter -from expo.experimenter.autosklearn import AutoSklearnExperimenter def get_args(): @@ -43,7 +43,7 @@ def get_di_args(parser): parser.add_argument("--reflection", dest="reflection", action="store_true") parser.add_argument("--no_reflection", dest="reflection", action="store_false") parser.add_argument("--num_experiments", type=int, default=1) - parser.add_argument("--special_instruction", type=str, default=None, choices=["ag", "stacking"]) + parser.add_argument("--special_instruction", type=str, default=None, choices=["ag", "stacking", "text", "image"]) parser.set_defaults(reflection=True) From 68c672d4381dfee21b15e682aba5801bdc3934a8 Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Wed, 25 Sep 2024 09:59:12 +0800 Subject: [PATCH 03/11] use transformers lib instead of torchvision --- expo/data/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expo/data/dataset.py b/expo/data/dataset.py index 8ad6f2854..f2f01f71b 100644 --- a/expo/data/dataset.py +++ b/expo/data/dataset.py @@ -26,7 +26,7 @@ TEXT_MODALITY = """ """ IMAGE_MODALITY = """ -7. You could use models from torchvision library for this image dataset. +7. You could use models from transformers library for this image dataset. 8. Use gpu if available for faster training. """ From 2b67355358a6ce9f490fd1b9b6970f7010789857 Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Thu, 26 Sep 2024 20:25:36 +0800 Subject: [PATCH 04/11] add step score --- expo/MCTS.py | 31 ++++++++++++++++++++++++++++++- expo/experimenter/mcts.py | 2 ++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/expo/MCTS.py b/expo/MCTS.py index aa4ade944..4564cd682 100644 --- a/expo/MCTS.py +++ b/expo/MCTS.py @@ -1,3 +1,4 @@ +import json import math import os import pickle @@ -240,6 +241,7 @@ class MCTS: max_depth: int = 5 c_explore: float = 1.4 c_unvisited: float = 0.8 + node_order: list = [] def __init__(self, root_node, max_depth, use_fixed_insights): self.root_node = root_node @@ -306,11 +308,32 @@ class MCTS: _, global_best_child = bfs(root, global_best_score, best_child, "test_score") _, dev_best_child = bfs(root, dev_best_score, best_child, "dev_score") - return {"dev_best": dev_best_child, "global_best": global_best_child} + return {"dev_best": dev_best_child, "global_best": global_best_child, "scores": self.get_score_order_dict()} def get_num_simulations(self): return self.root_node.visited + def save_node_order(self, node_id): + self.node_order.append(node_id) + with open(os.path.join(self.root_node.state["node_dir"], "node_order.json"), "w") as f: + json.dump(self.node_order, f) + + def load_node_order(self): + with open(os.path.join(self.root_node.state["node_dir"], "node_order.json"), "r") as f: + self.node_order = json.load(f) + + def get_score_order_dict(self): + scores = {"dev": [], "test": [], "dev_raw": [], "test_raw": []} + for node_id in self.node_order: + node = Node(parent=None, state=self.root_node.state, action=None, value=0) + node.id = node_id + node = node.load_node() + scores["dev"].append(node.normalized_reward["dev_score"]) + scores["test"].append(node.normalized_reward["test_score"]) + scores["dev_raw"].append(node.raw_reward["dev_score"]) + scores["test_raw"].append(node.raw_reward["test_score"]) + return scores + async def search(self, state, rollouts, load_tree=False, reflection=False): role, root = initialize_di_root_node(state, reflection=reflection) self.root_node = root @@ -329,8 +352,12 @@ class MCTS: self.backpropagate(root, reward) node, reward = await self.expand_and_simulate(root) # self.backpropagate(node, reward) + self.save_node_order(root.id) + self.save_node_order(node.id) else: root = self.root_node + self.load_node_order() + for _ in range(rollouts): # number of rollouts mcts_logger.log("MCTS", f"Start the next rollout {_+1}") node = self.select(root) @@ -344,6 +371,7 @@ class MCTS: else: node, reward = await self.expand_and_simulate(node) # self.backpropagate(node, reward) + self.save_node_order(node.id) return self.best_path(root) async def expand_and_simulate(self, node): @@ -373,6 +401,7 @@ class MCTS: self.root_node = pickle.load(f) self.children[self.root_node] = self.root_node.children load_children_node(self.root_node) + if self.children: return True return False diff --git a/expo/experimenter/mcts.py b/expo/experimenter/mcts.py index 5fb00ca8d..bd803bff1 100644 --- a/expo/experimenter/mcts.py +++ b/expo/experimenter/mcts.py @@ -29,6 +29,7 @@ class MCTSExperimenter(Experimenter): ) best_node = best_nodes["global_best"] dev_best_node = best_nodes["dev_best"] + score_dict = best_nodes["scores"] self.copy_notebook(best_node, "best") self.copy_notebook(dev_best_node, "dev_best") @@ -50,6 +51,7 @@ class MCTSExperimenter(Experimenter): "user_requirement": best_node.state["requirement"], "tree_text": text, "args": vars(self.args), + "scores": score_dict, } ] self.save_result(results) From 24536df24c24b80b870c96248bfc46fdd07ec929 Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Fri, 27 Sep 2024 12:54:22 +0800 Subject: [PATCH 05/11] update_save_order --- expo/experimenter/mcts.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/expo/experimenter/mcts.py b/expo/experimenter/mcts.py index bd803bff1..fa42cb070 100644 --- a/expo/experimenter/mcts.py +++ b/expo/experimenter/mcts.py @@ -31,9 +31,6 @@ class MCTSExperimenter(Experimenter): dev_best_node = best_nodes["dev_best"] score_dict = best_nodes["scores"] - self.copy_notebook(best_node, "best") - self.copy_notebook(dev_best_node, "dev_best") - text, num_generated_codes = get_tree_text(mcts.root_node) text += f"Generated {num_generated_codes} unique codes.\n" text += f"Best node: {best_node.id}, score: {best_node.raw_reward}\n" @@ -55,6 +52,8 @@ class MCTSExperimenter(Experimenter): } ] self.save_result(results) + self.copy_notebook(best_node, "best") + self.copy_notebook(dev_best_node, "dev_best") def copy_notebook(self, node, name): node_dir = node.get_node_dir() From af844693b1510655223edc8c879cc9f14f9140e0 Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Fri, 27 Sep 2024 15:13:23 +0800 Subject: [PATCH 06/11] change label column --- expo/data/hf_data.py | 4 ++-- expo/datasets.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/expo/data/hf_data.py b/expo/data/hf_data.py index df3a6ed20..a43fcd415 100644 --- a/expo/data/hf_data.py +++ b/expo/data/hf_data.py @@ -18,7 +18,7 @@ HFDATSETS = [ "name": "oxford-iiit-pet", "dataset_name": "timm/oxford-iiit-pet", "image_col": "image", - "target_col": "label_cat_dog", + "target_col": "label", "modality": "image", }, { @@ -115,7 +115,7 @@ class HFExpDataset(ExpDataset): if __name__ == "__main__": dataset_dir = "D:/work/automl/datasets" - save_analysis_pool = False + save_analysis_pool = True force_update = False datasets_dict = {"datasets": {}} solution_designer = SolutionDesigner() diff --git a/expo/datasets.yaml b/expo/datasets.yaml index 92e004c6d..e58e717b5 100644 --- a/expo/datasets.yaml +++ b/expo/datasets.yaml @@ -202,7 +202,7 @@ datasets: oxford-iiit-pet: dataset: oxford-iiit-pet metric: f1 - target_col: label_cat_dog + target_col: label user_requirement: "This is a oxford-iiit-pet dataset. Your goal is to predict\ \ the target column `label_cat_dog`.\nPerform data analysis, data preprocessing,\ \ feature engineering, and modeling to predict the target. \nReport f1 on the\ From bd26e900e6b33c9e16c6246958e977b356dd0a5a Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Sat, 28 Sep 2024 10:07:35 +0800 Subject: [PATCH 07/11] update target label --- expo/datasets.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/expo/datasets.yaml b/expo/datasets.yaml index e58e717b5..016daf7ec 100644 --- a/expo/datasets.yaml +++ b/expo/datasets.yaml @@ -201,10 +201,10 @@ datasets: \ Do not plot or make any visualizations.\n" oxford-iiit-pet: dataset: oxford-iiit-pet - metric: f1 + metric: f1 weighted target_col: label user_requirement: "This is a oxford-iiit-pet dataset. Your goal is to predict\ - \ the target column `label_cat_dog`.\nPerform data analysis, data preprocessing,\ + \ the target column `label`.\nPerform data analysis, data preprocessing,\ \ feature engineering, and modeling to predict the target. \nReport f1 on the\ \ eval data. Do not plot or make any visualizations.\n" sms_spam: From 06702db6d15deca1722027af9399173b9b647259 Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Sat, 28 Sep 2024 10:39:12 +0800 Subject: [PATCH 08/11] update pet --- expo/datasets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expo/datasets.yaml b/expo/datasets.yaml index 016daf7ec..2d02951d4 100644 --- a/expo/datasets.yaml +++ b/expo/datasets.yaml @@ -205,7 +205,7 @@ datasets: target_col: label user_requirement: "This is a oxford-iiit-pet dataset. Your goal is to predict\ \ the target column `label`.\nPerform data analysis, data preprocessing,\ - \ feature engineering, and modeling to predict the target. \nReport f1 on the\ + \ feature engineering, and modeling to predict the target. \nReport f1 weighted on the\ \ eval data. Do not plot or make any visualizations.\n" sms_spam: dataset: sms_spam From db247d9ff4dd8a10279bd8e1d232d552c8a2b324 Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Sat, 28 Sep 2024 15:01:01 +0800 Subject: [PATCH 09/11] save result order --- expo/experimenter/mcts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/expo/experimenter/mcts.py b/expo/experimenter/mcts.py index fa42cb070..22b480caf 100644 --- a/expo/experimenter/mcts.py +++ b/expo/experimenter/mcts.py @@ -36,8 +36,6 @@ class MCTSExperimenter(Experimenter): text += f"Best node: {best_node.id}, score: {best_node.raw_reward}\n" text += f"Dev best node: {dev_best_node.id}, score: {dev_best_node.raw_reward}\n" print(text) - self.save_tree(text) - results = [ { "best_node": best_node.id, @@ -54,6 +52,7 @@ class MCTSExperimenter(Experimenter): self.save_result(results) self.copy_notebook(best_node, "best") self.copy_notebook(dev_best_node, "dev_best") + self.save_tree(text) def copy_notebook(self, node, name): node_dir = node.get_node_dir() From 1589a04cdbf14ef14549bd09141483191b85929b Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Sat, 28 Sep 2024 18:27:23 +0800 Subject: [PATCH 10/11] clarify prediction saving prompt --- expo/data/dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/expo/data/dataset.py b/expo/data/dataset.py index f2f01f71b..dd4cb4543 100644 --- a/expo/data/dataset.py +++ b/expo/data/dataset.py @@ -26,7 +26,7 @@ TEXT_MODALITY = """ """ IMAGE_MODALITY = """ -7. You could use models from transformers library for this image dataset. +7. You could use models from transformers/torchvision library for this image dataset. 8. Use gpu if available for faster training. """ @@ -50,8 +50,8 @@ DI_INSTRUCTION = """ ## Saving Dev and Test Predictions 1. Save the prediction results of BOTH the dev set and test set in `dev_predictions.csv` and `test_predictions.csv` respectively in the output directory. - Both files should contain a single column named `target` with the predicted values. -2. Make sure the prediction results are in the same format as the target column in the training set. -- For instance, if the target column is categorical, the prediction results should be categorical as well. +2. Make sure the prediction results are in the same format as the target column in the original training set. +- For instance, if the original target column is a list of string, the prediction results should also be strings. ## Output Performance Print the train and dev set performance in the last step. From 788e42ea55e80682221e10b7f7cf56daa3c102fe Mon Sep 17 00:00:00 2001 From: Yizhou Chi Date: Mon, 30 Sep 2024 16:06:48 +0800 Subject: [PATCH 11/11] update model list --- metagpt/prompts/task_type.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/prompts/task_type.py b/metagpt/prompts/task_type.py index 97666874d..74286a28f 100644 --- a/metagpt/prompts/task_type.py +++ b/metagpt/prompts/task_type.py @@ -35,8 +35,8 @@ The current task is about feature engineering. when performing it, please adhere MODEL_TRAIN_PROMPT = """ The current task is about training a model, please ensure high performance: - For tabular datasets - you have access to XGBoost, CatBoost, random forest, extremely randomized trees, k-nearest neighbors, linear regression, etc. -- For image datasets - you have access to ResNet, VGG, Inception, MobileNet, DenseNet, EfficientNet, etc. -- For text datasets - you have access to BERT, GPT-2, RoBERTa, DistilBERT, T5, etc. +- For image datasets - you have access to Swin Transformer, ViT, ResNet, EfficientNet, etc. +- For text datasets - you have access to Electra, DeBERTa, GPT-2, BERT, etc. - Avoid the use of SVM because of its high training time. - Keep in mind that your user prioritizes results and is highly focused on model performance. So, when needed, feel free to use models of any complexity to improve effectiveness, such as XGBoost, CatBoost, etc. - If non-numeric columns exist, perform label encode together with all steps.