Merge branch 'autogluon' into 'expo'

implement autuglu exp

See merge request agents/exp_optimizer!7

.gitignore

@@ -29,6 +29,7 @@ share/python-wheels/
 MANIFEST
 metagpt/tools/schemas/
 examples/data/search_kb/*.json
+expo/AutogluonModels
 
 # PyInstaller
 #  Usually these files are written by a python scripts from a template

expo/experimenter/autogluon.py

@@ -1,27 +1,29 @@
+from datetime import datetime
 from autogluon.tabular import TabularDataset, TabularPredictor
-
 from expo.experimenter.custom import CustomExperimenter
 
 
 class AGRunner:
     preset = "best_quality"
-    time_limit = 500
+    time_limit = 1000 # 1000s
 
-    def __init__(self, datasets):
-        self.datasets = datasets
+    def __init__(self, state=None):
+        self.state = state
+        self.datasets = self.state["datasets_dir"]
 
     def run(self):
         train_path = self.datasets["train"]
-        test_wo_target_path = self.datasets["test_wo_target"]
         dev_wo_target_path = self.datasets["dev_wo_target"]
+        test_wo_target_path = self.datasets["test_wo_target"]
         target_col = self.state["dataset_config"]["target_col"]
         train_data = TabularDataset(train_path)
-        test_data = TabularDataset(test_wo_target_path)
         dev_data = TabularDataset(dev_wo_target_path)
-
-        predictor = TabularPredictor(label=target_col).fit(train_data, presets=self.preset, time_limit=self.time_limit)
-        test_preds = predictor.predict(test_data)
+        test_data = TabularDataset(test_wo_target_path)
+        eval_metric = self.state["dataset_config"]["metric"].replace(" ", "_")
+        # predictor = TabularPredictor(label=target_col, eval_metric=eval_metric, path="AutogluonModels/ag-{}-{}".format(self.state['task'], datetime.now().strftime("%y%m%d_%H%M"))).fit(train_data, presets=self.preset, time_limit=self.time_limit, fit_weighted_ensemble=False, num_gpus=1)
+        predictor = TabularPredictor(label=target_col, eval_metric=eval_metric, path="AutogluonModels/ag-{}-{}".format(self.state['task'], datetime.now().strftime("%y%m%d_%H%M"))).fit(train_data, num_gpus=1)
         dev_preds = predictor.predict(dev_data)
+        test_preds = predictor.predict(test_data)
         return {"test_preds": test_preds, "dev_preds": dev_preds}
 
 
@@ -30,4 +32,16 @@ class GluonExperimenter(CustomExperimenter):
 
     def __init__(self, args, **kwargs):
         super().__init__(args, **kwargs)
-        self.framework = AGRunner(self.datasets)
+        self.framework = AGRunner(self.state)
+
+    async def run_experiment(self):
+        result = self.framework.run()
+        user_requirement = self.state["requirement"]
+        dev_preds = result["dev_preds"]
+        test_preds = result["test_preds"]
+        score_dict = {
+            "dev_score": self.evaluate_predictions(dev_preds, "dev"),
+            "test_score": self.evaluate_predictions(test_preds, "test"),
+        }
+        results = [0, {"score_dict": score_dict, "user_requirement": user_requirement, "args": vars(self.args)}]
+        self.save_result(results)

expo/experimenter/custom.py

@@ -12,7 +12,7 @@ class CustomExperimenter(Experimenter):
 
     def __init__(self, args, **kwargs):
         super().__init__(args, **kwargs)
-        self.framework = kwargs["framework"]  # todo
+        self.framework = kwargs.get("framework", None)  # todo
         self.task = kwargs.get("task", self.args.task)
         self.low_is_better = kwargs.get("low_is_better", self.args.low_is_better)
         self.name = kwargs.get("name", "")

expo/run_experiment.py

@@ -5,12 +5,13 @@ from expo.experimenter.aug import AugExperimenter
 from expo.experimenter.custom import CustomExperimenter
 from expo.experimenter.experimenter import Experimenter
 from expo.experimenter.mcts import MCTSExperimenter
+from expo.experimenter.autogluon import GluonExperimenter
 
 
 def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("--name", type=str, default="")
-    parser.add_argument("--exp_mode", type=str, default="mcts", choices=["mcts", "aug", "base", "custom", "greedy"])
+    parser.add_argument("--exp_mode", type=str, default="mcts", choices=["mcts", "aug", "base", "custom", "greedy", "autogluon"])
     get_di_args(parser)
     get_mcts_args(parser)
     get_aug_exp_args(parser)
@@ -47,6 +48,8 @@ async def main(args):
         experimenter = AugExperimenter(args)
     elif args.exp_mode == "base":
         experimenter = Experimenter(args)
+    elif args.exp_mode == "autogluon":
+        experimenter = GluonExperimenter(args)
     elif args.exp_mode == "custom":
         experimenter = CustomExperimenter(args)
     else: