From 852fbc58ee9eaa77160dcd1bcdcd533c896ed6a9 Mon Sep 17 00:00:00 2001
From: Yizhou Chi <chiyizhou@fuzhi.ai>
Date: Thu, 17 Oct 2024 17:47:41 +0800
Subject: [PATCH] automatically update args.low_is_better for mle-bench

---
 expo/data/custom_task.py | 9 +++++++++
 expo/run_experiment.py   | 4 +++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/expo/data/custom_task.py b/expo/data/custom_task.py
index c4dd0012c..f3cd433f5 100644
--- a/expo/data/custom_task.py
+++ b/expo/data/custom_task.py
@@ -35,6 +35,15 @@ def get_mle_task_id(dataset_dir):
     return dataset_dir.split("/")[-3]
 
 
+def get_mle_is_lower_better(task):
+    from mlebench.data import get_leaderboard
+    from mlebench.registry import registry
+
+    competition = registry.get_competition(task)
+    competition_leaderboard = get_leaderboard(competition)
+    return competition.grader.is_lower_better(competition_leaderboard)
+
+
 def get_mle_bench_requirements(dataset_dir, data_config, special_instruction, obfuscated=False):
     work_dir = data_config["work_dir"]
     task = get_mle_task_id(dataset_dir)
diff --git a/expo/run_experiment.py b/expo/run_experiment.py
index be891814d..7b49e6738 100644
--- a/expo/run_experiment.py
+++ b/expo/run_experiment.py
@@ -1,7 +1,7 @@
 import argparse
 import asyncio
 
-from expo.data.custom_task import get_mle_task_id
+from expo.data.custom_task import get_mle_is_lower_better, get_mle_task_id
 from expo.experimenter.aug import AugExperimenter
 from expo.experimenter.autogluon import GluonExperimenter
 from expo.experimenter.autosklearn import AutoSklearnExperimenter
@@ -33,6 +33,8 @@ def get_args(cmd=True):
         args.eval_func = "mlebench"
         args.from_scratch = True
         args.task = get_mle_task_id(args.custom_dataset_dir)
+        args.low_is_better = get_mle_is_lower_better(args.task)
+        print("low_is_better:", args.low_is_better)
     return args