apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-05-24 14:15:17 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

automatically update args.low_is_better for mle-bench

Browse source
This commit is contained in:
Yizhou Chi 2024-10-17 17:47:41 +08:00
parent 06710fbc18
commit 852fbc58ee
2 changed files with 12 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

9
expo/data/custom_task.py
View file

@ -35,6 +35,15 @@ def get_mle_task_id(dataset_dir):
return dataset_dir.split("/")[-3]
def get_mle_is_lower_better(task):
from mlebench.data import get_leaderboard
from mlebench.registry import registry
competition = registry.get_competition(task)
competition_leaderboard = get_leaderboard(competition)
return competition.grader.is_lower_better(competition_leaderboard)
def get_mle_bench_requirements(dataset_dir, data_config, special_instruction, obfuscated=False):
work_dir = data_config["work_dir"]
task = get_mle_task_id(dataset_dir)