1. add eval_func for sela and compatibility to others

2. llm extract score (use all code block and execution results)
3. add argument for custom dataset dir
4. dataset custom requirement support
This commit is contained in:
Yizhou Chi 2024-10-12 17:16:51 +08:00
parent eda9322361
commit 3a57060e25
11 changed files with 202 additions and 38 deletions

View file

@ -37,12 +37,18 @@ class InstructionGenerator:
def __init__(self, state, use_fixed_insights, from_scratch):
self.state = state
self.file_path = state["exp_pool_path"]
self.dataset_info_path = f"{self.data_config['datasets_dir']}/{state['task']}/dataset_info.json"
with open(self.dataset_info_path, "r") as file:
self.dataset_info = json.load(file)
if state["custom_dataset_dir"]:
self.dataset_info = "xxx"
else:
dataset_info_path = f"{self.data_config['datasets_dir']}/{state['task']}/dataset_info.json"
with open(dataset_info_path, "r") as file:
self.dataset_info = json.load(file)
self.use_fixed_insights = use_fixed_insights
self.proposer = SolutionDesigner()
self.from_scratch = from_scratch
if self.file_path is None:
self.from_scratch = True
else:
self.from_scratch = from_scratch
async def initialize(self):
if self.from_scratch: