From a2743d2b1fe47761db9be24ca6a49e526b9289eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Tue, 26 Dec 2023 15:48:04 +0800 Subject: [PATCH] resolve cr in MR17. --- metagpt/actions/write_analysis_code.py | 34 ++++++----- metagpt/roles/ml_engineer.py | 80 ++++++++++---------------- tests/metagpt/roles/test_daml.py | 4 ++ 3 files changed, 55 insertions(+), 63 deletions(-) diff --git a/metagpt/actions/write_analysis_code.py b/metagpt/actions/write_analysis_code.py index 3e912ace5..9691f888f 100644 --- a/metagpt/actions/write_analysis_code.py +++ b/metagpt/actions/write_analysis_code.py @@ -249,6 +249,7 @@ class MakeTools(WriteCodeByGenerate): super().__init__(name, context, llm) self.workspace = workspace or str(Path(__file__).parents[1].joinpath("./tools/functions/libs/udf")) self.file_suffix: str = '.py' + self.context = [] def parse_function_name(self, function_code: str) -> str: # 定义正则表达式模式 @@ -270,11 +271,14 @@ class MakeTools(WriteCodeByGenerate): saved_path.write_text(tool_code, encoding='utf-8') @retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) - async def run(self, code: str, code_desc: str = None, **kwargs) -> str: + async def run(self, code: str | List[dict], code_desc: str = None, **kwargs) -> str: # 拼接code prompt code_prompt = f"The following code is about {code_desc}, convert it to be a General Function, {code}" - msgs = self.process_msg(code_prompt, self.DEFAULT_SYSTEM_MSG) - logger.info(f"\n\nAsk to Make tools:\n{'-'*60}\n {msgs[-1]}") + if not self.context: + self.context = self.process_msg(code_prompt) + else: + self.context.append(self.process_msg(code_prompt)[-1]) + logger.info(f"\n\nAsk to Make tools:\n{'-'*60}\n {self.context[-1]}") # 更新kwargs if 'code' in kwargs: @@ -282,17 +286,21 @@ class MakeTools(WriteCodeByGenerate): if 'code_desc' in kwargs: kwargs.pop('code_desc') - tool_code = await self.llm.aask_code(msgs, **kwargs) - max_tries, current_try = 3, 1 - func_name = self.parse_function_name(tool_code['code']) - while current_try < max_tries and func_name is None: - logger.info(f"\n\nTools Respond\n{'-'*60}\n: {tool_code}") - logger.warning(f"No function name found in code, we will retry make tools. \n\n{tool_code['code']}\n") - msgs.append({'role': 'assistant', 'content': 'We need a general function in above code,but not found function.'}) - tool_code = await self.llm.aask_code(msgs, **kwargs) - current_try += 1 + max_tries, current_try = 3, 0 + while True: + tool_code = await self.llm.aask_code(self.context, **kwargs) func_name = self.parse_function_name(tool_code['code']) - if func_name is not None: + current_try += 1 + # make tools failed, add error message to context. + if not func_name: + logger.info(f"\n\nTools Respond\n{'-'*60}\n: {tool_code}") + logger.error(f"No function name found in code, we will retry make tools.\n{tool_code['code']}\n") + self.context.append({'role': 'user', 'content': 'We need a general function in above code,but not found function.'}) + # end make tools + if func_name is not None or current_try >= max_tries: + if current_try >= max_tries: + logger.error(f"We have tried the maximum number of attempts {max_tries}\ + and still have not created tools successfully, we will skip it.") break logger.info(f"\n\nTools Respond\n{'-'*60}\n: {tool_code}") self.save(tool_code['code']) diff --git a/metagpt/roles/ml_engineer.py b/metagpt/roles/ml_engineer.py index b991d9329..cec572991 100644 --- a/metagpt/roles/ml_engineer.py +++ b/metagpt/roles/ml_engineer.py @@ -286,65 +286,45 @@ class MLEngineer(Role): logger.warning(f"Making tools for task_id {self.plan.current_task_id}: \ `{self.plan.current_task.instruction}` \n code: \n {code}") make_tools = MakeTools() - tool_code = await make_tools.run(code, self.plan.current_task.instruction) - # check tool_code by execute_code - logger.info(f"Checking task_id {self.plan.current_task_id} tool code by executor...") - _, success = await self.execute_code.run(tool_code) - make_tool_retries, make_tool_current_retry = 3, 1 - while not success: - # tool_code = await make_tools.run(code_prompt) - tool_code = await make_tools.run(code) - _, success = await self.execute_code.run(tool_code) + make_tool_retries, make_tool_current_retry = 3, 0 + while True: + # start make tools + tool_code = await make_tools.run(code, self.plan.current_task.instruction) make_tool_current_retry += 1 - if make_tool_current_retry > make_tool_retries: - logger.error(f"We have tried the maximum number of attempts {make_tool_retries}\ - and still have not created tools for task_id {self.plan.current_task_id} successfully,\ - we will skip it.") + + # check tool_code by execute_code + logger.info(f"Checking task_id {self.plan.current_task_id} tool code by executor...") + execute_result, execute_success = await self.execute_code.run(tool_code) + if not execute_success: + logger.error(f"Tool code faild to execute, \n{execute_result}\n.We will try to fix it ...") + # end make tools + if execute_success or make_tool_current_retry >= make_tool_retries: + if make_tool_current_retry >= make_tool_retries: + logger.error(f"We have tried the maximum number of attempts {make_tool_retries}\ + and still have not created tools for task_id {self.plan.current_task_id} successfully,\ + we will skip it.") break # save successful tool code in udf - if success: + if execute_success: make_tools.save(tool_code) if __name__ == "__main__": - requirement = "Run data analysis on sklearn Iris dataset, include a plot" - # requirement = "Run data analysis on sklearn Diabetes dataset, include a plot" - # requirement = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy" - # requirement = "Run data analysis on sklearn Wisconsin Breast Cancer dataset, include a plot, train a model to predict targets (20% as validation), and show validation accuracy" - # requirement = "Run EDA and visualization on this dataset, train a model to predict survival, report metrics on validation set (20%), dataset: workspace/titanic/train.csv" - - async def run_udfs(requirement: str = requirement, auto_run: bool = True): - role = MLEngineer(goal=requirement, auto_run=auto_run) - # make udfs - role.use_tools = False - role.use_code_steps = False - role.make_udfs = True - role.use_udfs = False - await role.run(requirement) - # use udfs - role.reset() - role.make_udfs = False - role.use_udfs = True - role.use_code_steps = False - role.use_tools = False - await role.run(requirement) - + requirement = "Perform data analysis on the provided data. Train a model to predict the target variable Survived. Include data preprocessing, feature engineering, and modeling in your pipeline. The metric is accuracy." - # requirement = "Perform data analysis on the provided data. Train a model to predict the target variable Survived. Include data preprocessing, feature engineering, and modeling in your pipeline. The metric is accuracy." + data_path = f"{DATA_PATH}/titanic" + requirement = f"This is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '{data_path}/split_train.csv', eval data path: '{data_path}/split_eval.csv'." + requirement = f"Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy" + data_path = f"{DATA_PATH}/icr-identify-age-related-conditions" + requirement = f"This is a medical dataset with over fifty anonymized health characteristics linked to three age-related conditions. Your goal is to predict whether a subject has or has not been diagnosed with one of these conditions.The target column is Class. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report f1 score on the eval data. Train data path: {data_path}/split_train.csv, eval data path: {data_path}/split_eval.csv." - # data_path = f"{DATA_PATH}/titanic" - # requirement = f"This is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '{data_path}/split_train.csv', eval data path: '{data_path}/split_eval.csv'." - # requirement = f"Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy" - # data_path = f"{DATA_PATH}/icr-identify-age-related-conditions" - # requirement = f"This is a medical dataset with over fifty anonymized health characteristics linked to three age-related conditions. Your goal is to predict whether a subject has or has not been diagnosed with one of these conditions.The target column is Class. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report f1 score on the eval data. Train data path: {data_path}/split_train.csv, eval data path: {data_path}/split_eval.csv." + data_path = f"{DATA_PATH}/santander-customer-transaction-prediction" + requirement = f"This is a customers financial dataset. Your goal is to predict which customers will make a specific transaction in the future. The target column is target. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report F1 Score on the eval data. Train data path: '{data_path}/split_train.csv', eval data path: '{data_path}/split_eval.csv' ." - # data_path = f"{DATA_PATH}/santander-customer-transaction-prediction" - # requirement = f"This is a customers financial dataset. Your goal is to predict which customers will make a specific transaction in the future. The target column is target. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report F1 Score on the eval data. Train data path: '{data_path}/split_train.csv', eval data path: '{data_path}/split_eval.csv' ." - - # data_path = f"{DATA_PATH}/house-prices-advanced-regression-techniques" - # requirement = f"This is a house price dataset, your goal is to predict the sale price of a property based on its features. The target column is SalePrice. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report RMSE between the logarithm of the predicted value and the logarithm of the observed sales price on the eval data. Train data path: '{data_path}/split_train.csv', eval data path: '{data_path}/split_eval.csv'." - # save_dir = "" - # # save_dir = DATA_PATH / "output" / "2023-12-14_20-40-34" + data_path = f"{DATA_PATH}/house-prices-advanced-regression-techniques" + requirement = f"This is a house price dataset, your goal is to predict the sale price of a property based on its features. The target column is SalePrice. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report RMSE between the logarithm of the predicted value and the logarithm of the observed sales price on the eval data. Train data path: '{data_path}/split_train.csv', eval data path: '{data_path}/split_eval.csv'." + save_dir = "" + # save_dir = DATA_PATH / "output" / "2023-12-14_20-40-34" async def main(requirement: str = requirement, auto_run: bool = True, use_tools: bool = False, use_code_steps: bool = False, save_dir: str = ""): """ @@ -377,4 +357,4 @@ if __name__ == "__main__": logger.exception(f"An error occurred: {e}, save trajectory here: {save_path}") - fire.Fire(run_udfs) + fire.Fire(main) diff --git a/tests/metagpt/roles/test_daml.py b/tests/metagpt/roles/test_daml.py index 672a3daed..55b425316 100644 --- a/tests/metagpt/roles/test_daml.py +++ b/tests/metagpt/roles/test_daml.py @@ -9,6 +9,8 @@ async def make_use_tools(requirement: str, auto_run: bool = True): """make and use tools for requirement.""" role = MLEngineer(goal=requirement, auto_run=auto_run) # make udfs + role.use_tools = False + role.use_code_steps = False role.make_udfs = True role.use_udfs = False await role.run(requirement) @@ -16,6 +18,8 @@ async def make_use_tools(requirement: str, auto_run: bool = True): role.reset() role.make_udfs = False role.use_udfs = True + role.use_code_steps = False + role.use_tools = False await role.run(requirement)