diff --git a/metagpt/actions/ml_da_action.py b/metagpt/actions/ml_action.py similarity index 52% rename from metagpt/actions/ml_da_action.py rename to metagpt/actions/ml_action.py index d4e77773f..a61233e5a 100644 --- a/metagpt/actions/ml_da_action.py +++ b/metagpt/actions/ml_action.py @@ -1,28 +1,64 @@ import json +from typing import List, Tuple from metagpt.actions import Action -from metagpt.prompts.ml_engineer import PRINT_DATA_COLUMNS, UPDATE_DATA_COLUMNS -from metagpt.schema import Plan +from metagpt.actions.write_analysis_code import WriteCodeWithTools +from metagpt.prompts.ml_action import ( + GENERATE_CODE_PROMPT, + ML_TOOL_USAGE_PROMPT, + PRINT_DATA_COLUMNS, + UPDATE_DATA_COLUMNS, +) +from metagpt.prompts.write_analysis_code import CODE_GENERATOR_WITH_TOOLS +from metagpt.schema import Message, Plan from metagpt.utils.common import CodeParser, create_func_config, remove_comments -class SummarizeAnalysis(Action): - PROMPT_TEMPLATE: str = """ - # Context - {context} - # Summary - Output a 30-word summary on analysis tool and modeling algorithms you have used, and the corresponding result. Make sure to announce the complete path to your test prediction file. Your summary: - """ +class WriteCodeWithToolsML(WriteCodeWithTools): + async def run( + self, + context: List[Message], + plan: Plan = None, + column_info: str = "", + **kwargs, + ) -> Tuple[List[Message], str]: + # prepare tool schemas and tool-type-specific instruction + tool_schemas, tool_type_usage_prompt = await self._prepare_tools(plan=plan) - async def run(self, conmpleted_plan: Plan) -> str: - tasks = json.dumps( - [task.dict() for task in conmpleted_plan.tasks], - indent=4, - ensure_ascii=False, - ) # all tasks finished, return all task outputs - prompt = self.PROMPT_TEMPLATE.format(context=tasks) - summary = await self._aask(prompt) - return summary + # ML-specific variables to be used in prompt + code_steps = plan.current_task.code_steps + finished_tasks = plan.get_finished_tasks() + code_context = [remove_comments(task.code) for task in finished_tasks] + code_context = "\n\n".join(code_context) + + # prepare prompt depending on tool availability & LLM call + if tool_schemas: + prompt = ML_TOOL_USAGE_PROMPT.format( + user_requirement=plan.goal, + history_code=code_context, + current_task=plan.current_task.instruction, + column_info=column_info, + tool_type_usage_prompt=tool_type_usage_prompt, + code_steps=code_steps, + tool_schemas=tool_schemas, + ) + + else: + prompt = GENERATE_CODE_PROMPT.format( + user_requirement=plan.goal, + history_code=code_context, + current_task=plan.current_task.instruction, + column_info=column_info, + tool_type_usage_prompt=tool_type_usage_prompt, + code_steps=code_steps, + ) + tool_config = create_func_config(CODE_GENERATOR_WITH_TOOLS) + rsp = await self.llm.aask_code(prompt, **tool_config) + + # Extra output to be used for potential debugging + context = [Message(content=prompt, role="user")] + + return context, rsp class Reflect(Action): diff --git a/metagpt/actions/write_analysis_code.py b/metagpt/actions/write_analysis_code.py index c6e504b9e..402f56ccc 100644 --- a/metagpt/actions/write_analysis_code.py +++ b/metagpt/actions/write_analysis_code.py @@ -4,19 +4,12 @@ @Author : orange-crow @File : write_code_v2.py """ -import re -from pathlib import Path from typing import Dict, List, Tuple, Union -from tenacity import retry, stop_after_attempt, wait_fixed - from metagpt.actions import Action -from metagpt.llm import LLM from metagpt.logs import logger -from metagpt.prompts.ml_engineer import ( +from metagpt.prompts.write_analysis_code import ( CODE_GENERATOR_WITH_TOOLS, - GENERATE_CODE_PROMPT, - ML_TOOL_USAGE_PROMPT, SELECT_FUNCTION_TOOLS, TOOL_RECOMMENDATION_PROMPT, TOOL_USAGE_PROMPT, @@ -24,7 +17,7 @@ from metagpt.prompts.ml_engineer import ( from metagpt.schema import Message, Plan from metagpt.tools import TOOL_REGISTRY from metagpt.tools.tool_registry import validate_tool_names -from metagpt.utils.common import create_func_config, remove_comments +from metagpt.utils.common import create_func_config class BaseWriteAnalysisCode(Action): @@ -195,133 +188,3 @@ class WriteCodeWithTools(BaseWriteAnalysisCode): rsp = await self.llm.aask_code(prompt, **tool_config) return rsp - - -class WriteCodeWithToolsML(WriteCodeWithTools): - async def run( - self, - context: List[Message], - plan: Plan = None, - column_info: str = "", - **kwargs, - ) -> Tuple[List[Message], str]: - # prepare tool schemas and tool-type-specific instruction - tool_schemas, tool_type_usage_prompt = await self._prepare_tools(plan=plan) - - # ML-specific variables to be used in prompt - code_steps = plan.current_task.code_steps - finished_tasks = plan.get_finished_tasks() - code_context = [remove_comments(task.code) for task in finished_tasks] - code_context = "\n\n".join(code_context) - - # prepare prompt depending on tool availability & LLM call - if tool_schemas: - prompt = ML_TOOL_USAGE_PROMPT.format( - user_requirement=plan.goal, - history_code=code_context, - current_task=plan.current_task.instruction, - column_info=column_info, - tool_type_usage_prompt=tool_type_usage_prompt, - code_steps=code_steps, - tool_schemas=tool_schemas, - ) - - else: - prompt = GENERATE_CODE_PROMPT.format( - user_requirement=plan.goal, - history_code=code_context, - current_task=plan.current_task.instruction, - column_info=column_info, - tool_type_usage_prompt=tool_type_usage_prompt, - code_steps=code_steps, - ) - tool_config = create_func_config(CODE_GENERATOR_WITH_TOOLS) - rsp = await self.llm.aask_code(prompt, **tool_config) - - # Extra output to be used for potential debugging - context = [Message(content=prompt, role="user")] - - return context, rsp - - -class MakeTools(WriteCodeByGenerate): - DEFAULT_SYSTEM_MSG: str = """Convert any codes provied for you to a very General Function Code startswith `def`.\n - **Notice: - 1. Your code must contain a general function start with `def`. - 2. Refactor your code to get the most efficient implementation for large input data in the shortest amount of time. - 3. Must use Google style for function docstring, and your docstring must be consistent with the code,without missing anything. - 4. Write example code after `if __name__ == '__main__':`by using old varibales in old code, - and make sure it could be execute in the user's machine. - 5. Only use the imported packages** - """ - - def __init__(self, name: str = "", context: list[Message] = None, llm: LLM = None, workspace: str = None): - """ - :param str name: name, defaults to '' - :param list[Message] context: context, defaults to None - :param LLM llm: llm, defaults to None - :param str workspace: tools code saved file path dir, defaults to None - """ - super().__init__(name, context, llm) - self.workspace = workspace or str(Path(__file__).parents[1].joinpath("./tools/functions/libs/udf")) - self.file_suffix: str = ".py" - self.context = [] - - def parse_function_name(self, function_code: str) -> str: - # 定义正则表达式模式 - pattern = r"\bdef\s+([a-zA-Z_]\w*)\s*\(" - # 在代码中搜索匹配的模式 - match = re.search(pattern, function_code) - # 如果找到匹配项，则返回匹配的函数名；否则返回None - if match: - return match.group(1) - else: - return None - - def save(self, tool_code: str) -> None: - func_name = self.parse_function_name(tool_code) - if func_name is None: - raise ValueError(f"No function name found in {tool_code}") - saved_path = Path(self.workspace).joinpath(func_name + self.file_suffix) - logger.info(f"Saved tool_code {func_name} in {str(saved_path)}.") - saved_path.write_text(tool_code, encoding="utf-8") - - @retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) - async def run(self, code: Union[str, List[dict]], code_desc: str = None, **kwargs) -> str: - # 拼接code prompt - code_prompt = f"The following code is about {code_desc}, convert it to be a General Function, {code}" - if not self.context: - self.context = self.process_msg(code_prompt) - else: - self.context.append(self.process_msg(code_prompt)[-1]) - logger.info(f"\n\nAsk to Make tools:\n{'-'*60}\n {self.context[-1]}") - - # 更新kwargs - if "code" in kwargs: - kwargs.pop("code") - if "code_desc" in kwargs: - kwargs.pop("code_desc") - - max_tries, current_try = 3, 0 - while True: - tool_code = await self.llm.aask_code(self.context, **kwargs) - func_name = self.parse_function_name(tool_code["code"]) - current_try += 1 - # make tools failed, add error message to context. - if not func_name: - logger.info(f"\n\nTools Respond\n{'-'*60}\n: {tool_code}") - logger.error(f"No function name found in code, we will retry make tools.\n{tool_code['code']}\n") - self.context.append( - {"role": "user", "content": "We need a general function in above code,but not found function."} - ) - # end make tools - if func_name is not None or current_try >= max_tries: - if current_try >= max_tries: - logger.error( - f"We have tried the maximum number of attempts {max_tries}\ - and still have not created tools successfully, we will skip it." - ) - break - logger.info(f"\n\nTools Respond\n{'-'*60}\n: {tool_code}") - self.save(tool_code["code"]) - return tool_code["code"] diff --git a/metagpt/actions/write_code_steps.py b/metagpt/actions/write_code_steps.py deleted file mode 100644 index 7ba22fde4..000000000 --- a/metagpt/actions/write_code_steps.py +++ /dev/null @@ -1,116 +0,0 @@ -import json - -from metagpt.actions import Action -from metagpt.schema import Plan -from metagpt.utils.common import CodeParser - -# CODE_STEPS_PROMPT_TEMPLATE = """ -# # Context -# {context} -# -# ----- -# Tasks are all code development tasks. -# You are a professional engineer, the main goal is to plan out concise solution steps for Current Task before coding. -# A planning process can reduce the difficulty and improve the quality of coding. -# You may be given some code plans for the tasks ahead, but you don't have to follow the existing plan when planning the current task. -# The output plan should following the subsequent principles: -# 1.The plan is a rough checklist of steps outlining the entire program's structure.Try to keep the number of steps fewer than 5. -# 2.The steps should be written concisely and at a high level, avoiding overly detailed implementation specifics. -# 3.The execution of the plan happens sequentially, but the plan can incorporate conditional (if) and looping(loop) keywords for more complex structures. -# -# Output the code steps in a JSON format, as shown in this example: -# ```json -# { -# "Step 1": "", -# "Step 2": "", -# "Step 3": "", -# ... -# } -# ``` -# """ - -CODE_STEPS_PROMPT_TEMPLATE = """ -# Context -{context} - ------ -Tasks are all code development tasks. -You are a professional engineer, the main goal is to plan out concise solution steps for Current Task before coding. -A planning process can reduce the difficulty and improve the quality of coding. -You may be given some code plans for the tasks ahead, but you don't have to follow the existing plan when planning the current task. -The output plan should following the subsequent principles: -1.The plan is a rough checklist of steps outlining the entire program's structure.Try to keep the number of steps fewer than 5. -2.The steps should be written concisely and at a high level, avoiding overly detailed implementation specifics. -3.The execution of the plan happens sequentially, but the plan can incorporate conditional (if) and looping(loop) keywords for more complex structures. -4.Design and provide code steps by following the code logic. Analyze the provided code step by step and reuse the imported library. - -Output the code steps in a JSON format, as shown in this example: -```json -{ - "Step 1": "", - "Step 2": "", - "Step 3": "", - ... -} -``` -""" - -# STRUCTURAL_CONTEXT = """ -# ## User Requirement -# {user_requirement} -# ## Current Plan -# {tasks} -# ## Current Task -# {current_task} -# """ - -STRUCTURAL_CONTEXT = """ -## User Requirement -{user_requirement} -## Plan -{tasks} -## Codes -{codes} -## Current Task -{current_task} -""" - - -class WriteCodeSteps(Action): - async def run(self, plan: Plan) -> str: - """Run of a task guide writing action, used in ml engineer - - Args: - plan (plan): task plan - useful_memories (list): useful_memories - Returns: - str: The dataset_descriptions string. - """ - - context = self.get_context(plan) - code_steps_prompt = CODE_STEPS_PROMPT_TEMPLATE.replace("{context}", context) - code_steps = await self._aask(code_steps_prompt) - code_steps = CodeParser.parse_code(block=None, text=code_steps) - return code_steps - - def get_context(self, plan: Plan): - user_requirement = plan.goal - # select_task_keys = ['task_id', 'instruction', 'is_finished', 'code'] - # select_task_keys = ['task_id','instruction'] - - def process_task(task): - task_dict = task.dict() - # ptask = {k: task_dict[k] for k in task_dict if k in select_task_keys } - ptask = f"task_id_{task_dict['task_id']}:{task_dict['instruction']}" - return ptask - - tasks = json.dumps([process_task(task) for task in plan.tasks], indent=4, ensure_ascii=False) - - code_lists = [task.code for task in plan.tasks if task.is_finished == True] - codes = "\n\n".join(code_lists) - current_task = json.dumps(process_task(plan.current_task)) if plan.current_task else {} - context = STRUCTURAL_CONTEXT.format( - user_requirement=user_requirement, tasks=tasks, codes=codes, current_task=current_task - ) - # print(context) - return context diff --git a/metagpt/actions/write_plan.py b/metagpt/actions/write_plan.py index 60dcef43b..335a09841 100644 --- a/metagpt/actions/write_plan.py +++ b/metagpt/actions/write_plan.py @@ -10,7 +10,10 @@ from typing import Dict, List, Tuple from metagpt.actions import Action from metagpt.logs import logger -from metagpt.prompts.ml_engineer import ASSIGN_TASK_TYPE_CONFIG, ASSIGN_TASK_TYPE_PROMPT +from metagpt.prompts.write_analysis_code import ( + ASSIGN_TASK_TYPE_CONFIG, + ASSIGN_TASK_TYPE_PROMPT, +) from metagpt.schema import Message, Plan, Task from metagpt.tools import TOOL_REGISTRY from metagpt.utils.common import CodeParser, create_func_config diff --git a/metagpt/prompts/ml_engineer.py b/metagpt/prompts/ml_action.py similarity index 64% rename from metagpt/prompts/ml_engineer.py rename to metagpt/prompts/ml_action.py index ac95e14bd..582b01146 100644 --- a/metagpt/prompts/ml_engineer.py +++ b/metagpt/prompts/ml_action.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # @Time : 2023/11/24 15:43 # @Author : lidanyang -# @File : ml_engineer +# @File : ml_action # @Desc : UPDATE_DATA_COLUMNS = """ # Background @@ -49,85 +49,6 @@ Output the information in a JSON format, as shown in this example: - Don't contain specific values or examples found in the data column. """ -ASSIGN_TASK_TYPE_PROMPT = """ -Please assign a task type to each task in the list below from the given categories: -{task_list} - -## All Task Type: -{task_type_desc} -""" - -ASSIGN_TASK_TYPE_CONFIG = { - "name": "assign_task_type", - "description": "Assign task type to each task by order.", - "parameters": { - "type": "object", - "properties": { - "task_type": { - "type": "array", - "description": "List of task type. The length should as long as task list", - "items": { - "type": "string", - }, - }, - }, - "required": ["task_type"], - }, -} - -TOOL_RECOMMENDATION_PROMPT = """ -## User Requirement: -{current_task} - -## Task -Recommend up to five tools from 'Available Tools' that can help solve the 'User Requirement'. -This is a detailed code steps for current task. You can refer to it when recommending tools. -{code_steps} - -## Available Tools: -{available_tools} - -## Tool Selection and Instructions: -- Select tools most relevant to completing the 'User Requirement'. -- If you believe that no tools are suitable, indicate with an empty list. -- Only list the names of the tools, not the full schema of each tool. -- Ensure selected tools are listed in 'Available Tools'. -""" - -SELECT_FUNCTION_TOOLS = { - "name": "select_function_tools", - "description": "For current task, select suitable tools for it.", - "parameters": { - "type": "object", - "properties": { - "recommend_tools": { - "type": "array", - "description": "List of tool names. Empty list if no tool is suitable.", - "items": { - "type": "string", - }, - }, - }, - "required": ["recommend_tools"], - }, -} - -CODE_GENERATOR_WITH_TOOLS = { - "name": "add_subtask_code", - "description": "Add new code cell of current task to the end of an active Jupyter notebook.", - "parameters": { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "The code to be added to a new cell in jupyter.", - }, - }, - "required": ["code"], - }, -} - - PRINT_DATA_COLUMNS = { "name": "print_column_info", "description": "Print the latest column information after 'Done Tasks' code if first read or data changed.", @@ -189,24 +110,6 @@ model.fit(train, y_train) - The output code should contain all steps implemented in 'Code Steps'. """ -TOOL_USAGE_PROMPT = """ -# Instruction -Write complete code for 'Current Task'. And avoid duplicating code from finished tasks, such as repeated import of packages, reading data, etc. -Specifically, {tool_type_usage_prompt} - -# Capabilities -- You can utilize pre-defined tools in any code lines from 'Available Tools' in the form of Python Class. -- You can freely combine the use of any other public packages, like sklearn, numpy, pandas, etc.. - -# Available Tools (can be empty): -Each Class tool is described in JSON format. When you call a tool, import the tool first. -{tool_schemas} - -# Constraints: -- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed. -- Always prioritize using pre-defined tools for the same functionality. -""" - ML_TOOL_USAGE_PROMPT = """ # Background As a data scientist, you need to help user to achieve their goal [{user_requirement}] step-by-step in an continuous Jupyter notebook. diff --git a/metagpt/prompts/write_analysis_code.py b/metagpt/prompts/write_analysis_code.py new file mode 100644 index 000000000..4c8a5081e --- /dev/null +++ b/metagpt/prompts/write_analysis_code.py @@ -0,0 +1,95 @@ +ASSIGN_TASK_TYPE_PROMPT = """ +Please assign a task type to each task in the list below from the given categories: +{task_list} + +## All Task Type: +{task_type_desc} +""" + +ASSIGN_TASK_TYPE_CONFIG = { + "name": "assign_task_type", + "description": "Assign task type to each task by order.", + "parameters": { + "type": "object", + "properties": { + "task_type": { + "type": "array", + "description": "List of task type. The length should as long as task list", + "items": { + "type": "string", + }, + }, + }, + "required": ["task_type"], + }, +} + +TOOL_RECOMMENDATION_PROMPT = """ +## User Requirement: +{current_task} + +## Task +Recommend up to five tools from 'Available Tools' that can help solve the 'User Requirement'. +This is a detailed code steps for current task. You can refer to it when recommending tools. +{code_steps} + +## Available Tools: +{available_tools} + +## Tool Selection and Instructions: +- Select tools most relevant to completing the 'User Requirement'. +- If you believe that no tools are suitable, indicate with an empty list. +- Only list the names of the tools, not the full schema of each tool. +- Ensure selected tools are listed in 'Available Tools'. +""" + +SELECT_FUNCTION_TOOLS = { + "name": "select_function_tools", + "description": "For current task, select suitable tools for it.", + "parameters": { + "type": "object", + "properties": { + "recommend_tools": { + "type": "array", + "description": "List of tool names. Empty list if no tool is suitable.", + "items": { + "type": "string", + }, + }, + }, + "required": ["recommend_tools"], + }, +} + +CODE_GENERATOR_WITH_TOOLS = { + "name": "add_subtask_code", + "description": "Add new code cell of current task to the end of an active Jupyter notebook.", + "parameters": { + "type": "object", + "properties": { + "code": { + "type": "string", + "description": "The code to be added to a new cell in jupyter.", + }, + }, + "required": ["code"], + }, +} + +TOOL_USAGE_PROMPT = """ +# Instruction +Write complete code for 'Current Task'. And avoid duplicating code from finished tasks, such as repeated import of packages, reading data, etc. +Specifically, {tool_type_usage_prompt} + +# Capabilities +- You can utilize pre-defined tools in any code lines from 'Available Tools' in the form of Python Class. +- You can freely combine the use of any other public packages, like sklearn, numpy, pandas, etc.. + +# Available Tools (can be empty): +Each Class tool is described in JSON format. When you call a tool, import the tool first. +{tool_schemas} + +# Constraints: +- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed. +- Always prioritize using pre-defined tools for the same functionality. +""" diff --git a/metagpt/roles/code_interpreter.py b/metagpt/roles/code_interpreter.py index d1136a1d4..b4f9622d3 100644 --- a/metagpt/roles/code_interpreter.py +++ b/metagpt/roles/code_interpreter.py @@ -3,7 +3,6 @@ from pydantic import Field from metagpt.actions.ask_review import ReviewConst from metagpt.actions.execute_code import ExecutePyCode from metagpt.actions.write_analysis_code import WriteCodeByGenerate, WriteCodeWithTools -from metagpt.actions.write_code_steps import WriteCodeSteps from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message, Task, TaskResult @@ -12,7 +11,6 @@ from metagpt.schema import Message, Task, TaskResult class CodeInterpreter(Role): auto_run: bool = True use_tools: bool = False - use_code_steps: bool = False execute_code: ExecutePyCode = Field(default_factory=ExecutePyCode, exclude=True) tools: list[str] = [] @@ -48,10 +46,6 @@ class CodeInterpreter(Role): return task_result async def _write_and_exec_code(self, max_retry: int = 3): - self.planner.current_task.code_steps = ( - await WriteCodeSteps().run(self.planner.plan) if self.use_code_steps else "" - ) - counter = 0 success = False diff --git a/metagpt/roles/kaggle_manager.py b/metagpt/roles/kaggle_manager.py deleted file mode 100644 index 3ef573a8c..000000000 --- a/metagpt/roles/kaggle_manager.py +++ /dev/null @@ -1,153 +0,0 @@ -import json -import os -import subprocess - -import fire -import pandas as pd - -from metagpt.actions import Action, UserRequirement -from metagpt.actions.ml_da_action import SummarizeAnalysis -from metagpt.config import CONFIG -from metagpt.logs import logger -from metagpt.roles import Role -from metagpt.schema import Message -from metagpt.utils.common import CodeParser - -os.environ["KAGGLE_USERNAME"] = CONFIG.kaggle_username -os.environ["KAGGLE_KEY"] = CONFIG.kaggle_key - - -def run_command(cmd): - print(cmd) - output = subprocess.run(cmd, shell=True, capture_output=True, text=True) - if output.returncode != 0: - print("Error output:", output.stderr) - exit() - else: - print(output.stdout) - return output.stdout - - -class DownloadData(Action): - async def run(self, competition, data_desc="") -> str: - data_path = CONFIG.workspace_path / competition - - output = run_command(f"kaggle competitions list --search {competition}") - assert output != "No competitions found", "You must provide the correct competition name" - - run_command(f"kaggle competitions download {competition} --path {WORKSPACE_ROOT}") - - if not os.path.exists(data_path): - # if True: - # run_command(f"rm -r {data_path / '*'}") - run_command(f"unzip -o {CONFIG.workspace_path / '*.zip'} -d {data_path}") # FIXME: not safe - - file_list = run_command(f"ls {data_path}") - - rsp = f""" - Location: - Data downloaded at {data_path} folder, including {file_list} - Data Description: - {data_desc} - """ - return rsp - - -class SubmitResult(Action): - PROMPT_TEMPLATE: str = """ - # Summary - __summary__ - # Your task - Extract the file path for test set prediction from the summary above, output a json following the format: - ```json - {"file_path": str = "the file path, for example, /path/to/the/prediction/file/xxx.csv, /path/to/the/prediction/file/xxx.xlsx"} - ``` - """ - - def __init__(self, name: str = "", context=None, llm=None) -> str: - super().__init__(name, context, llm) - - async def _parse_submit_file_path(self, context) -> str: - prompt = self.PROMPT_TEMPLATE.replace("__summary__", context) - rsp = await self._aask(prompt) - rsp = CodeParser.parse_code(block=None, text=rsp) - file_path = json.loads(rsp)["file_path"] - return file_path - - async def run(self, competition, submit_message="") -> str: - submit_file_path = await self._parse_submit_file_path(submit_message) - - data_path = CONFIG.workspace_path / competition - submit_message = submit_message.replace("'", "") - - run_command(f"kaggle competitions submit {competition} -f {submit_file_path} -m '{submit_message}'") - run_command(f"kaggle competitions leaderboard --show --csv {competition} > {data_path / 'leaderboard.csv'}") - run_command(f"kaggle competitions submissions --csv {competition} > {data_path / 'submission.csv'}") - - leaderboard = pd.read_csv(data_path / "leaderboard.csv") - submission = pd.read_csv(data_path / "submission.csv") - print(submission) # submission.to_json(orient="records") - - submission_score = submission.loc[0, "publicScore"] - best_score = max(submission["publicScore"]) # might be min - rank = leaderboard.loc[leaderboard["score"] == best_score].index[0] - rank_pct = round(rank / len(leaderboard), 4) * 100 - - submission_summary = f""" - # All histories: - {submission.head(5).to_string()} - # Current - Current submission score: {submission_score}, best score: {best_score}, best rank: {rank} (top {rank_pct}%) - """ - logger.info(submission_summary) - return submission_summary - - -class KaggleManager(Role): - def __init__(self, name="ABC", profile="KaggleManager", goal="", competition="titanic", data_desc=""): - super().__init__(name=name, profile=profile, goal=goal) - self._init_actions([DownloadData, SubmitResult]) - self._watch([UserRequirement, SummarizeAnalysis]) - self.competition = competition - self.data_desc = data_desc # currently passed in, later can be scrapped down from web by another Role - - async def _think(self): - observed = self.get_memories()[-1].cause_by - if observed == UserRequirement: - self._set_state(0) # DownloadData, get competition of interest from human, download datasets - elif observed == SummarizeAnalysis: - self._set_state(1) # SubmitResult, get prediction from MLEngineer and submit it to Kaggle - - async def _act(self): - todo = self.rc.todo - logger.info(f"{self._setting}: ready to {self.rc.todo}") - - if isinstance(todo, DownloadData): - rsp = await todo.run(self.competition, self.data_desc) - - elif isinstance(todo, SubmitResult): - submit_message = self.get_memories()[ - -1 - ].content # use analysis summary from MLEngineer as submission message - rsp = await todo.run(competition=self.competition, submit_message=submit_message) - - msg = Message(content=rsp, role="user", cause_by=type(todo)) - - return msg - - -if __name__ == "__main__": - competition, data_desc, requirement = ( - "titanic", - "Training set is train.csv.\nTest set is test.csv. We also include gender_submission.csv, a set of predictions that assume all and only female passengers survive, as an example of what a submission file should look like.", - "Run EDA on the train dataset, train a model to predict survival (20% as validation) and save it, predict the test set using saved model, save the test result according to format", - ) - - summary = "I used Python with pandas for data preprocessing, sklearn's RandomForestClassifier for modeling, and achieved 82.12% accuracy on validation. Predictions saved at '/Users/gary/Desktop/data_agents_opt/workspace/titanic/gender_submission.csv'." - - async def main(requirement: str = requirement): - role = KaggleManager(competition=competition, data_desc=data_desc) - # await role.run(Message(content="", cause_by=UserRequirement)) - await role.run(Message(content=summary, cause_by=SummarizeAnalysis)) - - fire.Fire(main) diff --git a/metagpt/roles/ml_engineer.py b/metagpt/roles/ml_engineer.py index d1a22b9d3..e7abee560 100644 --- a/metagpt/roles/ml_engineer.py +++ b/metagpt/roles/ml_engineer.py @@ -1,7 +1,6 @@ from metagpt.actions.debug_code import DebugCode from metagpt.actions.execute_code import ExecutePyCode -from metagpt.actions.ml_da_action import UpdateDataColumns -from metagpt.actions.write_analysis_code import WriteCodeWithToolsML +from metagpt.actions.ml_action import UpdateDataColumns, WriteCodeWithToolsML from metagpt.logs import logger from metagpt.roles.code_interpreter import CodeInterpreter from metagpt.tools.tool_data_type import ToolTypeEnum diff --git a/metagpt/roles/tool_maker.py b/metagpt/roles/tool_maker.py deleted file mode 100644 index 68d84b1e6..000000000 --- a/metagpt/roles/tool_maker.py +++ /dev/null @@ -1,53 +0,0 @@ -from pydantic import Field - -from metagpt.actions.ask_review import AskReview -from metagpt.actions.execute_code import ExecutePyCode -from metagpt.actions.write_analysis_code import MakeTools -from metagpt.logs import logger -from metagpt.roles import Role -from metagpt.utils.common import remove_comments - - -class ToolMaker(Role): - execute_code: ExecutePyCode = Field(default_factory=ExecutePyCode, exclude=True) - - async def make_tool(self, code: str, instruction: str, task_id: str = "", auto_run=True): - if len(remove_comments(code).split("\n")) < 5: # no need to consider trivial codes with fewer than 5 lines - return - - logger.warning( - f"Making tools for task_id {task_id}: \ - `{instruction}` \n code: \n {code}" - ) - make_tools = MakeTools() - make_tool_retries, make_tool_current_retry = 3, 0 - while True: - # start make tools - tool_code = await make_tools.run(code, instruction) - make_tool_current_retry += 1 - - # check tool_code by execute_code - logger.info(f"Checking task_id {task_id} tool code by executor...") - execute_result, execute_success = await self.execute_code.run(tool_code) - if not execute_success: - logger.error(f"Tool code faild to execute, \n{execute_result}\n.We will try to fix it ...") - # end make tools - if execute_success or make_tool_current_retry >= make_tool_retries: - if make_tool_current_retry >= make_tool_retries: - logger.error( - f"We have tried the maximum number of attempts {make_tool_retries}\ - and still have not created tools for task_id {task_id} successfully,\ - we will skip it." - ) - break - # save successful tool code in udf - if execute_success: - _, confirmed = await self.ask_review(auto_run=auto_run) - if confirmed: - make_tools.save(tool_code) - - async def ask_review(self, auto_run: bool = True): - if not auto_run: - review, confirmed = await AskReview().run() - return review, confirmed - return "", True diff --git a/tests/data/rsp_cache.json b/tests/data/rsp_cache.json index db452f676..cb3c1642c 100644 --- a/tests/data/rsp_cache.json +++ b/tests/data/rsp_cache.json @@ -141,5 +141,71 @@ "\nRole: You are a senior development and qa engineer, your role is summarize the code running result.\nIf the running result does not include an error, you should explicitly approve the result.\nOn the other hand, if the running result indicates some error, you should point out which part, the development code or the test code, produces the error,\nand give specific instructions on fixing the errors. Here is the code info:\n\n## Development Code File Name\na.txt\n## Development Code\n```python\nprint('Hello, World')\n```\n## Test File Name\n\n## Test Code\n```python\nNone\n```\n## Running Command\n\n## Running Output\nstandard output: \n```text\n\n```\nstandard errors: \n```text\n\n```\n\nNow you should begin your analysis\n---\n## instruction:\nPlease summarize the cause of the errors and give correction instruction\n## File To Rewrite:\nDetermine the ONE file to rewrite in order to fix the error, for example, xyz.py, or test_xyz.py\n## Status:\nDetermine if all of the code works fine, if so write PASS, else FAIL,\nWRITE ONLY ONE WORD, PASS OR FAIL, IN THIS SECTION\n## Send To:\nPlease write Engineer if the errors are due to problematic development codes, and QaEngineer to problematic test codes, and NoOne if there are no errors,\nWRITE ONLY ONE WORD, Engineer OR QaEngineer OR NoOne, IN THIS SECTION.\n---\nYou should fill in necessary instruction, status, send to, and finally return all content between the --- segment line.\n": "---\ninstruction: There are no errors in the provided code.\n\nFile To Rewrite: N/A\n\nStatus: PASS\n\nSend To: NoOne\n---", "\nRole: You are a senior development and qa engineer, your role is summarize the code running result.\nIf the running result does not include an error, you should explicitly approve the result.\nOn the other hand, if the running result indicates some error, you should point out which part, the development code or the test code, produces the error,\nand give specific instructions on fixing the errors. Here is the code info:\n\n## Development Code File Name\na.sh\n## Development Code\n```python\necho 'Hello World'\n```\n## Test File Name\n\n## Test Code\n```python\nNone\n```\n## Running Command\necho Hello World\n## Running Output\nstandard output: \n```text\nHello World\n\n```\nstandard errors: \n```text\n\n```\n\nNow you should begin your analysis\n---\n## instruction:\nPlease summarize the cause of the errors and give correction instruction\n## File To Rewrite:\nDetermine the ONE file to rewrite in order to fix the error, for example, xyz.py, or test_xyz.py\n## Status:\nDetermine if all of the code works fine, if so write PASS, else FAIL,\nWRITE ONLY ONE WORD, PASS OR FAIL, IN THIS SECTION\n## Send To:\nPlease write Engineer if the errors are due to problematic development codes, and QaEngineer to problematic test codes, and NoOne if there are no errors,\nWRITE ONLY ONE WORD, Engineer OR QaEngineer OR NoOne, IN THIS SECTION.\n---\nYou should fill in necessary instruction, status, send to, and finally return all content between the --- segment line.\n": "The running result indicates no error. The code works fine.\n\n## File To Rewrite:\nNo file needs to be rewritten.\n\n## Status:\nPASS\n\n## Send To:\nNoOne\n\n---", "\nRole: You are a senior development and qa engineer, your role is summarize the code running result.\nIf the running result does not include an error, you should explicitly approve the result.\nOn the other hand, if the running result indicates some error, you should point out which part, the development code or the test code, produces the error,\nand give specific instructions on fixing the errors. Here is the code info:\n\n## Development Code File Name\na.py\n## Development Code\n```python\npython -c \"print(1/0)\"\n```\n## Test File Name\n\n## Test Code\n```python\nNone\n```\n## Running Command\npython -c print(1/0)\n## Running Output\nstandard output: \n```text\n\n```\nstandard errors: \n```text\nTraceback (most recent call last):\n File \"\", line 1, in \nZeroDivisionError: division by zero\n\n```\n\nNow you should begin your analysis\n---\n## instruction:\nPlease summarize the cause of the errors and give correction instruction\n## File To Rewrite:\nDetermine the ONE file to rewrite in order to fix the error, for example, xyz.py, or test_xyz.py\n## Status:\nDetermine if all of the code works fine, if so write PASS, else FAIL,\nWRITE ONLY ONE WORD, PASS OR FAIL, IN THIS SECTION\n## Send To:\nPlease write Engineer if the errors are due to problematic development codes, and QaEngineer to problematic test codes, and NoOne if there are no errors,\nWRITE ONLY ONE WORD, Engineer OR QaEngineer OR NoOne, IN THIS SECTION.\n---\nYou should fill in necessary instruction, status, send to, and finally return all content between the --- segment line.\n": "---\nThe error is caused by the development code in file a.py, which attempts to divide by zero. To fix the error, the development code in a.py should be modified to handle the ZeroDivisionError, for example by using a try-except block.\n\nFile To Rewrite:\na.py\n\nStatus:\nFAIL\n\nSend To:\nEngineer\n---", - "\nNOTICE\n1. Role: You are a Development Engineer or QA engineer;\n2. Task: You received this message from another Development Engineer or QA engineer who ran or tested your code. \nBased on the message, first, figure out your own role, i.e. Engineer or QaEngineer,\nthen rewrite the development code or the test code based on your role, the error, and the summary, such that all bugs are fixed and the code performs well.\nAttention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes.\nThe message is as follows:\n# Legacy Code\n```python\n\nfrom typing import List\nfrom deck import Deck\nfrom card import Card\n\nclass Player:\n \"\"\"\n A class representing a player in the Black Jack game.\n \"\"\"\n\n def __init__(self, name: str):\n \"\"\"\n Initialize a Player object.\n \n Args:\n name (str): The name of the player.\n \"\"\"\n self.name = name\n self.hand: List[Card] = []\n self.score = 0\n\n def draw(self, deck: Deck):\n \"\"\"\n Draw a card from the deck and add it to the player's hand.\n \n Args:\n deck (Deck): The deck of cards.\n \"\"\"\n card = deck.draw_card()\n self.hand.append(card)\n self.calculate_score()\n\n def calculate_score(self) -> int:\n \"\"\"\n Calculate the score of the player's hand.\n \n Returns:\n int: The score of the player's hand.\n \"\"\"\n self.score = sum(card.value for card in self.hand)\n # Handle the case where Ace is counted as 11 and causes the score to exceed 21\n if self.score > 21 and any(card.rank == 'A' for card in self.hand):\n self.score -= 10\n return self.score\n\n```\n---\n# Unit Test Code\n```python\n\nimport unittest\nfrom blackjack_game.player import Player\nfrom blackjack_game.deck import Deck\nfrom blackjack_game.card import Card\n\nclass TestPlayer(unittest.TestCase):\n ## Test the Player's initialization\n def test_player_initialization(self):\n player = Player(\"Test Player\")\n self.assertEqual(player.name, \"Test Player\")\n self.assertEqual(player.hand, [])\n self.assertEqual(player.score, 0)\n\n ## Test the Player's draw method\n def test_player_draw(self):\n deck = Deck()\n player = Player(\"Test Player\")\n player.draw(deck)\n self.assertEqual(len(player.hand), 1)\n self.assertEqual(player.score, player.hand[0].value)\n\n ## Test the Player's calculate_score method\n def test_player_calculate_score(self):\n deck = Deck()\n player = Player(\"Test Player\")\n player.draw(deck)\n player.draw(deck)\n self.assertEqual(player.score, sum(card.value for card in player.hand))\n\n ## Test the Player's calculate_score method with Ace card\n def test_player_calculate_score_with_ace(self):\n deck = Deck()\n player = Player(\"Test Player\")\n player.hand.append(Card('A', 'Hearts', 11))\n player.hand.append(Card('K', 'Hearts', 10))\n player.calculate_score()\n self.assertEqual(player.score, 21)\n\n ## Test the Player's calculate_score method with multiple Aces\n def test_player_calculate_score_with_multiple_aces(self):\n deck = Deck()\n player = Player(\"Test Player\")\n player.hand.append(Card('A', 'Hearts', 11))\n player.hand.append(Card('A', 'Diamonds', 11))\n player.calculate_score()\n self.assertEqual(player.score, 12)\n\nif __name__ == '__main__':\n unittest.main()\n\n\n```\n---\n# Console logs\n```text\n\n```\n---\nNow you should start rewriting the code:\n## file name of the code to rewrite: Write code with triple quote. Do your best to implement THIS IN ONLY ONE FILE.\n": "```python\n## Development Engineer\n\nfrom typing import List\nfrom deck import Deck\nfrom card import Card\n\nclass Player:\n \"\"\"\n A class representing a player in the Black Jack game.\n \"\"\"\n\n def __init__(self, name: str):\n \"\"\"\n Initialize a Player object.\n \n Args:\n name (str): The name of the player.\n \"\"\"\n self.name = name\n self.hand: List[Card] = []\n self.score = 0\n\n def draw(self, deck: Deck):\n \"\"\"\n Draw a card from the deck and add it to the player's hand.\n \n Args:\n deck (Deck): The deck of cards.\n \"\"\"\n card = deck.draw_card()\n self.hand.append(card)\n self.calculate_score()\n\n def calculate_score(self) -> int:\n \"\"\"\n Calculate the score of the player's hand.\n \n Returns:\n int: The score of the player's hand.\n \"\"\"\n self.score = sum(card.value for card in self.hand)\n # Handle the case where Ace is counted as 11 and causes the score to exceed 21\n ace_count = sum(1 for card in self.hand if card.rank == 'A')\n while self.score > 21 and ace_count > 0:\n self.score -= 10\n ace_count -= 1\n return self.score\n```\n\n```python\n## QaEngineer\n\nimport unittest\nfrom player import Player\nfrom deck import Deck\nfrom card import Card\n\nclass TestPlayer(unittest.TestCase):\n ## Test the Player's initialization\n def test_player_initialization(self):\n player = Player(\"Test Player\")\n self.assertEqual(player.name, \"Test Player\")\n self.assertEqual(player.hand, [])\n self.assertEqual(player.score, 0)\n\n ## Test the Player's draw method\n def test_player_draw(self):\n deck = Deck()\n player = Player(\"Test Player\")\n player.draw(deck)\n self.assertEqual(len(player.hand), 1)\n self.assertEqual(player.score, player.hand[0].value)\n\n ## Test the Player's calculate_score method\n def test_player_calculate_score(self):\n deck = Deck()\n player = Player(\"Test Player\")\n player.draw(deck)\n player.draw(deck)\n self.assertEqual(player.score, sum(card.value for card in player.hand))\n\n ## Test the Player's calculate_score method with Ace card\n def test_player_calculate_score_with_ace(self):\n player = Player(\"Test Player\")\n player.hand.append(Card('A', 'Hearts', 11))\n player.hand.append(Card('K', 'Hearts', 10))\n player.calculate_score()\n self.assertEqual(player.score, 21)\n\n ## Test the Player's calculate_score method with multiple Aces\n def test_player_calculate_score_with_multiple_aces(self):\n player = Player(\"Test Player\")\n player.hand.append(Card('A', 'Hearts', 11))\n player.hand.append(Card('A', 'Diamonds', 11))\n player.calculate_score()\n self.assertEqual(player.score, 12)\n\nif __name__ == '__main__':\n unittest.main()\n```" + "\nNOTICE\n1. Role: You are a Development Engineer or QA engineer;\n2. Task: You received this message from another Development Engineer or QA engineer who ran or tested your code. \nBased on the message, first, figure out your own role, i.e. Engineer or QaEngineer,\nthen rewrite the development code or the test code based on your role, the error, and the summary, such that all bugs are fixed and the code performs well.\nAttention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes.\nThe message is as follows:\n# Legacy Code\n```python\n\nfrom typing import List\nfrom deck import Deck\nfrom card import Card\n\nclass Player:\n \"\"\"\n A class representing a player in the Black Jack game.\n \"\"\"\n\n def __init__(self, name: str):\n \"\"\"\n Initialize a Player object.\n \n Args:\n name (str): The name of the player.\n \"\"\"\n self.name = name\n self.hand: List[Card] = []\n self.score = 0\n\n def draw(self, deck: Deck):\n \"\"\"\n Draw a card from the deck and add it to the player's hand.\n \n Args:\n deck (Deck): The deck of cards.\n \"\"\"\n card = deck.draw_card()\n self.hand.append(card)\n self.calculate_score()\n\n def calculate_score(self) -> int:\n \"\"\"\n Calculate the score of the player's hand.\n \n Returns:\n int: The score of the player's hand.\n \"\"\"\n self.score = sum(card.value for card in self.hand)\n # Handle the case where Ace is counted as 11 and causes the score to exceed 21\n if self.score > 21 and any(card.rank == 'A' for card in self.hand):\n self.score -= 10\n return self.score\n\n```\n---\n# Unit Test Code\n```python\n\nimport unittest\nfrom blackjack_game.player import Player\nfrom blackjack_game.deck import Deck\nfrom blackjack_game.card import Card\n\nclass TestPlayer(unittest.TestCase):\n ## Test the Player's initialization\n def test_player_initialization(self):\n player = Player(\"Test Player\")\n self.assertEqual(player.name, \"Test Player\")\n self.assertEqual(player.hand, [])\n self.assertEqual(player.score, 0)\n\n ## Test the Player's draw method\n def test_player_draw(self):\n deck = Deck()\n player = Player(\"Test Player\")\n player.draw(deck)\n self.assertEqual(len(player.hand), 1)\n self.assertEqual(player.score, player.hand[0].value)\n\n ## Test the Player's calculate_score method\n def test_player_calculate_score(self):\n deck = Deck()\n player = Player(\"Test Player\")\n player.draw(deck)\n player.draw(deck)\n self.assertEqual(player.score, sum(card.value for card in player.hand))\n\n ## Test the Player's calculate_score method with Ace card\n def test_player_calculate_score_with_ace(self):\n deck = Deck()\n player = Player(\"Test Player\")\n player.hand.append(Card('A', 'Hearts', 11))\n player.hand.append(Card('K', 'Hearts', 10))\n player.calculate_score()\n self.assertEqual(player.score, 21)\n\n ## Test the Player's calculate_score method with multiple Aces\n def test_player_calculate_score_with_multiple_aces(self):\n deck = Deck()\n player = Player(\"Test Player\")\n player.hand.append(Card('A', 'Hearts', 11))\n player.hand.append(Card('A', 'Diamonds', 11))\n player.calculate_score()\n self.assertEqual(player.score, 12)\n\nif __name__ == '__main__':\n unittest.main()\n\n\n```\n---\n# Console logs\n```text\n\n```\n---\nNow you should start rewriting the code:\n## file name of the code to rewrite: Write code with triple quote. Do your best to implement THIS IN ONLY ONE FILE.\n": "```python\n## Development Engineer\n\nfrom typing import List\nfrom deck import Deck\nfrom card import Card\n\nclass Player:\n \"\"\"\n A class representing a player in the Black Jack game.\n \"\"\"\n\n def __init__(self, name: str):\n \"\"\"\n Initialize a Player object.\n \n Args:\n name (str): The name of the player.\n \"\"\"\n self.name = name\n self.hand: List[Card] = []\n self.score = 0\n\n def draw(self, deck: Deck):\n \"\"\"\n Draw a card from the deck and add it to the player's hand.\n \n Args:\n deck (Deck): The deck of cards.\n \"\"\"\n card = deck.draw_card()\n self.hand.append(card)\n self.calculate_score()\n\n def calculate_score(self) -> int:\n \"\"\"\n Calculate the score of the player's hand.\n \n Returns:\n int: The score of the player's hand.\n \"\"\"\n self.score = sum(card.value for card in self.hand)\n # Handle the case where Ace is counted as 11 and causes the score to exceed 21\n ace_count = sum(1 for card in self.hand if card.rank == 'A')\n while self.score > 21 and ace_count > 0:\n self.score -= 10\n ace_count -= 1\n return self.score\n```\n\n```python\n## QaEngineer\n\nimport unittest\nfrom player import Player\nfrom deck import Deck\nfrom card import Card\n\nclass TestPlayer(unittest.TestCase):\n ## Test the Player's initialization\n def test_player_initialization(self):\n player = Player(\"Test Player\")\n self.assertEqual(player.name, \"Test Player\")\n self.assertEqual(player.hand, [])\n self.assertEqual(player.score, 0)\n\n ## Test the Player's draw method\n def test_player_draw(self):\n deck = Deck()\n player = Player(\"Test Player\")\n player.draw(deck)\n self.assertEqual(len(player.hand), 1)\n self.assertEqual(player.score, player.hand[0].value)\n\n ## Test the Player's calculate_score method\n def test_player_calculate_score(self):\n deck = Deck()\n player = Player(\"Test Player\")\n player.draw(deck)\n player.draw(deck)\n self.assertEqual(player.score, sum(card.value for card in player.hand))\n\n ## Test the Player's calculate_score method with Ace card\n def test_player_calculate_score_with_ace(self):\n player = Player(\"Test Player\")\n player.hand.append(Card('A', 'Hearts', 11))\n player.hand.append(Card('K', 'Hearts', 10))\n player.calculate_score()\n self.assertEqual(player.score, 21)\n\n ## Test the Player's calculate_score method with multiple Aces\n def test_player_calculate_score_with_multiple_aces(self):\n player = Player(\"Test Player\")\n player.hand.append(Card('A', 'Hearts', 11))\n player.hand.append(Card('A', 'Diamonds', 11))\n player.calculate_score()\n self.assertEqual(player.score, 12)\n\nif __name__ == '__main__':\n unittest.main()\n```", + "\n # Context:\n user: \n## User Requirement\nThis is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv', eval data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv'.\n## Context\n\n## Current Plan\n[]\n## Current Task\n{}\n\n # Task:\n Based on the context, write a plan or modify an existing plan of what you should do to achieve the goal. A plan consists of one to 3 tasks.\n If you are modifying an existing plan, carefully follow the instruction, don't make unnecessary changes. Give the whole plan unless instructed to modify only one task of the plan.\n If you encounter errors on the current task, revise and output the current single task only.\n Output a list of jsons following the format:\n ```json\n [\n {\n \"task_id\": str = \"unique identifier for a task in plan, can be an ordinal\",\n \"dependent_task_ids\": list[str] = \"ids of tasks prerequisite to this task\",\n \"instruction\": \"what you should do in this task, one short phrase or sentence\",\n },\n ...\n ]\n ```\n ": "```json\n[\n {\n \"task_id\": \"1\",\n \"dependent_task_ids\": [],\n \"instruction\": \"Perform exploratory data analysis on the train dataset to understand the features and target variable.\"\n },\n {\n \"task_id\": \"2\",\n \"dependent_task_ids\": [\"1\"],\n \"instruction\": \"Preprocess the data by handling missing values, encoding categorical variables, and scaling features.\"\n },\n {\n \"task_id\": \"3\",\n \"dependent_task_ids\": [\"2\"],\n \"instruction\": \"Conduct feature engineering to create new features that may help improve model performance.\"\n },\n {\n \"task_id\": \"4\",\n \"dependent_task_ids\": [\"3\"],\n \"instruction\": \"Select and train a machine learning model using the processed train dataset.\"\n },\n {\n \"task_id\": \"5\",\n \"dependent_task_ids\": [\"4\"],\n \"instruction\": \"Evaluate the model's accuracy using the eval dataset and report the results.\"\n }\n]\n```", + "[{\"role\": \"user\", \"content\": \"\\nPlease assign a task type to each task in the list below from the given categories:\\nTask 1: Perform exploratory data analysis on the train dataset to understand the features and target variable.\\nTask 2: Preprocess the data by handling missing values, encoding categorical variables, and scaling features.\\nTask 3: Conduct feature engineering to create new features that may help improve model performance.\\nTask 4: Select and train a machine learning model using the processed train dataset.\\nTask 5: Evaluate the model's accuracy using the eval dataset and report the results.\\n\\n## All Task Type:\\n- **eda**: For performing exploratory data analysis\\n- **data_preprocess**: Only for changing value inplace.\\n- **feature_engineering**: Only for creating new columns for input data.\\n- **model_train**: Only for training model.\\n- **model_evaluate**: Only for evaluating model.\\n- **stable_diffusion**: Related to text2image, image2image using stable diffusion model.\\n- **image2webpage**: For converting image into webpage code.\\n- **web_scraping**: For scraping data from web pages.\\n- **other**: Any tools not in the defined categories\\n\"}]": { + "task_type": [ + "eda", + "data_preprocess", + "feature_engineering", + "model_train", + "model_evaluate" + ] + }, + "[{\"role\": \"user\", \"content\": \"\\n# Background\\nAs a data scientist, you need to help user to achieve their goal [This is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv', eval data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv'.] step-by-step in an continuous Jupyter notebook.\\n\\n## Done Tasks\\n```python\\n\\n```end\\n\\n## Current Task\\nPerform exploratory data analysis on the train dataset to understand the features and target variable.\\n\\n# Latest Data Info\\nLatest data info after previous tasks:\\n\\n\\n# Task\\nWrite complete code for 'Current Task'. And avoid duplicating code from 'Done Tasks', such as repeated import of packages, reading data, etc.\\nSpecifically, \\n\\n# Code Steps:\\nStrictly follow steps below when you writing code if it's convenient.\\n\\n\\n# Output Example:\\nwhen current task is \\\"train a lightgbm model on training data\\\", and their are two steps in 'Code Steps', the code be like:\\n```python\\n# Step 1: check data type and convert to numeric\\nojb_cols = train.select_dtypes(include='object').columns.tolist()\\n\\nfor col in obj_cols:\\n encoder = LabelEncoder()\\n train[col] = encoder.fit_transform(train[col].unique().tolist() + ['unknown'])\\n test[col] = test[col].apply(lambda x: x if x in encoder.classes_ else 'unknown')\\n test[col] = encoder.transform(test[col])\\n\\n# Step 2: train lightgbm model\\nmodel = LGBMClassifier()\\nmodel.fit(train, y_train)\\n```end\\n\\n# Constraints:\\n- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed.\\n- The output code should contain all steps implemented in 'Code Steps'.\\n\"}]": { + "code": "# Step 1: Load the training dataset\nimport pandas as pd\ntrain_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv')\n\n# Step 2: Display the first few rows of the dataset\ndisplay(train_data.head())\n\n# Step 3: Get a summary of the dataset\ntrain_data.info()\n\n# Step 4: Describe the numerical features\ntrain_data.describe()\n\n# Step 5: Describe the categorical features\ntrain_data.describe(include=['O'])\n\n# Step 6: Check for missing values\nmissing_values = train_data.isnull().sum()\nmissing_values[missing_values > 0]\n\n# Step 7: Visualize the distribution of the target variable 'Survived'\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nsns.countplot(x='Survived', data=train_data)\nplt.title('Distribution of Survival')\nplt.show()\n\n# Step 8: Explore the relationship between features and the target variable\n# Visualize the survival rate by sex\nsns.barplot(x='Sex', y='Survived', data=train_data)\nplt.title('Survival Rate by Sex')\nplt.show()\n\n# Visualize the survival rate by class\nsns.barplot(x='Pclass', y='Survived', data=train_data)\nplt.title('Survival Rate by Class')\nplt.show()\n\n# Visualize the survival rate by embarkation port\nsns.barplot(x='Embarked', y='Survived', data=train_data)\nplt.title('Survival Rate by Embarkation Port')\nplt.show()" + }, + "[{\"role\": \"user\", \"content\": \"\\n# Background\\nKeep dataset column information updated before model train.\\n## Done Tasks\\n```python\\nimport pandas as pd\\ntrain_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv')\\ndisplay(train_data.head())\\ntrain_data.info()\\ntrain_data.describe()\\ntrain_data.describe(include=['O'])\\nmissing_values = train_data.isnull().sum()\\nmissing_values[missing_values > 0]\\nimport seaborn as sns\\nimport matplotlib.pyplot as plt\\nsns.countplot(x='Survived', data=train_data)\\nplt.title('Distribution of Survival')\\nplt.show()\\nsns.barplot(x='Sex', y='Survived', data=train_data)\\nplt.title('Survival Rate by Sex')\\nplt.show()\\nsns.barplot(x='Pclass', y='Survived', data=train_data)\\nplt.title('Survival Rate by Class')\\nplt.show()\\nsns.barplot(x='Embarked', y='Survived', data=train_data)\\nplt.title('Survival Rate by Embarkation Port')\\nplt.show()\\n```end\\n\\n# Task\\nUpdate and print the dataset's column information only if the train or test data has changed. Use the following code:\\n```python\\nfrom metagpt.tools.libs.data_preprocess import get_column_info\\n\\ncolumn_info = get_column_info(df)\\nprint(\\\"column_info\\\")\\nprint(column_info)\\n```end\\n\\n# Constraints:\\n- Use the DataFrame variable from 'Done Tasks' in place of df.\\n- Import `get_column_info` only if it's not already imported.\\n\"}]": { + "code": "from metagpt.tools.libs.data_preprocess import get_column_info\n\ncolumn_info = get_column_info(train_data)\nprint(\"column_info\")\nprint(column_info)" + }, + "[{\"role\": \"user\", \"content\": \"\\n## User Requirement:\\nPreprocess the data by handling missing values, encoding categorical variables, and scaling features.\\n\\n## Task\\nRecommend up to five tools from 'Available Tools' that can help solve the 'User Requirement'. \\nThis is a detailed code steps for current task. You can refer to it when recommending tools.\\n\\n\\n## Available Tools:\\n{'FillMissingValue': 'Completing missing values with simple strategies'}\\n\\n## Tool Selection and Instructions:\\n- Select tools most relevant to completing the 'User Requirement'.\\n- If you believe that no tools are suitable, indicate with an empty list.\\n- Only list the names of the tools, not the full schema of each tool.\\n- Ensure selected tools are listed in 'Available Tools'.\\n\"}]": { + "recommend_tools": [ + "FillMissingValue" + ] + }, + "[{\"role\": \"user\", \"content\": \"\\n# Background\\nAs a data scientist, you need to help user to achieve their goal [This is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv', eval data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv'.] step-by-step in an continuous Jupyter notebook.\\n\\n## Done Tasks\\n```python\\nimport pandas as pd\\ntrain_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv')\\ndisplay(train_data.head())\\ntrain_data.info()\\ntrain_data.describe()\\ntrain_data.describe(include=['O'])\\nmissing_values = train_data.isnull().sum()\\nmissing_values[missing_values > 0]\\nimport seaborn as sns\\nimport matplotlib.pyplot as plt\\nsns.countplot(x='Survived', data=train_data)\\nplt.title('Distribution of Survival')\\nplt.show()\\nsns.barplot(x='Sex', y='Survived', data=train_data)\\nplt.title('Survival Rate by Sex')\\nplt.show()\\nsns.barplot(x='Pclass', y='Survived', data=train_data)\\nplt.title('Survival Rate by Class')\\nplt.show()\\nsns.barplot(x='Embarked', y='Survived', data=train_data)\\nplt.title('Survival Rate by Embarkation Port')\\nplt.show()\\n```end\\n\\n## Current Task\\nPreprocess the data by handling missing values, encoding categorical variables, and scaling features.\\n\\n# Latest Data Info\\nLatest data info after previous tasks:\\ncolumn_info\\n{'Category': ['Name', 'Sex', 'Ticket', 'Cabin', 'Embarked'], 'Numeric': ['PassengerId', 'Survived', 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare'], 'Datetime': [], 'Others': []}\\n\\n\\n# Task\\nWrite complete code for 'Current Task'. And avoid duplicating code from 'Done Tasks', such as repeated import of packages, reading data, etc.\\nSpecifically, \\nThe current task is about data preprocessing, please note the following:\\n- Monitor data types per column, applying appropriate methods.\\n- Ensure operations are on existing dataset columns.\\n- Avoid writing processed data to files.\\n- Avoid any change to label column, such as standardization, etc.\\n- Prefer alternatives to one-hot encoding for categorical data.\\n- Only encode or scale necessary columns to allow for potential feature-specific engineering tasks (like time_extract, binning, extraction, etc.) later.\\n- Each step do data preprocessing to train, must do same for test separately at the same time.\\n\\n\\n# Code Steps:\\nStrictly follow steps below when you writing code if it's convenient.\\n\\n\\n# Capabilities\\n- You can utilize pre-defined tools in any code lines from 'Available Tools' in the form of Python Class.\\n- You can freely combine the use of any other public packages, like sklearn, numpy, pandas, etc..\\n\\n# Available Tools:\\nEach Class tool is described in JSON format. When you call a tool, import the tool from its path first.\\n{'FillMissingValue': {'type': 'class', 'description': 'Completing missing values with simple strategies', 'methods': {'__init__': {'description': 'Initialize self.', 'parameters': {'properties': {'features': {'type': 'list', 'description': 'columns to be processed'}, 'strategy': {'type': 'str', 'description': 'the imputation strategy, notice mean/median can only be used for numeric features', 'default': 'mean', 'enum': ['mean', 'median', 'most_frequent', 'constant']}, 'fill_value': {'type': 'int', 'description': 'fill_value is used to replace all occurrences of missing_values', 'default': None}}, 'required': ['features']}}, 'fit': {'description': 'Fit the FillMissingValue model.', 'parameters': {'properties': {'df': {'type': 'DataFrame', 'description': 'The input DataFrame.'}}, 'required': ['df']}}, 'transform': {'description': 'Transform the input DataFrame with the fitted model.', 'parameters': {'properties': {'df': {'type': 'DataFrame', 'description': 'The input DataFrame.'}}, 'required': ['df']}, 'returns': {'df': {'type': 'DataFrame', 'description': 'The transformed DataFrame.'}}}, 'fit_transform': {'description': 'Fit and transform the input DataFrame.', 'parameters': {'properties': {'df': {'type': 'DataFrame', 'description': 'The input DataFrame.'}}, 'required': ['df']}, 'returns': {'df': {'type': 'DataFrame', 'description': 'The transformed DataFrame.'}}}}, 'tool_path': 'metagpt/tools/libs/data_preprocess.py'}}\\n\\n# Output Example:\\nwhen current task is \\\"do data preprocess, like fill missing value, handle outliers, etc.\\\", and their are two steps in 'Code Steps', the code be like:\\n```python\\n# Step 1: fill missing value\\n# Tools used: ['FillMissingValue']\\nfrom metagpt.tools.libs.data_preprocess import FillMissingValue\\n\\ntrain_processed = train.copy()\\ntest_processed = test.copy()\\nnum_cols = train_processed.select_dtypes(include='number').columns.tolist()\\nif 'label' in num_cols:\\n num_cols.remove('label')\\nfill_missing_value = FillMissingValue(features=num_cols, strategy='mean')\\nfill_missing_value.fit(train_processed)\\ntrain_processed = fill_missing_value.transform(train_processed)\\ntest_processed = fill_missing_value.transform(test_processed)\\n\\n# Step 2: handle outliers\\nfor col in num_cols:\\n low, high = train_processed[col].quantile([0.01, 0.99])\\n train_processed[col] = train_processed[col].clip(low, high)\\n test_processed[col] = test_processed[col].clip(low, high)\\n```end\\n\\n# Constraints:\\n- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed.\\n- Always prioritize using pre-defined tools for the same functionality.\\n- Always copy the DataFrame before processing it and use the copy to process.\\n- The output code should contain all steps implemented correctly in 'Code Steps'.\\n\"}]": { + "code": "# Step 1: Handle missing values\nfrom metagpt.tools.libs.data_preprocess import FillMissingValue\n\n# Create copies of the datasets to avoid changing the original data\ntrain_processed = train_data.copy()\neval_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv')\n\n# Fill missing values for 'Age' with median and 'Embarked' with most frequent value\nfill_missing_age = FillMissingValue(features=['Age'], strategy='median')\ntrain_processed = fill_missing_age.fit_transform(train_processed)\neval_processed = fill_missing_age.transform(eval_data)\n\nfill_missing_embarked = FillMissingValue(features=['Embarked'], strategy='most_frequent')\ntrain_processed = fill_missing_embarked.fit_transform(train_processed)\neval_processed = fill_missing_embarked.transform(eval_data)\n\n# Since 'Cabin' has a lot of missing values, we'll create a binary feature indicating whether cabin info is missing\ntrain_processed['Cabin_Ind'] = train_processed['Cabin'].isnull().astype(int)\neval_processed['Cabin_Ind'] = eval_processed['Cabin'].isnull().astype(int)\n\n# Step 2: Encode categorical variables\nfrom sklearn.preprocessing import LabelEncoder\n\nlabel_encoder = LabelEncoder()\n\n# Encode 'Sex'\ntrain_processed['Sex'] = label_encoder.fit_transform(train_processed['Sex'])\neval_processed['Sex'] = label_encoder.transform(eval_processed['Sex'])\n\n# Encode 'Embarked'\ntrain_processed['Embarked'] = label_encoder.fit_transform(train_processed['Embarked'])\neval_processed['Embarked'] = label_encoder.transform(eval_processed['Embarked'])\n\n# Step 3: Scale features\nfrom sklearn.preprocessing import StandardScaler\n\nscaler = StandardScaler()\n\n# Scale 'Age' and 'Fare'\nfor feature in ['Age', 'Fare']:\n train_processed[[feature]] = scaler.fit_transform(train_processed[[feature]])\n eval_processed[[feature]] = scaler.transform(eval_processed[[feature]])\n\n# Display the first few rows of the processed train dataset\ndisplay(train_processed.head())" + }, + "[{\"role\": \"user\", \"content\": \"\\n# Background\\nKeep dataset column information updated before model train.\\n## Done Tasks\\n```python\\nimport pandas as pd\\ntrain_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv')\\ndisplay(train_data.head())\\ntrain_data.info()\\ntrain_data.describe()\\ntrain_data.describe(include=['O'])\\nmissing_values = train_data.isnull().sum()\\nmissing_values[missing_values > 0]\\nimport seaborn as sns\\nimport matplotlib.pyplot as plt\\nsns.countplot(x='Survived', data=train_data)\\nplt.title('Distribution of Survival')\\nplt.show()\\nsns.barplot(x='Sex', y='Survived', data=train_data)\\nplt.title('Survival Rate by Sex')\\nplt.show()\\nsns.barplot(x='Pclass', y='Survived', data=train_data)\\nplt.title('Survival Rate by Class')\\nplt.show()\\nsns.barplot(x='Embarked', y='Survived', data=train_data)\\nplt.title('Survival Rate by Embarkation Port')\\nplt.show()\\n\\nfrom metagpt.tools.libs.data_preprocess import FillMissingValue\\ntrain_processed = train_data.copy()\\neval_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv')\\nfill_missing_age = FillMissingValue(features=['Age'], strategy='median')\\ntrain_processed = fill_missing_age.fit_transform(train_processed)\\neval_processed = fill_missing_age.transform(eval_data)\\nfill_missing_embarked = FillMissingValue(features=['Embarked'], strategy='most_frequent')\\ntrain_processed = fill_missing_embarked.fit_transform(train_processed)\\neval_processed = fill_missing_embarked.transform(eval_data)\\ntrain_processed['Cabin_Ind'] = train_processed['Cabin'].isnull().astype(int)\\neval_processed['Cabin_Ind'] = eval_processed['Cabin'].isnull().astype(int)\\nfrom sklearn.preprocessing import LabelEncoder\\nlabel_encoder = LabelEncoder()\\ntrain_processed['Sex'] = label_encoder.fit_transform(train_processed['Sex'])\\neval_processed['Sex'] = label_encoder.transform(eval_processed['Sex'])\\ntrain_processed['Embarked'] = label_encoder.fit_transform(train_processed['Embarked'])\\neval_processed['Embarked'] = label_encoder.transform(eval_processed['Embarked'])\\nfrom sklearn.preprocessing import StandardScaler\\nscaler = StandardScaler()\\nfor feature in ['Age', 'Fare']:\\n train_processed[[feature]] = scaler.fit_transform(train_processed[[feature]])\\n eval_processed[[feature]] = scaler.transform(eval_processed[[feature]])\\ndisplay(train_processed.head())\\n```end\\n\\n# Task\\nUpdate and print the dataset's column information only if the train or test data has changed. Use the following code:\\n```python\\nfrom metagpt.tools.libs.data_preprocess import get_column_info\\n\\ncolumn_info = get_column_info(df)\\nprint(\\\"column_info\\\")\\nprint(column_info)\\n```end\\n\\n# Constraints:\\n- Use the DataFrame variable from 'Done Tasks' in place of df.\\n- Import `get_column_info` only if it's not already imported.\\n\"}]": { + "code": "from metagpt.tools.libs.data_preprocess import get_column_info\ncolumn_info = get_column_info(train_processed)\nprint(\"column_info\")\nprint(column_info)" + }, + "[{\"role\": \"user\", \"content\": \"\\n## User Requirement:\\nConduct feature engineering to create new features that may help improve model performance.\\n\\n## Task\\nRecommend up to five tools from 'Available Tools' that can help solve the 'User Requirement'. \\nThis is a detailed code steps for current task. You can refer to it when recommending tools.\\n\\n\\n## Available Tools:\\n{'CatCross': 'Add pairwise crossed features and convert them to numerical features.'}\\n\\n## Tool Selection and Instructions:\\n- Select tools most relevant to completing the 'User Requirement'.\\n- If you believe that no tools are suitable, indicate with an empty list.\\n- Only list the names of the tools, not the full schema of each tool.\\n- Ensure selected tools are listed in 'Available Tools'.\\n\"}]": { + "recommend_tools": [ + "CatCross" + ] + }, + "[{\"role\": \"user\", \"content\": \"\\n# Background\\nAs a data scientist, you need to help user to achieve their goal [This is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv', eval data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv'.] step-by-step in an continuous Jupyter notebook.\\n\\n## Done Tasks\\n```python\\nimport pandas as pd\\ntrain_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv')\\ndisplay(train_data.head())\\ntrain_data.info()\\ntrain_data.describe()\\ntrain_data.describe(include=['O'])\\nmissing_values = train_data.isnull().sum()\\nmissing_values[missing_values > 0]\\nimport seaborn as sns\\nimport matplotlib.pyplot as plt\\nsns.countplot(x='Survived', data=train_data)\\nplt.title('Distribution of Survival')\\nplt.show()\\nsns.barplot(x='Sex', y='Survived', data=train_data)\\nplt.title('Survival Rate by Sex')\\nplt.show()\\nsns.barplot(x='Pclass', y='Survived', data=train_data)\\nplt.title('Survival Rate by Class')\\nplt.show()\\nsns.barplot(x='Embarked', y='Survived', data=train_data)\\nplt.title('Survival Rate by Embarkation Port')\\nplt.show()\\n\\nfrom metagpt.tools.libs.data_preprocess import FillMissingValue\\ntrain_processed = train_data.copy()\\neval_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv')\\nfill_missing_age = FillMissingValue(features=['Age'], strategy='median')\\ntrain_processed = fill_missing_age.fit_transform(train_processed)\\neval_processed = fill_missing_age.transform(eval_data)\\nfill_missing_embarked = FillMissingValue(features=['Embarked'], strategy='most_frequent')\\ntrain_processed = fill_missing_embarked.fit_transform(train_processed)\\neval_processed = fill_missing_embarked.transform(eval_data)\\ntrain_processed['Cabin_Ind'] = train_processed['Cabin'].isnull().astype(int)\\neval_processed['Cabin_Ind'] = eval_processed['Cabin'].isnull().astype(int)\\nfrom sklearn.preprocessing import LabelEncoder\\nlabel_encoder = LabelEncoder()\\ntrain_processed['Sex'] = label_encoder.fit_transform(train_processed['Sex'])\\neval_processed['Sex'] = label_encoder.transform(eval_processed['Sex'])\\ntrain_processed['Embarked'] = label_encoder.fit_transform(train_processed['Embarked'])\\neval_processed['Embarked'] = label_encoder.transform(eval_processed['Embarked'])\\nfrom sklearn.preprocessing import StandardScaler\\nscaler = StandardScaler()\\nfor feature in ['Age', 'Fare']:\\n train_processed[[feature]] = scaler.fit_transform(train_processed[[feature]])\\n eval_processed[[feature]] = scaler.transform(eval_processed[[feature]])\\ndisplay(train_processed.head())\\n```end\\n\\n## Current Task\\nConduct feature engineering to create new features that may help improve model performance.\\n\\n# Latest Data Info\\nLatest data info after previous tasks:\\ncolumn_info\\n{'Category': ['Name', 'Ticket', 'Cabin'], 'Numeric': ['PassengerId', 'Survived', 'Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Cabin_Ind'], 'Datetime': [], 'Others': []}\\n\\n\\n# Task\\nWrite complete code for 'Current Task'. And avoid duplicating code from 'Done Tasks', such as repeated import of packages, reading data, etc.\\nSpecifically, \\nThe current task is about feature engineering. when performing it, please adhere to the following principles:\\n- Generate as diverse features as possible to improve the model's performance step-by-step. \\n- If potential impactful features are not included in 'Code Steps', add new steps to generate them.\\n- Avoid creating redundant or excessively numerous features in one step.\\n- Exclude ID columns from feature generation and remove them.\\n- Each step do feature engineering to train, must do same for test separately at the same time.\\n- Avoid using the label column to create features, except for cat encoding.\\n- Use the data from previous task result if exist, do not mock or reload data yourself.\\n\\n\\n# Code Steps:\\nStrictly follow steps below when you writing code if it's convenient.\\n\\n\\n# Capabilities\\n- You can utilize pre-defined tools in any code lines from 'Available Tools' in the form of Python Class.\\n- You can freely combine the use of any other public packages, like sklearn, numpy, pandas, etc..\\n\\n# Available Tools:\\nEach Class tool is described in JSON format. When you call a tool, import the tool from its path first.\\n{'CatCross': {'type': 'class', 'description': 'Add pairwise crossed features and convert them to numerical features.', 'methods': {'__init__': {'description': 'Initialize self.', 'parameters': {'properties': {'cols': {'type': 'list', 'description': 'Columns to be pairwise crossed, at least 2 columns.'}, 'max_cat_num': {'type': 'int', 'description': 'Maximum unique categories per crossed feature.', 'default': 100}}}, 'required': ['cols']}, 'fit': {'description': 'Fit the CatCross model.', 'parameters': {'properties': {'df': {'type': 'DataFrame', 'description': 'The input DataFrame.'}}, 'required': ['df']}}, 'transform': {'description': 'Transform the input DataFrame with the fitted model.', 'parameters': {'properties': {'df': {'type': 'DataFrame', 'description': 'The input DataFrame.'}}, 'required': ['df']}, 'returns': {'df': {'type': 'DataFrame', 'description': 'The transformed DataFrame.'}}}, 'fit_transform': {'description': 'Fit and transform the input DataFrame.', 'parameters': {'properties': {'df': {'type': 'DataFrame', 'description': 'The input DataFrame.'}}, 'required': ['df']}, 'returns': {'df': {'type': 'DataFrame', 'description': 'The transformed DataFrame.'}}}}, 'tool_path': 'metagpt/tools/libs/feature_engineering.py'}}\\n\\n# Output Example:\\nwhen current task is \\\"do data preprocess, like fill missing value, handle outliers, etc.\\\", and their are two steps in 'Code Steps', the code be like:\\n```python\\n# Step 1: fill missing value\\n# Tools used: ['FillMissingValue']\\nfrom metagpt.tools.libs.data_preprocess import FillMissingValue\\n\\ntrain_processed = train.copy()\\ntest_processed = test.copy()\\nnum_cols = train_processed.select_dtypes(include='number').columns.tolist()\\nif 'label' in num_cols:\\n num_cols.remove('label')\\nfill_missing_value = FillMissingValue(features=num_cols, strategy='mean')\\nfill_missing_value.fit(train_processed)\\ntrain_processed = fill_missing_value.transform(train_processed)\\ntest_processed = fill_missing_value.transform(test_processed)\\n\\n# Step 2: handle outliers\\nfor col in num_cols:\\n low, high = train_processed[col].quantile([0.01, 0.99])\\n train_processed[col] = train_processed[col].clip(low, high)\\n test_processed[col] = test_processed[col].clip(low, high)\\n```end\\n\\n# Constraints:\\n- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed.\\n- Always prioritize using pre-defined tools for the same functionality.\\n- Always copy the DataFrame before processing it and use the copy to process.\\n- The output code should contain all steps implemented correctly in 'Code Steps'.\\n\"}]": { + "code": "# Step 1: Create a feature for the number of family members on board\ntrain_processed['Family_Size'] = train_processed['SibSp'] + train_processed['Parch']\neval_processed['Family_Size'] = eval_processed['SibSp'] + eval_processed['Parch']\n\n# Step 2: Create a feature to indicate whether a passenger was alone or not\ntrain_processed['Is_Alone'] = (train_processed['Family_Size'] == 0).astype(int)\neval_processed['Is_Alone'] = (eval_processed['Family_Size'] == 0).astype(int)\n\n# Step 3: Create a feature for the title extracted from the passenger's name\ntrain_processed['Title'] = train_processed['Name'].str.extract(' ([A-Za-z]+)\\.', expand=False)\neval_processed['Title'] = eval_processed['Name'].str.extract(' ([A-Za-z]+)\\.', expand=False)\n\n# Step 4: Create a feature for the deck extracted from the cabin number\ntrain_processed['Deck'] = train_processed['Cabin'].str[0]\neval_processed['Deck'] = eval_processed['Cabin'].str[0]\n\n# Step 5: Create a feature for the ticket class and sex cross\n# Tools used: ['CatCross']\nfrom metagpt.tools.libs.feature_engineering import CatCross\n\n# Initialize CatCross with the desired columns\ncat_cross = CatCross(cols=['Pclass', 'Sex'])\n\n# Fit and transform the train data\ntrain_processed = cat_cross.fit_transform(train_processed)\n\n# Transform the eval data\neval_processed = cat_cross.transform(eval_processed)\n\n# Remove unnecessary columns\nfor df in [train_processed, eval_processed]:\n df.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1, inplace=True)" + }, + "[{\"role\": \"user\", \"content\": \"\\n# Background\\nKeep dataset column information updated before model train.\\n## Done Tasks\\n```python\\nimport pandas as pd\\ntrain_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv')\\ndisplay(train_data.head())\\ntrain_data.info()\\ntrain_data.describe()\\ntrain_data.describe(include=['O'])\\nmissing_values = train_data.isnull().sum()\\nmissing_values[missing_values > 0]\\nimport seaborn as sns\\nimport matplotlib.pyplot as plt\\nsns.countplot(x='Survived', data=train_data)\\nplt.title('Distribution of Survival')\\nplt.show()\\nsns.barplot(x='Sex', y='Survived', data=train_data)\\nplt.title('Survival Rate by Sex')\\nplt.show()\\nsns.barplot(x='Pclass', y='Survived', data=train_data)\\nplt.title('Survival Rate by Class')\\nplt.show()\\nsns.barplot(x='Embarked', y='Survived', data=train_data)\\nplt.title('Survival Rate by Embarkation Port')\\nplt.show()\\n\\nfrom metagpt.tools.libs.data_preprocess import FillMissingValue\\ntrain_processed = train_data.copy()\\neval_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv')\\nfill_missing_age = FillMissingValue(features=['Age'], strategy='median')\\ntrain_processed = fill_missing_age.fit_transform(train_processed)\\neval_processed = fill_missing_age.transform(eval_data)\\nfill_missing_embarked = FillMissingValue(features=['Embarked'], strategy='most_frequent')\\ntrain_processed = fill_missing_embarked.fit_transform(train_processed)\\neval_processed = fill_missing_embarked.transform(eval_data)\\ntrain_processed['Cabin_Ind'] = train_processed['Cabin'].isnull().astype(int)\\neval_processed['Cabin_Ind'] = eval_processed['Cabin'].isnull().astype(int)\\nfrom sklearn.preprocessing import LabelEncoder\\nlabel_encoder = LabelEncoder()\\ntrain_processed['Sex'] = label_encoder.fit_transform(train_processed['Sex'])\\neval_processed['Sex'] = label_encoder.transform(eval_processed['Sex'])\\ntrain_processed['Embarked'] = label_encoder.fit_transform(train_processed['Embarked'])\\neval_processed['Embarked'] = label_encoder.transform(eval_processed['Embarked'])\\nfrom sklearn.preprocessing import StandardScaler\\nscaler = StandardScaler()\\nfor feature in ['Age', 'Fare']:\\n train_processed[[feature]] = scaler.fit_transform(train_processed[[feature]])\\n eval_processed[[feature]] = scaler.transform(eval_processed[[feature]])\\ndisplay(train_processed.head())\\n\\ntrain_processed['Family_Size'] = train_processed['SibSp'] + train_processed['Parch']\\neval_processed['Family_Size'] = eval_processed['SibSp'] + eval_processed['Parch']\\ntrain_processed['Is_Alone'] = (train_processed['Family_Size'] == 0).astype(int)\\neval_processed['Is_Alone'] = (eval_processed['Family_Size'] == 0).astype(int)\\ntrain_processed['Title'] = train_processed['Name'].str.extract(' ([A-Za-z]+)\\\\.', expand=False)\\neval_processed['Title'] = eval_processed['Name'].str.extract(' ([A-Za-z]+)\\\\.', expand=False)\\ntrain_processed['Deck'] = train_processed['Cabin'].str[0]\\neval_processed['Deck'] = eval_processed['Cabin'].str[0]\\nfrom metagpt.tools.libs.feature_engineering import CatCross\\ncat_cross = CatCross(cols=['Pclass', 'Sex'])\\ntrain_processed = cat_cross.fit_transform(train_processed)\\neval_processed = cat_cross.transform(eval_processed)\\nfor df in [train_processed, eval_processed]:\\n df.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1, inplace=True)\\n```end\\n\\n# Task\\nUpdate and print the dataset's column information only if the train or test data has changed. Use the following code:\\n```python\\nfrom metagpt.tools.libs.data_preprocess import get_column_info\\n\\ncolumn_info = get_column_info(df)\\nprint(\\\"column_info\\\")\\nprint(column_info)\\n```end\\n\\n# Constraints:\\n- Use the DataFrame variable from 'Done Tasks' in place of df.\\n- Import `get_column_info` only if it's not already imported.\\n\"}]": { + "code": "from metagpt.tools.libs.data_preprocess import get_column_info\ncolumn_info = get_column_info(train_processed)\nprint(\"column_info\")\nprint(column_info)" + }, + "[{\"role\": \"user\", \"content\": \"\\n# Background\\nAs a data scientist, you need to help user to achieve their goal [This is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv', eval data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv'.] step-by-step in an continuous Jupyter notebook.\\n\\n## Done Tasks\\n```python\\nimport pandas as pd\\ntrain_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv')\\ndisplay(train_data.head())\\ntrain_data.info()\\ntrain_data.describe()\\ntrain_data.describe(include=['O'])\\nmissing_values = train_data.isnull().sum()\\nmissing_values[missing_values > 0]\\nimport seaborn as sns\\nimport matplotlib.pyplot as plt\\nsns.countplot(x='Survived', data=train_data)\\nplt.title('Distribution of Survival')\\nplt.show()\\nsns.barplot(x='Sex', y='Survived', data=train_data)\\nplt.title('Survival Rate by Sex')\\nplt.show()\\nsns.barplot(x='Pclass', y='Survived', data=train_data)\\nplt.title('Survival Rate by Class')\\nplt.show()\\nsns.barplot(x='Embarked', y='Survived', data=train_data)\\nplt.title('Survival Rate by Embarkation Port')\\nplt.show()\\n\\nfrom metagpt.tools.libs.data_preprocess import FillMissingValue\\ntrain_processed = train_data.copy()\\neval_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv')\\nfill_missing_age = FillMissingValue(features=['Age'], strategy='median')\\ntrain_processed = fill_missing_age.fit_transform(train_processed)\\neval_processed = fill_missing_age.transform(eval_data)\\nfill_missing_embarked = FillMissingValue(features=['Embarked'], strategy='most_frequent')\\ntrain_processed = fill_missing_embarked.fit_transform(train_processed)\\neval_processed = fill_missing_embarked.transform(eval_data)\\ntrain_processed['Cabin_Ind'] = train_processed['Cabin'].isnull().astype(int)\\neval_processed['Cabin_Ind'] = eval_processed['Cabin'].isnull().astype(int)\\nfrom sklearn.preprocessing import LabelEncoder\\nlabel_encoder = LabelEncoder()\\ntrain_processed['Sex'] = label_encoder.fit_transform(train_processed['Sex'])\\neval_processed['Sex'] = label_encoder.transform(eval_processed['Sex'])\\ntrain_processed['Embarked'] = label_encoder.fit_transform(train_processed['Embarked'])\\neval_processed['Embarked'] = label_encoder.transform(eval_processed['Embarked'])\\nfrom sklearn.preprocessing import StandardScaler\\nscaler = StandardScaler()\\nfor feature in ['Age', 'Fare']:\\n train_processed[[feature]] = scaler.fit_transform(train_processed[[feature]])\\n eval_processed[[feature]] = scaler.transform(eval_processed[[feature]])\\ndisplay(train_processed.head())\\n\\ntrain_processed['Family_Size'] = train_processed['SibSp'] + train_processed['Parch']\\neval_processed['Family_Size'] = eval_processed['SibSp'] + eval_processed['Parch']\\ntrain_processed['Is_Alone'] = (train_processed['Family_Size'] == 0).astype(int)\\neval_processed['Is_Alone'] = (eval_processed['Family_Size'] == 0).astype(int)\\ntrain_processed['Title'] = train_processed['Name'].str.extract(' ([A-Za-z]+)\\\\.', expand=False)\\neval_processed['Title'] = eval_processed['Name'].str.extract(' ([A-Za-z]+)\\\\.', expand=False)\\ntrain_processed['Deck'] = train_processed['Cabin'].str[0]\\neval_processed['Deck'] = eval_processed['Cabin'].str[0]\\nfrom metagpt.tools.libs.feature_engineering import CatCross\\ncat_cross = CatCross(cols=['Pclass', 'Sex'])\\ntrain_processed = cat_cross.fit_transform(train_processed)\\neval_processed = cat_cross.transform(eval_processed)\\nfor df in [train_processed, eval_processed]:\\n df.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1, inplace=True)\\n```end\\n\\n## Current Task\\nSelect and train a machine learning model using the processed train dataset.\\n\\n# Latest Data Info\\nLatest data info after previous tasks:\\ncolumn_info\\n{'Category': ['Title', 'Deck'], 'Numeric': ['Survived', 'Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Cabin_Ind', 'Family_Size', 'Is_Alone', 'Pclass_Sex'], 'Datetime': [], 'Others': []}\\n\\n\\n# Task\\nWrite complete code for 'Current Task'. And avoid duplicating code from 'Done Tasks', such as repeated import of packages, reading data, etc.\\nSpecifically, \\nThe current task is about training a model, please ensure high performance:\\n- Keep in mind that your user prioritizes results and is highly focused on model performance. So, when needed, feel free to use models of any complexity to improve effectiveness, such as XGBoost, CatBoost, etc.\\n- If non-numeric columns exist, perform label encode together with all steps.\\n- Use the data from previous task result directly, do not mock or reload data yourself.\\n- Set suitable hyperparameters for the model, make metrics as high as possible.\\n\\n\\n# Code Steps:\\nStrictly follow steps below when you writing code if it's convenient.\\n\\n\\n# Output Example:\\nwhen current task is \\\"train a lightgbm model on training data\\\", and their are two steps in 'Code Steps', the code be like:\\n```python\\n# Step 1: check data type and convert to numeric\\nojb_cols = train.select_dtypes(include='object').columns.tolist()\\n\\nfor col in obj_cols:\\n encoder = LabelEncoder()\\n train[col] = encoder.fit_transform(train[col].unique().tolist() + ['unknown'])\\n test[col] = test[col].apply(lambda x: x if x in encoder.classes_ else 'unknown')\\n test[col] = encoder.transform(test[col])\\n\\n# Step 2: train lightgbm model\\nmodel = LGBMClassifier()\\nmodel.fit(train, y_train)\\n```end\\n\\n# Constraints:\\n- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed.\\n- The output code should contain all steps implemented in 'Code Steps'.\\n\"}]": { + "code": "from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom xgboost import XGBClassifier\n\n# Step 1: Prepare the data\n# Separate features and target variable\nX = train_processed.drop('Survived', axis=1)\ny = train_processed['Survived']\n\n# Step 2: Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Step 3: Initialize and train the XGBoost model\n# Note: Hyperparameters are set to values that generally perform well, but further tuning could be done.\nmodel = XGBClassifier(n_estimators=100, learning_rate=0.05, max_depth=4, subsample=0.8, colsample_bytree=0.8, random_state=42)\nmodel.fit(X_train, y_train)\n\n# Step 4: Evaluate the model\ny_pred = model.predict(X_val)\naccuracy = accuracy_score(y_val, y_pred)\nprint(f'Validation Accuracy: {accuracy:.4f}')" + }, + "[{\"role\": \"system\", \"content\": \"You are an AI Python assistant. You will be given your previous implementation code of a task, runtime error results, and a hint to change the implementation appropriately. Write your full implementation \"}, {\"role\": \"user\", \"content\": \"\\nHere is an example for you.\\n\\nExample 1:\\n[previous impl]:\\n```python\\ndef add(a: int, b: int) -> int:\\n \\\"\\\"\\\"\\n Given integers a and b, return the total value of a and b.\\n \\\"\\\"\\\"\\n return a - b\\n```\\n\\n[runtime Error]:\\nTested passed:\\n\\nTests failed:\\nassert add(1, 2) == 3 # output: -1\\nassert add(1, 2) == 4 # output: -1\\n\\n[reflection on previous impl]:\\nThe implementation failed the test cases where the input integers are 1 and 2. The issue arises because the code does not add the two integers together, but instead subtracts the second integer from the first. To fix this issue, we should change the operator from `-` to `+` in the return statement. This will ensure that the function returns the correct output for the given input.\\n\\n[improved impl]:\\n```python\\ndef add(a: int, b: int) -> int:\\n \\\"\\\"\\\"\\n Given integers a and b, return the total value of a and b.\\n \\\"\\\"\\\"\\n return a + b\\n```\\n\\n[context]\\n[user: \\n# Background\\nAs a data scientist, you need to help user to achieve their goal [This is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv', eval data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv'.] step-by-step in an continuous Jupyter notebook.\\n\\n## Done Tasks\\n```python\\nimport pandas as pd\\ntrain_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv')\\ndisplay(train_data.head())\\ntrain_data.info()\\ntrain_data.describe()\\ntrain_data.describe(include=['O'])\\nmissing_values = train_data.isnull().sum()\\nmissing_values[missing_values > 0]\\nimport seaborn as sns\\nimport matplotlib.pyplot as plt\\nsns.countplot(x='Survived', data=train_data)\\nplt.title('Distribution of Survival')\\nplt.show()\\nsns.barplot(x='Sex', y='Survived', data=train_data)\\nplt.title('Survival Rate by Sex')\\nplt.show()\\nsns.barplot(x='Pclass', y='Survived', data=train_data)\\nplt.title('Survival Rate by Class')\\nplt.show()\\nsns.barplot(x='Embarked', y='Survived', data=train_data)\\nplt.title('Survival Rate by Embarkation Port')\\nplt.show()\\n\\nfrom metagpt.tools.libs.data_preprocess import FillMissingValue\\ntrain_processed = train_data.copy()\\neval_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv')\\nfill_missing_age = FillMissingValue(features=['Age'], strategy='median')\\ntrain_processed = fill_missing_age.fit_transform(train_processed)\\neval_processed = fill_missing_age.transform(eval_data)\\nfill_missing_embarked = FillMissingValue(features=['Embarked'], strategy='most_frequent')\\ntrain_processed = fill_missing_embarked.fit_transform(train_processed)\\neval_processed = fill_missing_embarked.transform(eval_data)\\ntrain_processed['Cabin_Ind'] = train_processed['Cabin'].isnull().astype(int)\\neval_processed['Cabin_Ind'] = eval_processed['Cabin'].isnull().astype(int)\\nfrom sklearn.preprocessing import LabelEncoder\\nlabel_encoder = LabelEncoder()\\ntrain_processed['Sex'] = label_encoder.fit_transform(train_processed['Sex'])\\neval_processed['Sex'] = label_encoder.transform(eval_processed['Sex'])\\ntrain_processed['Embarked'] = label_encoder.fit_transform(train_processed['Embarked'])\\neval_processed['Embarked'] = label_encoder.transform(eval_processed['Embarked'])\\nfrom sklearn.preprocessing import StandardScaler\\nscaler = StandardScaler()\\nfor feature in ['Age', 'Fare']:\\n train_processed[[feature]] = scaler.fit_transform(train_processed[[feature]])\\n eval_processed[[feature]] = scaler.transform(eval_processed[[feature]])\\ndisplay(train_processed.head())\\n\\ntrain_processed['Family_Size'] = train_processed['SibSp'] + train_processed['Parch']\\neval_processed['Family_Size'] = eval_processed['SibSp'] + eval_processed['Parch']\\ntrain_processed['Is_Alone'] = (train_processed['Family_Size'] == 0).astype(int)\\neval_processed['Is_Alone'] = (eval_processed['Family_Size'] == 0).astype(int)\\ntrain_processed['Title'] = train_processed['Name'].str.extract(' ([A-Za-z]+)\\\\.', expand=False)\\neval_processed['Title'] = eval_processed['Name'].str.extract(' ([A-Za-z]+)\\\\.', expand=False)\\ntrain_processed['Deck'] = train_processed['Cabin'].str[0]\\neval_processed['Deck'] = eval_processed['Cabin'].str[0]\\nfrom metagpt.tools.libs.feature_engineering import CatCross\\ncat_cross = CatCross(cols=['Pclass', 'Sex'])\\ntrain_processed = cat_cross.fit_transform(train_processed)\\neval_processed = cat_cross.transform(eval_processed)\\nfor df in [train_processed, eval_processed]:\\n df.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1, inplace=True)\\n```end\\n\\n## Current Task\\nSelect and train a machine learning model using the processed train dataset.\\n\\n# Latest Data Info\\nLatest data info after previous tasks:\\ncolumn_info\\n{'Category': ['Title', 'Deck'], 'Numeric': ['Survived', 'Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Cabin_Ind', 'Family_Size', 'Is_Alone', 'Pclass_Sex'], 'Datetime': [], 'Others': []}\\n\\n\\n# Task\\nWrite complete code for 'Current Task'. And avoid duplicating code from 'Done Tasks', such as repeated import of packages, reading data, etc.\\nSpecifically, \\nThe current task is about training a model, please ensure high performance:\\n- Keep in mind that your user prioritizes results and is highly focused on model performance. So, when needed, feel free to use models of any complexity to improve effectiveness, such as XGBoost, CatBoost, etc.\\n- If non-numeric columns exist, perform label encode together with all steps.\\n- Use the data from previous task result directly, do not mock or reload data yourself.\\n- Set suitable hyperparameters for the model, make metrics as high as possible.\\n\\n\\n# Code Steps:\\nStrictly follow steps below when you writing code if it's convenient.\\n\\n\\n# Output Example:\\nwhen current task is \\\"train a lightgbm model on training data\\\", and their are two steps in 'Code Steps', the code be like:\\n```python\\n# Step 1: check data type and convert to numeric\\nojb_cols = train.select_dtypes(include='object').columns.tolist()\\n\\nfor col in obj_cols:\\n encoder = LabelEncoder()\\n train[col] = encoder.fit_transform(train[col].unique().tolist() + ['unknown'])\\n test[col] = test[col].apply(lambda x: x if x in encoder.classes_ else 'unknown')\\n test[col] = encoder.transform(test[col])\\n\\n# Step 2: train lightgbm model\\nmodel = LGBMClassifier()\\nmodel.fit(train, y_train)\\n```end\\n\\n# Constraints:\\n- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed.\\n- The output code should contain all steps implemented in 'Code Steps'.\\n]\\n\\n[previous impl]\\nfrom sklearn.model_selection import train_test_split\\nfrom sklearn.metrics import accuracy_score\\nfrom xgboost import XGBClassifier\\n\\n# Step 1: Prepare the data\\n# Separate features and target variable\\nX = train_processed.drop('Survived', axis=1)\\ny = train_processed['Survived']\\n\\n# Step 2: Split the data into training and validation sets\\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\\n\\n# Step 3: Initialize and train the XGBoost model\\n# Note: Hyperparameters are set to values that generally perform well, but further tuning could be done.\\nmodel = XGBClassifier(n_estimators=100, learning_rate=0.05, max_depth=4, subsample=0.8, colsample_bytree=0.8, random_state=42)\\nmodel.fit(X_train, y_train)\\n\\n# Step 4: Evaluate the model\\ny_pred = model.predict(X_val)\\naccuracy = accuracy_score(y_val, y_pred)\\nprint(f'Validation Accuracy: {accuracy:.4f}')\\n[runtime Error]\\n[assistant: from sklearn.model_selection import train_test_split\\nfrom sklearn.metrics import accuracy_score\\nfrom xgboost import XGBClassifier\\n\\n# Step 1: Prepare the data\\n# Separate features and target variable\\nX = train_processed.drop('Survived', axis=1)\\ny = train_processed['Survived']\\n\\n# Step 2: Split the data into training and validation sets\\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\\n\\n# Step 3: Initialize and train the XGBoost model\\n# Note: Hyperparameters are set to values that generally perform well, but further tuning could be done.\\nmodel = XGBClassifier(n_estimators=100, learning_rate=0.05, max_depth=4, subsample=0.8, colsample_bytree=0.8, random_state=42)\\nmodel.fit(X_train, y_train)\\n\\n# Step 4: Evaluate the model\\ny_pred = model.predict(X_val)\\naccuracy = accuracy_score(y_val, y_pred)\\nprint(f'Validation Accuracy: {accuracy:.4f}'), user: Executed code failed, please reflect the cause of bug and then debug. Truncated to show only last 2000 characters\\n= self._temporary_data\\n 622 else:\\n--> 623 new, cat_codes, feature_names, feature_types = _proxy_transform(\\n 624 data,\\n 625 feature_names,\\n 626 feature_types,\\n 627 self._enable_categorical,\\n 628 )\\n 629 # Stage the data, meta info are copied inside C++ MetaInfo.\\n 630 self._temporary_data = (new, cat_codes, feature_names, feature_types)\\n\\nFile ~/miniconda3/envs/mg_temp/lib/python3.9/site-packages/xgboost/data.py:1315, in _proxy_transform(data, feature_names, feature_types, enable_categorical)\\n 1313 data = pd.DataFrame(data)\\n 1314 if _is_pandas_df(data):\\n-> 1315 arr, feature_names, feature_types = _transform_pandas_df(\\n 1316 data, enable_categorical, feature_names, feature_types\\n 1317 )\\n 1318 arr, _ = _ensure_np_dtype(arr, arr.dtype)\\n 1319 return arr, None, feature_names, feature_types\\n\\nFile ~/miniconda3/envs/mg_temp/lib/python3.9/site-packages/xgboost/data.py:490, in _transform_pandas_df(data, enable_categorical, feature_names, feature_types, meta, meta_type)\\n 483 for dtype in data.dtypes:\\n 484 if not (\\n 485 (dtype.name in _pandas_dtype_mapper)\\n 486 or is_pd_sparse_dtype(dtype)\\n 487 or (is_pd_cat_dtype(dtype) and enable_categorical)\\n 488 or is_pa_ext_dtype(dtype)\\n 489 ):\\n--> 490 _invalid_dataframe_dtype(data)\\n 491 if is_pa_ext_dtype(dtype):\\n 492 pyarrow_extension = True\\n\\nFile ~/miniconda3/envs/mg_temp/lib/python3.9/site-packages/xgboost/data.py:308, in _invalid_dataframe_dtype(data)\\n 306 type_err = \\\"DataFrame.dtypes for data must be int, float, bool or category.\\\"\\n 307 msg = f\\\"\\\"\\\"{type_err} {_ENABLE_CAT_ERR} {err}\\\"\\\"\\\"\\n--> 308 raise ValueError(msg)\\n\\nValueError: DataFrame.dtypes for data must be int, float, bool or category. When categorical type is supplied, The experimental DMatrix parameter`enable_categorical` must be set to `True`. Invalid columns:Title: object, Deck: object\\n\\n]\\n\\nAnalysis the error step by step, provide me improve method and code. Remember to follow [context] rerquirement. Don't forget write code for steps behind the error step.\\n[reflection on previous impl]:\\nxxx\\n\"}]": { + "reflection": "The error message indicates that the XGBoost model cannot handle columns with 'object' data types directly. The columns 'Title' and 'Deck' are of 'object' type and need to be encoded as numeric values before they can be used for training the XGBoost model. The error also suggests that if we want to use categorical data, we must set the `enable_categorical` parameter to `True` when creating the DMatrix. However, a more common practice is to convert these object types to numeric types using techniques such as label encoding or one-hot encoding. To resolve this issue, we should encode the 'Title' and 'Deck' columns using label encoding before training the model. Additionally, we should ensure that any other non-numeric columns are also converted to numeric types if they exist.", + "improved_impl": "from sklearn.preprocessing import LabelEncoder\n\n# Step 1: Encode non-numeric columns\nencoder = LabelEncoder()\ntrain_processed['Title'] = encoder.fit_transform(train_processed['Title'])\neval_processed['Title'] = encoder.transform(eval_processed['Title'])\ntrain_processed['Deck'] = encoder.fit_transform(train_processed['Deck'].astype(str))\neval_processed['Deck'] = encoder.transform(eval_processed['Deck'].astype(str))\n\n# Step 2: Prepare the data\n# Separate features and target variable\nX = train_processed.drop('Survived', axis=1)\ny = train_processed['Survived']\n\n# Step 3: Train the XGBoost model\nmodel = XGBClassifier(n_estimators=100, learning_rate=0.05, max_depth=4, subsample=0.8, colsample_bytree=0.8, random_state=42)\nmodel.fit(X, y)\n\n# Step 4: Evaluate the model on evaluation data\nX_eval = eval_processed.drop('Survived', axis=1)\ny_eval = eval_processed['Survived']\ny_pred_eval = model.predict(X_eval)\naccuracy_eval = accuracy_score(y_eval, y_pred_eval)\nprint(f'Evaluation Accuracy: {accuracy_eval:.4f}')" + }, + "[{\"role\": \"user\", \"content\": \"\\n# Background\\nAs a data scientist, you need to help user to achieve their goal [This is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv', eval data path: '/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv'.] step-by-step in an continuous Jupyter notebook.\\n\\n## Done Tasks\\n```python\\nimport pandas as pd\\ntrain_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_train.csv')\\ndisplay(train_data.head())\\ntrain_data.info()\\ntrain_data.describe()\\ntrain_data.describe(include=['O'])\\nmissing_values = train_data.isnull().sum()\\nmissing_values[missing_values > 0]\\nimport seaborn as sns\\nimport matplotlib.pyplot as plt\\nsns.countplot(x='Survived', data=train_data)\\nplt.title('Distribution of Survival')\\nplt.show()\\nsns.barplot(x='Sex', y='Survived', data=train_data)\\nplt.title('Survival Rate by Sex')\\nplt.show()\\nsns.barplot(x='Pclass', y='Survived', data=train_data)\\nplt.title('Survival Rate by Class')\\nplt.show()\\nsns.barplot(x='Embarked', y='Survived', data=train_data)\\nplt.title('Survival Rate by Embarkation Port')\\nplt.show()\\n\\nfrom metagpt.tools.libs.data_preprocess import FillMissingValue\\ntrain_processed = train_data.copy()\\neval_data = pd.read_csv('/Users/gary/Desktop/data_agents_opt/data/titanic/split_eval.csv')\\nfill_missing_age = FillMissingValue(features=['Age'], strategy='median')\\ntrain_processed = fill_missing_age.fit_transform(train_processed)\\neval_processed = fill_missing_age.transform(eval_data)\\nfill_missing_embarked = FillMissingValue(features=['Embarked'], strategy='most_frequent')\\ntrain_processed = fill_missing_embarked.fit_transform(train_processed)\\neval_processed = fill_missing_embarked.transform(eval_data)\\ntrain_processed['Cabin_Ind'] = train_processed['Cabin'].isnull().astype(int)\\neval_processed['Cabin_Ind'] = eval_processed['Cabin'].isnull().astype(int)\\nfrom sklearn.preprocessing import LabelEncoder\\nlabel_encoder = LabelEncoder()\\ntrain_processed['Sex'] = label_encoder.fit_transform(train_processed['Sex'])\\neval_processed['Sex'] = label_encoder.transform(eval_processed['Sex'])\\ntrain_processed['Embarked'] = label_encoder.fit_transform(train_processed['Embarked'])\\neval_processed['Embarked'] = label_encoder.transform(eval_processed['Embarked'])\\nfrom sklearn.preprocessing import StandardScaler\\nscaler = StandardScaler()\\nfor feature in ['Age', 'Fare']:\\n train_processed[[feature]] = scaler.fit_transform(train_processed[[feature]])\\n eval_processed[[feature]] = scaler.transform(eval_processed[[feature]])\\ndisplay(train_processed.head())\\n\\ntrain_processed['Family_Size'] = train_processed['SibSp'] + train_processed['Parch']\\neval_processed['Family_Size'] = eval_processed['SibSp'] + eval_processed['Parch']\\ntrain_processed['Is_Alone'] = (train_processed['Family_Size'] == 0).astype(int)\\neval_processed['Is_Alone'] = (eval_processed['Family_Size'] == 0).astype(int)\\ntrain_processed['Title'] = train_processed['Name'].str.extract(' ([A-Za-z]+)\\\\.', expand=False)\\neval_processed['Title'] = eval_processed['Name'].str.extract(' ([A-Za-z]+)\\\\.', expand=False)\\ntrain_processed['Deck'] = train_processed['Cabin'].str[0]\\neval_processed['Deck'] = eval_processed['Cabin'].str[0]\\nfrom metagpt.tools.libs.feature_engineering import CatCross\\ncat_cross = CatCross(cols=['Pclass', 'Sex'])\\ntrain_processed = cat_cross.fit_transform(train_processed)\\neval_processed = cat_cross.transform(eval_processed)\\nfor df in [train_processed, eval_processed]:\\n df.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1, inplace=True)\\n\\nfrom sklearn.preprocessing import LabelEncoder\\nencoder = LabelEncoder()\\ntrain_processed['Title'] = encoder.fit_transform(train_processed['Title'])\\neval_processed['Title'] = encoder.transform(eval_processed['Title'])\\ntrain_processed['Deck'] = encoder.fit_transform(train_processed['Deck'].astype(str))\\neval_processed['Deck'] = encoder.transform(eval_processed['Deck'].astype(str))\\nX = train_processed.drop('Survived', axis=1)\\ny = train_processed['Survived']\\nmodel = XGBClassifier(n_estimators=100, learning_rate=0.05, max_depth=4, subsample=0.8, colsample_bytree=0.8, random_state=42)\\nmodel.fit(X, y)\\nX_eval = eval_processed.drop('Survived', axis=1)\\ny_eval = eval_processed['Survived']\\ny_pred_eval = model.predict(X_eval)\\naccuracy_eval = accuracy_score(y_eval, y_pred_eval)\\nprint(f'Evaluation Accuracy: {accuracy_eval:.4f}')\\n```end\\n\\n## Current Task\\nEvaluate the model's accuracy using the eval dataset and report the results.\\n\\n# Latest Data Info\\nLatest data info after previous tasks:\\n\\n\\n# Task\\nWrite complete code for 'Current Task'. And avoid duplicating code from 'Done Tasks', such as repeated import of packages, reading data, etc.\\nSpecifically, \\nThe current task is about evaluating a model, please note the following:\\n- Ensure that the evaluated data is same processed as the training data. If not, remember use object in 'Done Tasks' to transform the data.\\n- Use trained model from previous task result directly, do not mock or reload model yourself.\\n\\n\\n# Code Steps:\\nStrictly follow steps below when you writing code if it's convenient.\\n\\n\\n# Output Example:\\nwhen current task is \\\"train a lightgbm model on training data\\\", and their are two steps in 'Code Steps', the code be like:\\n```python\\n# Step 1: check data type and convert to numeric\\nojb_cols = train.select_dtypes(include='object').columns.tolist()\\n\\nfor col in obj_cols:\\n encoder = LabelEncoder()\\n train[col] = encoder.fit_transform(train[col].unique().tolist() + ['unknown'])\\n test[col] = test[col].apply(lambda x: x if x in encoder.classes_ else 'unknown')\\n test[col] = encoder.transform(test[col])\\n\\n# Step 2: train lightgbm model\\nmodel = LGBMClassifier()\\nmodel.fit(train, y_train)\\n```end\\n\\n# Constraints:\\n- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed.\\n- The output code should contain all steps implemented in 'Code Steps'.\\n\"}]": { + "code": "# Evaluate the model's accuracy using the eval dataset\nfrom sklearn.metrics import accuracy_score\n\n# Predict the target for the evaluation set\ny_pred_eval = model.predict(X_eval)\n\n# Calculate the accuracy for the evaluation set\naccuracy_eval = accuracy_score(y_eval, y_pred_eval)\n\n# Report the accuracy\nprint(f'Evaluation Accuracy: {accuracy_eval:.4f}')" + }, + "\n # Context:\n user: \n## User Requirement\nRun data analysis on sklearn Iris dataset, include a plot\n## Context\n\n## Current Plan\n[]\n## Current Task\n{}\n\n # Task:\n Based on the context, write a plan or modify an existing plan of what you should do to achieve the goal. A plan consists of one to 3 tasks.\n If you are modifying an existing plan, carefully follow the instruction, don't make unnecessary changes. Give the whole plan unless instructed to modify only one task of the plan.\n If you encounter errors on the current task, revise and output the current single task only.\n Output a list of jsons following the format:\n ```json\n [\n {\n \"task_id\": str = \"unique identifier for a task in plan, can be an ordinal\",\n \"dependent_task_ids\": list[str] = \"ids of tasks prerequisite to this task\",\n \"instruction\": \"what you should do in this task, one short phrase or sentence\",\n },\n ...\n ]\n ```\n ": "```json\n[\n {\n \"task_id\": \"1\",\n \"dependent_task_ids\": [],\n \"instruction\": \"Load the sklearn Iris dataset.\"\n },\n {\n \"task_id\": \"2\",\n \"dependent_task_ids\": [\"1\"],\n \"instruction\": \"Perform exploratory data analysis on the Iris dataset.\"\n },\n {\n \"task_id\": \"3\",\n \"dependent_task_ids\": [\"2\"],\n \"instruction\": \"Create a plot visualizing the Iris dataset features.\"\n }\n]\n```", + "[{\"role\": \"user\", \"content\": \"\\nPlease assign a task type to each task in the list below from the given categories:\\nTask 1: Load the sklearn Iris dataset.\\nTask 2: Perform exploratory data analysis on the Iris dataset.\\nTask 3: Create a plot visualizing the Iris dataset features.\\n\\n## All Task Type:\\n- **eda**: For performing exploratory data analysis\\n- **data_preprocess**: Only for changing value inplace.\\n- **feature_engineering**: Only for creating new columns for input data.\\n- **model_train**: Only for training model.\\n- **model_evaluate**: Only for evaluating model.\\n- **stable_diffusion**: Related to text2image, image2image using stable diffusion model.\\n- **image2webpage**: For converting image into webpage code.\\n- **web_scraping**: For scraping data from web pages.\\n- **other**: Any tools not in the defined categories\\n\"}]": { + "task_type": [ + "other", + "eda", + "other" + ] + }, + "[{\"role\": \"system\", \"content\": \"You are Code Interpreter, a world-class programmer that can complete any goal by executing code. Strictly follow the plan and generate code step by step. Each step of the code will be executed on the user's machine, and the user will provide the code execution results to you.**Notice: The code for the next step depends on the code for the previous step. Must reuse variables in the lastest other code directly, dont creat it again, it is very import for you. Use !pip install in a standalone block to install missing packages.Usually the libraries you need are already installed.Dont check if packages already imported.**\"}, {\"role\": \"user\", \"content\": \"\\n## User Requirement\\nRun data analysis on sklearn Iris dataset, include a plot\\n## Context\\n\\n## Current Plan\\n[\\n {\\n \\\"task_id\\\": \\\"1\\\",\\n \\\"dependent_task_ids\\\": [],\\n \\\"instruction\\\": \\\"Load the sklearn Iris dataset.\\\",\\n \\\"task_type\\\": \\\"other\\\",\\n \\\"code\\\": \\\"\\\",\\n \\\"result\\\": \\\"\\\",\\n \\\"is_success\\\": false,\\n \\\"is_finished\\\": false\\n },\\n {\\n \\\"task_id\\\": \\\"2\\\",\\n \\\"dependent_task_ids\\\": [\\n \\\"1\\\"\\n ],\\n \\\"instruction\\\": \\\"Perform exploratory data analysis on the Iris dataset.\\\",\\n \\\"task_type\\\": \\\"eda\\\",\\n \\\"code\\\": \\\"\\\",\\n \\\"result\\\": \\\"\\\",\\n \\\"is_success\\\": false,\\n \\\"is_finished\\\": false\\n },\\n {\\n \\\"task_id\\\": \\\"3\\\",\\n \\\"dependent_task_ids\\\": [\\n \\\"2\\\"\\n ],\\n \\\"instruction\\\": \\\"Create a plot visualizing the Iris dataset features.\\\",\\n \\\"task_type\\\": \\\"other\\\",\\n \\\"code\\\": \\\"\\\",\\n \\\"result\\\": \\\"\\\",\\n \\\"is_success\\\": false,\\n \\\"is_finished\\\": false\\n }\\n]\\n## Current Task\\n{\\\"task_id\\\":\\\"1\\\",\\\"dependent_task_ids\\\":[],\\\"instruction\\\":\\\"Load the sklearn Iris dataset.\\\",\\\"task_type\\\":\\\"other\\\",\\\"code_steps\\\":\\\"\\\",\\\"code\\\":\\\"\\\",\\\"result\\\":\\\"\\\",\\\"is_success\\\":false,\\\"is_finished\\\":false}\\n\"}, {\"role\": \"user\", \"content\": \"\\n# Instruction\\nWrite complete code for 'Current Task'. And avoid duplicating code from finished tasks, such as repeated import of packages, reading data, etc.\\nSpecifically, \\n\\n# Capabilities\\n- You can utilize pre-defined tools in any code lines from 'Available Tools' in the form of Python Class.\\n- You can freely combine the use of any other public packages, like sklearn, numpy, pandas, etc..\\n\\n# Available Tools (can be empty):\\nEach Class tool is described in JSON format. When you call a tool, import the tool first.\\n{}\\n\\n# Constraints:\\n- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed.\\n- Always prioritize using pre-defined tools for the same functionality.\\n\"}]": { + "code": "from sklearn.datasets import load_iris\niris_data = load_iris()" + }, + "[{\"role\": \"system\", \"content\": \"You are Code Interpreter, a world-class programmer that can complete any goal by executing code. Strictly follow the plan and generate code step by step. Each step of the code will be executed on the user's machine, and the user will provide the code execution results to you.**Notice: The code for the next step depends on the code for the previous step. Must reuse variables in the lastest other code directly, dont creat it again, it is very import for you. Use !pip install in a standalone block to install missing packages.Usually the libraries you need are already installed.Dont check if packages already imported.**\"}, {\"role\": \"user\", \"content\": \"\\n## User Requirement\\nRun data analysis on sklearn Iris dataset, include a plot\\n## Context\\n\\n## Current Plan\\n[\\n {\\n \\\"task_id\\\": \\\"1\\\",\\n \\\"dependent_task_ids\\\": [],\\n \\\"instruction\\\": \\\"Load the sklearn Iris dataset.\\\",\\n \\\"task_type\\\": \\\"other\\\",\\n \\\"code\\\": \\\"from sklearn.datasets import load_iris\\\\niris_data = load_iris()\\\",\\n \\\"result\\\": \\\"\\\",\\n \\\"is_success\\\": true,\\n \\\"is_finished\\\": true\\n },\\n {\\n \\\"task_id\\\": \\\"2\\\",\\n \\\"dependent_task_ids\\\": [\\n \\\"1\\\"\\n ],\\n \\\"instruction\\\": \\\"Perform exploratory data analysis on the Iris dataset.\\\",\\n \\\"task_type\\\": \\\"eda\\\",\\n \\\"code\\\": \\\"\\\",\\n \\\"result\\\": \\\"\\\",\\n \\\"is_success\\\": false,\\n \\\"is_finished\\\": false\\n },\\n {\\n \\\"task_id\\\": \\\"3\\\",\\n \\\"dependent_task_ids\\\": [\\n \\\"2\\\"\\n ],\\n \\\"instruction\\\": \\\"Create a plot visualizing the Iris dataset features.\\\",\\n \\\"task_type\\\": \\\"other\\\",\\n \\\"code\\\": \\\"\\\",\\n \\\"result\\\": \\\"\\\",\\n \\\"is_success\\\": false,\\n \\\"is_finished\\\": false\\n }\\n]\\n## Current Task\\n{\\\"task_id\\\":\\\"2\\\",\\\"dependent_task_ids\\\":[\\\"1\\\"],\\\"instruction\\\":\\\"Perform exploratory data analysis on the Iris dataset.\\\",\\\"task_type\\\":\\\"eda\\\",\\\"code_steps\\\":\\\"\\\",\\\"code\\\":\\\"\\\",\\\"result\\\":\\\"\\\",\\\"is_success\\\":false,\\\"is_finished\\\":false}\\n\"}, {\"role\": \"user\", \"content\": \"\\n# Instruction\\nWrite complete code for 'Current Task'. And avoid duplicating code from finished tasks, such as repeated import of packages, reading data, etc.\\nSpecifically, \\n\\n# Capabilities\\n- You can utilize pre-defined tools in any code lines from 'Available Tools' in the form of Python Class.\\n- You can freely combine the use of any other public packages, like sklearn, numpy, pandas, etc..\\n\\n# Available Tools (can be empty):\\nEach Class tool is described in JSON format. When you call a tool, import the tool first.\\n{}\\n\\n# Constraints:\\n- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed.\\n- Always prioritize using pre-defined tools for the same functionality.\\n\"}]": { + "code": "import pandas as pd\n\n# Convert the loaded Iris dataset into a DataFrame for easier manipulation\niris_df = pd.DataFrame(data=iris_data.data, columns=iris_data.feature_names)\niris_df['target'] = iris_data.target\n\n# Display basic statistical details like percentile, mean, std etc. of a data frame\niris_stats = iris_df.describe()\n\n# Display the first few rows of the DataFrame\niris_head = iris_df.head()\n\n# Display the class distribution\niris_target_counts = iris_df['target'].value_counts()\n\n# Output the results\nprint('Basic Statistical Details:\\n', iris_stats)\nprint('\\nFirst Five Rows:\\n', iris_head)\nprint('\\nClass Distribution:\\n', iris_target_counts)" + }, + "[{\"role\": \"system\", \"content\": \"You are Code Interpreter, a world-class programmer that can complete any goal by executing code. Strictly follow the plan and generate code step by step. Each step of the code will be executed on the user's machine, and the user will provide the code execution results to you.**Notice: The code for the next step depends on the code for the previous step. Must reuse variables in the lastest other code directly, dont creat it again, it is very import for you. Use !pip install in a standalone block to install missing packages.Usually the libraries you need are already installed.Dont check if packages already imported.**\"}, {\"role\": \"user\", \"content\": \"\\n## User Requirement\\nRun data analysis on sklearn Iris dataset, include a plot\\n## Context\\n\\n## Current Plan\\n[\\n {\\n \\\"task_id\\\": \\\"1\\\",\\n \\\"dependent_task_ids\\\": [],\\n \\\"instruction\\\": \\\"Load the sklearn Iris dataset.\\\",\\n \\\"task_type\\\": \\\"other\\\",\\n \\\"code\\\": \\\"from sklearn.datasets import load_iris\\\\niris_data = load_iris()\\\",\\n \\\"result\\\": \\\"\\\",\\n \\\"is_success\\\": true,\\n \\\"is_finished\\\": true\\n },\\n {\\n \\\"task_id\\\": \\\"2\\\",\\n \\\"dependent_task_ids\\\": [\\n \\\"1\\\"\\n ],\\n \\\"instruction\\\": \\\"Perform exploratory data analysis on the Iris dataset.\\\",\\n \\\"task_type\\\": \\\"eda\\\",\\n \\\"code\\\": \\\"import pandas as pd\\\\n\\\\n# Convert the loaded Iris dataset into a DataFrame for easier manipulation\\\\niris_df = pd.DataFrame(data=iris_data.data, columns=iris_data.feature_names)\\\\niris_df['target'] = iris_data.target\\\\n\\\\n# Display basic statistical details like percentile, mean, std etc. of a data frame\\\\niris_stats = iris_df.describe()\\\\n\\\\n# Display the first few rows of the DataFrame\\\\niris_head = iris_df.head()\\\\n\\\\n# Display the class distribution\\\\niris_target_counts = iris_df['target'].value_counts()\\\\n\\\\n# Output the results\\\\nprint('Basic Statistical Details:\\\\\\\\n', iris_stats)\\\\nprint('\\\\\\\\nFirst Five Rows:\\\\\\\\n', iris_head)\\\\nprint('\\\\\\\\nClass Distribution:\\\\\\\\n', iris_target_counts)\\\",\\n \\\"result\\\": \\\"Basic Statistical Details:\\\\n sepal length (cm) sepal width (cm) petal length (cm) \\\\\\\\\\\\ncount 150.000000 150.000000 150.000000 \\\\nmean 5.843333 3.057333 3.758000 \\\\nstd 0.828066 0.435866 1.765298 \\\\nmin 4.300000 2.000000 1.000000 \\\\n25% 5.100000 2.800000 1.600000 \\\\n50% 5.800000 3.000000 4.350000 \\\\n75% 6.400000 3.300000 5.100000 \\\\nmax 7.900000 4.400000 6.900000 \\\\n\\\\n petal width (cm) target \\\\ncount 150.000000 150.000000 \\\\nmean 1.199333 1.000000 \\\\nstd 0.762238 0.819232 \\\\nmin 0.100000 0.000000 \\\\n25% 0.300000 0.000000 \\\\n50% 1.300000 1.000000 \\\\n75% 1.800000 2.000000 \\\\nmax 2.500000 2.000000 \\\\n\\\\nFirst Five Rows:\\\\n sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) \\\\\\\\\\\\n0 5.1 3.5 1.4 0.2 \\\\n1 4.9 3.0 1.4 0.2 \\\\n2 4.7 3.2 1.3 0.2 \\\\n3 4.6 3.1 1.5 0.2 \\\\n4 5.0 3.6 1.4 0.2 \\\\n\\\\n target \\\\n0 0 \\\\n1 0 \\\\n2 0 \\\\n3 0 \\\\n4 0 \\\\n\\\\nClass Distribution:\\\\n target\\\\n0 50\\\\n1 50\\\\n2 50\\\\nName: count, dtype: int64\\\\n\\\",\\n \\\"is_success\\\": true,\\n \\\"is_finished\\\": true\\n },\\n {\\n \\\"task_id\\\": \\\"3\\\",\\n \\\"dependent_task_ids\\\": [\\n \\\"2\\\"\\n ],\\n \\\"instruction\\\": \\\"Create a plot visualizing the Iris dataset features.\\\",\\n \\\"task_type\\\": \\\"other\\\",\\n \\\"code\\\": \\\"\\\",\\n \\\"result\\\": \\\"\\\",\\n \\\"is_success\\\": false,\\n \\\"is_finished\\\": false\\n }\\n]\\n## Current Task\\n{\\\"task_id\\\":\\\"3\\\",\\\"dependent_task_ids\\\":[\\\"2\\\"],\\\"instruction\\\":\\\"Create a plot visualizing the Iris dataset features.\\\",\\\"task_type\\\":\\\"other\\\",\\\"code_steps\\\":\\\"\\\",\\\"code\\\":\\\"\\\",\\\"result\\\":\\\"\\\",\\\"is_success\\\":false,\\\"is_finished\\\":false}\\n\"}, {\"role\": \"user\", \"content\": \"\\n# Instruction\\nWrite complete code for 'Current Task'. And avoid duplicating code from finished tasks, such as repeated import of packages, reading data, etc.\\nSpecifically, \\n\\n# Capabilities\\n- You can utilize pre-defined tools in any code lines from 'Available Tools' in the form of Python Class.\\n- You can freely combine the use of any other public packages, like sklearn, numpy, pandas, etc..\\n\\n# Available Tools (can be empty):\\nEach Class tool is described in JSON format. When you call a tool, import the tool first.\\n{}\\n\\n# Constraints:\\n- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed.\\n- Always prioritize using pre-defined tools for the same functionality.\\n\"}]": { + "code": "import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Set the style of seaborn\nsns.set(style=\"whitegrid\")\n\n# Create a pairplot to visualize the relationships between features\npair_plot = sns.pairplot(iris_df, hue='target', markers=['o', 's', 'D'], palette='husl')\n\n# Show the plot\nplt.show()" + }, + "\n # Context:\n user: run analysis on sklearn iris dataset\n # Task:\n Based on the context, write a plan or modify an existing plan of what you should do to achieve the goal. A plan consists of one to 5 tasks.\n If you are modifying an existing plan, carefully follow the instruction, don't make unnecessary changes. Give the whole plan unless instructed to modify only one task of the plan.\n If you encounter errors on the current task, revise and output the current single task only.\n Output a list of jsons following the format:\n ```json\n [\n {\n \"task_id\": str = \"unique identifier for a task in plan, can be an ordinal\",\n \"dependent_task_ids\": list[str] = \"ids of tasks prerequisite to this task\",\n \"instruction\": \"what you should do in this task, one short phrase or sentence\",\n },\n ...\n ]\n ```\n ": "```json\n[\n {\n \"task_id\": \"1\",\n \"dependent_task_ids\": [],\n \"instruction\": \"Import the Iris dataset from scikit-learn.\"\n },\n {\n \"task_id\": \"2\",\n \"dependent_task_ids\": [\"1\"],\n \"instruction\": \"Perform exploratory data analysis to understand the dataset.\"\n },\n {\n \"task_id\": \"3\",\n \"dependent_task_ids\": [\"2\"],\n \"instruction\": \"Preprocess the data if necessary (e.g., scaling, encoding).\"\n },\n {\n \"task_id\": \"4\",\n \"dependent_task_ids\": [\"3\"],\n \"instruction\": \"Split the dataset into training and testing sets.\"\n },\n {\n \"task_id\": \"5\",\n \"dependent_task_ids\": [\"4\"],\n \"instruction\": \"Choose a suitable model and train it on the dataset.\"\n },\n {\n \"task_id\": \"6\",\n \"dependent_task_ids\": [\"5\"],\n \"instruction\": \"Evaluate the model's performance on the test set.\"\n },\n {\n \"task_id\": \"7\",\n \"dependent_task_ids\": [\"6\"],\n \"instruction\": \"Report the results of the analysis.\"\n }\n]\n```" } \ No newline at end of file diff --git a/tests/metagpt/actions/test_write_analysis_code.py b/tests/metagpt/actions/test_write_analysis_code.py index e64b4a551..3e20a8bfb 100644 --- a/tests/metagpt/actions/test_write_analysis_code.py +++ b/tests/metagpt/actions/test_write_analysis_code.py @@ -3,11 +3,8 @@ import asyncio import pytest from metagpt.actions.execute_code import ExecutePyCode -from metagpt.actions.write_analysis_code import ( - WriteCodeByGenerate, - WriteCodeWithTools, - WriteCodeWithToolsML, -) +from metagpt.actions.ml_action import WriteCodeWithToolsML +from metagpt.actions.write_analysis_code import WriteCodeByGenerate, WriteCodeWithTools from metagpt.logs import logger from metagpt.plan.planner import STRUCTURAL_CONTEXT from metagpt.schema import Message, Plan, Task diff --git a/tests/metagpt/roles/run_code_interpreter.py b/tests/metagpt/roles/run_code_interpreter.py index 379194534..1c5b2873f 100644 --- a/tests/metagpt/roles/run_code_interpreter.py +++ b/tests/metagpt/roles/run_code_interpreter.py @@ -9,7 +9,7 @@ from metagpt.schema import Plan from metagpt.utils.recovery_util import load_history, save_history -async def run_code_interpreter(role_class, requirement, auto_run, use_tools, use_code_steps, save_dir, tools): +async def run_code_interpreter(role_class, requirement, auto_run, use_tools, save_dir, tools): """ The main function to run the MLEngineer with optional history loading. @@ -28,7 +28,6 @@ async def run_code_interpreter(role_class, requirement, auto_run, use_tools, use role = MLEngineer( auto_run=auto_run, use_tools=use_tools, - use_code_steps=use_code_steps, tools=tools, ) @@ -75,10 +74,9 @@ if __name__ == "__main__": requirement: str = requirement, auto_run: bool = auto_run, use_tools: bool = use_tools, - use_code_steps: bool = False, save_dir: str = save_dir, tools=tools, ): - await run_code_interpreter(role_class, requirement, auto_run, use_tools, use_code_steps, save_dir, tools) + await run_code_interpreter(role_class, requirement, auto_run, use_tools, save_dir, tools) fire.Fire(main) diff --git a/tests/metagpt/roles/test_code_interpreter.py b/tests/metagpt/roles/test_code_interpreter.py index 8595b9b15..aeb7070fd 100644 --- a/tests/metagpt/roles/test_code_interpreter.py +++ b/tests/metagpt/roles/test_code_interpreter.py @@ -3,11 +3,24 @@ import pytest from metagpt.logs import logger from metagpt.roles.code_interpreter import CodeInterpreter +# from metagpt.const import DATA_PATH + @pytest.mark.asyncio -async def test_code_interpreter(): +@pytest.mark.parametrize("use_tools", [(True)]) +async def test_code_interpreter(use_tools): requirement = "Run data analysis on sklearn Iris dataset, include a plot" - ci = CodeInterpreter(goal=requirement, auto_run=True, use_tools=False) + # requirement = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy" + # data_path = f"{DATA_PATH}/titanic" + # requirement = f"This is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '{data_path}/split_train.csv', eval data path: '{data_path}/split_eval.csv'." + # data_path = f"{DATA_PATH}/icr-identify-age-related-conditions" + # requirement = f"This is a medical dataset with over fifty anonymized health characteristics linked to three age-related conditions. Your goal is to predict whether a subject has or has not been diagnosed with one of these conditions.The target column is Class. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report f1 score on the eval data. Train data path: {data_path}/split_train.csv, eval data path: {data_path}/split_eval.csv." + # data_path = f"{DATA_PATH}/house-prices-advanced-regression-techniques" + # requirement = f"This is a house price dataset, your goal is to predict the sale price of a property based on its features. The target column is SalePrice. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report RMSE between the logarithm of the predicted value and the logarithm of the observed sales price on the eval data. Train data path: '{data_path}/split_train.csv', eval data path: '{data_path}/split_eval.csv'." + tools = [] + # tools = ["FillMissingValue", "CatCross", "a"] + + ci = CodeInterpreter(auto_run=True, use_tools=use_tools, tools=tools) rsp = await ci.run(requirement) logger.info(rsp) assert len(rsp.content) > 0 diff --git a/tests/metagpt/roles/test_daml.py b/tests/metagpt/roles/test_daml.py deleted file mode 100644 index 2e2c003d9..000000000 --- a/tests/metagpt/roles/test_daml.py +++ /dev/null @@ -1,50 +0,0 @@ -import pytest -from tqdm import tqdm - -from metagpt.logs import logger -from metagpt.roles.ml_engineer import ExecutePyCode, MLEngineer -from metagpt.schema import Plan - - -def reset(role): - """Restart role with the same goal.""" - role.working_memory.clear() - role.planner.plan = Plan(goal=role.planner.plan.goal) - role.execute_code = ExecutePyCode() - - -async def make_use_tools(requirement: str, auto_run: bool = True): - """make and use tools for requirement.""" - role = MLEngineer(goal=requirement, auto_run=auto_run) - # make udfs - role.use_tools = False - role.use_code_steps = False - role.make_udfs = True - role.use_udfs = False - await role.run(requirement) - # use udfs - reset(role) - role.make_udfs = False - role.use_udfs = True - role.use_code_steps = False - role.use_tools = False - await role.run(requirement) - - -@pytest.mark.asyncio -async def test_make_use_tools(): - requirements = [ - "Run data analysis on sklearn Iris dataset, include a plot", - "Run data analysis on sklearn Diabetes dataset, include a plot", - "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy", - "Run data analysis on sklearn Wisconsin Breast Cancer dataset, include a plot, train a model to predict targets (20% as validation), and show validation accuracy", - "Run EDA and visualization on this dataset, train a model to predict survival, report metrics on validation set (20%), dataset: tests/data/titanic.csv", - ] - success = 0 - for requirement in tqdm(requirements, total=len(requirements)): - try: - await make_use_tools(requirement) - success += 1 - except Exception as e: - logger.error(f"Found Error in {requirement}, {e}") - logger.info(f"success: {round(success/len(requirements), 1)*100}%") diff --git a/tests/metagpt/roles/test_ml_engineer.py b/tests/metagpt/roles/test_ml_engineer.py new file mode 100644 index 000000000..23570b0f1 --- /dev/null +++ b/tests/metagpt/roles/test_ml_engineer.py @@ -0,0 +1,31 @@ +import pytest + +from metagpt.const import DATA_PATH +from metagpt.logs import logger +from metagpt.roles.ml_engineer import MLEngineer + + +def test_mle_init(): + ci = MLEngineer(goal="test", auto_run=True, use_tools=True, tools=["tool1", "tool2"]) + assert ci.tools == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_tools", [(True)]) +async def test_code_interpreter(use_tools): + # requirement = "Run data analysis on sklearn Iris dataset, include a plot" + # requirement = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy" + data_path = f"{DATA_PATH}/titanic" + requirement = f"This is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '{data_path}/split_train.csv', eval data path: '{data_path}/split_eval.csv'." + # data_path = f"{DATA_PATH}/icr-identify-age-related-conditions" + # requirement = f"This is a medical dataset with over fifty anonymized health characteristics linked to three age-related conditions. Your goal is to predict whether a subject has or has not been diagnosed with one of these conditions.The target column is Class. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report f1 score on the eval data. Train data path: {data_path}/split_train.csv, eval data path: {data_path}/split_eval.csv." + # data_path = f"{DATA_PATH}/santander-customer-transaction-prediction" + # requirement = f"This is a customers financial dataset. Your goal is to predict which customers will make a specific transaction in the future. The target column is target. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report AUC Score on the eval data. Train data path: '{data_path}/split_train.csv', eval data path: '{data_path}/split_eval.csv' ." + # data_path = f"{DATA_PATH}/house-prices-advanced-regression-techniques" + # requirement = f"This is a house price dataset, your goal is to predict the sale price of a property based on its features. The target column is SalePrice. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report RMSE between the logarithm of the predicted value and the logarithm of the observed sales price on the eval data. Train data path: '{data_path}/split_train.csv', eval data path: '{data_path}/split_eval.csv'." + tools = ["FillMissingValue", "CatCross", "dummy_tool"] + + mle = MLEngineer(goal=requirement, auto_run=True, use_tools=use_tools, tools=tools) + rsp = await mle.run(requirement) + logger.info(rsp) + assert len(rsp.content) > 0 diff --git a/tests/metagpt/tools/libs/test_udf.py b/tests/metagpt/tools/libs/test_udf.py deleted file mode 100644 index 19e523448..000000000 --- a/tests/metagpt/tools/libs/test_udf.py +++ /dev/null @@ -1,49 +0,0 @@ -import json - -import yaml - -from metagpt.logs import logger -from metagpt.tools.libs.udf import UDFS, UDFS_YAML, docstring_to_yaml - - -def test_udfs(): - assert len(UDFS) > 0 - assert "udf_name" in UDFS[0] - assert "udf_doc" in UDFS[0] - logger.info(UDFS) - - -def test_docstring2yaml(): - docstring = """Calculate the duration in hours between two datetime columns. - - Args: - dataframe (pd.DataFrame): The dataframe containing the datetime columns. - - Returns: - pd.DataFrame: The dataframe with an additional column 'duration_hour' added. - """ - - yaml_result = docstring_to_yaml(docstring, return_vars="dataframe") - assert "parameters" in yaml_result - assert "properties" in yaml_result["parameters"] - assert "dataframe" in yaml_result["parameters"]["properties"] - - -def test_UDFS_YAML(): - assert len(UDFS_YAML) > 0 - logger.info(f"\n\n{json.dumps(UDFS_YAML, indent=2, ensure_ascii=False)}") - function_schema = UDFS_YAML - assert "description" in function_schema[list(function_schema.keys())[0]] - assert "type" in function_schema[list(function_schema.keys())[0]] - assert "parameters" in function_schema[list(function_schema.keys())[0]] - assert "properties" in function_schema[list(function_schema.keys())[0]]["parameters"] - assert "required" in function_schema[list(function_schema.keys())[0]]["parameters"] - assert "returns" in function_schema[list(function_schema.keys())[0]] - # 指定要保存的文件路径 - file_path = "./tests/data/function_schema.yaml" - - # 使用 PyYAML 将字典保存为 YAML 文件 - with open(file_path, "w") as file: - yaml.dump(function_schema, file, default_flow_style=False) - - print(f"Data has been saved to {file_path}") diff --git a/tests/metagpt/utils/test_save_code.py b/tests/metagpt/utils/test_save_code.py index 278d9a539..0674315d0 100644 --- a/tests/metagpt/utils/test_save_code.py +++ b/tests/metagpt/utils/test_save_code.py @@ -9,7 +9,6 @@ import nbformat import pytest from metagpt.actions.execute_code import ExecutePyCode -from metagpt.actions.write_analysis_code import WriteCodeByGenerate from metagpt.utils.save_code import DATA_PATH, save_code_file @@ -17,11 +16,6 @@ def test_save_code_file_python(): save_code_file("example", "print('Hello, World!')") file_path = DATA_PATH / "output" / "example" / "code.py" assert os.path.exists(file_path), f"File does not exist: {file_path}" - - -def test_save_code_file_python(): - save_code_file("example", "print('Hello, World!')") - file_path = DATA_PATH / "output" / "example" / "code.py" with open(file_path, "r", encoding="utf-8") as fp: content = fp.read() assert "print('Hello, World!')" in content, "File content does not match" @@ -38,7 +32,7 @@ def test_save_code_file_json(): @pytest.mark.asyncio async def test_save_code_file_notebook(): - code = await WriteCodeByGenerate().run(context="basic python, hello world", plan="", code_steps="", temperature=0.0) + code = "print('Hello, World!')" executor = ExecutePyCode() await executor.run(code) # Save as a Notebook file