From 8d7657f347d51feb3048d6774bdbe17308ecf2ee Mon Sep 17 00:00:00 2001 From: yzlin Date: Mon, 4 Dec 2023 14:29:47 +0800 Subject: [PATCH] update reflect on previous plan --- config/config.yaml | 4 ++-- kaggle_team.py | 7 ++++--- metagpt/actions/ml_da_action.py | 37 ++++++++++++++++++++------------- metagpt/roles/kaggle_manager.py | 4 ++-- metagpt/roles/ml_engineer.py | 19 +++++++++++------ 5 files changed, 44 insertions(+), 27 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 52a8eb036..bf998def7 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -96,5 +96,5 @@ MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k PROMPT_FORMAT: json #json or markdown -KAGGLE_USERNAME: "" -KAGGLE_KEY: "" \ No newline at end of file +# KAGGLE_USERNAME: "" +# KAGGLE_KEY: "" \ No newline at end of file diff --git a/kaggle_team.py b/kaggle_team.py index 659c4a495..e8ab3ec41 100644 --- a/kaggle_team.py +++ b/kaggle_team.py @@ -13,20 +13,21 @@ async def main( # data_desc: str, # requirement: str, investment: float = 5.0, - n_round: int = 5, + n_round: int = 10, + auto_run: bool = False, ): competition, data_desc, requirement = ( "titanic", "Training set is train.csv.\nTest set is test.csv. We also include gender_submission.csv, a set of predictions that assume all and only female passengers survive, as an example of what a submission file should look like.", "Run EDA on the train dataset, train a model to predict survival (20% as validation) and save it, predict the test set using saved model, save the test result according to format", - # "generate a random prediction of the same shape as gender_submission.csv and save", + # "generate a random prediction, replace the Survived column of gender_submission.csv, and save the prediction to a new submission file", ) team = Team() team.hire( [ KaggleManager(competition=competition, data_desc=data_desc), - MLEngineer(goal=requirement), + MLEngineer(goal=requirement, auto_run=auto_run), ] ) diff --git a/metagpt/actions/ml_da_action.py b/metagpt/actions/ml_da_action.py index 9f903fd22..a4537dad9 100644 --- a/metagpt/actions/ml_da_action.py +++ b/metagpt/actions/ml_da_action.py @@ -3,6 +3,7 @@ from typing import Dict, List, Union from metagpt.actions import Action from metagpt.schema import Message, Plan +from metagpt.utils.common import CodeParser from metagpt.logs import logger @@ -98,22 +99,30 @@ class SummarizeAnalysis(Action): class Reflect(Action): PROMPT_TEMPLATE = """ - # User Requirement - {user_requirement} # Context - {context} + __context__ + # Latest User Requirement + __user_requirement__ # Summary Above is all your attempts to tackle the user requirement. You plan, act, submit your output, and get the result and feedback. - First, summarize each of your previous trial in a triple of (your methods, the corresponding result, potential improvement), list them out. - # Takeaways - Second, carefully find key takeaways from your summarization in a step-by-step thinking process - # Guidance - Finally, make a concise one-sentence guidance for improving your future plan. - Your response: + Output a json following the format: + ```json + { + "summary": str = "summarize each of your previous trial in a triple of (your methods, the corresponding result, potential improvement), list them out", + "takeaways": str = "carefully find key takeaways from your summarization in a step-by-step thinking process", + "reflection": "in one sentence, state executable actions for improving your future plan", + } + ``` """ + REWRITE_PLAN_INSTRUCTION = """When taking this reflection for rewriting plan, modify the current plan in place, replace, add, or delete tasks in the plan, + only make necessary change to the current plan, keep reusable tasks unchanged, provide the complete new plan.""" - async def run(self, context: str) -> str: - user_requirement = "Score as high as possible in a data modeling competition" - prompt = self.PROMPT_TEMPLATE.format(context=context, user_requirement=user_requirement) - rsp = await self._aask(prompt) - return rsp + async def run(self, context: str, user_requirement: str = "") -> str: + user_requirement = user_requirement or "Score as high as possible in a data modeling competition" + # prompt = self.PROMPT_TEMPLATE.format(context=context, user_requirement=user_requirement) + prompt = self.PROMPT_TEMPLATE.replace("__context__", context).replace("__user_requirement__", user_requirement) + rsp_json = await self._aask(prompt) + rsp = CodeParser.parse_code(block=None, text=rsp_json) + reflection = json.loads(rsp)["reflection"] + reflection += self.REWRITE_PLAN_INSTRUCTION + return reflection diff --git a/metagpt/roles/kaggle_manager.py b/metagpt/roles/kaggle_manager.py index d20769b92..354289975 100644 --- a/metagpt/roles/kaggle_manager.py +++ b/metagpt/roles/kaggle_manager.py @@ -38,8 +38,8 @@ class DownloadData(Action): run_command(f"kaggle competitions download {competition} --path {WORKSPACE_ROOT}") - # if not os.path.exists(data_path): - if True: + if not os.path.exists(data_path): + # if True: # run_command(f"rm -r {data_path / '*'}") run_command(f"unzip -o {WORKSPACE_ROOT / '*.zip'} -d {data_path}") # FIXME: not safe diff --git a/metagpt/roles/ml_engineer.py b/metagpt/roles/ml_engineer.py index 4536395ba..abd14c7fb 100644 --- a/metagpt/roles/ml_engineer.py +++ b/metagpt/roles/ml_engineer.py @@ -48,13 +48,11 @@ class MLEngineer(Role): if latest_event == DownloadData: self.plan.context = memories[-1].content elif latest_event == SubmitResult: + # self reflect on previous plan outcomes and think about how to improve the plan, add to working memory + await self._reflect() + # get feedback for improvement from human, add to working memory await self._ask_review(trigger=ReviewConst.TASK_REVIEW_TRIGGER) - # self reflect on previous plan outcomes and think about how to improve the plan, add to working memory - prev_plan_outcomes = memories[-1].content - reflection = await Reflect().run(context=prev_plan_outcomes) - self.working_memory.add(Message(content=reflection, role="assistant")) - ### Common Procedure in both single- and multi-agent setting ### # create initial plan and update until confirmation @@ -172,7 +170,16 @@ class MLEngineer(Role): self.plan.replace_task(tasks[0]) else: self.plan.add_tasks(tasks) - self.working_memory.clear() + self.working_memory.clear() + + async def _reflect(self): + context = self.get_memories() + context = "\n".join([str(msg) for msg in context]) + # print("*" * 10) + # print(context) + # print("*" * 10) + reflection = await Reflect().run(context=context) + self.working_memory.add(Message(content=reflection, role="assistant")) def get_useful_memories(self) -> List[Message]: """find useful memories only to reduce context length and improve performance"""