update reflect on previous plan

config/config.yaml

@@ -96,5 +96,5 @@ MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k
 
 PROMPT_FORMAT: json #json or markdown
 
-KAGGLE_USERNAME: ""
-KAGGLE_KEY: ""
+# KAGGLE_USERNAME: ""
+# KAGGLE_KEY: ""

kaggle_team.py

@@ -13,20 +13,21 @@ async def main(
     # data_desc: str,
     # requirement: str,
     investment: float = 5.0,
-    n_round: int = 5,
+    n_round: int = 10,
+    auto_run: bool = False,
 ):
     competition, data_desc, requirement = (
         "titanic",
         "Training set is train.csv.\nTest set is test.csv. We also include gender_submission.csv, a set of predictions that assume all and only female passengers survive, as an example of what a submission file should look like.",
         "Run EDA on the train dataset, train a model to predict survival (20% as validation) and save it, predict the test set using saved model, save the test result according to format",
-        # "generate a random prediction of the same shape as gender_submission.csv and save",
+        # "generate a random prediction, replace the Survived column of gender_submission.csv, and save the prediction to a new submission file",
     )
 
     team = Team()
     team.hire(
         [
             KaggleManager(competition=competition, data_desc=data_desc),
-            MLEngineer(goal=requirement),
+            MLEngineer(goal=requirement, auto_run=auto_run),
         ]
     )
 

metagpt/actions/ml_da_action.py

@@ -3,6 +3,7 @@ from typing import Dict, List, Union
 
 from metagpt.actions import Action
 from metagpt.schema import Message, Plan
+from metagpt.utils.common import CodeParser
 from metagpt.logs import logger
 
 
@@ -98,22 +99,30 @@ class SummarizeAnalysis(Action):
 
 class Reflect(Action):
     PROMPT_TEMPLATE = """
-    # User Requirement
-    {user_requirement}
     # Context
-    {context}
+    __context__
+    # Latest User Requirement
+    __user_requirement__
     # Summary
     Above is all your attempts to tackle the user requirement. You plan, act, submit your output, and get the result and feedback.
-    First, summarize each of your previous trial in a triple of (your methods, the corresponding result, potential improvement), list them out.
-    # Takeaways
-    Second, carefully find key takeaways from your summarization in a step-by-step thinking process
-    # Guidance
-    Finally, make a concise one-sentence guidance for improving your future plan.
-    Your response:
+    Output a json following the format:
+    ```json
+    {
+        "summary": str = "summarize each of your previous trial in a triple of (your methods, the corresponding result, potential improvement), list them out",
+        "takeaways": str = "carefully find key takeaways from your summarization in a step-by-step thinking process",
+        "reflection": "in one sentence, state executable actions for improving your future plan",
+    }
+    ```
     """
+    REWRITE_PLAN_INSTRUCTION = """When taking this reflection for rewriting plan, modify the current plan in place, replace, add, or delete tasks in the plan,
+    only make necessary change to the current plan, keep reusable tasks unchanged, provide the complete new plan."""
 
-    async def run(self, context: str) -> str:
-        user_requirement = "Score as high as possible in a data modeling competition"
-        prompt = self.PROMPT_TEMPLATE.format(context=context, user_requirement=user_requirement)
-        rsp = await self._aask(prompt)
-        return rsp
+    async def run(self, context: str, user_requirement: str = "") -> str:
+        user_requirement = user_requirement or "Score as high as possible in a data modeling competition"
+        # prompt = self.PROMPT_TEMPLATE.format(context=context, user_requirement=user_requirement)
+        prompt = self.PROMPT_TEMPLATE.replace("__context__", context).replace("__user_requirement__", user_requirement)
+        rsp_json = await self._aask(prompt)
+        rsp = CodeParser.parse_code(block=None, text=rsp_json)
+        reflection = json.loads(rsp)["reflection"]
+        reflection += self.REWRITE_PLAN_INSTRUCTION
+        return reflection

metagpt/roles/kaggle_manager.py

@@ -38,8 +38,8 @@ class DownloadData(Action):
         
         run_command(f"kaggle competitions download {competition} --path {WORKSPACE_ROOT}")
         
-        # if not os.path.exists(data_path):
-        if True:
+        if not os.path.exists(data_path):
+        # if True:
             # run_command(f"rm -r {data_path / '*'}")
             run_command(f"unzip -o {WORKSPACE_ROOT / '*.zip'} -d {data_path}")  # FIXME: not safe
         

metagpt/roles/ml_engineer.py

@@ -48,13 +48,11 @@ class MLEngineer(Role):
             if latest_event == DownloadData:
                 self.plan.context = memories[-1].content
             elif latest_event == SubmitResult:
+                # self reflect on previous plan outcomes and think about how to improve the plan, add to working  memory
+                await self._reflect()
+
                 # get feedback for improvement from human, add to working memory
                 await self._ask_review(trigger=ReviewConst.TASK_REVIEW_TRIGGER)
-                # self reflect on previous plan outcomes and think about how to improve the plan, add to working  memory
-                prev_plan_outcomes = memories[-1].content
-                reflection = await Reflect().run(context=prev_plan_outcomes)
-                self.working_memory.add(Message(content=reflection, role="assistant"))
-
 
         ### Common Procedure in both single- and multi-agent setting ###
         # create initial plan and update until confirmation
@@ -172,7 +170,16 @@ class MLEngineer(Role):
             self.plan.replace_task(tasks[0])
         else:
             self.plan.add_tasks(tasks)
-        self.working_memory.clear()        
+        self.working_memory.clear()
+    
+    async def _reflect(self):
+        context = self.get_memories()
+        context = "\n".join([str(msg) for msg in context])
+        # print("*" * 10)
+        # print(context)
+        # print("*" * 10)
+        reflection = await Reflect().run(context=context)
+        self.working_memory.add(Message(content=reflection, role="assistant"))
 
     def get_useful_memories(self) -> List[Message]:
         """find useful memories only to reduce context length and improve performance"""