From a2f809263a4232367ce03397788bf0dc89108953 Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Thu, 27 Jun 2024 11:22:12 +0800
Subject: [PATCH 01/22] ignore warning info

---
 metagpt/actions/di/execute_nb_code.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/metagpt/actions/di/execute_nb_code.py b/metagpt/actions/di/execute_nb_code.py
index 64620d9cc..9a9f0483a 100644
--- a/metagpt/actions/di/execute_nb_code.py
+++ b/metagpt/actions/di/execute_nb_code.py
@@ -30,6 +30,12 @@ from metagpt.logs import logger
 from metagpt.utils.report import NotebookReporter
 
 INSTALL_KEEPLEN = 500
+INI_CODE = """import warnings
+import logging
+
+root_logger = logging.getLogger()
+root_logger.setLevel(logging.ERROR)
+warnings.filterwarnings('ignore')"""
 
 
 class RealtimeOutputNotebookClient(NotebookClient):
@@ -79,6 +85,10 @@ class ExecuteNbCode(Action):
         )
         self.reporter = NotebookReporter()
         self.set_nb_client()
+        asyncio.run(self._init_code())
+
+    async def _init_code(self):
+        await self.run(INI_CODE)
 
     def set_nb_client(self):
         self.nb_client = RealtimeOutputNotebookClient(
@@ -175,6 +185,8 @@ class ExecuteNbCode(Action):
                 is_success = False
 
             output_text = remove_escape_and_color_codes(output_text)
+            if is_success:
+                output_text = remove_log_and_warning_lines(output_text)
             # The useful information of the exception is at the end,
             # the useful information of normal output is at the begining.
             output_text = output_text[:keep_len] if is_success else output_text[-keep_len:]
@@ -268,6 +280,18 @@ class ExecuteNbCode(Action):
             return outputs, success
 
 
+def remove_log_and_warning_lines(input_str: str) -> str:
+    delete_lines = ["[warning]", "warning:", "[cv]", "[info]"]
+    result = "\n".join(
+        [
+            line
+            for line in input_str.split("\n")
+            if not any(dl in line.lower() for dl in delete_lines)
+        ]
+    ).strip()
+    return result
+
+
 def remove_escape_and_color_codes(input_str: str):
     # 使用正则表达式去除jupyter notebook输出结果中的转义字符和颜色代码
     # Use regular expressions to get rid of escape characters and color codes in jupyter notebook output.

From ddaecf12eb86574d93f49f28c5f4be89ecaee55d Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Thu, 27 Jun 2024 11:23:00 +0800
Subject: [PATCH 02/22] apply data_analyst to role_zero

---
 metagpt/prompts/di/role_zero.py           |   2 +-
 metagpt/prompts/di/write_analysis_code.py |   5 +-
 metagpt/roles/di/data_analyst.py          | 178 ++++++++--------------
 metagpt/roles/di/role_zero.py             |  11 +-
 metagpt/tools/tool_recommend.py           |   4 +-
 5 files changed, 83 insertions(+), 117 deletions(-)

diff --git a/metagpt/prompts/di/role_zero.py b/metagpt/prompts/di/role_zero.py
index 4d52476aa..04344fa1e 100644
--- a/metagpt/prompts/di/role_zero.py
+++ b/metagpt/prompts/di/role_zero.py
@@ -5,7 +5,7 @@ When presented a current task, tackle the task using the available commands.
 Pay close attention to new user message, review the conversation history, use RoleZero.reply_to_human to respond to new user requirement.
 Note:
 1. If you keeping encountering errors, unexpected situation, or you are not sure of proceeding, use RoleZero.ask_human to ask for help.
-2. Carefully review your progress at the current task, if your actions so far has not fulfilled the task instruction, you should continue with current task. Otherwise, finish current task.
+2. Carefully review your progress at the current task, if your actions so far has not fulfilled the task instruction, you should continue with current task. Otherwise, finish current task by Plan.finish_current_task.
 3. Each time you finish a task, use RoleZero.reply_to_human to report your progress.
 """
 
diff --git a/metagpt/prompts/di/write_analysis_code.py b/metagpt/prompts/di/write_analysis_code.py
index af941808d..1d743a719 100644
--- a/metagpt/prompts/di/write_analysis_code.py
+++ b/metagpt/prompts/di/write_analysis_code.py
@@ -28,7 +28,10 @@ your code
 ```
 """
 
-REFLECTION_SYSTEM_MSG = """You are an AI Python assistant. You will be given your previous implementation code of a task, runtime error results, and a hint to change the implementation appropriately. Write your full implementation."""
+REFLECTION_SYSTEM_MSG = """
+You are an AI Python assistant. You will be given your previous implementation code of a task, runtime error results, and a hint to change the implementation appropriately. Write your full implementation.
+When occuring ModuleNotFoundError, always install the required package. And use Terminal tool if available.
+"""
 
 DEBUG_REFLECTION_EXAMPLE = '''
 [previous impl]:
diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py
index d4d67742b..f13fc78fb 100644
--- a/metagpt/roles/di/data_analyst.py
+++ b/metagpt/roles/di/data_analyst.py
@@ -1,134 +1,88 @@
 from __future__ import annotations
 
-import json
-from typing import Literal
+from pydantic import Field
 
-from pydantic import model_validator
-
-from metagpt.actions import Action
+from metagpt.actions.di.execute_nb_code import ExecuteNbCode
 from metagpt.actions.di.write_analysis_code import WriteAnalysisCode
 from metagpt.logs import logger
-from metagpt.prompts.di.data_analyst import CMD_PROMPT
-from metagpt.roles.di.data_interpreter import DataInterpreter
-from metagpt.schema import Message, TaskResult
-from metagpt.strategy.experience_retriever import KeywordExpRetriever
-from metagpt.strategy.planner import Planner
-from metagpt.strategy.thinking_command import (
-    Command,
-    prepare_command_prompt,
-    run_commands,
-)
-from metagpt.tools.tool_recommend import BM25ToolRecommender
-from metagpt.utils.common import CodeParser
-from metagpt.utils.report import ThoughtReporter
+from metagpt.roles.di.role_zero import RoleZero
+from metagpt.schema import TaskResult, Message
+from metagpt.tools.tool_registry import register_tool
 
 
-class DataAnalyst(DataInterpreter):
+@register_tool(include_functions=["write_and_exec_code"])
+class DataAnalyst(RoleZero):
     name: str = "David"
     profile: str = "DataAnalyst"
     goal: str = "Take on any data-related tasks, such as data analysis, machine learning, deep learning, web browsing, web scraping, web searching, web deployment, terminal operation, git and github operation, etc."
-    react_mode: Literal["react"] = "react"
-    max_react_loop: int = 20  # used for react mode
+
+    tools: list[str] = ["Plan", "DataAnalyst", "RoleZero"]
+    custom_tools: list[str] = ["machine learning", "web scraping", "Terminal"]
+
+    use_reflection: bool = True
+    write_code: WriteAnalysisCode = Field(default_factory=WriteAnalysisCode, exclude=True)
+    execute_code: ExecuteNbCode = Field(default_factory=ExecuteNbCode, exclude=True)
     task_result: TaskResult = None
-    available_commands: list[Command] = [
-        Command.APPEND_TASK,
-        Command.RESET_TASK,
-        Command.REPLACE_TASK,
-        Command.FINISH_CURRENT_TASK,
-        # Command.PUBLISH_MESSAGE,
-        Command.ASK_HUMAN,
-        Command.REPLY_TO_HUMAN,
-        # Command.PASS,
-    ]
-    commands: list[dict] = []  # issued commands to be executed
-    user_requirement: str = ""
 
-    @model_validator(mode="after")
-    def set_plan_and_tool(self) -> "DataInterpreter":
-        # We force using this parameter for DataAnalyst
-        assert self.react_mode == "react"
-        assert self.auto_run
-        assert self.use_plan
+    def _update_tool_execution(self):
+        self.tool_execution_map = {
+            "Plan.append_task": self.planner.plan.append_task,
+            "Plan.reset_task": self.planner.plan.reset_task,
+            "Plan.replace_task": self.planner.plan.replace_task,
+            "DataAnalyst.write_and_exec_code": self.write_and_exec_code,
+            "RoleZero.ask_human": self.ask_human,
+            "RoleZero.reply_to_human": self.reply_to_human,
+        }
 
-        # Roughly the same part as DataInterpreter.set_plan_and_tool
-        self._set_react_mode(react_mode=self.react_mode, max_react_loop=self.max_react_loop, auto_run=self.auto_run)
-        if self.tools and not self.tool_recommender:
-            self.tool_recommender = BM25ToolRecommender(tools=self.tools)
-        self.set_actions([WriteAnalysisCode])
+    async def write_and_exec_code(self):
+        """Write a code block for current task and execute it in an interactive notebook environment."""
+        counter = 0
+        success = False
 
-        # HACK: Init Planner, control it through dynamic thinking; Consider formalizing as a react mode
-        self.planner = Planner(goal="", working_memory=self.rc.working_memory, auto_run=True)
+        # plan info
+        plan_status = self.planner.get_plan_status()
 
-        return self
+        # tool info
+        if self.custom_tool_recommender:
+            plan = self.planner.plan
+            fix = ["Terminal"] if "Terminal" in self.custom_tools else None
+            tool_info = await self.custom_tool_recommender.get_recommended_tool_info(fix=fix, plan=plan)
+        else:
+            tool_info = ""
 
-    async def _think(self) -> bool:
-        """Useful in 'react' mode. Use LLM to decide whether and what to do next."""
-        self._set_state(0)
-        example = ""
-        if not self.planner.plan.goal:
-            self.user_requirement = self.get_memories()[-1].content
-            self.planner.plan.goal = self.user_requirement
-            example = KeywordExpRetriever().retrieve(self.user_requirement)
+        while not success and counter < 3:
+            ### write code ###
+            logger.info(f"ready to WriteAnalysisCode")
+            use_reflection = (counter > 0 and self.use_reflection)  # only use reflection after the first trial
 
-        plan_status = self.planner.plan.model_dump(include=["goal", "tasks"])
-        # for task in plan_status["tasks"]:
-        #     task.pop("code")
-        #     task.pop("result")
-        prompt = CMD_PROMPT.format(
-            plan_status=plan_status,
-            example=example,
-            available_commands=prepare_command_prompt(self.available_commands),
-        )
-        context = self.llm.format_msg(self.working_memory.get() + [Message(content=prompt, role="user")])
-        # print(*context, sep="\n" + "*" * 5 + "\n")
-        async with ThoughtReporter(enable_llm_stream=True):
-            rsp = await self.llm.aask(context)
-        self.commands = json.loads(CodeParser.parse_code(block=None, lang='json', text=rsp))
-        self.rc.working_memory.add(Message(content=rsp, role="assistant"))
+            code = await self.write_code.run(
+                user_requirement=self.planner.plan.goal,
+                plan_status=plan_status,
+                tool_info=tool_info,
+                working_memory=self.rc.working_memory.get() if use_reflection else None,
+                use_reflection=use_reflection,
+            )
+            self.rc.working_memory.add(Message(content=code, role="assistant", cause_by=WriteAnalysisCode))
 
-        await run_commands(self, self.commands, self.rc.working_memory)
+            ### execute code ###
+            result, success = await self.execute_code.run(code)
+            print(result)
 
-        return bool(self.rc.todo)
+            self.rc.working_memory.add(Message(content=result, role="user", cause_by=ExecuteNbCode))
 
-    async def _act(self) -> Message:
-        """Useful in 'react' mode. Return a Message conforming to Role._act interface."""
-        logger.info(f"ready to take on task {self.planner.plan.current_task}")
+            ### process execution result ###
+            counter += 1
+            self.task_result = TaskResult(code=code, result=result, is_success=success)
 
-        # TODO: Consider an appropriate location to insert task experience formally
-        experience = KeywordExpRetriever().retrieve(self.planner.plan.current_task.instruction, exp_type="task")
-        if experience and experience not in [msg.content for msg in self.rc.working_memory.get()]:
-            exp_msg = Message(content=experience, role="assistant")
-            self.rc.working_memory.add(exp_msg)
+        output = f"""
+        Code written:
+        {code}
+        Execution status:{'Success' if success else 'Failed'}
+        Execution result: {result}
+        """
+        self.rc.working_memory.clear()
+        return output
 
-        code, result, is_success = await self._write_and_exec_code()
-        self.planner.plan.current_task.is_success = (
-            is_success  # mark is_success, determine is_finished later in thinking
-        )
-
-        # FIXME: task result is always overwritten by the last act, whereas it can be made of of multiple acts
-        self.task_result = TaskResult(code=code, result=result, is_success=is_success)
-        return Message(content="Task completed", role="assistant", sent_from=self._setting, cause_by=WriteAnalysisCode)
-
-    async def _react(self) -> Message:
-        # NOTE: Diff 1: Each time landing here means observing news, set todo to allow news processing in _think
-        self._set_state(0)
-
-        actions_taken = 0
-        rsp = Message(content="No actions taken yet", cause_by=Action)  # will be overwritten after Role _act
-        while actions_taken < self.rc.max_react_loop:
-            # NOTE: Diff 2: Keep observing within _react, news will go into memory, allowing adapting to new info
-            # add news from self._observe, the one called in self.run, consider removing when switching from working_memory to memory
-            self.working_memory.add_batch(self.rc.news)
-            await self._observe()
-            # add news from this self._observe, we need twice because _observe rewrites rc.news
-            self.working_memory.add_batch(self.rc.news)
-
-            # think
-            has_todo = await self._think()
-            if not has_todo:
-                break
-            # act
-            logger.debug(f"{self._setting}: {self.rc.state=}, will do {self.rc.todo}")
-            rsp = await self._act()
-            actions_taken += 1
-        return rsp  # return output from the last action
+    def _finish_current_task(self):
+        self.planner.current_task.update_task_result(self.task_result)
+        super()._finish_current_task()
diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py
index 39338471a..4225d1f76 100644
--- a/metagpt/roles/di/role_zero.py
+++ b/metagpt/roles/di/role_zero.py
@@ -41,8 +41,10 @@ class RoleZero(Role):
     max_react_loop: int = 20  # used for react mode
 
     # Tools
-    tools: list[str] = []  # Use special symbol ["<all>"] to indicate use of all registered tools
+    tools: list[str] = []
     tool_recommender: ToolRecommender = None
+    custom_tools: list[str] = []
+    custom_tool_recommender: ToolRecommender = None
     tool_execution_map: dict[str, Callable] = {}
     special_tool_commands: list[str] = ["Plan.finish_current_task", "end"]
     # Equipped with three basic tools by default for optional use
@@ -68,6 +70,8 @@ class RoleZero(Role):
         self._set_react_mode(react_mode=self.react_mode, max_react_loop=self.max_react_loop)
         if self.tools and not self.tool_recommender:
             self.tool_recommender = BM25ToolRecommender(tools=self.tools, force=True)
+        if self.custom_tools and not self.custom_tool_recommender:
+            self.custom_tool_recommender = BM25ToolRecommender(tools=self.custom_tools)
         self.set_actions([RunCommand])
 
         # HACK: Init Planner, control it through dynamic thinking; Consider formalizing as a react mode
@@ -235,13 +239,16 @@ class RoleZero(Role):
         if cmd["command_name"] == "Plan.finish_current_task" and not self.planner.plan.is_plan_finished():
             # task_result = TaskResult(code=str(commands), result=outputs, is_success=is_success)
             # self.planner.plan.current_task.update_task_result(task_result=task_result)
-            self.planner.plan.finish_current_task()
+            self._finish_current_task()
 
         elif cmd["command_name"] == "end":
             self._set_state(-1)
 
         return is_special_cmd
 
+    def _finish_current_task(self):
+        self.planner.plan.finish_current_task()
+
     def _get_plan_status(self) -> Tuple[str, str]:
         plan_status = self.planner.plan.model_dump(include=["goal", "tasks"])
         for task in plan_status["tasks"]:
diff --git a/metagpt/tools/tool_recommend.py b/metagpt/tools/tool_recommend.py
index 05e8e1400..ab847d10e 100644
--- a/metagpt/tools/tool_recommend.py
+++ b/metagpt/tools/tool_recommend.py
@@ -101,7 +101,7 @@ class ToolRecommender(BaseModel):
 
         return ranked_tools
 
-    async def get_recommended_tool_info(self, **kwargs) -> str:
+    async def get_recommended_tool_info(self, fix: list[str] = None, **kwargs) -> str:
         """
         Wrap recommended tools with their info in a string, which can be used directly in a prompt.
         """
@@ -109,6 +109,8 @@ class ToolRecommender(BaseModel):
         if not recommended_tools:
             return ""
         tool_schemas = {tool.name: tool.schemas for tool in recommended_tools}
+        if fix:
+            tool_schemas.update({tool.name: tool.schemas for tool in self.tools.values() if tool.name in fix})
         return TOOL_INFO_PROMPT.format(tool_schemas=tool_schemas)
 
     async def recall_tools(self, context: str = "", plan: Plan = None, topk: int = 20) -> list[Tool]:

From a5b94af82fcd5e63172578451bfb7f72cf7cc95f Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Thu, 27 Jun 2024 13:41:10 +0800
Subject: [PATCH 03/22] remove custom tools to data analyst

---
 metagpt/roles/di/data_analyst.py | 18 ++++++++++--------
 metagpt/roles/di/role_zero.py    |  6 +-----
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py
index f13fc78fb..b3144a100 100644
--- a/metagpt/roles/di/data_analyst.py
+++ b/metagpt/roles/di/data_analyst.py
@@ -1,12 +1,13 @@
 from __future__ import annotations
 
-from pydantic import Field
+from pydantic import Field, model_validator
 
 from metagpt.actions.di.execute_nb_code import ExecuteNbCode
 from metagpt.actions.di.write_analysis_code import WriteAnalysisCode
 from metagpt.logs import logger
 from metagpt.roles.di.role_zero import RoleZero
 from metagpt.schema import TaskResult, Message
+from metagpt.tools.tool_recommend import BM25ToolRecommender, ToolRecommender
 from metagpt.tools.tool_registry import register_tool
 
 
@@ -18,21 +19,22 @@ class DataAnalyst(RoleZero):
 
     tools: list[str] = ["Plan", "DataAnalyst", "RoleZero"]
     custom_tools: list[str] = ["machine learning", "web scraping", "Terminal"]
+    custom_tool_recommender: ToolRecommender = None
 
     use_reflection: bool = True
     write_code: WriteAnalysisCode = Field(default_factory=WriteAnalysisCode, exclude=True)
     execute_code: ExecuteNbCode = Field(default_factory=ExecuteNbCode, exclude=True)
     task_result: TaskResult = None
 
+    @model_validator(mode="after")
+    def set_custom_tool(self):
+        if self.custom_tools and not self.custom_tool_recommender:
+            self.custom_tool_recommender = BM25ToolRecommender(tools=self.custom_tools)
+
     def _update_tool_execution(self):
-        self.tool_execution_map = {
-            "Plan.append_task": self.planner.plan.append_task,
-            "Plan.reset_task": self.planner.plan.reset_task,
-            "Plan.replace_task": self.planner.plan.replace_task,
+        self.tool_execution_map.update({
             "DataAnalyst.write_and_exec_code": self.write_and_exec_code,
-            "RoleZero.ask_human": self.ask_human,
-            "RoleZero.reply_to_human": self.reply_to_human,
-        }
+        })
 
     async def write_and_exec_code(self):
         """Write a code block for current task and execute it in an interactive notebook environment."""
diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py
index 4225d1f76..93abe8c02 100644
--- a/metagpt/roles/di/role_zero.py
+++ b/metagpt/roles/di/role_zero.py
@@ -41,10 +41,8 @@ class RoleZero(Role):
     max_react_loop: int = 20  # used for react mode
 
     # Tools
-    tools: list[str] = []
+    tools: list[str] = []  # Use special symbol ["<all>"] to indicate use of all registered tools
     tool_recommender: ToolRecommender = None
-    custom_tools: list[str] = []
-    custom_tool_recommender: ToolRecommender = None
     tool_execution_map: dict[str, Callable] = {}
     special_tool_commands: list[str] = ["Plan.finish_current_task", "end"]
     # Equipped with three basic tools by default for optional use
@@ -70,8 +68,6 @@ class RoleZero(Role):
         self._set_react_mode(react_mode=self.react_mode, max_react_loop=self.max_react_loop)
         if self.tools and not self.tool_recommender:
             self.tool_recommender = BM25ToolRecommender(tools=self.tools, force=True)
-        if self.custom_tools and not self.custom_tool_recommender:
-            self.custom_tool_recommender = BM25ToolRecommender(tools=self.custom_tools)
         self.set_actions([RunCommand])
 
         # HACK: Init Planner, control it through dynamic thinking; Consider formalizing as a react mode

From f2cf8dd74ba54fc2e37ddcf59b9fe7bcab1bd94b Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Thu, 27 Jun 2024 16:22:00 +0800
Subject: [PATCH 04/22] fix bug of ExecuteNbCode._init_code was never awaited

---
 metagpt/actions/di/execute_nb_code.py | 6 ++++--
 metagpt/roles/di/data_analyst.py      | 1 +
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/metagpt/actions/di/execute_nb_code.py b/metagpt/actions/di/execute_nb_code.py
index 9a9f0483a..cc2c05341 100644
--- a/metagpt/actions/di/execute_nb_code.py
+++ b/metagpt/actions/di/execute_nb_code.py
@@ -85,10 +85,12 @@ class ExecuteNbCode(Action):
         )
         self.reporter = NotebookReporter()
         self.set_nb_client()
-        asyncio.run(self._init_code())
+        self._init_called = False
 
     async def _init_code(self):
-        await self.run(INI_CODE)
+        if not self._init_called:
+            await self.run(INI_CODE)
+            self._init_called = True
 
     def set_nb_client(self):
         self.nb_client = RealtimeOutputNotebookClient(
diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py
index b3144a100..491968d14 100644
--- a/metagpt/roles/di/data_analyst.py
+++ b/metagpt/roles/di/data_analyst.py
@@ -40,6 +40,7 @@ class DataAnalyst(RoleZero):
         """Write a code block for current task and execute it in an interactive notebook environment."""
         counter = 0
         success = False
+        await self.execute_code._init_code()
 
         # plan info
         plan_status = self.planner.get_plan_status()

From 898ee44bec5ff8e737a913cadd92ae2059435818 Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Mon, 1 Jul 2024 10:52:50 +0800
Subject: [PATCH 05/22] add supprot of task_type and add output of special
 command

---
 metagpt/prompts/di/role_zero.py  |  5 ++++-
 metagpt/roles/di/data_analyst.py | 16 ++++++----------
 metagpt/roles/di/role_zero.py    | 11 ++++++-----
 metagpt/schema.py                | 10 +++++++---
 metagpt/tools/libs/__init__.py   | 12 ++++++------
 5 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/metagpt/prompts/di/role_zero.py b/metagpt/prompts/di/role_zero.py
index 04344fa1e..2b4fe105a 100644
--- a/metagpt/prompts/di/role_zero.py
+++ b/metagpt/prompts/di/role_zero.py
@@ -22,6 +22,9 @@ class Task(BaseModel):
 {available_commands}
 Special Command: Use {{"command_name": "end"}} to do nothing or indicate completion of all requirements and the end of actions.
 
+# Available Task Types
+{task_type_desc}
+
 # Current Plan
 {plan_status}
 
@@ -38,7 +41,7 @@ Pay close attention to the Example provided, you can reuse the example for your
 You may use any of the available commands to create a plan or update the plan. You may output mutiple commands, they will be executed sequentially.
 If you finish current task, you will automatically take the next task in the existing plan, use Plan.finish_task, DON'T append a new task.
 
-# Your commands in a json array, in the following output format. If there is nothing to do, use the pass or end command:
+# Your commands in a json array, in the following output format with command_name and args. If there is nothing to do, use the pass or end command:
 Some text indicating your thoughts, such as how you should update the plan status, respond to inquiry, or seek for help. Then a json array of commands. You must output ONE and ONLY ONE json array. DON'T output multiple json arrays with thoughts between them.
 ```json
 [
diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py
index 491968d14..f3586a6f5 100644
--- a/metagpt/roles/di/data_analyst.py
+++ b/metagpt/roles/di/data_analyst.py
@@ -24,7 +24,6 @@ class DataAnalyst(RoleZero):
     use_reflection: bool = True
     write_code: WriteAnalysisCode = Field(default_factory=WriteAnalysisCode, exclude=True)
     execute_code: ExecuteNbCode = Field(default_factory=ExecuteNbCode, exclude=True)
-    task_result: TaskResult = None
 
     @model_validator(mode="after")
     def set_custom_tool(self):
@@ -75,17 +74,14 @@ class DataAnalyst(RoleZero):
 
             ### process execution result ###
             counter += 1
-            self.task_result = TaskResult(code=code, result=result, is_success=success)
-
+            if success:
+                task_result = TaskResult(code=code, result=result, is_success=success)
+                self.planner.current_task.update_task_result(task_result)
         output = f"""
-        Code written:
+        **Code written**:
         {code}
-        Execution status:{'Success' if success else 'Failed'}
-        Execution result: {result}
+        **Execution status**:{'Success' if success else 'Failed'}
+        **Execution result**: {result}
         """
         self.rc.working_memory.clear()
         return output
-
-    def _finish_current_task(self):
-        self.planner.current_task.update_task_result(self.task_result)
-        super()._finish_current_task()
diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py
index 93abe8c02..b179f5a00 100644
--- a/metagpt/roles/di/role_zero.py
+++ b/metagpt/roles/di/role_zero.py
@@ -6,6 +6,7 @@ import re
 import traceback
 from typing import Callable, Literal, Tuple
 
+from metagpt.strategy.task_type import TaskType
 from pydantic import model_validator
 
 from metagpt.actions import Action
@@ -130,6 +131,7 @@ class RoleZero(Role):
 
         ### 2. Plan Status ###
         plan_status, current_task = self._get_plan_status()
+        task_type_desc = "\n".join([f"- **{tt.type_name}**: {tt.value.desc}" for tt in TaskType])
 
         ### 3. Tool/Command Info ###
         tools = await self.tool_recommender.recommend_tools()
@@ -142,6 +144,7 @@ class RoleZero(Role):
             example=example,
             available_commands=tool_info,
             instruction=self.instruction.strip(),
+            task_type_desc=task_type_desc,
         )
         memory = self.rc.memory.get(self.memory_k)
         if not self.browser.is_empty_page:
@@ -201,13 +204,14 @@ class RoleZero(Role):
     async def _run_commands(self, commands) -> str:
         outputs = []
         for cmd in commands:
+            output = f"Command {cmd['command_name']} executed"
             # handle special command first
             if await self._run_special_command(cmd):
+                outputs.append(output)
                 continue
             # run command as specified by tool_execute_map
             if cmd["command_name"] in self.tool_execution_map:
                 tool_obj = self.tool_execution_map[cmd["command_name"]]
-                output = f"Command {cmd['command_name']} executed"
                 try:
                     if inspect.iscoroutinefunction(tool_obj):
                         tool_output = await tool_obj(**cmd["args"])
@@ -235,16 +239,13 @@ class RoleZero(Role):
         if cmd["command_name"] == "Plan.finish_current_task" and not self.planner.plan.is_plan_finished():
             # task_result = TaskResult(code=str(commands), result=outputs, is_success=is_success)
             # self.planner.plan.current_task.update_task_result(task_result=task_result)
-            self._finish_current_task()
+            self.planner.plan.finish_current_task()
 
         elif cmd["command_name"] == "end":
             self._set_state(-1)
 
         return is_special_cmd
 
-    def _finish_current_task(self):
-        self.planner.plan.finish_current_task()
-
     def _get_plan_status(self) -> Tuple[str, str]:
         plan_status = self.planner.plan.model_dump(include=["goal", "tasks"])
         for task in plan_status["tasks"]:
diff --git a/metagpt/schema.py b/metagpt/schema.py
index 69c7a519b..11610b6c3 100644
--- a/metagpt/schema.py
+++ b/metagpt/schema.py
@@ -464,7 +464,7 @@ class Task(BaseModel):
         self.is_finished = False
 
     def update_task_result(self, task_result: TaskResult):
-        self.code = task_result.code
+        self.code = task_result.code + "\n" + task_result.code
         self.result = task_result.result
         self.is_success = task_result.is_success
 
@@ -669,10 +669,14 @@ class Plan(BaseModel):
         """
         return [task for task in self.tasks if task.is_finished]
 
-    def append_task(self, task_id: str, dependent_task_ids: list[str], instruction: str, assignee: str):
+    def append_task(self, task_id: str, dependent_task_ids: list[str], instruction: str, assignee: str, task_type: str):
         """Append a new task with task_id (number) to the end of existing task sequences. If dependent_task_ids is not empty, the task will depend on the tasks with the ids in the list."""
         new_task = Task(
-            task_id=task_id, dependent_task_ids=dependent_task_ids, instruction=instruction, assignee=assignee
+            task_id=task_id,
+            dependent_task_ids=dependent_task_ids,
+            instruction=instruction,
+            assignee=assignee,
+            task_type=task_type
         )
         return self._append_task(new_task)
 
diff --git a/metagpt/tools/libs/__init__.py b/metagpt/tools/libs/__init__.py
index 725ab73c9..c9de6bd21 100644
--- a/metagpt/tools/libs/__init__.py
+++ b/metagpt/tools/libs/__init__.py
@@ -5,11 +5,11 @@
 # @File    : __init__.py
 # @Desc    :
 from metagpt.tools.libs import (
-    # data_preprocess,
-    # feature_engineering,
+    data_preprocess,
+    feature_engineering,
     sd_engine,
     gpt_v_generator,
-    # web_scraping,
+    web_scraping,
     # email_login,
     terminal,
     editor,
@@ -20,11 +20,11 @@ from metagpt.tools.libs import (
 from metagpt.tools.libs.env import get_env, set_get_env_entry, default_get_env, get_env_description
 
 _ = (
-    # data_preprocess,
-    # feature_engineering,
+    data_preprocess,
+    feature_engineering,
     sd_engine,
     gpt_v_generator,
-    # web_scraping,
+    web_scraping,
     # email_login,
     terminal,
     editor,

From 5a83c4539ae86eb5d4dd46ef4f9fd97de51851b9 Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Tue, 9 Jul 2024 16:51:58 +0800
Subject: [PATCH 06/22] avoid truncate output of html content

---
 metagpt/actions/di/execute_nb_code.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/metagpt/actions/di/execute_nb_code.py b/metagpt/actions/di/execute_nb_code.py
index cc2c05341..91480d3a7 100644
--- a/metagpt/actions/di/execute_nb_code.py
+++ b/metagpt/actions/di/execute_nb_code.py
@@ -191,7 +191,8 @@ class ExecuteNbCode(Action):
                 output_text = remove_log_and_warning_lines(output_text)
             # The useful information of the exception is at the end,
             # the useful information of normal output is at the begining.
-            output_text = output_text[:keep_len] if is_success else output_text[-keep_len:]
+            if '<!DOCTYPE html>' not in output_text:
+                output_text = output_text[:keep_len] if is_success else output_text[-keep_len:]
 
             parsed_output.append(output_text)
         return is_success, ",".join(parsed_output)

From 3d32a5d621c93667a1945df7086f223f972d035e Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Tue, 9 Jul 2024 16:57:36 +0800
Subject: [PATCH 07/22] add browser memory to write code

---
 metagpt/actions/di/write_analysis_code.py | 10 ++++++++--
 metagpt/roles/di/role_zero.py             |  7 ++++++-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/metagpt/actions/di/write_analysis_code.py b/metagpt/actions/di/write_analysis_code.py
index 548555196..19afab711 100644
--- a/metagpt/actions/di/write_analysis_code.py
+++ b/metagpt/actions/di/write_analysis_code.py
@@ -16,6 +16,7 @@ from metagpt.prompts.di.write_analysis_code import (
     REFLECTION_PROMPT,
     REFLECTION_SYSTEM_MSG,
     STRUCTUAL_PROMPT,
+    BROWSER_INFO,
 )
 from metagpt.schema import Message, Plan
 from metagpt.utils.common import CodeParser, remove_comments
@@ -41,6 +42,7 @@ class WriteAnalysisCode(Action):
         tool_info: str = "",
         working_memory: list[Message] = None,
         use_reflection: bool = False,
+        browser_memory: list[dict] = None,
         **kwargs,
     ) -> str:
         structual_prompt = STRUCTUAL_PROMPT.format(
@@ -48,16 +50,20 @@ class WriteAnalysisCode(Action):
             plan_status=plan_status,
             tool_info=tool_info,
         )
+        message = [Message(content=structual_prompt, role="user")]
+        if browser_memory:
+            browser_prompt = BROWSER_INFO.format(browser_memory=browser_memory)
+            message = [Message(content=browser_prompt, role="user")] + message
 
         working_memory = working_memory or []
-        context = self.llm.format_msg([Message(content=structual_prompt, role="user")] + working_memory)
+        context = self.llm.format_msg(message + working_memory)
 
         # LLM call
         if use_reflection:
             code = await self._debug_with_reflection(context=context, working_memory=working_memory)
         else:
             rsp = await self.llm.aask(context, system_msgs=[INTERPRETER_SYSTEM_MSG], **kwargs)
-            code = CodeParser.parse_code(text=rsp)
+            code = CodeParser.parse_code(text=rsp, lang="python")
 
         return code
 
diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py
index b179f5a00..01f792ed0 100644
--- a/metagpt/roles/di/role_zero.py
+++ b/metagpt/roles/di/role_zero.py
@@ -49,6 +49,7 @@ class RoleZero(Role):
     # Equipped with three basic tools by default for optional use
     editor: Editor = Editor()
     browser: Browser = Browser()
+    browser_memory: list[dict] = []  # store the memory of browser
     # terminal: Terminal = Terminal()  # FIXME: TypeError: cannot pickle '_thread.lock' object
 
     # Experience
@@ -151,7 +152,11 @@ class RoleZero(Role):
             pattern = re.compile(r"Command Browser\.(\w+) executed")
             for index, msg in zip(range(len(memory), 0, -1), memory[::-1]):
                 if pattern.match(msg.content):
-                    memory.insert(index, UserMessage(cause_by="browser", content=await self.browser.view()))
+                    content = await self.browser.view()
+                    memory.insert(index, UserMessage(cause_by="browser", content=content))
+                    browser_url = re.search('URL: (.*?)\\n', content).group(1)
+                    browser_action = {'command': pattern.match(msg.content).group(1), 'current url': browser_url}
+                    self.browser_memory.append(browser_action)
                     break
         context = self.llm.format_msg(memory + [UserMessage(content=prompt)])
         # print(*context, sep="\n" + "*" * 5 + "\n")

From 79744803fd9951269c50284e8c27f664651e4c1e Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Tue, 9 Jul 2024 16:59:19 +0800
Subject: [PATCH 08/22] refine prompt

---
 metagpt/prompts/di/data_analyst.py        | 7 +++++++
 metagpt/prompts/di/role_zero.py           | 4 ++--
 metagpt/prompts/di/write_analysis_code.py | 6 ++++++
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/metagpt/prompts/di/data_analyst.py b/metagpt/prompts/di/data_analyst.py
index 8450b2fe1..27b247fb3 100644
--- a/metagpt/prompts/di/data_analyst.py
+++ b/metagpt/prompts/di/data_analyst.py
@@ -41,3 +41,10 @@ Some text indicating your thoughts, such as how you should update the plan statu
 ]
 ```
 """
+
+BROWSER_INSTRUCTION = """
+4. Carefully choose to use or not use the browser tool to assist you in web tasks. 
+    - When no click action is required, no need to use the browser tool to navigate to the webpage before scraping.
+    - If you need detail HTML content, write code to get it but not to use the browser tool.
+    - Make sure the command_name are certainly in Available Commands when you use the browser tool.
+"""
diff --git a/metagpt/prompts/di/role_zero.py b/metagpt/prompts/di/role_zero.py
index 2b4fe105a..ea25aab82 100644
--- a/metagpt/prompts/di/role_zero.py
+++ b/metagpt/prompts/di/role_zero.py
@@ -5,7 +5,7 @@ When presented a current task, tackle the task using the available commands.
 Pay close attention to new user message, review the conversation history, use RoleZero.reply_to_human to respond to new user requirement.
 Note:
 1. If you keeping encountering errors, unexpected situation, or you are not sure of proceeding, use RoleZero.ask_human to ask for help.
-2. Carefully review your progress at the current task, if your actions so far has not fulfilled the task instruction, you should continue with current task. Otherwise, finish current task by Plan.finish_current_task.
+2. Carefully review your progress at the current task, if your actions so far has not fulfilled the task instruction, you should continue with current task. Otherwise, finish current task by Plan.finish_current_task explicitly.
 3. Each time you finish a task, use RoleZero.reply_to_human to report your progress.
 """
 
@@ -41,7 +41,7 @@ Pay close attention to the Example provided, you can reuse the example for your
 You may use any of the available commands to create a plan or update the plan. You may output mutiple commands, they will be executed sequentially.
 If you finish current task, you will automatically take the next task in the existing plan, use Plan.finish_task, DON'T append a new task.
 
-# Your commands in a json array, in the following output format with command_name and args. If there is nothing to do, use the pass or end command:
+# Your commands in a json array, in the following output format with correct command_name and args. If there is nothing to do, use the pass or end command:
 Some text indicating your thoughts, such as how you should update the plan status, respond to inquiry, or seek for help. Then a json array of commands. You must output ONE and ONLY ONE json array. DON'T output multiple json arrays with thoughts between them.
 ```json
 [
diff --git a/metagpt/prompts/di/write_analysis_code.py b/metagpt/prompts/di/write_analysis_code.py
index 1d743a719..bf67d8ba0 100644
--- a/metagpt/prompts/di/write_analysis_code.py
+++ b/metagpt/prompts/di/write_analysis_code.py
@@ -119,3 +119,9 @@ DATA_INFO = """
 Latest data info after previous tasks:
 {info}
 """
+
+BROWSER_INFO = """
+Here are ordered web actions in the browser environment, note that you can not use the browser tool in the current environment.
+{browser_memory}
+The latest url is the one you should use to view the page. If view page has been done, directly use the variable and html content in executing result.
+"""

From 0b7d7bdf559976ef6f060204bd3ddcff78bc2533 Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Tue, 9 Jul 2024 17:00:08 +0800
Subject: [PATCH 09/22] add prompt for scraping task

---
 metagpt/prompts/task_type.py  | 6 ++++++
 metagpt/strategy/task_type.py | 3 ++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/metagpt/prompts/task_type.py b/metagpt/prompts/task_type.py
index 5b1ffc744..312421c21 100644
--- a/metagpt/prompts/task_type.py
+++ b/metagpt/prompts/task_type.py
@@ -53,3 +53,9 @@ The current task is about converting image into webpage code. please note the fo
 - Single-Step Code Generation: Execute the entire code generation process in a single step, encompassing HTML, CSS, and JavaScript. Avoid fragmenting the code generation into multiple separate steps to maintain consistency and simplify the development workflow.
 - Save webpages: Be sure to use the save method provided.
 """
+
+# Prompt for taking on "web_scraping" tasks
+WEB_SCRAPING_PROMPT = """
+- Remember to view and print the necessary HTML content in a separate task to understand the structure first before scraping data.
+- Since the data required by user may not correspond directly to the actual HTML element names, you should thoroughly analyze the HTML structure and meanings of all elements in the executing result first. Ensure the `class_` in your code should derived from the actual HTML structure directly, not based on your knowledge. To ensure it, analyse the most suitable location of the 'class_' in the actual HTML content before code.
+"""
diff --git a/metagpt/strategy/task_type.py b/metagpt/strategy/task_type.py
index 2bc53b964..b44cc3ac0 100644
--- a/metagpt/strategy/task_type.py
+++ b/metagpt/strategy/task_type.py
@@ -8,7 +8,7 @@ from metagpt.prompts.task_type import (
     FEATURE_ENGINEERING_PROMPT,
     IMAGE2WEBPAGE_PROMPT,
     MODEL_EVALUATE_PROMPT,
-    MODEL_TRAIN_PROMPT,
+    MODEL_TRAIN_PROMPT, WEB_SCRAPING_PROMPT,
 )
 
 
@@ -62,6 +62,7 @@ class TaskType(Enum):
     WEBSCRAPING = TaskTypeDef(
         name="web scraping",
         desc="For scraping data from web pages.",
+        guidance=WEB_SCRAPING_PROMPT,
     )
     EMAIL_LOGIN = TaskTypeDef(
         name="email login",

From 32fc2762458180cc73999b2b1e64414be8b72613 Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Tue, 9 Jul 2024 17:01:07 +0800
Subject: [PATCH 10/22] add experience examples for scraping task

---
 metagpt/roles/di/data_analyst.py         |   8 +-
 metagpt/strategy/experience_retriever.py | 126 +++++++++++++++++++++++
 2 files changed, 133 insertions(+), 1 deletion(-)

diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py
index f3586a6f5..647196433 100644
--- a/metagpt/roles/di/data_analyst.py
+++ b/metagpt/roles/di/data_analyst.py
@@ -5,8 +5,11 @@ from pydantic import Field, model_validator
 from metagpt.actions.di.execute_nb_code import ExecuteNbCode
 from metagpt.actions.di.write_analysis_code import WriteAnalysisCode
 from metagpt.logs import logger
+from metagpt.prompts.di.data_analyst import BROWSER_INSTRUCTION
+from metagpt.prompts.di.role_zero import ROLE_INSTRUCTION
 from metagpt.roles.di.role_zero import RoleZero
 from metagpt.schema import TaskResult, Message
+from metagpt.strategy.experience_retriever import ExpRetriever, WebExpRetriever
 from metagpt.tools.tool_recommend import BM25ToolRecommender, ToolRecommender
 from metagpt.tools.tool_registry import register_tool
 
@@ -16,10 +19,12 @@ class DataAnalyst(RoleZero):
     name: str = "David"
     profile: str = "DataAnalyst"
     goal: str = "Take on any data-related tasks, such as data analysis, machine learning, deep learning, web browsing, web scraping, web searching, web deployment, terminal operation, git and github operation, etc."
+    instruction: str = ROLE_INSTRUCTION + BROWSER_INSTRUCTION
 
-    tools: list[str] = ["Plan", "DataAnalyst", "RoleZero"]
+    tools: list[str] = ["Plan", "DataAnalyst", "RoleZero", "Browser"]
     custom_tools: list[str] = ["machine learning", "web scraping", "Terminal"]
     custom_tool_recommender: ToolRecommender = None
+    experience_retriever: ExpRetriever = WebExpRetriever()
 
     use_reflection: bool = True
     write_code: WriteAnalysisCode = Field(default_factory=WriteAnalysisCode, exclude=True)
@@ -63,6 +68,7 @@ class DataAnalyst(RoleZero):
                 tool_info=tool_info,
                 working_memory=self.rc.working_memory.get() if use_reflection else None,
                 use_reflection=use_reflection,
+                browser_memory=self.browser_memory
             )
             self.rc.working_memory.add(Message(content=code, role="assistant", cause_by=WriteAnalysisCode))
 
diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py
index 5e85b056a..6356a0faf 100644
--- a/metagpt/strategy/experience_retriever.py
+++ b/metagpt/strategy/experience_retriever.py
@@ -448,3 +448,129 @@ Explanation: to review the code, call ReviewAndRewriteCode.run.
 ]
 ```
 """
+
+
+WEB_SCRAPING_EXAMPLE = """
+## action 1
+User Requirement: Scrap and list the restaurant names of first page by searching for the keyword `beef` on the website https://www.yelp.com/.
+Explanation: The requirement is to scrape data from a website and extract information about restaurants. The process involves searching for restaurants with a specific keyword, retrieving and presenting the data in a structured format.
+
+```json
+[
+    {
+        "command_name": "Plan.append_task",
+        "args": {
+            "task_id": "1",
+            "dependent_task_ids": [],
+            "instruction": "Navigate to the yelp website.",
+            "assignee": "Browser"
+        }
+    },
+    {
+        "command_name": "Plan.append_task",
+        "args": {
+            "task_id": "2",
+            "dependent_task_ids": ["1"],
+            "instruction": "Search for restaurants with the keyword 'beef'.",
+            "assignee": "Browser"
+        }
+    },
+    {
+        "command_name": "Plan.append_task",
+        "args": {
+            "task_id": "3",
+            "dependent_task_ids": ["2"],
+            "instruction": "View the html content of the search result page before scrap data to understand the structure.",
+            "assignee": "DataAnalyst"
+        }
+    },
+    {
+        "command_name": "Plan.append_task",
+        "args": {
+            "task_id": "4",
+            "dependent_task_ids": ["3"],
+            "instruction": "Parse the html content to scrape the restaurant names and print it.",
+            "assignee": "DataAnalyst"
+        }
+    }
+]
+```
+
+## action 2
+Explanation: To search for restaurants, I will now go to the website https://www.yelp.com/ first.
+Here is the command to navigate to the website:
+
+```json
+[
+    {
+        "command_name": "Browser.goto",
+        "args": {
+            "url": "https://www.yelp.com/"
+        }
+    }
+]
+```
+
+## action 3
+Explanation: Since the Browser has successfully navigated to the website, and I find that the element id of the search box is 53. I will finish the current task and then use the Browser tool to type the keyword `beef` in the search box and press enter.
+Here is the command to finish the current task and type the keyword:
+
+```json
+[
+    {
+        "command_name": "Plan.finish_current_task",
+        "args": {}
+    },
+    {
+        "command_name": "Browser.type",
+        "args": {
+            "element_id": 53,
+            "content": "beef",
+            "press_enter_after": true
+        }
+    }
+]
+```
+
+## action 4
+Explanation: Since the Browser has successfully search the keyword `beef`, I will finish the current task and then write code to view the html content of the page.
+Here is the command to finish the current task and view the html content:
+
+```json
+[
+    {
+        "command_name": "Plan.finish_current_task",
+        "args": {}
+    },
+    {
+        "command_name": "DataAnalyst.write_and_exec_code",
+        "args": {}
+    }
+]
+```
+
+## action 5
+Explanation: Since the DataAnalyst has successfully viewed the html content of the page, I will finish the current task and then write code to parse the html content and extract the restaurant names.
+Here is the command to finish the current task and parse the html content:
+
+```json
+[
+    {
+        "command_name": "Plan.finish_current_task",
+        "args": {}
+    },
+    {
+        "command_name": "DataAnalyst.write_and_exec_code",
+        "args": {}
+    }
+]
+
+...
+"""
+
+
+class WebExpRetriever(ExpRetriever):
+    """A simple experience retriever that returns manually crafted examples."""
+
+    def retrieve(self, context: str = "") -> str:
+        return WEB_SCRAPING_EXAMPLE

From 4c4d9547ff553f42fce1ba01a27c418804a3c044 Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Tue, 9 Jul 2024 17:01:54 +0800
Subject: [PATCH 11/22] support multi write_code steps for one task

---
 metagpt/schema.py               |  4 ++--
 metagpt/strategy/planner.py     | 10 +++++++++-
 metagpt/tools/tool_recommend.py |  4 ++--
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/metagpt/schema.py b/metagpt/schema.py
index 11610b6c3..94e64d7fa 100644
--- a/metagpt/schema.py
+++ b/metagpt/schema.py
@@ -464,8 +464,8 @@ class Task(BaseModel):
         self.is_finished = False
 
     def update_task_result(self, task_result: TaskResult):
-        self.code = task_result.code + "\n" + task_result.code
-        self.result = task_result.result
+        self.code = self.code + "\n" + task_result.code
+        self.result = self.result + "\n" + task_result.result
         self.is_success = task_result.is_success
 
 
diff --git a/metagpt/strategy/planner.py b/metagpt/strategy/planner.py
index 427e41562..95ad1f5cc 100644
--- a/metagpt/strategy/planner.py
+++ b/metagpt/strategy/planner.py
@@ -40,8 +40,14 @@ PLAN_STATUS = """
 ## Current Task
 {current_task}
 
+## Finished Section of Current Task
+### code
+{current_task_code}
+### execution result
+{current_task_result}
+
 ## Task Guidance
-Write complete code for 'Current Task'. And avoid duplicating code from 'Finished Tasks', such as repeated import of packages, reading data, etc.
+Write code for the incomplete sections of 'Current Task'. And avoid duplicating code from 'Finished Tasks', such as repeated import of packages, reading data, etc.
 Specifically, {guidance}
 """
 
@@ -173,6 +179,8 @@ class Planner(BaseModel):
             code_written=code_written,
             task_results=task_results,
             current_task=self.current_task.instruction,
+            current_task_code=self.current_task.code if self.current_task.code else "",
+            current_task_result=self.current_task.result if self.current_task.result else "",
             guidance=guidance,
         )
 
diff --git a/metagpt/tools/tool_recommend.py b/metagpt/tools/tool_recommend.py
index ab847d10e..0c596707a 100644
--- a/metagpt/tools/tool_recommend.py
+++ b/metagpt/tools/tool_recommend.py
@@ -106,11 +106,11 @@ class ToolRecommender(BaseModel):
         Wrap recommended tools with their info in a string, which can be used directly in a prompt.
         """
         recommended_tools = await self.recommend_tools(**kwargs)
+        if fix:
+            recommended_tools.extend([self.tools[tool_name] for tool_name in fix if tool_name in self.tools])
         if not recommended_tools:
             return ""
         tool_schemas = {tool.name: tool.schemas for tool in recommended_tools}
-        if fix:
-            tool_schemas.update({tool.name: tool.schemas for tool in self.tools.values() if tool.name in fix})
         return TOOL_INFO_PROMPT.format(tool_schemas=tool_schemas)
 
     async def recall_tools(self, context: str = "", plan: Plan = None, topk: int = 20) -> list[Tool]:

From ae861d99cdbb0fdcb1b666ce57883ae6576910de Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Tue, 9 Jul 2024 17:02:44 +0800
Subject: [PATCH 12/22] refine web_scraping tool

---
 metagpt/tools/libs/web_scraping.py | 8 +++++---
 metagpt/utils/parse_html.py        | 6 +++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/metagpt/tools/libs/web_scraping.py b/metagpt/tools/libs/web_scraping.py
index 489c3a472..9e7a8041c 100644
--- a/metagpt/tools/libs/web_scraping.py
+++ b/metagpt/tools/libs/web_scraping.py
@@ -8,13 +8,15 @@ from metagpt.utils.parse_html import simplify_html
 
 
 @register_tool(tags=["web scraping"])
-async def view_page_element_to_scrape(url: str, requirement: str, keep_links: bool = False) -> None:
-    """view the HTML content of current page to understand the structure. When executed, the content will be printed out
+async def view_page_element_to_scrape(url: str, requirement: str, keep_links: bool = False) -> str:
+    """view the HTML content of current page to understand the structure.
 
     Args:
         url (str): The URL of the web page to scrape.
         requirement (str): Providing a clear and detailed requirement helps in focusing the inspection on the desired elements.
         keep_links (bool): Whether to keep the hyperlinks in the HTML content. Set to True if links are required
+    Returns:
+        str: The HTML content of the page.
     """
     async with Browser() as browser:
         await browser.goto(url)
@@ -36,7 +38,7 @@ async def view_page_element_to_scrape(url: str, requirement: str, keep_links: bo
         html = "\n".join(i.text for i in nodes)
 
     mem_fs.rm_file(filename)
-    print(html)
+    return html
 
 
 # async def get_elements_outerhtml(self, element_ids: list[int]):
diff --git a/metagpt/utils/parse_html.py b/metagpt/utils/parse_html.py
index 1ed3a620c..031393501 100644
--- a/metagpt/utils/parse_html.py
+++ b/metagpt/utils/parse_html.py
@@ -41,13 +41,13 @@ class WebPage(BaseModel):
 
     def get_slim_soup(self, keep_links: bool = False):
         soup = _get_soup(self.html)
-        keep_attrs = ["class"]
+        keep_attrs = ["class", "id"]
         if keep_links:
-            keep_attrs.append("href")
+            keep_attrs.extend(["href", "title"])
 
         for i in soup.find_all(True):
             for name in list(i.attrs):
-                if i[name] and name not in keep_attrs:
+                if i[name] and name not in keep_attrs and not name.startswith("data-"):
                     del i[name]
 
         for i in soup.find_all(["svg", "img", "video", "audio"]):

From 46cd961ebaeafb3d28fc11f0921e3ffe7c26ac66 Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Tue, 9 Jul 2024 17:03:31 +0800
Subject: [PATCH 13/22] add test for DataAnalyst

---
 tests/metagpt/roles/di/run_data_analyst.py | 53 ++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 tests/metagpt/roles/di/run_data_analyst.py

diff --git a/tests/metagpt/roles/di/run_data_analyst.py b/tests/metagpt/roles/di/run_data_analyst.py
new file mode 100644
index 000000000..9c1f72394
--- /dev/null
+++ b/tests/metagpt/roles/di/run_data_analyst.py
@@ -0,0 +1,53 @@
+from metagpt.roles.di.data_analyst import DataAnalyst
+
+HOUSE_PRICE_TRAIN_PATH = '/data/house-prices-advanced-regression-techniques/split_train.csv'
+HOUSE_PRICE_EVAL_PATH = '/data/house-prices-advanced-regression-techniques/split_eval.csv'
+HOUSE_PRICE_REQ = f"""
+This is a house price dataset, your goal is to predict the sale price of a property based on its features. The target column is SalePrice. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report RMSE between the logarithm of the predicted value and the logarithm of the observed sales price on the eval data. Train data path: '{HOUSE_PRICE_TRAIN_PATH}', eval data path: '{HOUSE_PRICE_EVAL_PATH}'.
+"""
+
+CALIFORNIA_HOUSING_REQ = """
+Analyze the 'Canifornia-housing-dataset' using https://scikit-learn.org/stable/modules/generated/sklearn.datasets.fetch_california_housing.html#sklearn.datasets.fetch_california_housing to predict the median house value. you need to perfrom data preprocessing, feature engineering and finally modeling to predict the target. Use machine learning techniques such as linear regression (including ridge regression and lasso regression), random forest, CatBoost, LightGBM, XGBoost or other appropriate method. You also need to report the MSE on the test dataset
+"""
+
+PAPER_LIST_REQ = """"
+Get data from `paperlist` table in https://papercopilot.com/statistics/iclr-statistics/iclr-2024-statistics/,
+and save it to a csv file. paper title must include `multiagent` or `large language model`.
+**Notice: view the page element before writing scraping code**
+"""
+
+ECOMMERCE_REQ = """
+Get products data from website https://scrapeme.live/shop/ and save it as a csv file.
+The first page product name, price, product URL, and image URL must be saved in the csv.
+**Notice: view the page element before writing scraping code**
+"""
+
+NEWS_36KR_REQ = """从36kr创投平台https://pitchhub.36kr.com/financing-flash 所有初创企业融资的信息, **注意: 这是一个中文网站**;
+下面是一个大致流程, 你会根据每一步的运行结果对当前计划中的任务做出适当调整:
+1. 爬取并本地保存html结构;
+2. 直接打印第7个*`快讯`*关键词后2000个字符的html内容, 作为*快讯的html内容示例*;
+3. 反思*快讯的html内容示例*中的规律, 设计正则匹配表达式来获取*`快讯`*的标题、链接、时间;
+4. 筛选最近3天的初创企业融资*`快讯`*, 以list[dict]形式打印前5个。
+5. 将全部结果存在本地csv中
+**Notice: view the page element before writing scraping code**
+"""
+
+WIKIPEDIA_SEARCH_REQ = """
+Search for `LLM` on Wikipedia and print all the meaningful significances of the entry.
+"""
+
+STACKOVERFLOW_CLICK_REQ = """
+Click the Questions tag in https://stackoverflow.com/ and scrap question name, votes, answers and views num to csv in the first result page.
+"""
+
+
+async def main():
+    di = DataAnalyst()
+    await di.browser.start()
+    await di.run(STACKOVERFLOW_CLICK_REQ)
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

From 46e7c2c05c13ac0ac6a1b90065f60f16ced9a43b Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Wed, 10 Jul 2024 11:25:34 +0800
Subject: [PATCH 14/22] use keyword retriever for DataAnalyst

---
 metagpt/strategy/experience_retriever.py   | 9 ++-------
 tests/metagpt/roles/di/run_data_analyst.py | 2 +-
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py
index 63ca4fcbd..04807ebec 100644
--- a/metagpt/strategy/experience_retriever.py
+++ b/metagpt/strategy/experience_retriever.py
@@ -638,6 +638,8 @@ class KeywordExpRetriever(ExpRetriever):
                 return DEPLOY_EXAMPLE
             elif "issue" in context.lower():
                 return FIX_ISSUE_EXAMPLE
+            elif "https:" or "http:" in context.lower():
+                return WEB_SCRAPING_EXAMPLE
         elif exp_type == "task":
             if "diagnose" in context.lower():
                 return SEARCH_SYMBOL_EXAMPLE
@@ -1018,10 +1020,3 @@ Here is the command to finish the current task and parse the html content:
 
 ...
 """
-
-
-class WebExpRetriever(ExpRetriever):
-    """A simple experience retriever that returns manually crafted examples."""
-
-    def retrieve(self, context: str = "") -> str:
-        return WEB_SCRAPING_EXAMPLE
diff --git a/tests/metagpt/roles/di/run_data_analyst.py b/tests/metagpt/roles/di/run_data_analyst.py
index 9c1f72394..445f8f800 100644
--- a/tests/metagpt/roles/di/run_data_analyst.py
+++ b/tests/metagpt/roles/di/run_data_analyst.py
@@ -33,7 +33,7 @@ NEWS_36KR_REQ = """从36kr创投平台https://pitchhub.36kr.com/financing-flash
 """
 
 WIKIPEDIA_SEARCH_REQ = """
-Search for `LLM` on Wikipedia and print all the meaningful significances of the entry.
+Search for `LLM` on https://www.wikipedia.org/ and print all the meaningful significances of the entry.
 """
 
 STACKOVERFLOW_CLICK_REQ = """

From be607ba3e3a4a00ed903f37a213a94870df31380 Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Wed, 10 Jul 2024 11:27:10 +0800
Subject: [PATCH 15/22] use keyword retriever for DataAnalyst

---
 tests/metagpt/roles/di/run_data_analyst.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/metagpt/roles/di/run_data_analyst.py b/tests/metagpt/roles/di/run_data_analyst.py
index 445f8f800..b7b48e0db 100644
--- a/tests/metagpt/roles/di/run_data_analyst.py
+++ b/tests/metagpt/roles/di/run_data_analyst.py
@@ -10,6 +10,7 @@ CALIFORNIA_HOUSING_REQ = """
 Analyze the 'Canifornia-housing-dataset' using https://scikit-learn.org/stable/modules/generated/sklearn.datasets.fetch_california_housing.html#sklearn.datasets.fetch_california_housing to predict the median house value. you need to perfrom data preprocessing, feature engineering and finally modeling to predict the target. Use machine learning techniques such as linear regression (including ridge regression and lasso regression), random forest, CatBoost, LightGBM, XGBoost or other appropriate method. You also need to report the MSE on the test dataset
 """
 
+# For web scraping task, please provide url begin with `https://` or `http://`
 PAPER_LIST_REQ = """"
 Get data from `paperlist` table in https://papercopilot.com/statistics/iclr-statistics/iclr-2024-statistics/,
 and save it to a csv file. paper title must include `multiagent` or `large language model`.

From b03ce58fde78117a7e11c235b0ecbe8cecaebdc2 Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Wed, 10 Jul 2024 11:29:27 +0800
Subject: [PATCH 16/22] remove task_type and parse_browser_action to
 DataAnalyst

---
 metagpt/actions/di/write_analysis_code.py |  6 +--
 metagpt/prompts/di/data_analyst.py        | 56 +++++------------------
 metagpt/prompts/di/write_analysis_code.py |  2 +-
 metagpt/roles/di/data_analyst.py          | 32 +++++++++----
 metagpt/roles/di/role_zero.py             | 17 ++++---
 5 files changed, 46 insertions(+), 67 deletions(-)

diff --git a/metagpt/actions/di/write_analysis_code.py b/metagpt/actions/di/write_analysis_code.py
index 19afab711..d0c4c016e 100644
--- a/metagpt/actions/di/write_analysis_code.py
+++ b/metagpt/actions/di/write_analysis_code.py
@@ -42,7 +42,7 @@ class WriteAnalysisCode(Action):
         tool_info: str = "",
         working_memory: list[Message] = None,
         use_reflection: bool = False,
-        browser_memory: list[dict] = None,
+        browser_actions: list[dict] = None,
         **kwargs,
     ) -> str:
         structual_prompt = STRUCTUAL_PROMPT.format(
@@ -51,8 +51,8 @@ class WriteAnalysisCode(Action):
             tool_info=tool_info,
         )
         message = [Message(content=structual_prompt, role="user")]
-        if browser_memory:
-            browser_prompt = BROWSER_INFO.format(browser_memory=browser_memory)
+        if browser_actions:
+            browser_prompt = BROWSER_INFO.format(browser_actions=browser_actions)
             message = [Message(content=browser_prompt, role="user")] + message
 
         working_memory = working_memory or []
diff --git a/metagpt/prompts/di/data_analyst.py b/metagpt/prompts/di/data_analyst.py
index 08b8d0df8..7abe0ac93 100644
--- a/metagpt/prompts/di/data_analyst.py
+++ b/metagpt/prompts/di/data_analyst.py
@@ -1,47 +1,4 @@
-CMD_PROMPT = """
-# Data Structure
-class Task(BaseModel):
-    task_id: str = ""
-    dependent_task_ids: list[str] = []
-    instruction: str = ""
-    task_type: str = ""
-    assignee: str = "David"
-
-# Available Commands
-{available_commands}
-
-# Current Plan
-{plan_status}
-
-# Example
-{example}
-
-# Instructions
-Based on the context, write a plan or modify an existing plan to achieve the goal. A plan consists of one to 3 tasks.
-If plan is created, you should track the progress and update the plan accordingly, such as finish_current_task, append_task, reset_task, replace_task, etc.
-Pay close attention to new user message, review the conversation history, use reply_to_human to respond to new user requirement.
-Note:
-1. If you keeping encountering errors, unexpected situation, or you are not sure of proceeding, use ask_human to ask for help.
-2. Carefully review your progress at the current task, if your actions so far has not fulfilled the task instruction, you should continue with current task. Otherwise, finish current task.
-3. Each time you finish a task, use reply_to_human to report your progress.
-Pay close attention to the Example provided, you can reuse the example for your current situation if it fits.
-
-You may use any of the available commands to create a plan or update the plan. You may output mutiple commands, they will be executed sequentially.
-If you finish current task, you will automatically take the next task in the existing plan, use finish_task, DON'T append a new task.
-
-# Your commands in a json array, in the following output format, always output a json array, if there is nothing to do, use the pass command:
-Some text indicating your thoughts, such as how you should update the plan status, respond to inquiry, or seek for help. Then a json array of commands.
-```json
-[
-    {{
-        "command_name": str,
-        "args": {{"arg_name": arg_value, ...}}
-    }},
-    ...
-]
-```
-Notice: your output JSON data section must start with **```json [**
-"""
+from metagpt.strategy.task_type import TaskType
 
 BROWSER_INSTRUCTION = """
 4. Carefully choose to use or not use the browser tool to assist you in web tasks. 
@@ -49,3 +6,14 @@ BROWSER_INSTRUCTION = """
     - If you need detail HTML content, write code to get it but not to use the browser tool.
     - Make sure the command_name are certainly in Available Commands when you use the browser tool.
 """
+
+TASK_TYPE_DESC = "\n".join([f"- **{tt.type_name}**: {tt.value.desc}" for tt in TaskType])
+
+
+CODE_STATUS = """
+**Code written**:
+{code}
+
+**Execution status**: {status}
+**Execution result**: {result}
+"""
\ No newline at end of file
diff --git a/metagpt/prompts/di/write_analysis_code.py b/metagpt/prompts/di/write_analysis_code.py
index bf67d8ba0..6105c4182 100644
--- a/metagpt/prompts/di/write_analysis_code.py
+++ b/metagpt/prompts/di/write_analysis_code.py
@@ -122,6 +122,6 @@ Latest data info after previous tasks:
 
 BROWSER_INFO = """
 Here are ordered web actions in the browser environment, note that you can not use the browser tool in the current environment.
-{browser_memory}
+{browser_actions}
 The latest url is the one you should use to view the page. If view page has been done, directly use the variable and html content in executing result.
 """
diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py
index 647196433..e6b734417 100644
--- a/metagpt/roles/di/data_analyst.py
+++ b/metagpt/roles/di/data_analyst.py
@@ -1,15 +1,18 @@
 from __future__ import annotations
 
+import re
+from typing import List
+
 from pydantic import Field, model_validator
 
 from metagpt.actions.di.execute_nb_code import ExecuteNbCode
 from metagpt.actions.di.write_analysis_code import WriteAnalysisCode
 from metagpt.logs import logger
-from metagpt.prompts.di.data_analyst import BROWSER_INSTRUCTION
+from metagpt.prompts.di.data_analyst import BROWSER_INSTRUCTION, TASK_TYPE_DESC, CODE_STATUS
 from metagpt.prompts.di.role_zero import ROLE_INSTRUCTION
 from metagpt.roles.di.role_zero import RoleZero
 from metagpt.schema import TaskResult, Message
-from metagpt.strategy.experience_retriever import ExpRetriever, WebExpRetriever
+from metagpt.strategy.experience_retriever import ExpRetriever, KeywordExpRetriever
 from metagpt.tools.tool_recommend import BM25ToolRecommender, ToolRecommender
 from metagpt.tools.tool_registry import register_tool
 
@@ -20,11 +23,12 @@ class DataAnalyst(RoleZero):
     profile: str = "DataAnalyst"
     goal: str = "Take on any data-related tasks, such as data analysis, machine learning, deep learning, web browsing, web scraping, web searching, web deployment, terminal operation, git and github operation, etc."
     instruction: str = ROLE_INSTRUCTION + BROWSER_INSTRUCTION
+    task_type_desc: str = TASK_TYPE_DESC
 
     tools: list[str] = ["Plan", "DataAnalyst", "RoleZero", "Browser"]
     custom_tools: list[str] = ["machine learning", "web scraping", "Terminal"]
     custom_tool_recommender: ToolRecommender = None
-    experience_retriever: ExpRetriever = WebExpRetriever()
+    experience_retriever: ExpRetriever = KeywordExpRetriever()
 
     use_reflection: bool = True
     write_code: WriteAnalysisCode = Field(default_factory=WriteAnalysisCode, exclude=True)
@@ -40,6 +44,17 @@ class DataAnalyst(RoleZero):
             "DataAnalyst.write_and_exec_code": self.write_and_exec_code,
         })
 
+    def parse_browser_actions(self, memory: List[Message]):
+        for index, msg in enumerate(memory):
+            if msg.cause_by == "browser":
+                browser_url = re.search('URL: (.*?)\\n', msg.content).group(1)
+                pattern = re.compile(r"Command Browser\.(\w+) executed")
+                browser_action = {
+                    'command': pattern.match(memory[index - 1].content).group(1),
+                    'current url': browser_url
+                }
+                self.browser_actions.append(browser_action)
+
     async def write_and_exec_code(self):
         """Write a code block for current task and execute it in an interactive notebook environment."""
         counter = 0
@@ -68,7 +83,7 @@ class DataAnalyst(RoleZero):
                 tool_info=tool_info,
                 working_memory=self.rc.working_memory.get() if use_reflection else None,
                 use_reflection=use_reflection,
-                browser_memory=self.browser_memory
+                browser_actions=self.browser_actions
             )
             self.rc.working_memory.add(Message(content=code, role="assistant", cause_by=WriteAnalysisCode))
 
@@ -83,11 +98,8 @@ class DataAnalyst(RoleZero):
             if success:
                 task_result = TaskResult(code=code, result=result, is_success=success)
                 self.planner.current_task.update_task_result(task_result)
-        output = f"""
-        **Code written**:
-        {code}
-        **Execution status**:{'Success' if success else 'Failed'}
-        **Execution result**: {result}
-        """
+
+        status = 'Success' if success else 'Failed'
+        output = CODE_STATUS.format(code=code, status=status, result=result)
         self.rc.working_memory.clear()
         return output
diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py
index a52d72c8e..fb89114c8 100644
--- a/metagpt/roles/di/role_zero.py
+++ b/metagpt/roles/di/role_zero.py
@@ -6,7 +6,6 @@ import re
 import traceback
 from typing import Callable, Dict, List, Literal, Tuple
 
-from metagpt.strategy.task_type import TaskType
 from pydantic import model_validator
 
 from metagpt.actions import Action
@@ -41,6 +40,7 @@ class RoleZero(Role):
     system_msg: list[str] = None  # Use None to conform to the default value at llm.aask
     cmd_prompt: str = CMD_PROMPT
     instruction: str = ROLE_INSTRUCTION
+    task_type_desc: str = None
 
     # React Mode
     react_mode: Literal["react"] = "react"
@@ -54,7 +54,7 @@ class RoleZero(Role):
     # Equipped with three basic tools by default for optional use
     editor: Editor = Editor()
     browser: Browser = Browser()
-    browser_memory: list[dict] = []  # store the memory of browser
+    browser_actions: list[dict] = []  # store the browser history actions
     # terminal: Terminal = Terminal()  # FIXME: TypeError: cannot pickle '_thread.lock' object
 
     # Experience
@@ -137,7 +137,6 @@ class RoleZero(Role):
 
         ### 2. Plan Status ###
         plan_status, current_task = self._get_plan_status()
-        task_type_desc = "\n".join([f"- **{tt.type_name}**: {tt.value.desc}" for tt in TaskType])
 
         ### 3. Tool/Command Info ###
         tools = await self.tool_recommender.recommend_tools()
@@ -150,19 +149,16 @@ class RoleZero(Role):
             example=example,
             available_commands=tool_info,
             instruction=self.instruction.strip(),
-            task_type_desc=task_type_desc,
+            task_type_desc=self.task_type_desc,
         )
         memory = self.rc.memory.get(self.memory_k)
         if not self.browser.is_empty_page:
             pattern = re.compile(r"Command Browser\.(\w+) executed")
             for index, msg in zip(range(len(memory), 0, -1), memory[::-1]):
                 if pattern.match(msg.content):
-                    content = await self.browser.view()
-                    memory.insert(index, UserMessage(cause_by="browser", content=content))
-                    browser_url = re.search('URL: (.*?)\\n', content).group(1)
-                    browser_action = {'command': pattern.match(msg.content).group(1), 'current url': browser_url}
-                    self.browser_memory.append(browser_action)
+                    memory.insert(index, UserMessage(cause_by="browser", content=await self.browser.view()))
                     break
+        self.parse_browser_actions(memory=memory)
         context = self.llm.format_msg(memory + [UserMessage(content=prompt)])
         # print(*context, sep="\n" + "*" * 5 + "\n")
         async with ThoughtReporter(enable_llm_stream=True):
@@ -171,6 +167,9 @@ class RoleZero(Role):
 
         return True
 
+    def parse_browser_actions(self, memory: List[Message]):
+        pass
+
     async def _act(self) -> Message:
         if self.use_fixed_sop:
             return await super()._act()

From 7508ff66cdadf72e2c349631ef2bed7dccaab1cf Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Fri, 12 Jul 2024 14:33:05 +0800
Subject: [PATCH 17/22] move loc of task_type

---
 metagpt/prompts/di/role_zero.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/metagpt/prompts/di/role_zero.py b/metagpt/prompts/di/role_zero.py
index 83ad5cd8e..b642df9c7 100644
--- a/metagpt/prompts/di/role_zero.py
+++ b/metagpt/prompts/di/role_zero.py
@@ -18,13 +18,13 @@ class Task(BaseModel):
     task_type: str = ""
     assignee: str = ""
 
+# Available Task Types
+{task_type_desc}
+
 # Available Commands
 {available_commands}
 Special Command: Use {{"command_name": "end"}} to do nothing or indicate completion of all requirements and the end of actions.
 
-# Available Task Types
-{task_type_desc}
-
 # Current Plan
 {plan_status}
 

From df340cb33084ad1d0b03a7fd688d2119d842955c Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Fri, 12 Jul 2024 15:55:52 +0800
Subject: [PATCH 18/22] remove browser_actions from role

---
 metagpt/actions/di/write_analysis_code.py |  4 ++--
 metagpt/roles/di/data_analyst.py          | 13 ++++++++-----
 metagpt/roles/di/role_zero.py             | 21 ++++++++++-----------
 3 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/metagpt/actions/di/write_analysis_code.py b/metagpt/actions/di/write_analysis_code.py
index d0c4c016e..06e0ba4e2 100644
--- a/metagpt/actions/di/write_analysis_code.py
+++ b/metagpt/actions/di/write_analysis_code.py
@@ -42,7 +42,6 @@ class WriteAnalysisCode(Action):
         tool_info: str = "",
         working_memory: list[Message] = None,
         use_reflection: bool = False,
-        browser_actions: list[dict] = None,
         **kwargs,
     ) -> str:
         structual_prompt = STRUCTUAL_PROMPT.format(
@@ -51,11 +50,12 @@ class WriteAnalysisCode(Action):
             tool_info=tool_info,
         )
         message = [Message(content=structual_prompt, role="user")]
+        browser_actions = [msg for msg in working_memory if msg.cause_by == "browser"]
         if browser_actions:
             browser_prompt = BROWSER_INFO.format(browser_actions=browser_actions)
             message = [Message(content=browser_prompt, role="user")] + message
 
-        working_memory = working_memory or []
+        working_memory = [msg for msg in working_memory if msg.cause_by != "browser"] if use_reflection else []
         context = self.llm.format_msg(message + working_memory)
 
         # LLM call
diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py
index e6b734417..066b82874 100644
--- a/metagpt/roles/di/data_analyst.py
+++ b/metagpt/roles/di/data_analyst.py
@@ -1,7 +1,7 @@
 from __future__ import annotations
 
+import json
 import re
-from typing import List
 
 from pydantic import Field, model_validator
 
@@ -44,7 +44,8 @@ class DataAnalyst(RoleZero):
             "DataAnalyst.write_and_exec_code": self.write_and_exec_code,
         })
 
-    def parse_browser_actions(self, memory: List[Message]):
+    async def parse_browser_actions(self):
+        memory = await super().parse_browser_actions()
         for index, msg in enumerate(memory):
             if msg.cause_by == "browser":
                 browser_url = re.search('URL: (.*?)\\n', msg.content).group(1)
@@ -53,7 +54,10 @@ class DataAnalyst(RoleZero):
                     'command': pattern.match(memory[index - 1].content).group(1),
                     'current url': browser_url
                 }
-                self.browser_actions.append(browser_action)
+                self.rc.working_memory.add(
+                    Message(content=json.dumps(browser_action), role="user", cause_by="browser")
+                )
+        return memory
 
     async def write_and_exec_code(self):
         """Write a code block for current task and execute it in an interactive notebook environment."""
@@ -81,9 +85,8 @@ class DataAnalyst(RoleZero):
                 user_requirement=self.planner.plan.goal,
                 plan_status=plan_status,
                 tool_info=tool_info,
-                working_memory=self.rc.working_memory.get() if use_reflection else None,
+                working_memory=self.rc.working_memory.get(),
                 use_reflection=use_reflection,
-                browser_actions=self.browser_actions
             )
             self.rc.working_memory.add(Message(content=code, role="assistant", cause_by=WriteAnalysisCode))
 
diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py
index fb89114c8..ed617eb81 100644
--- a/metagpt/roles/di/role_zero.py
+++ b/metagpt/roles/di/role_zero.py
@@ -54,7 +54,6 @@ class RoleZero(Role):
     # Equipped with three basic tools by default for optional use
     editor: Editor = Editor()
     browser: Browser = Browser()
-    browser_actions: list[dict] = []  # store the browser history actions
     # terminal: Terminal = Terminal()  # FIXME: TypeError: cannot pickle '_thread.lock' object
 
     # Experience
@@ -151,14 +150,7 @@ class RoleZero(Role):
             instruction=self.instruction.strip(),
             task_type_desc=self.task_type_desc,
         )
-        memory = self.rc.memory.get(self.memory_k)
-        if not self.browser.is_empty_page:
-            pattern = re.compile(r"Command Browser\.(\w+) executed")
-            for index, msg in zip(range(len(memory), 0, -1), memory[::-1]):
-                if pattern.match(msg.content):
-                    memory.insert(index, UserMessage(cause_by="browser", content=await self.browser.view()))
-                    break
-        self.parse_browser_actions(memory=memory)
+        memory = await self.parse_browser_actions()
         context = self.llm.format_msg(memory + [UserMessage(content=prompt)])
         # print(*context, sep="\n" + "*" * 5 + "\n")
         async with ThoughtReporter(enable_llm_stream=True):
@@ -167,8 +159,15 @@ class RoleZero(Role):
 
         return True
 
-    def parse_browser_actions(self, memory: List[Message]):
-        pass
+    async def parse_browser_actions(self):
+        memory = self.rc.memory.get(self.memory_k)
+        if not self.browser.is_empty_page:
+            pattern = re.compile(r"Command Browser\.(\w+) executed")
+            for index, msg in zip(range(len(memory), 0, -1), memory[::-1]):
+                if pattern.match(msg.content):
+                    memory.insert(index, UserMessage(cause_by="browser", content=await self.browser.view()))
+                    break
+        return memory
 
     async def _act(self) -> Message:
         if self.use_fixed_sop:

From df29e16e2b78b2a1c351f0b689109ef9b41a2e05 Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Fri, 12 Jul 2024 16:44:05 +0800
Subject: [PATCH 19/22] refine code

---
 metagpt/actions/di/write_analysis_code.py | 10 ++--------
 metagpt/prompts/di/data_analyst.py        |  9 ++++++++-
 metagpt/prompts/di/write_analysis_code.py |  6 ------
 metagpt/roles/di/data_analyst.py          | 19 ++++++++++---------
 metagpt/roles/di/role_zero.py             |  6 +++---
 5 files changed, 23 insertions(+), 27 deletions(-)

diff --git a/metagpt/actions/di/write_analysis_code.py b/metagpt/actions/di/write_analysis_code.py
index 06e0ba4e2..00e6d174d 100644
--- a/metagpt/actions/di/write_analysis_code.py
+++ b/metagpt/actions/di/write_analysis_code.py
@@ -16,7 +16,6 @@ from metagpt.prompts.di.write_analysis_code import (
     REFLECTION_PROMPT,
     REFLECTION_SYSTEM_MSG,
     STRUCTUAL_PROMPT,
-    BROWSER_INFO,
 )
 from metagpt.schema import Message, Plan
 from metagpt.utils.common import CodeParser, remove_comments
@@ -49,14 +48,9 @@ class WriteAnalysisCode(Action):
             plan_status=plan_status,
             tool_info=tool_info,
         )
-        message = [Message(content=structual_prompt, role="user")]
-        browser_actions = [msg for msg in working_memory if msg.cause_by == "browser"]
-        if browser_actions:
-            browser_prompt = BROWSER_INFO.format(browser_actions=browser_actions)
-            message = [Message(content=browser_prompt, role="user")] + message
 
-        working_memory = [msg for msg in working_memory if msg.cause_by != "browser"] if use_reflection else []
-        context = self.llm.format_msg(message + working_memory)
+        working_memory = working_memory or []
+        context = self.llm.format_msg([Message(content=structual_prompt, role="user")] + working_memory)
 
         # LLM call
         if use_reflection:
diff --git a/metagpt/prompts/di/data_analyst.py b/metagpt/prompts/di/data_analyst.py
index 7abe0ac93..56ae0b68b 100644
--- a/metagpt/prompts/di/data_analyst.py
+++ b/metagpt/prompts/di/data_analyst.py
@@ -16,4 +16,11 @@ CODE_STATUS = """
 
 **Execution status**: {status}
 **Execution result**: {result}
-"""
\ No newline at end of file
+"""
+
+
+BROWSER_INFO = """
+Here are ordered web actions in the browser environment, note that you can not use the browser tool in the current environment.
+{browser_actions}
+The latest url is the one you should use to view the page. If view page has been done, directly use the variable and html content in executing result.
+"""
diff --git a/metagpt/prompts/di/write_analysis_code.py b/metagpt/prompts/di/write_analysis_code.py
index 6105c4182..1d743a719 100644
--- a/metagpt/prompts/di/write_analysis_code.py
+++ b/metagpt/prompts/di/write_analysis_code.py
@@ -119,9 +119,3 @@ DATA_INFO = """
 Latest data info after previous tasks:
 {info}
 """
-
-BROWSER_INFO = """
-Here are ordered web actions in the browser environment, note that you can not use the browser tool in the current environment.
-{browser_actions}
-The latest url is the one you should use to view the page. If view page has been done, directly use the variable and html content in executing result.
-"""
diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py
index 066b82874..2e5315f33 100644
--- a/metagpt/roles/di/data_analyst.py
+++ b/metagpt/roles/di/data_analyst.py
@@ -1,14 +1,14 @@
 from __future__ import annotations
 
-import json
 import re
+from typing import List
 
 from pydantic import Field, model_validator
 
 from metagpt.actions.di.execute_nb_code import ExecuteNbCode
 from metagpt.actions.di.write_analysis_code import WriteAnalysisCode
 from metagpt.logs import logger
-from metagpt.prompts.di.data_analyst import BROWSER_INSTRUCTION, TASK_TYPE_DESC, CODE_STATUS
+from metagpt.prompts.di.data_analyst import BROWSER_INSTRUCTION, TASK_TYPE_DESC, CODE_STATUS, BROWSER_INFO
 from metagpt.prompts.di.role_zero import ROLE_INSTRUCTION
 from metagpt.roles.di.role_zero import RoleZero
 from metagpt.schema import TaskResult, Message
@@ -44,19 +44,20 @@ class DataAnalyst(RoleZero):
             "DataAnalyst.write_and_exec_code": self.write_and_exec_code,
         })
 
-    async def parse_browser_actions(self):
-        memory = await super().parse_browser_actions()
+    async def parse_browser_actions(self, memory: List[Message]) -> List[Message]:
+        memory = await super().parse_browser_actions(memory)
+        browser_actions = []
         for index, msg in enumerate(memory):
             if msg.cause_by == "browser":
                 browser_url = re.search('URL: (.*?)\\n', msg.content).group(1)
                 pattern = re.compile(r"Command Browser\.(\w+) executed")
-                browser_action = {
+                browser_actions.append({
                     'command': pattern.match(memory[index - 1].content).group(1),
                     'current url': browser_url
-                }
-                self.rc.working_memory.add(
-                    Message(content=json.dumps(browser_action), role="user", cause_by="browser")
-                )
+                })
+        if browser_actions:
+            browser_actions = BROWSER_INFO.format(browser_actions=browser_actions)
+            self.rc.working_memory.add(Message(content=browser_actions, role="user", cause_by="browser"))
         return memory
 
     async def write_and_exec_code(self):
diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py
index ed617eb81..671cddc79 100644
--- a/metagpt/roles/di/role_zero.py
+++ b/metagpt/roles/di/role_zero.py
@@ -150,7 +150,8 @@ class RoleZero(Role):
             instruction=self.instruction.strip(),
             task_type_desc=self.task_type_desc,
         )
-        memory = await self.parse_browser_actions()
+        memory = self.rc.memory.get(self.memory_k)
+        memory = await self.parse_browser_actions(memory)
         context = self.llm.format_msg(memory + [UserMessage(content=prompt)])
         # print(*context, sep="\n" + "*" * 5 + "\n")
         async with ThoughtReporter(enable_llm_stream=True):
@@ -159,8 +160,7 @@ class RoleZero(Role):
 
         return True
 
-    async def parse_browser_actions(self):
-        memory = self.rc.memory.get(self.memory_k)
+    async def parse_browser_actions(self, memory: List[Message]) -> List[Message]:
         if not self.browser.is_empty_page:
             pattern = re.compile(r"Command Browser\.(\w+) executed")
             for index, msg in zip(range(len(memory), 0, -1), memory[::-1]):

From 857085e8592ee3f6d49db8d23138823f53d40090 Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Fri, 12 Jul 2024 17:09:11 +0800
Subject: [PATCH 20/22] rename fix to fixed

---
 metagpt/roles/di/data_analyst.py | 4 ++--
 metagpt/tools/tool_recommend.py  | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py
index 2e5315f33..5040385ea 100644
--- a/metagpt/roles/di/data_analyst.py
+++ b/metagpt/roles/di/data_analyst.py
@@ -72,8 +72,8 @@ class DataAnalyst(RoleZero):
         # tool info
         if self.custom_tool_recommender:
             plan = self.planner.plan
-            fix = ["Terminal"] if "Terminal" in self.custom_tools else None
-            tool_info = await self.custom_tool_recommender.get_recommended_tool_info(fix=fix, plan=plan)
+            fixed = ["Terminal"] if "Terminal" in self.custom_tools else None
+            tool_info = await self.custom_tool_recommender.get_recommended_tool_info(fixed=fixed, plan=plan)
         else:
             tool_info = ""
 
diff --git a/metagpt/tools/tool_recommend.py b/metagpt/tools/tool_recommend.py
index b0fd0f39b..4bea137c3 100644
--- a/metagpt/tools/tool_recommend.py
+++ b/metagpt/tools/tool_recommend.py
@@ -104,13 +104,13 @@ class ToolRecommender(BaseModel):
 
         return ranked_tools
 
-    async def get_recommended_tool_info(self, fix: list[str] = None, **kwargs) -> str:
+    async def get_recommended_tool_info(self, fixed: list[str] = None, **kwargs) -> str:
         """
         Wrap recommended tools with their info in a string, which can be used directly in a prompt.
         """
         recommended_tools = await self.recommend_tools(**kwargs)
-        if fix:
-            recommended_tools.extend([self.tools[tool_name] for tool_name in fix if tool_name in self.tools])
+        if fixed:
+            recommended_tools.extend([self.tools[tool_name] for tool_name in fixed if tool_name in self.tools])
         if not recommended_tools:
             return ""
         tool_schemas = {tool.name: tool.schemas for tool in recommended_tools}

From 27e8fdf32065f44f98f72ae2718a7923be387bc7 Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Fri, 12 Jul 2024 18:40:43 +0800
Subject: [PATCH 21/22] recover

---
 metagpt/utils/parse_html.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/metagpt/utils/parse_html.py b/metagpt/utils/parse_html.py
index 031393501..985e54d96 100644
--- a/metagpt/utils/parse_html.py
+++ b/metagpt/utils/parse_html.py
@@ -43,11 +43,11 @@ class WebPage(BaseModel):
         soup = _get_soup(self.html)
         keep_attrs = ["class", "id"]
         if keep_links:
-            keep_attrs.extend(["href", "title"])
+            keep_attrs.append("href")
 
         for i in soup.find_all(True):
             for name in list(i.attrs):
-                if i[name] and name not in keep_attrs and not name.startswith("data-"):
+                if i[name] and name not in keep_attrs:
                     del i[name]
 
         for i in soup.find_all(["svg", "img", "video", "audio"]):

From b214e49733bc2b21375eb3aae7e99c194080c3cc Mon Sep 17 00:00:00 2001
From: lidanyang <lidanyang@fuzhi.ai>
Date: Fri, 12 Jul 2024 18:41:29 +0800
Subject: [PATCH 22/22] change _init_code to public

---
 metagpt/actions/di/execute_nb_code.py | 8 ++++----
 metagpt/roles/di/data_analyst.py      | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/metagpt/actions/di/execute_nb_code.py b/metagpt/actions/di/execute_nb_code.py
index 91480d3a7..f3dfd1601 100644
--- a/metagpt/actions/di/execute_nb_code.py
+++ b/metagpt/actions/di/execute_nb_code.py
@@ -85,12 +85,12 @@ class ExecuteNbCode(Action):
         )
         self.reporter = NotebookReporter()
         self.set_nb_client()
-        self._init_called = False
+        self.init_called = False
 
-    async def _init_code(self):
-        if not self._init_called:
+    async def init_code(self):
+        if not self.init_called:
             await self.run(INI_CODE)
-            self._init_called = True
+            self.init_called = True
 
     def set_nb_client(self):
         self.nb_client = RealtimeOutputNotebookClient(
diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py
index 5040385ea..2b1bb10b1 100644
--- a/metagpt/roles/di/data_analyst.py
+++ b/metagpt/roles/di/data_analyst.py
@@ -64,7 +64,7 @@ class DataAnalyst(RoleZero):
         """Write a code block for current task and execute it in an interactive notebook environment."""
         counter = 0
         success = False
-        await self.execute_code._init_code()
+        await self.execute_code.init_code()
 
         # plan info
         plan_status = self.planner.get_plan_status()