Minecraft game add action_developer

Temp.md

@@ -31,3 +31,13 @@ ### 0926： 环境信息获取和更新 on_event()实际内容
 
 <img src="docs/resources/workspace/minecraft_tests/on_event.jpeg" style="zoom:67%;" />
 
+
+
+### 0927：Action_developer 更新
+
+对应需实现 GenerateActionCode ，完成对应的和 GameEnvironment 的交
+互和 Environment 的信息传递
+
+测试结果  
+
+![action_developer](docs/resources/workspace/minecraft_tests/action_developer.png)

mc_requirements.txt

@@ -0,0 +1,3 @@
+javascript
+requests
+psutil

metagpt/actions/minecraft/generate_actions.py

@@ -4,6 +4,7 @@
 # @Desc    :
 from metagpt.logs import logger
 from metagpt.actions import Action
+from metagpt.utils.minecraft import parse_action_response
 
 
 class GenerateActionCode(Action):
@@ -11,23 +12,33 @@ class GenerateActionCode(Action):
     Action class for generating action code.
     Refer to the code in the voyager/agents/action.py for implementation details.
     """
-    
+
     def __init__(self, name="", context=None, llm=None):
         super().__init__(name, context, llm)
-    
-    async def generate_code(self):
+
+    async def generate_code(self, human_msg, system_msg=[]):
         """
         Generate action code logic.
 
         Implement the logic for generating action code here.
         """
-        return ""
-    
-    async def run(self, human_msg, system_msg=[], *args, **kwargs):
+        rsp = await self._aask(prompt=human_msg, system_msgs=system_msg)
+        parsed_result = parse_action_response(rsp)
+        # logger.info(f"parsed_result is HERE: {parsed_result}")
+
+        try:
+            return parsed_result["program_code"] + "\n" + parsed_result["exec_code"]
+        except:
+            logger.error(f"Failed to parse response: {parsed_result}")
+            return None
+
+    async def run(self, msg, *args, **kwargs):
         logger.info(f"run {self.__repr__()}")
         # Generate action code.
-        generated_code = await self.generate_code()
-        
+        generated_code = await self.generate_code(
+            human_msg=msg['human_msg'], system_msg=msg['system_msg']
+        )
+
         # Return the generated code.
         return generated_code
 
@@ -37,10 +48,10 @@ class SummarizeLog(Action):
     Action class for parsing and summarizing logs.
     Refer to the code in the voyager/agents/action.py for implementation details.
     """
-    
+
     def __init__(self, name="", context=None, llm=None):
         super().__init__(name, context, llm)
-    
+
     async def summarize_logs(self):
         """
         Summarize chatlogs.
@@ -48,10 +59,10 @@ class SummarizeLog(Action):
         Implement the logic for summarizing chatlogs here.
         """
         return ""
-    
+
     async def run(self, *args, **kwargs):
         # Summarize chatlogs.
         summary = await self.summarize_logs()
-        
+
         # Return the summary.
         return summary

metagpt/minecraft_team.py

@@ -6,6 +6,7 @@ from typing import Iterable, Dict, Any
 from pydantic import BaseModel, Field
 import requests
 import json
+import asyncio
 
 from metagpt.logs import logger
 from metagpt.roles import Role
@@ -15,44 +16,53 @@ from metagpt.software_company import SoftwareCompany
 from metagpt.actions.minecraft.player_action import PlayerActions
 from metagpt.roles.minecraft.minecraft_base import Minecraft
 from metagpt.environment import Environment
-from .mineflayer_environment import MineflayerEnv
+from metagpt.mineflayer_environment import MineflayerEnv
+
 
 class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
     """
     游戏环境的记忆，用于多个agent进行信息的共享和缓存，而不需要重复在自己的角色内维护缓存
     """
+
     event: dict[str, Any] = Field(default_factory=dict)
     current_task: str = Field(default="Craft 4 wooden planks")
     task_execution_time: float = Field(default=float)
     context: str = Field(default="")
-
-    code: str = Field(default="")
+    code: str = Field(default=None)
     programs: str = Field(default="")
+    critique: str = Field(default="")
+    skills: list[str] = Field(default_factory=list)
 
-    mf_instance : MineflayerEnv = Field(default_factory=MineflayerEnv)
+    mf_instance: MineflayerEnv = Field(default_factory=MineflayerEnv)
 
     def set_mc_port(self, mc_port):
         self.mf_instance.set_mc_port(mc_port)
-    
+
     def register_roles(self, roles: Iterable[Minecraft]):
         for role in roles:
             role.set_memory(self)
-    
+
     def update_event(self, event: Dict):
         self.event = event
-    
+
     def update_task(self, task: str):
         self.current_task = task
-    
+
     def update_context(self, context: str):
         self.context = context
 
     def update_code(self, code: str):
-        self.code = code
+        self.code = code  # action_developer.gen to HERE
 
     def update_programs(self, programs: str):
         self.programs = programs
 
+    def update_critique(self, critique: str):
+        self.critique = critique  # critic_agent.check_task_success to HERE
+
+    def update_skills(self, skills: list):
+        self.skills = skills  # skill_manager.retrieve_skills to HERE
+
     async def on_event(self, *args):
         """
         Retrieve Minecraft events.
@@ -70,10 +80,12 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
             if not self.mf_instance.has_reset:
                 # TODO Modify
                 logger.info("Environment has not been reset yet, is resetting")
-                self.mf_instance.reset(options={
-                    "mode": "soft",
-                    "wait_ticks": 20,
-                })
+                self.mf_instance.reset(
+                    options={
+                        "mode": "soft",
+                        "wait_ticks": 20,
+                    }
+                )
                 # raise {}
             self.mf_instance.check_process()
             self.mf_instance.unpause()
@@ -82,7 +94,9 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
                 "programs": self.programs,
             }
             res = requests.post(
-                f"{self.mf_instance.server}/step", json=data, timeout=self.mf_instance.request_timeout
+                f"{self.mf_instance.server}/step",
+                json=data,
+                timeout=self.mf_instance.request_timeout,
             )
             if res.status_code != 200:
                 logger.error("Failed to step Minecraft server")
@@ -96,33 +110,37 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
             logger.error(f"Failed to retrieve Minecraft events: {str(e)}")
             raise {}
 
+
 class MinecraftPlayer(SoftwareCompany):
     """
     Software Company: Possesses a team, SOP (Standard Operating Procedures), and a platform for instant messaging,
     dedicated to writing executable code.
     """
+
     environment: Environment = Field(default_factory=Environment)
     game_memory: GameEnvironment = Field(default_factory=GameEnvironment)
     investment: float = Field(default=50.0)
     task: str = Field(default="")
     game_info: dict = Field(default={})
-    
+
     def set_port(self, mc_port):
         self.game_memory.set_mc_port(mc_port)
 
     def hire(self, roles: list[Role]):
         self.environment.add_roles(roles)
         self.game_memory.register_roles(roles)
-    
+
     def start(self, task):
         """Start a project from publishing boss requirement."""
         self.task = task
-        self.environment.publish_message(Message(role="Player", content=task, cause_by=PlayerActions))
+        self.environment.publish_message(
+            Message(role="Player", content=task, cause_by=PlayerActions)
+        )
         logger.info(self.game_info)
-    
+
     def _save(self):
         logger.info(self.json())
-    
+
     async def run(self, n_round=3):
         """Run company until target round or no money"""
         while n_round > 0:
@@ -131,13 +149,19 @@ class MinecraftPlayer(SoftwareCompany):
             logger.debug(f"{n_round=}")
             self._check_balance()
             await self.environment.run()
-        
+
         return self.environment.history
 
-if "__name__" == "__main__":
+
+async def main():
     test_code = "bot.chat(`/time set ${getNextTime()}`);"
-    mc_port = 1960
+    mc_port = 6286
     ge = GameEnvironment()
     ge.set_mc_port(mc_port)
     ge.update_code(test_code)
-    logger.info(ge.on_event())
+    result = await ge.on_event()
+    logger.info("On event test done")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

metagpt/mineflayer_environment.py

@@ -11,6 +11,7 @@ from metagpt.logs import logger
 import metagpt.utils.minecraft as U
 from metagpt.utils.minecraft.process_monitor import SubprocessMonitor
 
+
 class MineflayerEnv:
     def __init__(
         self,
@@ -66,18 +67,21 @@ class MineflayerEnv:
             )
             if res.status_code != 200:
                 self.mineflayer.stop()
-                logger.error(
-                    f"Minecraft server reply with code {res.status_code}"
-                )
+                logger.error(f"Minecraft server reply with code {res.status_code}")
                 raise {}
             return res.json()
 
-    def reset(self, *, seed=None, options=None, ):
+    def reset(
+        self,
+        *,
+        seed=None,
+        options=None,
+    ):
         if options is None:
             options = {}
         if options.get("inventory", {}) and options.get("mode", "hard") != "hard":
             logger.error("inventory can only be set when options is hard")
-            raise{}
+            raise {}
 
         self.reset_options = {
             "port": self.mc_port,
@@ -100,7 +104,7 @@ class MineflayerEnv:
         self.reset_options["reset"] = "soft"
         self.pause()
         return json.loads(returned_data)
-    
+
     def close(self):
         self.unpause()
         if self.connected:

metagpt/prompts/minecraft/action_response_format.txt

@@ -1,15 +0,0 @@
-Explain: ...
-Plan:
-1) ...
-2) ...
-3) ...
-...
-Code:
-```javascript
-// helper functions (only if needed, try to avoid them)
-...
-// main function after the helper functions
-async function yourMainFunctionName(bot) {
-  // ...
-}
-```

metagpt/prompts/minecraft/critic.txt

@@ -1,127 +0,0 @@
-You are an assistant that assesses my progress of playing Minecraft and provides useful guidance.
-
-You are required to evaluate if I have met the task requirements. Exceeding the task requirements is also considered a success while failing to meet them requires you to provide critique to help me improve.
-
-I will give you the following information:
-
-Biome: The biome after the task execution.
-Time: The current time.
-Nearby blocks: The surrounding blocks. These blocks are not collected yet. However, this is useful for some placing or planting tasks.
-Health: My current health.
-Hunger: My current hunger level. For eating task, if my hunger level is 20.0, then I successfully ate the food.
-Position: My current position.
-Equipment: My final equipment. For crafting tasks, I sometimes equip the crafted item.
-Inventory (xx/36): My final inventory. For mining and smelting tasks, you only need to check inventory.
-Chests: If the task requires me to place items in a chest, you can find chest information here.
-Task: The objective I need to accomplish.
-Context: The context of the task.
-
-You should only respond in JSON format as described below:
-{
-    "reasoning": "reasoning",
-    "success": boolean,
-    "critique": "critique",
-}
-Ensure the response can be parsed by Python `json.loads`, e.g.: no trailing commas, no single quotes, etc.
-
-Here are some examples:
-INPUT:
-Inventory (2/36): {'oak_log':2, 'spruce_log':2}
-
-Task: Mine 3 wood logs
-
-RESPONSE:
-{
-    "reasoning": "You need to mine 3 wood logs. You have 2 oak logs and 2 spruce logs, which add up to 4 wood logs.",
-    "success": true,
-    "critique": ""
-}
-
-INPUT:
-Inventory (3/36): {'crafting_table': 1, 'spruce_planks': 6, 'stick': 4}
-
-Task: Craft a wooden pickaxe
-
-RESPONSE:
-{
-    "reasoning": "You have enough materials to craft a wooden pickaxe, but you didn't craft it.",
-    "success": false,
-    "critique": "Craft a wooden pickaxe with a crafting table using 3 spruce planks and 2 sticks."
-}
-
-INPUT:
-Inventory (2/36): {'raw_iron': 5, 'stone_pickaxe': 1}
-
-Task: Mine 5 iron_ore
-
-RESPONSE:
-{
-    "reasoning": "Mining iron_ore in Minecraft will get raw_iron. You have 5 raw_iron in your inventory.",
-    "success": true,
-    "critique": ""
-}
-
-INPUT:
-Biome: plains
-
-Nearby blocks: stone, dirt, grass_block, grass, farmland, wheat
-
-Inventory (26/36): ...
-
-Task:  Plant 1 wheat seed.
-
-RESPONSE:
-{
-    "reasoning": "For planting tasks, inventory information is useless. In nearby blocks, there is farmland and wheat, which means you succeed to plant the wheat seed.",
-    "success": true,
-    "critique": ""
-}
-
-INPUT:
-Inventory (11/36): {... ,'rotten_flesh': 1}
-
-Task: Kill 1 zombie
-
-Context: ...
-
-RESPONSE
-{
-    "reasoning": "You have rotten flesh in your inventory, which means you successfully killed one zombie.",
-    "success": true,
-    "critique": ""
-}
-
-INPUT:
-Hunger: 20.0/20.0
-
-Inventory (11/36): ...
-
-Task: Eat 1 ...
-
-Context: ...
-
-RESPONSE
-{
-    "reasoning": "For all eating task, if the player's hunger is 20.0, then the player successfully ate the food.",
-    "success": true,
-    "critique": ""
-}
-
-INPUT:
-Nearby blocks: chest
-
-Inventory (28/36): {'rail': 1, 'coal': 2, 'oak_planks': 13, 'copper_block': 1, 'diorite': 7, 'cooked_beef': 4, 'granite': 22, 'cobbled_deepslate': 23, 'feather': 4, 'leather': 2, 'cooked_chicken': 3, 'white_wool': 2, 'stick': 3, 'black_wool': 1, 'stone_sword': 2, 'stone_hoe': 1, 'stone_axe': 2, 'stone_shovel': 2, 'cooked_mutton': 4, 'cobblestone_wall': 18, 'crafting_table': 1, 'furnace': 1, 'iron_pickaxe': 1, 'stone_pickaxe': 1, 'raw_copper': 12}
-
-Chests:
-(81, 131, 16): {'andesite': 2, 'dirt': 2, 'cobblestone': 75, 'wooden_pickaxe': 1, 'wooden_sword': 1}
-
-Task: Deposit useless items into the chest at (81, 131, 16)
-
-Context: ...
-
-RESPONSE
-{
-    "reasoning": "You have 28 items in your inventory after depositing, which is more than 20. You need to deposit more items from your inventory to the chest.",
-    "success": false,
-    "critique": "Deposit more useless items such as copper_block, diorite, granite, cobbled_deepslate, feather, and leather to meet the requirement of having only 20 occupied slots in your inventory."
-}

metagpt/prompts/minecraft/curriculum.txt

@@ -1,42 +0,0 @@
-You are a helpful assistant that tells me the next immediate task to do in Minecraft. My ultimate goal is to discover as many diverse things as possible, accomplish as many diverse tasks as possible and become the best Minecraft player in the world.
-
-I will give you the following information:
-Question 1: ...
-Answer: ...
-Question 2: ...
-Answer: ...
-Question 3: ...
-Answer: ...
-...
-Biome: ...
-Time: ...
-Nearby blocks: ...
-Other blocks that are recently seen: ...
-Nearby entities (nearest to farthest): ...
-Health: Higher than 15 means I'm healthy.
-Hunger: Higher than 15 means I'm not hungry.
-Position: ...
-Equipment: If I have better armor in my inventory, you should ask me to equip it.
-Inventory (xx/36): ...
-Chests: You can ask me to deposit or take items from these chests. There also might be some unknown chest, you should ask me to open and check items inside the unknown chest.
-Completed tasks so far: ...
-Failed tasks that are too hard: ...
-
-You must follow the following criteria:
-1) You should act as a mentor and guide me to the next task based on my current learning progress.
-2) Please be very specific about what resources I need to collect, what I need to craft, or what mobs I need to kill.
-3) The next task should follow a concise format, such as "Mine [quantity] [block]", "Craft [quantity] [item]", "Smelt [quantity] [item]", "Kill [quantity] [mob]", "Cook [quantity] [food]", "Equip [item]" etc. It should be a single phrase. Do not propose multiple tasks at the same time. Do not mention anything else.
-4) The next task should not be too hard since I may not have the necessary resources or have learned enough skills to complete it yet.
-5) The next task should be novel and interesting. I should look for rare resources, upgrade my equipment and tools using better materials, and discover new things. I should not be doing the same thing over and over again.
-6) I may sometimes need to repeat some tasks if I need to collect more resources to complete more difficult tasks. Only repeat tasks if necessary.
-7) Do not ask me to build or dig shelter even if it's at night. I want to explore the world and discover new things. I don't want to stay in one place.
-8) Tasks that require information beyond the player's status to verify should be avoided. For instance, "Placing 4 torches" and "Dig a 2x1x2 hole" are not ideal since they require visual confirmation from the screen. All the placing, building, planting, and trading tasks should be avoided. Do not propose task starting with these keywords.
-
-You should only respond in the format as described below:
-RESPONSE FORMAT:
-Reasoning: Based on the information I listed above, do reasoning about what the next task should be.
-Task: The next task.
-
-Here's an example response:
-Reasoning: The inventory is empty now, chop down a tree to get some wood.
-Task: Obtain a wood log.

metagpt/prompts/minecraft/curriculum_qa_step2_answer_questions.txt

@@ -1,8 +0,0 @@
-You are a helpful assistant that answer my question about Minecraft.
-
-I will give you the following information:
-Question: ...
-
-You will answer the question based on the context (only if available and helpful) and your own knowledge of Minecraft.
-1) Start your answer with "Answer: ".
-2) Answer "Answer: Unknown" if you don't know the answer.

metagpt/prompts/minecraft/curriculum_task_decomposition.txt

@@ -1,12 +0,0 @@
-You are a helpful assistant that generates a curriculum of subgoals to complete any Minecraft task specified by me.
-
-I'll give you a final task and my current inventory, you need to decompose the task into a list of subgoals based on my inventory.
-
-You must follow the following criteria:
-1) Return a Python list of subgoals that can be completed in order to complete the specified task.
-2) Each subgoal should follow a concise format, such as "Mine [quantity] [block]", "Craft [quantity] [item]", "Smelt [quantity] [item]", "Kill [quantity] [mob]", "Cook [quantity] [food]", "Equip [item]".
-3) Include each level of necessary tools as a subgoal, such as wooden, stone, iron, diamond, etc.
-
-You should only respond in JSON format as described below:
-["subgoal1", "subgoal2", "subgoal3", ...]
-Ensure the response can be parsed by Python `json.loads`, e.g.: no trailing commas, no single quotes, etc.

metagpt/prompts/minecraft/skill.txt

@@ -1,51 +0,0 @@
-You are a helpful assistant that writes a description of the given function written in Mineflayer javascript code.
-
-1) Do not mention the function name.
-2) Do not mention anything about `bot.chat` or helper functions.
-3) There might be some helper functions before the main function, but you only need to describe the main function.
-4) Try to summarize the function in no more than 6 sentences.
-5) Your response should be a single line of text.
-
-For example, if the function is:
-
-async function mineCobblestone(bot) {
-  // Check if the wooden pickaxe is in the inventory, if not, craft one
-  let woodenPickaxe = bot.inventory.findInventoryItem(mcData.itemsByName["wooden_pickaxe"].id);
-  if (!woodenPickaxe) {
-    bot.chat("Crafting a wooden pickaxe.");
-    await craftWoodenPickaxe(bot);
-    woodenPickaxe = bot.inventory.findInventoryItem(mcData.itemsByName["wooden_pickaxe"].id);
-  }
-
-  // Equip the wooden pickaxe if it exists
-  if (woodenPickaxe) {
-    await bot.equip(woodenPickaxe, "hand");
-
-    // Explore until we find a stone block
-    await exploreUntil(bot, new Vec3(1, -1, 1), 60, () => {
-      const stone = bot.findBlock({
-        matching: mcData.blocksByName["stone"].id,
-        maxDistance: 32
-      });
-      if (stone) {
-        return true;
-      }
-    });
-
-    // Mine 8 cobblestone blocks using the wooden pickaxe
-    bot.chat("Found a stone block. Mining 8 cobblestone blocks.");
-    await mineBlock(bot, "stone", 8);
-    bot.chat("Successfully mined 8 cobblestone blocks.");
-
-    // Save the event of mining 8 cobblestone
-    bot.save("cobblestone_mined");
-  } else {
-    bot.chat("Failed to craft a wooden pickaxe. Cannot mine cobblestone.");
-  }
-}
-
-The main function is `mineCobblestone`.
-
-Then you would write:
-
-The function is about mining 8 cobblestones using a wooden pickaxe. First check if a wooden pickaxe is in the inventory. If not, craft one. If the wooden pickaxe is available, equip the wooden pickaxe in the hand. Next, explore the environment until finding a stone block. Once a stone block is found, mine a total of 8 cobblestone blocks using the wooden pickaxe.

metagpt/roles/minecraft/action_developer.py

@@ -2,13 +2,22 @@
 # @Date    : 2023/9/23 12:45
 # @Author  : stellahong (stellahong@fuzhi.ai)
 # @Desc    :
+import asyncio
+
 from metagpt.logs import logger
 from metagpt.roles.minecraft.minecraft_base import Minecraft as Base
 from metagpt.schema import Message, HumanMessage, SystemMessage
 from metagpt.roles.minecraft.minecraft_base import agent_registry
 from metagpt.actions.minecraft.generate_actions import GenerateActionCode
 from metagpt.actions.minecraft.design_curriculumn import DesignCurriculum
-from metagpt.actions.minecraft.manage_skills import GenerateSkillDescription, RetrieveSkills, AddNewSkills
+from metagpt.actions.minecraft.manage_skills import (
+    GenerateSkillDescription,
+    RetrieveSkills,
+    AddNewSkills,
+)
+import metagpt.utils.minecraft as utils
+from metagpt.config import CONFIG
+from metagpt.minecraft_team import GameEnvironment
 
 
 @agent_registry.register("action_developer")
@@ -17,22 +26,153 @@ class ActionDeveloper(Base):
     iterative prompting mechanism in paper.
     generate action code based on environment observation and plan, as well as skills retrieval results
     """
-    
+
     def __init__(
-            self,
-            name: str = "Bob",
-            profile: str = "Generate code for specified tasks",
-            goal: str = "Produce accurate and efficient code solutions in Python and JavaScript",
-            constraints: str = "Adhere to coding best practices and style guidelines",
+        self,
+        name: str = "Bob",
+        profile: str = "Generate code for specified tasks",
+        goal: str = "Produce accurate and efficient code solutions in Python and JavaScript",
+        constraints: str = "Adhere to coding best practices and style guidelines",
     ) -> None:
         super().__init__(name, profile, goal, constraints)
         # Initialize actions specific to the Action role
         self._init_actions([GenerateActionCode])
-        
+
         # Set events or actions the ActionAgent should watch or be aware of
         # 需要根据events进行自己chest_observation的更新
         self._watch([RetrieveSkills])
-    
+
+    def render_system_message(self, skills=[], *args, **kwargs):
+        """
+        According to basic skills context files to genenarate js skill codes.
+        Refer to @ https://github.com/MineDojo/Voyager/blob/main/voyager/agents/action.py
+        """
+
+        action_template = utils.load_prompt("action_template")
+        base_skills = [
+            "exploreUntil",
+            "mineBlock",
+            "craftItem",
+            "placeItem",
+            "smeltItem",
+            "killMob",
+        ]
+        if not CONFIG.openai_api_model == "gpt-3.5-turbo":
+            base_skills += [
+                "useChest",
+                "mineflayer",
+            ]
+        programs = "\n\n".join(utils.load_skills_code_context(base_skills) + skills)
+        response_format = utils.load_prompt("action_response_format")
+        system_action_prompt = action_template.format(
+            programs=programs, response_format=response_format
+        )
+        system_action_message = SystemMessage(content=system_action_prompt)
+        assert isinstance(system_action_message, SystemMessage)
+        return system_action_message
+
+    def render_human_message(
+        self, events, code="", task="", context="", critique="", *args, **kwargs
+    ):
+        """
+        Integrate observation about the environment(especially events), add to HumanMessage.
+        Refer to @ https://github.com/MineDojo/Voyager/blob/main/voyager/agents/action.py
+        """
+
+        # Deal with events info
+        chat_messages = []
+        error_messages = []
+        # damage_messages = [] # TODO: try to add damage_messages into prompt later
+        assert events[-1][0] == "observe", "Last event must be observe"
+
+        for i, (event_type, event) in enumerate(events):
+            if event_type == "onChat":
+                chat_messages.append(event["onChat"])
+            elif event_type == "onError":
+                error_messages.append(event["onError"])
+            elif event_type == "observe":
+                biome = event["status"]["biome"]
+                time_of_day = event["status"]["timeOfDay"]
+                voxels = event["voxels"]
+                entities = event["status"]["entities"]
+                health = event["status"]["health"]
+                hunger = event["status"]["food"]
+                position = event["status"]["position"]
+                equipment = event["status"]["equipment"]
+                inventory_used = event["status"]["inventoryUsed"]
+                inventory = event["inventory"]
+                assert i == len(events) - 1, "observe must be the last event"
+
+        # Collect all the environment information into a str: observation
+        observation = ""
+
+        observation = (
+            f"Code from the last round:\n{code or 'No code in the first round'}\n\n"
+        )
+
+        if error_messages:
+            error = "\n".join(error_messages)
+            observation += f"Execution error:\n{error}\n\n"
+        else:
+            observation += f"Execution error: No error\n\n"
+
+        if chat_messages:
+            chat_log = "\n".join(chat_messages)
+            observation += f"Chat log: {chat_log}\n\n"
+        else:
+            observation += f"Chat log: None\n\n"
+
+        observation += f"Biome: {biome}\n\n"
+        observation += f"Time: {time_of_day}\n\n"
+        observation += f"Nearby blocks: {', '.join(voxels) if voxels else 'None'}\n\n"
+
+        if entities:
+            nearby_entities = [
+                k for k, v in sorted(entities.items(), key=lambda x: x[1])
+            ]
+            observation += f"Nearby entities (nearest to farthest): {', '.join(nearby_entities)}\n\n"
+        else:
+            observation += f"Nearby entities (nearest to farthest): None\n\n"
+
+        observation += f"Health: {health:.1f}/20\n\n"
+        observation += f"Hunger: {hunger:.1f}/20\n\n"
+        observation += f"Position: x={position['x']:.1f}, y={position['y']:.1f}, z={position['z']:.1f}\n\n"
+        observation += f"Equipment: {equipment}\n\n"
+        observation += f"Inventory ({inventory_used}/36): {'Empty' if not inventory else ', '.join(inventory)}\n\n"
+
+        if not (
+            task == "Place and deposit useless items into a chest"
+            or task.startswith("Deposit useless items into the chest at")
+        ):
+            # TODO: observation += self.render_chest_observation()
+            logger.warning("chest_observation will add later")
+
+        observation += f"Task: {task}\n\n"
+        observation += f"Context: {context or 'None'}\n\n"
+        observation += f"Critique: {critique or 'None'}\n\n"
+
+        return HumanMessage(content=observation)
+
+    def encapsule_message(
+        self,
+        events,
+        code="",
+        task="",
+        context="",
+        critique="",
+        skills=[],
+        *args,
+        **kwargs,
+    ):
+        system_message = self.render_system_message(skills=skills)
+        human_message = self.render_human_message(
+            events=events, code=code, task=task, context=context, critique=critique
+        )
+        return {
+            "system_msg": [system_message.content],
+            "human_msg": human_message.content,
+        }
+
     async def _observe(self) -> int:
         await super()._observe()
         for msg in self._rc.news:
@@ -42,26 +182,42 @@ class ActionDeveloper(Base):
         ]  # only relevant msgs count as observed news
         logger.info(len(self._rc.news))
         return len(self._rc.news)
-    
-    async def generate_action_code(self, human_msg, system_msg, *args, **kwargs):
-        code = await GenerateActionCode().run(human_msg)
-        logger.info(code)
-        msg = Message(content=f"test_action", instruct_content="generate_action_code", role=self.profile)
+
+    async def generate_action_code(self, msg, *args, **kwargs):
+        code = await GenerateActionCode().run(msg, *args, **kwargs)
+        # logger.warning(type(code))
+        # logger.info(f"Code is Here:{code}")
+        self.perform_game_info_callback(code, self.game_memory.update_code)
+        msg = Message(
+            content=f"{code}",
+            instruct_content="generate_action_code",
+            role=self.profile,
+        )
         logger.info(msg)
         return msg
-        
+
     async def _act(self) -> Message:
         todo = self._rc.todo
         logger.debug(f"Todo is {todo}")
-        
+
         # 获取最新的游戏周边信息
+        events = await self._obtain_events()
         context = self.game_memory.context
         task = self.game_memory.current_task
+        code = self.game_memory.code
+        critique = self.game_memory.critique
+        skills = self.game_memory.skills
 
-        message = self.encapsule_message(task, context)
+        message = self.encapsule_message(
+            events=events,
+            code=code,
+            task=task,
+            context=context,
+            critique=critique,
+            skills=skills,
+        )
         logger.info(todo)
         handler_map = {
-
             GenerateActionCode: self.generate_action_code,
         }
         handler = handler_map.get(type(todo))
@@ -69,10 +225,95 @@ class ActionDeveloper(Base):
 
         if handler:
             msg = await handler(**message)
-            logger.info(msg)
             msg.cause_by = type(todo)
             logger.info(msg.send_to)
             self._publish_message(msg)
             return msg
-        
-        raise ValueError(f"Unknown todo type: {type(todo)}")
+
+        raise ValueError(f"Unknown todo type: {type(todo)}")
+
+
+async def main():
+    events = [
+        [
+            "observe",
+            {
+                "voxels": ["grass_block", "dirt", "grass"],
+                "status": {
+                    "health": 20,
+                    "food": 20,
+                    "saturation": 5,
+                    "oxygen": 20,
+                    "position": {"x": 0.5, "y": 84, "z": -207.5},
+                    "velocity": {"x": 0, "y": -0.0784000015258789, "z": 0},
+                    "yaw": 3.141592653589793,
+                    "pitch": 0,
+                    "onGround": True,
+                    "equipment": [None, None, None, None, None, None],
+                    "name": "bot",
+                    "isInWater": False,
+                    "isInLava": False,
+                    "isCollidedHorizontally": False,
+                    "isCollidedVertically": True,
+                    "biome": "plains",
+                    "entities": {
+                        "chicken": 29.071822119730644,
+                        "sheep": 20.361212992763768,
+                    },
+                    "timeOfDay": "day",
+                    "inventoryUsed": 0,
+                    "elapsedTime": 41,
+                },
+                "inventory": {},
+                "nearbyChests": {},
+                "blockRecords": ["grass_block", "dirt", "grass"],
+            },
+        ]
+    ]
+    code = """
+    collectBamboo.code	async function collectBamboo(bot) {
+    // Equip the iron sword
+    const ironSword = bot.inventory.findInventoryItem(mcData.itemsByName.iron_sword.id);
+    await bot.equip(ironSword, "hand");
+
+    // Find bamboo plants using the exploreUntil function
+    const bambooPlants = await exploreUntil(bot, new Vec3(1, 0, 1), 60, () => {
+        const bambooPlants = bot.findBlocks({
+        matching: block => block.name === "bamboo",
+        maxDistance: 32,
+        count: 10
+        });
+        return bambooPlants.length >= 10 ? bambooPlants : null;
+    });
+    if (!bambooPlants) {
+        bot.chat("Could not find enough bamboo plants.");
+        return;
+    }
+
+    // Break 10 bamboo plants using the iron sword
+    for (const bambooPlant of bambooPlants) {
+        const block = bot.blockAt(bambooPlant);
+        await bot.dig(block);
+    }
+    bot.chat("Broke 10 bamboo plants.");
+
+    // Collect the dropped bamboo items
+    for (const bambooPlant of bambooPlants) {
+        await bot.pathfinder.goto(new GoalBlock(bambooPlant.x, bambooPlant.y, bambooPlant.z));
+    }
+    bot.chat("Collected 10 bamboo.");
+    }
+    """
+    ad = ActionDeveloper()
+    ge = GameEnvironment()
+    ad.set_memory(shared_memory=ge)
+    msg = ad.encapsule_message(events=events, code=code)
+    logger.info(f"Encapsuled_message: {msg}")
+
+    parsed_result = await ad.generate_action_code(msg)
+
+    logger.info(f"Parsed_code_updating: {parsed_result}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

metagpt/roles/minecraft/minecraft_base.py

@@ -8,7 +8,6 @@ import json
 from metagpt.logs import logger
 from metagpt.roles.role import Role
 from metagpt.schema import HumanMessage, SystemMessage
-
 from typing import Dict
 
 from pydantic import BaseModel

metagpt/utils/minecraft/__init__.py

@@ -4,4 +4,6 @@
 # @Desc    :
 from .load_prompts import load_prompt
 from .json_utils import *
-from .file_utils import *
+from .file_utils import *
+from .load_skills_code_context import load_skills_code_context
+from .action_rsp_parser import parse_action_response

metagpt/utils/minecraft/action_rsp_parser.py

@@ -0,0 +1,69 @@
+import re
+import time
+from javascript import require
+
+
+def parse_action_response(msg: str):
+    """
+    Return:
+    {
+        "program_code": program_code,
+        "program_name": main_function["name"],
+        "exec_code": exec_code,
+    }
+    Refer to @ https://github.com/MineDojo/Voyager/blob/main/voyager/agents/action.py
+    """
+
+    retry = 3
+    error = None
+    while retry > 0:
+        try:
+            babel = require("@babel/core")
+            babel_generator = require("@babel/generator").default
+
+            code_pattern = re.compile(r"```(?:javascript|js)(.*?)```", re.DOTALL)
+            code = "\n".join(code_pattern.findall(msg))
+            parsed = babel.parse(code)
+            functions = []
+            assert len(list(parsed.program.body)) > 0, "No functions found"
+            for i, node in enumerate(parsed.program.body):
+                if node.type != "FunctionDeclaration":
+                    continue
+                node_type = (
+                    "AsyncFunctionDeclaration"
+                    if node["async"]
+                    else "FunctionDeclaration"
+                )
+                functions.append(
+                    {
+                        "name": node.id.name,
+                        "type": node_type,
+                        "body": babel_generator(node).code,
+                        "params": list(node["params"]),
+                    }
+                )
+            # find the last async function
+            main_function = None
+            for function in reversed(functions):
+                if function["type"] == "AsyncFunctionDeclaration":
+                    main_function = function
+                    break
+            assert (
+                main_function is not None
+            ), "No async function found. Your main function must be async."
+            assert (
+                len(main_function["params"]) == 1
+                and main_function["params"][0].name == "bot"
+            ), f"Main function {main_function['name']} must take a single argument named 'bot'"
+            program_code = "\n\n".join(function["body"] for function in functions)
+            exec_code = f"await {main_function['name']}(bot);"
+            return {
+                "program_code": program_code,
+                "program_name": main_function["name"],
+                "exec_code": exec_code,
+            }
+        except Exception as e:
+            retry -= 1
+            error = e
+            time.sleep(1)
+    return f"Error parsing action response (before program execution): {error}"

metagpt/utils/minecraft/file_utils.py

@@ -15,6 +15,7 @@ is_dir = os.path.isdir
 
 get_dir = os.path.dirname
 
+
 def is_sequence(obj):
     """
     Returns:
@@ -78,7 +79,8 @@ def load_text(*fpaths, by_lines=False):
 def load_text_lines(*fpaths):
     return load_text(*fpaths, by_lines=True)
 
+
 # aliases to be consistent with other load_* and dump_*
 text_load = load_text
 read_text = load_text
-read_text_lines = load_text_lines
+read_text_lines = load_text_lines

metagpt/utils/minecraft/json_utils.py

@@ -8,6 +8,7 @@ import re
 from typing import Any, Dict, Union
 from .file_utils import f_join
 
+
 def json_load(*file_path, **kwargs):
     file_path = f_join(file_path)
     with open(file_path, "r") as fp:
@@ -144,6 +145,7 @@ def correct_json(json_str: str) -> str:
             return balanced_str
     return json_str
 
+
 def fix_and_parse_json(
     json_str: str, try_to_fix_with_gpt: bool = True
 ) -> Union[str, Dict[Any, Any]]:
@@ -164,4 +166,4 @@ def fix_and_parse_json(
         json_str = json_str[: last_brace_index + 1]
         return json.loads(json_str)
     except json.JSONDecodeError as e:  # noqa: F841
-        raise e
+        raise e

metagpt/utils/minecraft/load_prompts.py

@@ -4,7 +4,8 @@
 # @Desc    :
 import pkg_resources
 from .file_utils import load_text
-        
+
+
 def load_prompt(prompt):
     package_path = pkg_resources.resource_filename("metagpt", "")
-    return load_text(f"{package_path}/prompts/minecraft/{prompt}.txt")
+    return load_text(f"{package_path}/prompts/minecraft/{prompt}.txt")

metagpt/utils/minecraft/load_skills_code_context.py

@@ -0,0 +1,21 @@
+import os
+import metagpt.utils.minecraft as utils
+from metagpt.logs import logger
+
+
+def load_skills_code_context(skill_names=None):
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    skills_dir = os.path.join(current_dir, "skills_code_context")
+    if skill_names is None:
+        skill_names = [
+            skill[:-3] for skill in os.listdir(f"{skills_dir}") if skill.endswith(".js")
+        ]
+    skills = [
+        utils.load_text(os.path.join(skills_dir, f"{skill_name}.js"))
+        for skill_name in skill_names
+    ]
+    return skills
+
+
+if __name__ == "__main__":
+    logger.info(load_skills_code_context(["craftItem", "exploreUntil"]))

metagpt/utils/minecraft/skills_code_context/.prettierrc.json

@@ -0,0 +1,3 @@
+{
+    "tabWidth": 4
+}

metagpt/utils/minecraft/skills_code_context/craftItem.js

@@ -0,0 +1,14 @@
+// Craft 8 oak_planks from 2 oak_log (do the recipe 2 times): craftItem(bot, "oak_planks", 2);
+// You must place a crafting table before calling this function
+async function craftItem(bot, name, count = 1) {
+    const item = mcData.itemsByName[name];
+    const craftingTable = bot.findBlock({
+        matching: mcData.blocksByName.crafting_table.id,
+        maxDistance: 32,
+    });
+    await bot.pathfinder.goto(
+        new GoalLookAtBlock(craftingTable.position, bot.world)
+    );
+    const recipe = bot.recipesFor(item.id, null, 1, craftingTable)[0];
+    await bot.craft(recipe, count, craftingTable);
+}

metagpt/utils/minecraft/skills_code_context/exploreUntil.js

@@ -0,0 +1,31 @@
+/*
+Explore until find an iron_ore, use Vec3(0, -1, 0) because iron ores are usually underground
+await exploreUntil(bot, new Vec3(0, -1, 0), 60, () => {
+    const iron_ore = bot.findBlock({
+        matching: mcData.blocksByName["iron_ore"].id,
+        maxDistance: 32,
+    });
+    return iron_ore;
+});
+
+Explore until find a pig, use Vec3(1, 0, 1) because pigs are usually on the surface
+let pig = await exploreUntil(bot, new Vec3(1, 0, 1), 60, () => {
+    const pig = bot.nearestEntity((entity) => {
+        return (
+            entity.name === "pig" &&
+            entity.position.distanceTo(bot.entity.position) < 32
+        );
+    });
+    return pig;
+});
+*/
+async function exploreUntil(bot, direction, maxTime = 60, callback) {
+    /*
+    Implementation of this function is omitted.
+    direction: Vec3, can only contain value of -1, 0 or 1
+    maxTime: number, the max time for exploration
+    callback: function, early stop condition, will be called each second, exploration will stop if return value is not null
+
+    Return: null if explore timeout, otherwise return the return value of callback
+    */
+}

metagpt/utils/minecraft/skills_code_context/killMob.js

@@ -0,0 +1,12 @@
+// Kill a pig and collect the dropped item: killMob(bot, "pig", 300);
+async function killMob(bot, mobName, timeout = 300) {
+    const entity = bot.nearestEntity(
+        (entity) =>
+            entity.name === mobName &&
+            entity.position.distanceTo(bot.entity.position) < 32
+    );
+    await bot.pvp.attack(entity);
+    await bot.pathfinder.goto(
+        new GoalBlock(entity.position.x, entity.position.y, entity.position.z)
+    );
+}

metagpt/utils/minecraft/skills_code_context/mineBlock.js

@@ -0,0 +1,15 @@
+// Mine 3 cobblestone: mineBlock(bot, "stone", 3);
+async function mineBlock(bot, name, count = 1) {
+    const blocks = bot.findBlocks({
+        matching: (block) => {
+            return block.name === name;
+        },
+        maxDistance: 32,
+        count: count,
+    });
+    const targets = [];
+    for (let i = 0; i < Math.min(blocks.length, count); i++) {
+        targets.push(bot.blockAt(blocks[i]));
+    }
+    await bot.collectBlock.collect(targets, { ignoreNoPath: true });
+}

metagpt/utils/minecraft/skills_code_context/mineflayer.js

@@ -0,0 +1,22 @@
+await bot.pathfinder.goto(goal); // A very useful function. This function may change your main-hand equipment.
+// Following are some Goals you can use:
+new GoalNear(x, y, z, range); // Move the bot to a block within the specified range of the specified block. `x`, `y`, `z`, and `range` are `number`
+new GoalXZ(x, z); // Useful for long-range goals that don't have a specific Y level. `x` and `z` are `number`
+new GoalGetToBlock(x, y, z); // Not get into the block, but get directly adjacent to it. Useful for fishing, farming, filling bucket, and beds. `x`, `y`, and `z` are `number`
+new GoalFollow(entity, range); // Follow the specified entity within the specified range. `entity` is `Entity`, `range` is `number`
+new GoalPlaceBlock(position, bot.world, {}); // Position the bot in order to place a block. `position` is `Vec3`
+new GoalLookAtBlock(position, bot.world, {}); // Path into a position where a blockface of the block at position is visible. `position` is `Vec3`
+
+// These are other Mineflayer functions you can use:
+bot.isABed(bedBlock); // Return true if `bedBlock` is a bed
+bot.blockAt(position); // Return the block at `position`. `position` is `Vec3`
+
+// These are other Mineflayer async functions you can use:
+await bot.equip(item, destination); // Equip the item in the specified destination. `item` is `Item`, `destination` can only be "hand", "head", "torso", "legs", "feet", "off-hand"
+await bot.consume(); // Consume the item in the bot's hand. You must equip the item to consume first. Useful for eating food, drinking potions, etc.
+await bot.fish(); // Let bot fish. Before calling this function, you must first get to a water block and then equip a fishing rod. The bot will automatically stop fishing when it catches a fish
+await bot.sleep(bedBlock); // Sleep until sunrise. You must get to a bed block first
+await bot.activateBlock(block); // This is the same as right-clicking a block in the game. Useful for buttons, doors, etc. You must get to the block first
+await bot.lookAt(position); // Look at the specified position. You must go near the position before you look at it. To fill bucket with water, you must lookAt first. `position` is `Vec3`
+await bot.activateItem(); // This is the same as right-clicking to use the item in the bot's hand. Useful for using buckets, etc. You must equip the item to activate first
+await bot.useOn(entity); // This is the same as right-clicking an entity in the game. Useful for shearing sheep, equipping harnesses, etc. You must get to the entity first

metagpt/utils/minecraft/skills_code_context/placeItem.js

@@ -0,0 +1,28 @@
+// Place a crafting_table near the player, Vec3(1, 0, 0) is just an example, you shouldn't always use that: placeItem(bot, "crafting_table", bot.entity.position.offset(1, 0, 0));
+async function placeItem(bot, name, position) {
+    const item = bot.inventory.findInventoryItem(mcData.itemsByName[name].id);
+    // find a reference block
+    const faceVectors = [
+        new Vec3(0, 1, 0),
+        new Vec3(0, -1, 0),
+        new Vec3(1, 0, 0),
+        new Vec3(-1, 0, 0),
+        new Vec3(0, 0, 1),
+        new Vec3(0, 0, -1),
+    ];
+    let referenceBlock = null;
+    let faceVector = null;
+    for (const vector of faceVectors) {
+        const block = bot.blockAt(position.minus(vector));
+        if (block?.name !== "air") {
+            referenceBlock = block;
+            faceVector = vector;
+            break;
+        }
+    }
+    // You must first go to the block position you want to place
+    await bot.pathfinder.goto(new GoalPlaceBlock(position, bot.world, {}));
+    // You must equip the item right before calling placeBlock
+    await bot.equip(item, "hand");
+    await bot.placeBlock(referenceBlock, faceVector);
+}

metagpt/utils/minecraft/skills_code_context/smeltItem.js

@@ -0,0 +1,22 @@
+// Smelt 1 raw_iron into 1 iron_ingot using 1 oak_planks as fuel: smeltItem(bot, "raw_iron", "oak_planks");
+// You must place a furnace before calling this function
+async function smeltItem(bot, itemName, fuelName, count = 1) {
+    const item = mcData.itemsByName[itemName];
+    const fuel = mcData.itemsByName[fuelName];
+    const furnaceBlock = bot.findBlock({
+        matching: mcData.blocksByName.furnace.id,
+        maxDistance: 32,
+    });
+    await bot.pathfinder.goto(
+        new GoalLookAtBlock(furnaceBlock.position, bot.world)
+    );
+    const furnace = await bot.openFurnace(furnaceBlock);
+    for (let i = 0; i < count; i++) {
+        await furnace.putFuel(fuel.id, null, 1);
+        await furnace.putInput(item.id, null, 1);
+        // Wait 12 seconds for the furnace to smelt the item
+        await bot.waitForTicks(12 * 20);
+        await furnace.takeOutput();
+    }
+    await furnace.close();
+}

metagpt/utils/minecraft/skills_code_context/useChest.js

@@ -0,0 +1,35 @@
+// Get a torch from chest at (30, 65, 100): getItemFromChest(bot, new Vec3(30, 65, 100), {"torch": 1});
+// This function will work no matter how far the bot is from the chest.
+async function getItemFromChest(bot, chestPosition, itemsToGet) {
+    await moveToChest(bot, chestPosition);
+    const chestBlock = bot.blockAt(chestPosition);
+    const chest = await bot.openContainer(chestBlock);
+    for (const name in itemsToGet) {
+        const itemByName = mcData.itemsByName[name];
+        const item = chest.findContainerItem(itemByName.id);
+        await chest.withdraw(item.type, null, itemsToGet[name]);
+    }
+    await closeChest(bot, chestBlock);
+}
+// Deposit a torch into chest at (30, 65, 100): depositItemIntoChest(bot, new Vec3(30, 65, 100), {"torch": 1});
+// This function will work no matter how far the bot is from the chest.
+async function depositItemIntoChest(bot, chestPosition, itemsToDeposit) {
+    await moveToChest(bot, chestPosition);
+    const chestBlock = bot.blockAt(chestPosition);
+    const chest = await bot.openContainer(chestBlock);
+    for (const name in itemsToDeposit) {
+        const itemByName = mcData.itemsByName[name];
+        const item = bot.inventory.findInventoryItem(itemByName.id);
+        await chest.deposit(item.type, null, itemsToDeposit[name]);
+    }
+    await closeChest(bot, chestBlock);
+}
+// Check the items inside the chest at (30, 65, 100): checkItemInsideChest(bot, new Vec3(30, 65, 100));
+// You only need to call this function once without any action to finish task of checking items inside the chest.
+async function checkItemInsideChest(bot, chestPosition) {
+    await moveToChest(bot, chestPosition);
+    const chestBlock = bot.blockAt(chestPosition);
+    await bot.openContainer(chestBlock);
+    // You must close the chest after opening it if you are asked to open a chest
+    await closeChest(bot, chestBlock);
+}