Minecraft game add skill_manager & ignore bug fix

.gitignore

@@ -16,7 +16,8 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
+lib/* 
+!/metagpt/mineflayer_env/mineflayer/lib/* # Mineflayer 
 lib64/
 parts/
 sdist/

Temp.md

@@ -2,11 +2,11 @@ ## MG-MC记录文档
 
 ### 0926： 环境信息获取和更新 on_event()实际内容
 
-1. Nodejs + Mineflayer配置
+1.Nodejs + Mineflayer配置
 
   A.自行安装[Node.js (nodejs.org)](https://nodejs.org/en)
 
-  B.Mineflayer配置
+  B.clone完之后，必须重新继续Mineflayer配置
 
   ```bash
   cd metagpt/mineflayer_env/mineflayer
@@ -19,9 +19,13 @@ ### 0926： 环境信息获取和更新 on_event()实际内容
   npm install
   ```
 
+2.在mg环境上额外执行
 
+```python
+pip install -r mc_requirement.txt
+```
 
-2.配置完游戏后，在 minecraft_run.py 下修改
+3.配置完游戏后，在 minecraft_run.py 下修改
 
 ```python
     mc_player.set_port(2465) # Modify this to your LAN port
@@ -41,3 +45,25 @@ ### 0927：Action_developer 更新
 测试结果  
 
 ![action_developer](docs/resources/workspace/minecraft_tests/action_developer.png)
+
+
+
+### 0930：Curriculum agent 更新
+
+对应需实现 DesignTask和DesignCurriculum，以及与Environment 的信息传递。
+
+
+
+**BUG FIX(0930)：**
+
+A.在前面的提交中，由于ignore了mineflayer下的lib，会造成如下报错
+
+```bash
+metagpt.minecraft_team:on_event:143 - Failed to retrieve Minecraft events: HTTPConnectionPool(host='127.0.0.1', port=3000): Max retries exceeded with url: /start (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fa7a0556130>: Failed to establish a new connection: [Errno 111] Connection refused'))
+```
+
+解决方法：
+
+1. 若本地已克隆项目不好更改可尝试：删除 metagpt/mineflayer_env/mineflayer + 重新copy voyager/env/mineflayer到目录下 + （npm install...0926.B命令）
+2. 重新拉取最新提交+重新配置
+

mc_requirements.txt

@@ -1,3 +1,4 @@
 javascript
 requests
-psutil
+psutil
+chromadb==0.3.29

metagpt/actions/minecraft/design_curriculumn.py

@@ -101,20 +101,19 @@ class DesignCurriculum(Action):
         # TODO: change to FaissStore
         # self.qa_cache_questions_vectordb = FaissStore( {CKPT_DIR}/ 'curriculum/vectordb')
 
-        # Check if qa_cache right using
-        assert self.qa_cache_questions_vectordb._collection.count() == len(
-            self.qa_cache
-        ), (
-            f"Curriculum Agent's qa cache question vectordb is not synced with qa_cache.json.\n"
-            f"There are {self.qa_cache_questions_vectordb._collection.count()} questions in vectordb "
-            f"but {len(self.qa_cache)} questions in qa_cache.json.\n"
-            f"Did you set resume=False when initializing the agent?\n"
-            f"You may need to manually delete the qa cache question vectordb directory for running from scratch.\n"
-        )
-
     @classmethod
     def set_qa_cache(cls, qa_cache):
         cls.qa_cache = qa_cache
+        # Check if qa_cache right using
+        assert cls.qa_cache_questions_vectordb._collection.count() == len(
+            cls.qa_cache
+        ), (
+            f"Curriculum Agent's qa cache question vectordb is not synced with qa_cache.json.\n"
+            f"There are {cls.qa_cache_questions_vectordb._collection.count()} questions in vectordb "
+            f"but {len(cls.qa_cache)} questions in qa_cache.json.\n"
+            f"Did you set resume=False when initializing the agent?\n"
+            f"You may need to manually delete the qa cache question vectordb directory for running from scratch.\n"
+        )
 
     @classmethod
     def generate_qa(cls, events, chest_observation):

metagpt/actions/minecraft/generate_actions.py

@@ -27,20 +27,23 @@ class GenerateActionCode(Action):
         # logger.info(f"parsed_result is HERE: {parsed_result}")
 
         try:
-            return parsed_result["program_code"] + "\n" + parsed_result["exec_code"]
+            return (
+                parsed_result["program_code"] + "\n" + parsed_result["exec_code"],
+                parsed_result["program_name"],
+            )
         except:
             logger.error(f"Failed to parse response: {parsed_result}")
-            return None
+            return None, None
 
     async def run(self, human_msg, system_msg, *args, **kwargs):
         logger.info(f"run {self.__repr__()}")
         # Generate action code.
-        generated_code = await self.generate_code(
+        generated_code, program_name = await self.generate_code(
             human_msg=human_msg, system_msg=system_msg
         )
 
         # Return the generated code.
-        return generated_code
+        return generated_code, program_name
 
 
 class SummarizeLog(Action):

metagpt/actions/minecraft/manage_skills.py

@@ -2,9 +2,15 @@
 # @Date    : 2023/9/23 14:56
 # @Author  : stellahong (stellahong@fuzhi.ai)
 # @Desc    :
+import os
+import json
 
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import Chroma
+from metagpt.document_store import FaissStore
 from metagpt.logs import logger
 from metagpt.actions import Action
+from metagpt.const import CKPT_DIR
 
 
 class RetrieveSkills(Action):
@@ -12,14 +18,44 @@ class RetrieveSkills(Action):
     Action class for retrieving skills.
     Refer to the code in the voyager/agents/skill.py for implementation details.
     """
-    
+
     def __init__(self, name="", context=None, llm=None):
         super().__init__(name, context, llm)
-        self.vect_db = ""
-    
-    async def run(self, *args, **kwargs):
+        # TODO: mv to PlayerAction
+        self.retrieval_top_k = 5
+        self.skills = {}
+        self.vectordb = Chroma(
+            collection_name="skill_vectordb",
+            embedding_function=OpenAIEmbeddings(),
+            persist_directory=f"{CKPT_DIR}/skill/vectordb",
+        )
+
+    @classmethod
+    def set_skills(cls, skills):
+        cls.skills = skills
+        # Check if skills right using
+        assert cls.vectordb._collection.count() == len(cls.skills), (
+            f"Skill Manager's vectordb is not synced with skills.json.\n"
+            f"There are {cls.vectordb._collection.count()} skills in vectordb but {len(cls.skills)} skills in skills.json.\n"
+            f"Did you set resume=False when initializing the manager?\n"
+            f"You may need to manually delete the vectordb directory for running from scratch."
+        )
+
+    async def run(self, query, *args, **kwargs):
         # Implement the logic for retrieving skills here.
-        return []
+        k = min(self.vectordb._collection.count(), self.retrieval_top_k)
+        if k == 0:
+            return []
+        logger.info(f"Skill Manager retrieving for {k} skills")
+        docs_and_scores = self.vectordb.similarity_search_with_score(query, k=k)
+        logger.info(
+            f"Skill Manager retrieved skills: "
+            f"{', '.join([doc.metadata['name'] for doc, _ in docs_and_scores])}"
+        )
+        skills = []
+        for doc, _ in docs_and_scores:
+            skills.append(self.skills[doc.metadata["name"]]["code"])
+        return skills
 
 
 class AddNewSkills(Action):
@@ -27,13 +63,71 @@ class AddNewSkills(Action):
     Action class for adding new skills.
     Refer to the code in the voyager/agents/skill.py for implementation details.
     """
-    
+
     def __init__(self, name="", context=None, llm=None):
         super().__init__(name, context, llm)
-    
-    async def run(self, *args, **kwargs):
+        # TODO: mv to PlayerAction
+        self.vectordb = Chroma(
+            collection_name="skill_vectordb",
+            embedding_function=OpenAIEmbeddings(),
+            persist_directory=f"{CKPT_DIR}/skill/vectordb",
+        )
+        # TODO: change to FaissStore
+        # self.qa_cache_questions_vectordb = FaissStore( {CKPT_DIR}/ 'skill/vectordb')
+
+    @classmethod
+    def set_skills(cls, skills):
+        cls.skills = skills
+        # Check if skills right using
+        assert cls.vectordb._collection.count() == len(cls.skills), (
+            f"Skill Manager's vectordb is not synced with skills.json.\n"
+            f"There are {cls.vectordb._collection.count()} skills in vectordb but {len(cls.skills)} skills in skills.json.\n"
+            f"Did you set resume=False when initializing the manager?\n"
+            f"You may need to manually delete the vectordb directory for running from scratch."
+        )
+
+    async def run(
+        self, task, program_name, program_code, skills, skill_desp, *args, **kwargs
+    ):
         # Implement the logic for adding new skills here.
-        pass
+        if task.startswith("Deposit useless items into the chest at"):
+            # No need to reuse the deposit skill
+            return {}
+        # TODO: Fix this
+        logger.info(
+            f"Skill Manager generated description for {program_name}:\n{skill_desp}\033[0m"
+        )
+        if program_name in skills:
+            logger.info(f"Skill {program_name} already exists. Rewriting!")
+            self.vectordb._collection.delete(ids=[program_name])
+            i = 2
+            while f"{program_name}V{i}.js" in os.listdir(f"{CKPT_DIR}/skill/code"):
+                i += 1
+            dumped_program_name = f"{program_name}V{i}"
+        else:
+            dumped_program_name = program_name
+        self.vectordb.add_texts(
+            texts=[skill_desp],
+            ids=[program_name],
+            metadatas=[{"name": program_name}],
+        )
+
+        # FIXME
+        # assert self.vectordb._collection.count() == len(
+        #     skills
+        # ), "vectordb is not synced with skills.json"
+
+        with open(f"{CKPT_DIR}/skill/code/{dumped_program_name}.js", "w") as f:
+            f.write(program_code)
+        with open(f"{CKPT_DIR}/skill/description/{dumped_program_name}.txt", "w") as f:
+            f.write(skill_desp)
+        with open(f"{CKPT_DIR}/skill/skills.json", "w") as f:
+            json.dump(skills, f)
+        self.vectordb.persist()
+        return {
+            "code": program_code,
+            "description": skill_desp,
+        }
 
 
 class GenerateSkillDescription(Action):
@@ -41,13 +135,12 @@ class GenerateSkillDescription(Action):
     Action class for generating skill descriptions.
     Refer to the code in the voyager/agents/skill.py for implementation details.
     """
-    
+
     def __init__(self, name="", context=None, llm=None):
         super().__init__(name, context, llm)
-    
-    async def run(self, *args, **kwargs):
-        # Implement the logic for generating skill descriptions here.
-        pass
 
-    
-    
+    async def run(self, program_name, human_message, system_message, *args, **kwargs):
+        # Implement the logic for generating skill descriptions here.
+        rsp = await self._aask(prompt=human_message, system_msgs=system_message)
+        skill_description = f"    // { rsp}"
+        return f"async function {program_name}(bot) {{\n{skill_description}\n}}"

metagpt/minecraft_team.py

@@ -6,6 +6,7 @@ from typing import Iterable, Dict, Any
 from pydantic import BaseModel, Field
 import requests
 import json
+import re
 
 from metagpt.logs import logger
 from metagpt.roles import Role
@@ -17,6 +18,7 @@ from metagpt.roles.minecraft.minecraft_base import Minecraft
 from metagpt.environment import Environment
 from metagpt.mineflayer_environment import MineflayerEnv
 from metagpt.const import CKPT_DIR
+from metagpt.actions.minecraft.control_primitives_context import load_skills_code_context
 
 
 class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
@@ -31,15 +33,17 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
         default="You can mine one of oak, birch, spruce, jungle, acacia, dark oak, or mangrove logs."
     )
     code: str = Field(default=None)
-    programs: str = Field(default="")
+    program_name: str = Field(default="")
     critique: str = Field(default=None)
-    skills: list[str] = Field(default_factory=list)
-    question: str = Field(default=None)
+    skills: dict = Field(default_factory=dict)
+    event_summary: str = Field(default="")
 
     qa_cache: dict[str, str] = Field(default_factory=dict)
     completed_tasks: list[str] = Field(default_factory=list)  # Critique things
     failed_tasks: list[str] = Field(default_factory=list)
 
+    skill_desp: str = Field(default="")
+
     chest_memory: dict[str, Any] = Field(
         default_factory=dict
     )  # eg: {'(1344, 64, 1381)': 'Unknown'}
@@ -51,6 +55,15 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
     def progress(self):
         # return len(self.completed_tasks) + 10 # Test only
         return len(self.completed_tasks)
+    
+    @property
+    def programs(self):
+        programs = ""
+        for skill_name, entry in self.skills.items():
+            programs += f"{entry['code']}\n\n"
+        for primitives in load_skills_code_context():
+            programs += f"{primitives}\n\n"
+        return programs
 
     @property
     def warm_up(self):
@@ -76,16 +89,21 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
                 self.failed_tasks = json.load(f)
             with open(f"{CKPT_DIR}/curriculum/qa_cache.json", "r") as f:
                 self.qa_cache = json.load(f)
-            # TODO: add skills resume
+
+            logger.info(f"Loading Skill Manager from {CKPT_DIR}/skill\033[0m")
+            with open(f"{CKPT_DIR}/skill/skills.json", "r") as f:
+                self.skills = json.load(f)
 
     def register_roles(self, roles: Iterable[Minecraft]):
         for role in roles:
             role.set_memory(self)
 
     def update_event(self, event: Dict):
+        if self.event == event:
+            return
         self.event = event
         self.update_chest_memory(event)
-        self.update_chest_observation()
+        self.event_summary = self.summarize_chatlog(event)
 
     def update_task(self, task: str):
         self.current_task = task
@@ -96,14 +114,17 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
     def update_code(self, code: str):
         self.code = code  # action_developer.gen_action_code to HERE
 
-    def update_programs(self, programs: str):
-        self.programs = programs
+    def update_program_name(self, program_name: str):
+        self.program_name = program_name
 
     def update_critique(self, critique: str):
         self.critique = critique  # critic_agent.check_task_success to HERE
 
-    def update_skills(self, skills: list):
-        self.skills = skills  # skill_manager.retrieve_skills to HERE
+    def append_skill(self, skill: dict):
+        self.skills[self.program_name] = skill  # skill_manager.retrieve_skills to HERE
+
+    def update_skill_desp(self, skill_desp: str):
+        self.skill_desp = skill_desp
 
     def update_chest_memory(self, events: Dict):
         """
@@ -149,6 +170,30 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
         else:
             self.chest_observation = f"Chests: None\n\n"
 
+    def summarize_chatlog(self, events):
+        def filter_item(message: str):
+            craft_pattern = r"I cannot make \w+ because I need: (.*)"
+            craft_pattern2 = (
+                r"I cannot make \w+ because there is no crafting table nearby"
+            )
+            mine_pattern = r"I need at least a (.*) to mine \w+!"
+            if re.match(craft_pattern, message):
+                return re.match(craft_pattern, message).groups()[0]
+            elif re.match(craft_pattern2, message):
+                return "a nearby crafting table"
+            elif re.match(mine_pattern, message):
+                return re.match(mine_pattern, message).groups()[0]
+            else:
+                return ""
+
+        chatlog = set()
+        for event_type, event in events:
+            if event_type == "onChat":
+                item = filter_item(event["onChat"])
+                if item:
+                    chatlog.add(item)
+        return "I also need " + ", ".join(chatlog) + "." if chatlog else ""
+
     async def on_event(self, *args):
         """
         Retrieve Minecraft events.

metagpt/mineflayer_env/mineflayer/lib/observation/base.js

@@ -0,0 +1,45 @@
+class Observation {
+    constructor(bot) {
+        if (new.target === Observation) {
+            throw new TypeError(
+                "Cannot instantiate abstract class Observation"
+            );
+        }
+
+        this.bot = bot;
+        this.name = "Observation";
+    }
+
+    observe() {
+        throw new TypeError("Method 'observe()' must be implemented.");
+    }
+
+    reset() {}
+}
+
+function inject(bot, obs_list) {
+    bot.obsList = [];
+    bot.cumulativeObs = [];
+    bot.eventMemory = {};
+    obs_list.forEach((obs) => {
+        bot.obsList.push(new obs(bot));
+    });
+    bot.event = function (event_name) {
+        let result = {};
+        bot.obsList.forEach((obs) => {
+            if (obs.name.startsWith("on") && obs.name !== event_name) {
+                return;
+            }
+            result[obs.name] = obs.observe();
+        });
+        bot.cumulativeObs.push([event_name, result]);
+    };
+    bot.observe = function () {
+        bot.event("observe");
+        const result = bot.cumulativeObs;
+        bot.cumulativeObs = [];
+        return JSON.stringify(result);
+    };
+}
+
+module.exports = { Observation, inject };

metagpt/mineflayer_env/mineflayer/lib/observation/chests.js

@@ -0,0 +1,31 @@
+const { Observation } = require("./base");
+
+class Chests extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "nearbyChests";
+        this.chestsItems = {};
+        bot.on("closeChest", (chestItems, position) => {
+            this.chestsItems[position] = chestItems;
+        });
+        bot.on("removeChest", (chestPosition) => {
+            this.chestsItems[chestPosition] = "Invalid";
+        });
+    }
+
+    observe() {
+        const chests = this.bot.findBlocks({
+            matching: this.bot.registry.blocksByName.chest.id,
+            maxDistance: 16,
+            count: 999,
+        });
+        chests.forEach((chest) => {
+            if (!this.chestsItems.hasOwnProperty(chest)) {
+                this.chestsItems[chest] = "Unknown";
+            }
+        });
+        return this.chestsItems;
+    }
+}
+
+module.exports = Chests;

metagpt/mineflayer_env/mineflayer/lib/observation/inventory.js

@@ -0,0 +1,39 @@
+const { Observation } = require("./base");
+
+class Inventory extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "inventory";
+    }
+
+    observe() {
+        return listItems(this.bot);
+    }
+}
+
+function listItems(bot) {
+    const items = getInventoryItems(bot);
+    return items.reduce(itemToDict, {});
+}
+
+function getInventoryItems(bot) {
+    const inventory = bot.currentWindow || bot.inventory;
+    return inventory.items();
+}
+
+function itemToDict(acc, cur) {
+    if (cur.name && cur.count) {
+        //if both name and count property are defined
+        if (acc[cur.name]) {
+            //if the item is already in the dict
+            acc[cur.name] += cur.count;
+        } else {
+            //if the item is not in the dict
+            acc[cur.name] = cur.count;
+        }
+    }
+    return acc;
+}
+
+//export modules
+module.exports = Inventory;

metagpt/mineflayer_env/mineflayer/lib/observation/onChat.js

@@ -0,0 +1,26 @@
+const Observation = require("./base.js").Observation;
+
+class onChat extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "onChat";
+        this.obs = "";
+        bot.on("chatEvent", (username, message) => {
+            // Save entity status to local variable
+            if (message.startsWith("/")) {
+                return;
+            }
+
+            this.obs += message;
+            this.bot.event(this.name);
+        });
+    }
+
+    observe() {
+        const result = this.obs;
+        this.obs = "";
+        return result;
+    }
+}
+
+module.exports = onChat;

metagpt/mineflayer_env/mineflayer/lib/observation/onError.js

@@ -0,0 +1,22 @@
+const Observation = require("./base.js").Observation;
+
+class onError extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "onError";
+        this.obs = null;
+        bot.on("error", (err) => {
+            // Save entity status to local variable
+            this.obs = err;
+            this.bot.event(this.name);
+        });
+    }
+
+    observe() {
+        const result = this.obs;
+        this.obs = null;
+        return result;
+    }
+}
+
+module.exports = onError;

metagpt/mineflayer_env/mineflayer/lib/observation/onSave.js

@@ -0,0 +1,22 @@
+const Observation = require("./base.js").Observation;
+
+class onSave extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "onSave";
+        this.obs = null;
+        bot.on("save", (eventName) => {
+            // Save entity status to local variable
+            this.obs = eventName;
+            this.bot.event(this.name);
+        });
+    }
+
+    observe() {
+        const result = this.obs;
+        this.obs = null;
+        return result;
+    }
+}
+
+module.exports = onSave;

metagpt/mineflayer_env/mineflayer/lib/observation/status.js

@@ -0,0 +1,103 @@
+const Observation = require("./base.js").Observation;
+
+class Status extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "status";
+    }
+
+    observe() {
+        return {
+            health: this.bot.health,
+            food: this.bot.food,
+            saturation: this.bot.foodSaturation,
+            oxygen: this.bot.oxygenLevel,
+            position: this.bot.entity.position,
+            velocity: this.bot.entity.velocity,
+            yaw: this.bot.entity.yaw,
+            pitch: this.bot.entity.pitch,
+            onGround: this.bot.entity.onGround,
+            equipment: this.getEquipment(),
+            name: this.bot.entity.username,
+            timeSinceOnGround: this.bot.entity.timeSinceOnGround,
+            isInWater: this.bot.entity.isInWater,
+            isInLava: this.bot.entity.isInLava,
+            isInWeb: this.bot.entity.isInWeb,
+            isCollidedHorizontally: this.bot.entity.isCollidedHorizontally,
+            isCollidedVertically: this.bot.entity.isCollidedVertically,
+            biome: this.bot.blockAt(this.bot.entity.position)
+                ? this.bot.blockAt(this.bot.entity.position).biome.name
+                : "None",
+            entities: this.getEntities(),
+            timeOfDay: this.getTime(),
+            inventoryUsed: this.bot.inventoryUsed(),
+            elapsedTime: this.bot.globalTickCounter,
+        };
+    }
+
+    itemToObs(item) {
+        if (!item) return null;
+        return item.name;
+    }
+
+    getTime() {
+        const timeOfDay = this.bot.time.timeOfDay;
+        let time = "";
+        if (timeOfDay < 1000) {
+            time = "sunrise";
+        } else if (timeOfDay < 6000) {
+            time = "day";
+        } else if (timeOfDay < 12000) {
+            time = "noon";
+        } else if (timeOfDay < 13000) {
+            time = "sunset";
+        } else if (timeOfDay < 18000) {
+            time = "night";
+        } else if (timeOfDay < 22000) {
+            time = "midnight";
+        } else {
+            time = "sunrise";
+        }
+        return time;
+    }
+
+    // For each item in equipment, if it exists, return the name of the item
+    // otherwise return null
+    getEquipment() {
+        const slots = this.bot.inventory.slots;
+        const mainHand = this.bot.heldItem;
+        return slots
+            .slice(5, 9)
+            .concat(mainHand, slots[45])
+            .map(this.itemToObs);
+    }
+
+    getEntities() {
+        const entities = this.bot.entities;
+        if (!entities) return {};
+        // keep all monsters in one list, keep other mobs in another list
+        const mobs = {};
+        for (const id in entities) {
+            const entity = entities[id];
+            if (!entity.displayName) continue;
+            if (entity.name === "player" || entity.name === "item") continue;
+            if (entity.position.distanceTo(this.bot.entity.position) < 32) {
+                if (!mobs[entity.name]) {
+                    mobs[entity.name] = entity.position.distanceTo(
+                        this.bot.entity.position
+                    );
+                } else if (
+                    mobs[entity.name] >
+                    entity.position.distanceTo(this.bot.entity.position)
+                ) {
+                    mobs[entity.name] = entity.position.distanceTo(
+                        this.bot.entity.position
+                    );
+                }
+            }
+        }
+        return mobs;
+    }
+}
+
+module.exports = Status;

metagpt/mineflayer_env/mineflayer/lib/observation/voxels.js

@@ -0,0 +1,67 @@
+// Blocks = require("./blocks")
+const { Observation } = require("./base");
+
+class Voxels extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "voxels";
+    }
+
+    observe() {
+        return Array.from(getSurroundingBlocks(this.bot, 8, 2, 8));
+    }
+}
+
+class BlockRecords extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "blockRecords";
+        this.records = new Set();
+        this.tick = 0;
+        bot.on("physicsTick", () => {
+            this.tick++;
+            if (this.tick >= 100) {
+                const items = getInventoryItems(this.bot);
+                getSurroundingBlocks(this.bot, 8, 2, 8).forEach((block) => {
+                    if (!items.has(block)) this.records.add(block);
+                });
+                this.tick = 0;
+            }
+        });
+    }
+
+    observe() {
+        return Array.from(this.records);
+    }
+
+    reset() {
+        this.records = new Set();
+    }
+}
+
+function getSurroundingBlocks(bot, x_distance, y_distance, z_distance) {
+    const surroundingBlocks = new Set();
+
+    for (let x = -x_distance; x <= x_distance; x++) {
+        for (let y = -y_distance; y <= y_distance; y++) {
+            for (let z = -z_distance; z <= z_distance; z++) {
+                const block = bot.blockAt(bot.entity.position.offset(x, y, z));
+                if (block && block.type !== 0) {
+                    surroundingBlocks.add(block.name);
+                }
+            }
+        }
+    }
+    // console.log(surroundingBlocks);
+    return surroundingBlocks;
+}
+
+function getInventoryItems(bot) {
+    const items = new Set();
+    bot.inventory.items().forEach((item) => {
+        if (item) items.add(item.name);
+    });
+    return items;
+}
+
+module.exports = { Voxels, BlockRecords };

metagpt/mineflayer_env/mineflayer/lib/skillLoader.js

@@ -0,0 +1,79 @@
+function inject(bot) {
+    bot._sleep = bot.sleep;
+    bot.sleep = async (bedBlock) => {
+        await bot.waitForTicks(20);
+        await bot._sleep(bedBlock);
+        await bot.waitForTicks(135);
+    };
+
+    bot._fish = bot.fish;
+    bot.fish = async () => {
+        if (bot.heldItem?.name !== "fishing_rod") {
+            bot.chat("I'm not holding a fishing rod!");
+            return;
+        }
+        let timeout = null;
+        await Promise.race([
+            bot._fish(),
+            new Promise(
+                (resolve, reject) =>
+                    (timeout = setTimeout(() => {
+                        bot.activateItem();
+                        reject(
+                            new Error(
+                                "Finishing timeout, make sure you get to and look at a water block!"
+                            )
+                        );
+                    }, 60000))
+            ),
+        ]);
+        clearTimeout(timeout);
+        await bot.waitForTicks(20);
+    };
+
+    bot._consume = bot.consume;
+    bot.consume = async () => {
+        // action_count.activateItem++;
+        await bot._consume();
+        await bot.waitForTicks(20);
+    };
+
+    bot._useOn = bot.useOn;
+    bot.useOn = async (entity) => {
+        if (entity.position.distanceTo(bot.entity.position) > 6) {
+            bot.chat("Please goto a place near the entity first!");
+            return;
+        }
+        await bot._useOn(entity);
+        await bot.waitForTicks(20);
+    };
+
+    bot._activateBlock = bot.activateBlock;
+    bot.activateBlock = async (block) => {
+        if (block.position.distanceTo(bot.entity.position) > 6) {
+            bot.chat("Please goto a place near the block first!");
+            return;
+        }
+        // action_count.activateBlock++;
+        await bot._activateBlock(block);
+    };
+
+    bot._chat = bot.chat;
+    bot.chat = (message) => {
+        // action_count.chat++;
+        bot.emit("chatEvent", "bot", message);
+        bot._chat(message);
+    };
+
+    bot.inventoryUsed = () => {
+        return bot.inventory.slots.slice(9, 45).filter((item) => item !== null)
+            .length;
+    };
+
+    bot.save = function (eventName) {
+        bot.emit("save", eventName);
+    };
+}
+
+// export all control_primitives
+module.exports = { inject };

metagpt/mineflayer_env/mineflayer/lib/utils.js

@@ -0,0 +1,31 @@
+let gameTimeCounter = 0;
+let gameTimeList = [];
+const initCounter = (bot) => {
+    gameTimeList = [];
+    for (let i = 0; i < 13000; i += 1000) {
+        gameTimeList.push(i);
+    }
+    for (let i = 13000; i < 24000; i += 2000) {
+        gameTimeList.push(i);
+    }
+    const timeOfDay = bot.time.timeOfDay;
+    for (let i = 0; i < gameTimeList.length; i++) {
+        if (gameTimeList[i] > timeOfDay) {
+            gameTimeCounter = i - 1;
+            break;
+        }
+    }
+};
+
+const getNextTime = () => {
+    gameTimeCounter++;
+    if (gameTimeCounter >= gameTimeList.length) {
+        gameTimeCounter = 0;
+    }
+    return gameTimeList[gameTimeCounter];
+};
+
+module.exports = {
+    initCounter,
+    getNextTime,
+};

metagpt/mineflayer_environment.py

@@ -38,6 +38,9 @@ class MineflayerEnv:
 
         os.makedirs(f"{CKPT_DIR}/curriculum/vectordb", exist_ok=True)
         os.makedirs(f"{CKPT_DIR}/action", exist_ok=True)
+        os.makedirs(f"{CKPT_DIR}/skill/code", exist_ok=True)
+        os.makedirs(f"{CKPT_DIR}/skill/description", exist_ok=True)
+        os.makedirs(f"{CKPT_DIR}/skill/vectordb", exist_ok=True)
 
     def _set_warmup(self):
         warm_up = DEFAULT_WARMUP

metagpt/roles/minecraft/action_developer.py

@@ -182,10 +182,15 @@ class ActionDeveloper(Base):
         return len(self._rc.news)
 
     async def generate_action_code(self, human_msg, system_msg, *args, **kwargs):
-        code = await GenerateActionCode().run(human_msg, system_msg, *args, **kwargs)
+        code, program_name = await GenerateActionCode().run(
+            human_msg, system_msg, *args, **kwargs
+        )
         # logger.warning(type(code))
         # logger.info(f"Code is Here:{code}")
         self.perform_game_info_callback(code, self.game_memory.update_code)
+        self.perform_game_info_callback(
+            program_name, self.game_memory.update_program_name
+        )
         msg = Message(
             content=f"{code}",
             instruct_content="generate_action_code",

metagpt/roles/minecraft/critic_agent.py

@@ -50,8 +50,9 @@ class CriticReviewer(Base):
 
         for i, (event_type, event) in enumerate(events):
             if event_type == "onError":
-                print(f"\033[31mCritic Agent: Error occurs {event['onError']}\033[0m")
-                return None
+                logger.info(f"\033[31mCritic Agent: Error occurs {event['onError']}\033[0m")
+                # return None
+                return HumanMessage(content="")
 
         observation = ""
 
@@ -119,6 +120,7 @@ class CriticReviewer(Base):
         self.maintain_actions(todo)
         # 获取最新的游戏周边信息
         events = await self._obtain_events()
+        self.perform_game_info_callback(events, self.game_memory.update_event) # update chest_memory / chest observation
         context = self.game_memory.context
         task = self.game_memory.current_task
         chest_observation = self.game_memory.chest_observation

metagpt/roles/minecraft/curriculum_agent.py

@@ -346,7 +346,7 @@ class CurriculumDesigner(Base):
                 updated_failed_tasks.remove(task)
 
         self.game_memory.completed_tasks = updated_completed_tasks
-        self.failed_tasks = updated_failed_tasks
+        self.game_memory.failed_tasks = updated_failed_tasks
 
         # dump to json
         with open(f"{CKPT_DIR}/curriculum/completed_tasks.json", "w") as f:

metagpt/roles/minecraft/skill_manager.py

@@ -6,55 +6,107 @@ from metagpt.logs import logger
 from metagpt.roles.minecraft.minecraft_base import Minecraft as Base
 from metagpt.roles.minecraft.minecraft_base import agent_registry
 from metagpt.schema import Message, HumanMessage, SystemMessage
-from metagpt.actions.minecraft.manage_skills import GenerateSkillDescription, RetrieveSkills, AddNewSkills
+from metagpt.actions.minecraft.manage_skills import (
+    GenerateSkillDescription,
+    RetrieveSkills,
+    AddNewSkills,
+)
 from metagpt.actions.minecraft.review_task import VerifyTask
 from metagpt.actions.minecraft.design_curriculumn import DesignCurriculum
-
+from metagpt.utils.minecraft import load_prompt
 
 
 @agent_registry.register("skill_manager")
 class SkillManager(Base):
     def __init__(
-            self,
-            name: str = "John",
-            profile: str = "Skills Management Specialist",
-            goal: str = "To oversee and optimize the acquisition, development, and utilization of skills within the organization, ensuring workforce competence and efficiency.",
-            constraints: str = "Resource allocation, training budgets, and alignment with organizational goals.",
+        self,
+        name: str = "John",
+        profile: str = "Skills Management Specialist",
+        goal: str = "To oversee and optimize the acquisition, development, and utilization of skills within the organization, ensuring workforce competence and efficiency.",
+        constraints: str = "Resource allocation, training budgets, and alignment with organizational goals.",
     ) -> None:
         super().__init__(name, profile, goal, constraints)
+
         # Initialize actions specific to the SkillManager role
         self._init_actions([RetrieveSkills, GenerateSkillDescription]) #AddNewSkills])#先去掉add
         
         # Set events or actions the SkillManager should watch or be aware of
-        self._watch([DesignCurriculum, VerifyTask, RetrieveSkills, GenerateSkillDescription])
-    
-    async def retrieve_skills(self, human_msg, system_msg, *args, **kwargs):
-        skills = await RetrieveSkills().run(human_msg)
-        logger.info(
-            f"\033[33mRender Action Agent system message with {len(skills)} skills\033[0m"
+        self._watch(
+            [DesignCurriculum, VerifyTask, RetrieveSkills, GenerateSkillDescription]
         )
-        return Message(content=f"{skills}", instruct_content="retrieve_skills", role=self.profile,
-                       send_to=agent_registry.entries["action_developer"]()._setting.name)
-    
+
+    def encapsule_message(self, program_code, program_name, *args, **kwargs):
+        human_msg = self.render_system_message(load_prompt("skill"))
+        system_msg = self.render_human_message(
+            program_code + "\n\n" + f"The main function is `{program_name}`."
+        )
+        return {"system_msg": [system_msg.content], "human_msg": human_msg.content}
+
+    async def retrieve_skills(self, query, *args, **kwargs):
+        skills = await RetrieveSkills().run(query)
+        logger.info(f"Render Action Agent system message with {len(skills)} skills")
+        return Message(content=f"{skills}", instruct_content="retrieve_skills", 
+                       role=self.profile, send_to=agent_registry.entries["action_developer"]()._setting.name)
+        # return Message(
+        #     content=f"{skills}", instruct_content="retrieve_skills", role=self.profile
+        # )  # Unit test only
+
     async def generate_skill_descp(self, human_msg, system_msg, *args, **kwargs):
-        desp = await GenerateSkillDescription().run(human_msg)
-        return Message(content=f"{desp}", instruct_content="generate_skill_descp", role=self.profile)
-    
-    async def handle_add_new_skills(self, human_msg, system_msg, *args, **kwargs):
-        new_skills = await AddNewSkills().run(human_msg)
-        return Message(content=f"", instruct_content="generate_skill_descp", role=self.profile)
-    
+        program_name = self.game_memory.program_name
+        desp = await GenerateSkillDescription().run(program_name, human_msg, system_msg)
+        self.perform_game_info_callback(desp, self.game_memory.update_skill_desp)
+        return Message(
+            content=f"{desp}",
+            instruct_content="generate_skill_descp",
+            role=self.profile,
+        )
+
+    async def handle_add_new_skills(
+        self, task, program_name, program_code, skills, *args, **kwargs
+    ):
+        skill_desp = self.game_memory.skill_desp
+        new_skills_info = await AddNewSkills().run(
+            task, program_name, program_code, skills, skill_desp
+        )
+        self.perform_game_info_callback(new_skills_info, self.game_memory.append_skill)
+        return Message(
+            content=f"{new_skills_info}",
+            instruct_content="handle_add_new_skills",
+            role=self.profile,
+        )
+
     async def _act(self) -> Message:
         todo = self._rc.todo
         logger.debug(f"Todo is {todo}")
         self.maintain_actions(todo)    
         # 获取最新的游戏周边信息
         context = self.game_memory.context
-        
-        msg = self._rc.memory.get(k=1)[0]
-        
-        message = self.encapsule_message(context)
-        
+        task = self.game_memory.current_task
+        event_summary = self.game_memory.event_summary
+        code = self.game_memory.code
+        program_code = code["program_code"]
+        program_name = self.game_memory.program_name
+        skills = self.game_memory.skills
+
+        # TODO: mv to PlayerAction
+        RetrieveSkills.set_skills(skills)
+        AddNewSkills.set_skills(skills)
+
+        # msg = self._rc.memory.get(k=1)[0]
+
+        retrieve_skills_message_step1 = {"query": context}
+
+        retrieve_skills_message_step2 = {"query": context + "\n\n" + event_summary}
+
+        generate_skill_message = self.encapsule_message(program_code, program_name)
+
+        add_new_skills_message = {
+            "task": task,
+            "program_name": program_name,
+            "program_code": program_code,
+            "skills": skills,
+        }
+
         handler_map = {
             DesignCurriculum: self.retrieve_skills,
             RetrieveSkills: self.retrieve_skills,
@@ -63,10 +115,18 @@ class SkillManager(Base):
         }
         handler = handler_map.get(type(todo))
         if handler:
-            msg = await handler(**message)
+            if type(todo) == "DesignCurriculum":
+                msg = await handler(**retrieve_skills_message_step1)
+            elif type(todo) == "RetrieveSkills":
+                msg = await handler(**retrieve_skills_message_step2)
+            elif type(todo) == "GenerateSkillDescription":
+                msg = await handler(**generate_skill_message)
+            else:
+                msg = await handler(**add_new_skills_message)
+
             msg.cause_by = type(todo)
             msg.round_id = self.round_id
             self._publish_message(msg)
             return msg
-        
+
         raise ValueError(f"Unknown todo type: {type(todo)}")

tests/metagpt/roles/minecraft/test_critic_agent.py

@@ -0,0 +1,64 @@
+import asyncio
+
+from metagpt.minecraft_team import GameEnvironment
+from metagpt.roles.minecraft.critic_agent import CriticReviewer
+from metagpt.logs import logger
+
+
+async def main():
+    events = [
+        [
+            "observe",
+            {
+                "voxels": ["grass_block", "dirt", "grass"],
+                "status": {
+                    "health": 20,
+                    "food": 20,
+                    "saturation": 5,
+                    "oxygen": 20,
+                    "position": {"x": 0.5, "y": 84, "z": -207.5},
+                    "velocity": {"x": 0, "y": -0.0784000015258789, "z": 0},
+                    "yaw": 3.141592653589793,
+                    "pitch": 0,
+                    "onGround": True,
+                    "equipment": [None, None, None, None, None, None],
+                    "name": "bot",
+                    "isInWater": False,
+                    "isInLava": False,
+                    "isCollidedHorizontally": False,
+                    "isCollidedVertically": True,
+                    "biome": "plains",
+                    "entities": {
+                        "chicken": 29.071822119730644,
+                        "sheep": 20.361212992763768,
+                    },
+                    "timeOfDay": "day",
+                    "inventoryUsed": 0,
+                    "elapsedTime": 41,
+                },
+                "inventory": {},
+                "nearbyChests": {"(1344, 64, 1381)": "Unknown"},
+                "blockRecords": ["grass_block", "dirt", "grass"],
+            },
+        ]
+    ]
+    task = "Obtain 3 more spruce logs"
+    chest_observation = "Chests: None\n\n"
+
+    context = "Question: How to obtain 3 more spruce logs in Minecraft?\nAnswer: You can obtain more spruce logs in Minecraft by finding and chopping down spruce trees in a spruce forest biome. If you have already chopped down all the spruce trees in the area, you can either explore further to find more spruce trees or plant saplings and wait for them to grow into trees."
+    cr = CriticReviewer()
+    ge = GameEnvironment()
+    ge.update_event(events)
+    cr.set_memory(shared_memory=ge)
+    msg = cr.encapsule_message(
+        events=ge.event, task=task, context=context, chest_observation=chest_observation
+    )
+    logger.info(f"Encapsuled_message: {msg}")
+
+    verify = await cr.verify_task(**msg)
+
+    logger.info(f"Parsed_code_updating: {verify}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/metagpt/roles/minecraft/test_curriculum_agent.py

@@ -58,8 +58,8 @@ async def main():
     context_msg = cd.encapsule_design_curriculum_message(
         events=ge.event, chest_observation=ge.chest_observation
     )
-    logger.info(f"Encapsuled_design_task_message: {context_msg}")
-    context = await cd.handle_curriculum_design(**task_msg)
+    logger.info(f"Encapsuled_design_context_message: {context_msg}")
+    context = await cd.handle_curriculum_design(**context_msg)
     logger.info(f"Design_context_updating: {context}")
 
 

tests/metagpt/roles/minecraft/test_skill_manager.py

@@ -0,0 +1,93 @@
+import asyncio
+
+from metagpt.minecraft_team import GameEnvironment
+from metagpt.roles.minecraft.skill_manager import SkillManager
+from metagpt.logs import logger
+from metagpt.actions.minecraft.manage_skills import (
+    GenerateSkillDescription,
+    RetrieveSkills,
+    AddNewSkills,
+)
+
+
+async def main():
+    events = [
+        [
+            "observe",
+            {
+                "voxels": ["grass_block", "dirt", "grass"],
+                "status": {
+                    "health": 20,
+                    "food": 20,
+                    "saturation": 5,
+                    "oxygen": 20,
+                    "position": {"x": 0.5, "y": 84, "z": -207.5},
+                    "velocity": {"x": 0, "y": -0.0784000015258789, "z": 0},
+                    "yaw": 3.141592653589793,
+                    "pitch": 0,
+                    "onGround": True,
+                    "equipment": [None, None, None, None, None, None],
+                    "name": "bot",
+                    "isInWater": False,
+                    "isInLava": False,
+                    "isCollidedHorizontally": False,
+                    "isCollidedVertically": True,
+                    "biome": "plains",
+                    "entities": {
+                        "chicken": 29.071822119730644,
+                        "sheep": 20.361212992763768,
+                    },
+                    "timeOfDay": "day",
+                    "inventoryUsed": 0,
+                    "elapsedTime": 41,
+                },
+                "inventory": {},
+                "nearbyChests": {"(1344, 64, 1381)": "Unknown"},
+                "blockRecords": ["grass_block", "dirt", "grass"],
+            },
+        ]
+    ]
+    program_code = 'async function obtainSpruceLogs(bot) {\n  // Find 3 spruce_log blocks\n  const spruceLogs = await exploreUntil(bot, new Vec3(1, 0, 1), 60, () => {\n    const spruceLog = bot.findBlock({\n      matching: mcData.blocksByName["spruce_log"].id,\n      maxDistance: 32,\n      count: 3\n    });\n    return spruceLog ? spruceLog : null;\n  });\n  if (spruceLogs) {\n    // Mine the spruce_log blocks\n    await mineBlock(bot, "spruce_log", 3);\n    bot.chat("3 spruce logs obtained.");\n  } else {\n    bot.chat("Could not find enough spruce logs.");\n  }\n}'
+    program_name = "obtainSpruceLogs"
+    task = "Obtain 3 more spruce logs"
+    skills = {
+        "mineWoodLog": {
+            "code": 'async function mineWoodLog(bot) {\n  const woodLogNames = ["oak_log", "birch_log", "spruce_log", "jungle_log", "acacia_log", "dark_oak_log", "mangrove_log"];\n\n  // Find a wood log block\n  const woodLog = await exploreUntil(bot, new Vec3(1, 0, 1), 60, () => {\n    for (const name of woodLogNames) {\n      const log = bot.findBlock({\n        matching: mcData.blocksByName[name].id,\n        maxDistance: 32\n      });\n      if (log) {\n        return log;\n      }\n    }\n    return null;\n  });\n  if (woodLog) {\n    // Mine the wood log block\n    await mineBlock(bot, woodLog.name, 1);\n    bot.chat("Wood log mined.");\n  } else {\n    bot.chat("Could not find a wood log.");\n  }\n}',
+            "description": "async function mineWoodLog(bot) {\n    // The function is about mining a wood log block. It searches for a wood log block by exploring the environment until it finds one of the seven types of wood logs. Once a wood log block is found, it is mined and a message is sent to the chat. If a wood log block is not found, a message is sent to the chat indicating that it could not be found.\n}",
+        },
+        "obtainSpruceLogs": {
+            "code": 'async function obtainSpruceLogs(bot) {\n  // Find 3 spruce_log blocks\n  const spruceLogs = await exploreUntil(bot, new Vec3(1, 0, 1), 60, () => {\n    const spruceLog = bot.findBlock({\n      matching: mcData.blocksByName["spruce_log"].id,\n      maxDistance: 32,\n      count: 3\n    });\n    return spruceLog ? spruceLog : null;\n  });\n  if (spruceLogs) {\n    // Mine the spruce_log blocks\n    await mineBlock(bot, "spruce_log", 3);\n    bot.chat("3 spruce logs obtained.");\n  } else {\n    bot.chat("Could not find enough spruce logs.");\n  }\n}',
+            "description": "async function obtainSpruceLogs(bot) {\n    // The function is about obtaining 3 spruce logs. It explores the environment until it finds 3 spruce_log blocks within a certain distance. Once the blocks are found, it mines them and sends a message indicating that 3 spruce logs have been obtained. If the blocks are not found, it sends a message indicating that it could not find enough spruce logs.\n}",
+        },
+    }
+    context = "Question: How to obtain 3 more spruce logs in Minecraft?\nAnswer: You can obtain more spruce logs in Minecraft by finding and chopping down spruce trees in a spruce forest biome. If you have already chopped down all the spruce trees in the area, you can either explore further to find more spruce trees or plant saplings and wait for them to grow into trees."
+
+    sm = SkillManager()
+    ge = GameEnvironment()
+    ge.update_event(events)
+    sm.set_memory(shared_memory=ge)
+
+    generate_skill_message = sm.encapsule_message(program_code, program_name)
+    logger.info(f"Generate_skill_message: {generate_skill_message}")
+    desp = await sm.generate_skill_descp(**generate_skill_message)
+    logger.info(f"Generate_skill_descp UPDATING: {desp}")
+
+    add_new_skills_message = {
+        "task": task,
+        "program_name": program_name,
+        "program_code": program_code,
+        "skills": skills,
+    }
+    logger.info(f"Handle_add_new_skills_message: {add_new_skills_message}")
+    new_skills_info = await sm.handle_add_new_skills(**add_new_skills_message)
+    logger.info(f"Handle_add_new_skills UPDATING: {new_skills_info}")
+
+    retrieve_skills_message_step1 = {"query": context}
+
+    logger.info(f"Retrieve_skills_message: {retrieve_skills_message_step1}")
+    skills = await sm.retrieve_skills(**retrieve_skills_message_step1)
+    logger.info(f"Retrieve_skills UPDATING: {skills}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())