Minecraft game add skill_manager & ignore bug fix

metagpt/actions/minecraft/design_curriculumn.py

@@ -101,20 +101,19 @@ class DesignCurriculum(Action):
         # TODO: change to FaissStore
         # self.qa_cache_questions_vectordb = FaissStore( {CKPT_DIR}/ 'curriculum/vectordb')
 
-        # Check if qa_cache right using
-        assert self.qa_cache_questions_vectordb._collection.count() == len(
-            self.qa_cache
-        ), (
-            f"Curriculum Agent's qa cache question vectordb is not synced with qa_cache.json.\n"
-            f"There are {self.qa_cache_questions_vectordb._collection.count()} questions in vectordb "
-            f"but {len(self.qa_cache)} questions in qa_cache.json.\n"
-            f"Did you set resume=False when initializing the agent?\n"
-            f"You may need to manually delete the qa cache question vectordb directory for running from scratch.\n"
-        )
-
     @classmethod
     def set_qa_cache(cls, qa_cache):
         cls.qa_cache = qa_cache
+        # Check if qa_cache right using
+        assert cls.qa_cache_questions_vectordb._collection.count() == len(
+            cls.qa_cache
+        ), (
+            f"Curriculum Agent's qa cache question vectordb is not synced with qa_cache.json.\n"
+            f"There are {cls.qa_cache_questions_vectordb._collection.count()} questions in vectordb "
+            f"but {len(cls.qa_cache)} questions in qa_cache.json.\n"
+            f"Did you set resume=False when initializing the agent?\n"
+            f"You may need to manually delete the qa cache question vectordb directory for running from scratch.\n"
+        )
 
     @classmethod
     def generate_qa(cls, events, chest_observation):

metagpt/actions/minecraft/generate_actions.py

@@ -27,20 +27,23 @@ class GenerateActionCode(Action):
         # logger.info(f"parsed_result is HERE: {parsed_result}")
 
         try:
-            return parsed_result["program_code"] + "\n" + parsed_result["exec_code"]
+            return (
+                parsed_result["program_code"] + "\n" + parsed_result["exec_code"],
+                parsed_result["program_name"],
+            )
         except:
             logger.error(f"Failed to parse response: {parsed_result}")
-            return None
+            return None, None
 
     async def run(self, human_msg, system_msg, *args, **kwargs):
         logger.info(f"run {self.__repr__()}")
         # Generate action code.
-        generated_code = await self.generate_code(
+        generated_code, program_name = await self.generate_code(
             human_msg=human_msg, system_msg=system_msg
         )
 
         # Return the generated code.
-        return generated_code
+        return generated_code, program_name
 
 
 class SummarizeLog(Action):

metagpt/actions/minecraft/manage_skills.py

@@ -2,9 +2,15 @@
 # @Date    : 2023/9/23 14:56
 # @Author  : stellahong (stellahong@fuzhi.ai)
 # @Desc    :
+import os
+import json
 
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import Chroma
+from metagpt.document_store import FaissStore
 from metagpt.logs import logger
 from metagpt.actions import Action
+from metagpt.const import CKPT_DIR
 
 
 class RetrieveSkills(Action):
@@ -12,14 +18,44 @@ class RetrieveSkills(Action):
     Action class for retrieving skills.
     Refer to the code in the voyager/agents/skill.py for implementation details.
     """
-    
+
     def __init__(self, name="", context=None, llm=None):
         super().__init__(name, context, llm)
-        self.vect_db = ""
-    
-    async def run(self, *args, **kwargs):
+        # TODO: mv to PlayerAction
+        self.retrieval_top_k = 5
+        self.skills = {}
+        self.vectordb = Chroma(
+            collection_name="skill_vectordb",
+            embedding_function=OpenAIEmbeddings(),
+            persist_directory=f"{CKPT_DIR}/skill/vectordb",
+        )
+
+    @classmethod
+    def set_skills(cls, skills):
+        cls.skills = skills
+        # Check if skills right using
+        assert cls.vectordb._collection.count() == len(cls.skills), (
+            f"Skill Manager's vectordb is not synced with skills.json.\n"
+            f"There are {cls.vectordb._collection.count()} skills in vectordb but {len(cls.skills)} skills in skills.json.\n"
+            f"Did you set resume=False when initializing the manager?\n"
+            f"You may need to manually delete the vectordb directory for running from scratch."
+        )
+
+    async def run(self, query, *args, **kwargs):
         # Implement the logic for retrieving skills here.
-        return []
+        k = min(self.vectordb._collection.count(), self.retrieval_top_k)
+        if k == 0:
+            return []
+        logger.info(f"Skill Manager retrieving for {k} skills")
+        docs_and_scores = self.vectordb.similarity_search_with_score(query, k=k)
+        logger.info(
+            f"Skill Manager retrieved skills: "
+            f"{', '.join([doc.metadata['name'] for doc, _ in docs_and_scores])}"
+        )
+        skills = []
+        for doc, _ in docs_and_scores:
+            skills.append(self.skills[doc.metadata["name"]]["code"])
+        return skills
 
 
 class AddNewSkills(Action):
@@ -27,13 +63,71 @@ class AddNewSkills(Action):
     Action class for adding new skills.
     Refer to the code in the voyager/agents/skill.py for implementation details.
     """
-    
+
     def __init__(self, name="", context=None, llm=None):
         super().__init__(name, context, llm)
-    
-    async def run(self, *args, **kwargs):
+        # TODO: mv to PlayerAction
+        self.vectordb = Chroma(
+            collection_name="skill_vectordb",
+            embedding_function=OpenAIEmbeddings(),
+            persist_directory=f"{CKPT_DIR}/skill/vectordb",
+        )
+        # TODO: change to FaissStore
+        # self.qa_cache_questions_vectordb = FaissStore( {CKPT_DIR}/ 'skill/vectordb')
+
+    @classmethod
+    def set_skills(cls, skills):
+        cls.skills = skills
+        # Check if skills right using
+        assert cls.vectordb._collection.count() == len(cls.skills), (
+            f"Skill Manager's vectordb is not synced with skills.json.\n"
+            f"There are {cls.vectordb._collection.count()} skills in vectordb but {len(cls.skills)} skills in skills.json.\n"
+            f"Did you set resume=False when initializing the manager?\n"
+            f"You may need to manually delete the vectordb directory for running from scratch."
+        )
+
+    async def run(
+        self, task, program_name, program_code, skills, skill_desp, *args, **kwargs
+    ):
         # Implement the logic for adding new skills here.
-        pass
+        if task.startswith("Deposit useless items into the chest at"):
+            # No need to reuse the deposit skill
+            return {}
+        # TODO: Fix this
+        logger.info(
+            f"Skill Manager generated description for {program_name}:\n{skill_desp}\033[0m"
+        )
+        if program_name in skills:
+            logger.info(f"Skill {program_name} already exists. Rewriting!")
+            self.vectordb._collection.delete(ids=[program_name])
+            i = 2
+            while f"{program_name}V{i}.js" in os.listdir(f"{CKPT_DIR}/skill/code"):
+                i += 1
+            dumped_program_name = f"{program_name}V{i}"
+        else:
+            dumped_program_name = program_name
+        self.vectordb.add_texts(
+            texts=[skill_desp],
+            ids=[program_name],
+            metadatas=[{"name": program_name}],
+        )
+
+        # FIXME
+        # assert self.vectordb._collection.count() == len(
+        #     skills
+        # ), "vectordb is not synced with skills.json"
+
+        with open(f"{CKPT_DIR}/skill/code/{dumped_program_name}.js", "w") as f:
+            f.write(program_code)
+        with open(f"{CKPT_DIR}/skill/description/{dumped_program_name}.txt", "w") as f:
+            f.write(skill_desp)
+        with open(f"{CKPT_DIR}/skill/skills.json", "w") as f:
+            json.dump(skills, f)
+        self.vectordb.persist()
+        return {
+            "code": program_code,
+            "description": skill_desp,
+        }
 
 
 class GenerateSkillDescription(Action):
@@ -41,13 +135,12 @@ class GenerateSkillDescription(Action):
     Action class for generating skill descriptions.
     Refer to the code in the voyager/agents/skill.py for implementation details.
     """
-    
+
     def __init__(self, name="", context=None, llm=None):
         super().__init__(name, context, llm)
-    
-    async def run(self, *args, **kwargs):
-        # Implement the logic for generating skill descriptions here.
-        pass
 
-    
-    
+    async def run(self, program_name, human_message, system_message, *args, **kwargs):
+        # Implement the logic for generating skill descriptions here.
+        rsp = await self._aask(prompt=human_message, system_msgs=system_message)
+        skill_description = f"    // { rsp}"
+        return f"async function {program_name}(bot) {{\n{skill_description}\n}}"

metagpt/minecraft_team.py

@@ -6,6 +6,7 @@ from typing import Iterable, Dict, Any
 from pydantic import BaseModel, Field
 import requests
 import json
+import re
 
 from metagpt.logs import logger
 from metagpt.roles import Role
@@ -17,6 +18,7 @@ from metagpt.roles.minecraft.minecraft_base import Minecraft
 from metagpt.environment import Environment
 from metagpt.mineflayer_environment import MineflayerEnv
 from metagpt.const import CKPT_DIR
+from metagpt.actions.minecraft.control_primitives_context import load_skills_code_context
 
 
 class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
@@ -31,15 +33,17 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
         default="You can mine one of oak, birch, spruce, jungle, acacia, dark oak, or mangrove logs."
     )
     code: str = Field(default=None)
-    programs: str = Field(default="")
+    program_name: str = Field(default="")
     critique: str = Field(default=None)
-    skills: list[str] = Field(default_factory=list)
-    question: str = Field(default=None)
+    skills: dict = Field(default_factory=dict)
+    event_summary: str = Field(default="")
 
     qa_cache: dict[str, str] = Field(default_factory=dict)
     completed_tasks: list[str] = Field(default_factory=list)  # Critique things
     failed_tasks: list[str] = Field(default_factory=list)
 
+    skill_desp: str = Field(default="")
+
     chest_memory: dict[str, Any] = Field(
         default_factory=dict
     )  # eg: {'(1344, 64, 1381)': 'Unknown'}
@@ -51,6 +55,15 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
     def progress(self):
         # return len(self.completed_tasks) + 10 # Test only
         return len(self.completed_tasks)
+    
+    @property
+    def programs(self):
+        programs = ""
+        for skill_name, entry in self.skills.items():
+            programs += f"{entry['code']}\n\n"
+        for primitives in load_skills_code_context():
+            programs += f"{primitives}\n\n"
+        return programs
 
     @property
     def warm_up(self):
@@ -76,16 +89,21 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
                 self.failed_tasks = json.load(f)
             with open(f"{CKPT_DIR}/curriculum/qa_cache.json", "r") as f:
                 self.qa_cache = json.load(f)
-            # TODO: add skills resume
+
+            logger.info(f"Loading Skill Manager from {CKPT_DIR}/skill\033[0m")
+            with open(f"{CKPT_DIR}/skill/skills.json", "r") as f:
+                self.skills = json.load(f)
 
     def register_roles(self, roles: Iterable[Minecraft]):
         for role in roles:
             role.set_memory(self)
 
     def update_event(self, event: Dict):
+        if self.event == event:
+            return
         self.event = event
         self.update_chest_memory(event)
-        self.update_chest_observation()
+        self.event_summary = self.summarize_chatlog(event)
 
     def update_task(self, task: str):
         self.current_task = task
@@ -96,14 +114,17 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
     def update_code(self, code: str):
         self.code = code  # action_developer.gen_action_code to HERE
 
-    def update_programs(self, programs: str):
-        self.programs = programs
+    def update_program_name(self, program_name: str):
+        self.program_name = program_name
 
     def update_critique(self, critique: str):
         self.critique = critique  # critic_agent.check_task_success to HERE
 
-    def update_skills(self, skills: list):
-        self.skills = skills  # skill_manager.retrieve_skills to HERE
+    def append_skill(self, skill: dict):
+        self.skills[self.program_name] = skill  # skill_manager.retrieve_skills to HERE
+
+    def update_skill_desp(self, skill_desp: str):
+        self.skill_desp = skill_desp
 
     def update_chest_memory(self, events: Dict):
         """
@@ -149,6 +170,30 @@ class GameEnvironment(BaseModel, arbitrary_types_allowed=True):
         else:
             self.chest_observation = f"Chests: None\n\n"
 
+    def summarize_chatlog(self, events):
+        def filter_item(message: str):
+            craft_pattern = r"I cannot make \w+ because I need: (.*)"
+            craft_pattern2 = (
+                r"I cannot make \w+ because there is no crafting table nearby"
+            )
+            mine_pattern = r"I need at least a (.*) to mine \w+!"
+            if re.match(craft_pattern, message):
+                return re.match(craft_pattern, message).groups()[0]
+            elif re.match(craft_pattern2, message):
+                return "a nearby crafting table"
+            elif re.match(mine_pattern, message):
+                return re.match(mine_pattern, message).groups()[0]
+            else:
+                return ""
+
+        chatlog = set()
+        for event_type, event in events:
+            if event_type == "onChat":
+                item = filter_item(event["onChat"])
+                if item:
+                    chatlog.add(item)
+        return "I also need " + ", ".join(chatlog) + "." if chatlog else ""
+
     async def on_event(self, *args):
         """
         Retrieve Minecraft events.

metagpt/mineflayer_env/mineflayer/lib/observation/base.js

@@ -0,0 +1,45 @@
+class Observation {
+    constructor(bot) {
+        if (new.target === Observation) {
+            throw new TypeError(
+                "Cannot instantiate abstract class Observation"
+            );
+        }
+
+        this.bot = bot;
+        this.name = "Observation";
+    }
+
+    observe() {
+        throw new TypeError("Method 'observe()' must be implemented.");
+    }
+
+    reset() {}
+}
+
+function inject(bot, obs_list) {
+    bot.obsList = [];
+    bot.cumulativeObs = [];
+    bot.eventMemory = {};
+    obs_list.forEach((obs) => {
+        bot.obsList.push(new obs(bot));
+    });
+    bot.event = function (event_name) {
+        let result = {};
+        bot.obsList.forEach((obs) => {
+            if (obs.name.startsWith("on") && obs.name !== event_name) {
+                return;
+            }
+            result[obs.name] = obs.observe();
+        });
+        bot.cumulativeObs.push([event_name, result]);
+    };
+    bot.observe = function () {
+        bot.event("observe");
+        const result = bot.cumulativeObs;
+        bot.cumulativeObs = [];
+        return JSON.stringify(result);
+    };
+}
+
+module.exports = { Observation, inject };

metagpt/mineflayer_env/mineflayer/lib/observation/chests.js

@@ -0,0 +1,31 @@
+const { Observation } = require("./base");
+
+class Chests extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "nearbyChests";
+        this.chestsItems = {};
+        bot.on("closeChest", (chestItems, position) => {
+            this.chestsItems[position] = chestItems;
+        });
+        bot.on("removeChest", (chestPosition) => {
+            this.chestsItems[chestPosition] = "Invalid";
+        });
+    }
+
+    observe() {
+        const chests = this.bot.findBlocks({
+            matching: this.bot.registry.blocksByName.chest.id,
+            maxDistance: 16,
+            count: 999,
+        });
+        chests.forEach((chest) => {
+            if (!this.chestsItems.hasOwnProperty(chest)) {
+                this.chestsItems[chest] = "Unknown";
+            }
+        });
+        return this.chestsItems;
+    }
+}
+
+module.exports = Chests;

metagpt/mineflayer_env/mineflayer/lib/observation/inventory.js

@@ -0,0 +1,39 @@
+const { Observation } = require("./base");
+
+class Inventory extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "inventory";
+    }
+
+    observe() {
+        return listItems(this.bot);
+    }
+}
+
+function listItems(bot) {
+    const items = getInventoryItems(bot);
+    return items.reduce(itemToDict, {});
+}
+
+function getInventoryItems(bot) {
+    const inventory = bot.currentWindow || bot.inventory;
+    return inventory.items();
+}
+
+function itemToDict(acc, cur) {
+    if (cur.name && cur.count) {
+        //if both name and count property are defined
+        if (acc[cur.name]) {
+            //if the item is already in the dict
+            acc[cur.name] += cur.count;
+        } else {
+            //if the item is not in the dict
+            acc[cur.name] = cur.count;
+        }
+    }
+    return acc;
+}
+
+//export modules
+module.exports = Inventory;

metagpt/mineflayer_env/mineflayer/lib/observation/onChat.js

@@ -0,0 +1,26 @@
+const Observation = require("./base.js").Observation;
+
+class onChat extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "onChat";
+        this.obs = "";
+        bot.on("chatEvent", (username, message) => {
+            // Save entity status to local variable
+            if (message.startsWith("/")) {
+                return;
+            }
+
+            this.obs += message;
+            this.bot.event(this.name);
+        });
+    }
+
+    observe() {
+        const result = this.obs;
+        this.obs = "";
+        return result;
+    }
+}
+
+module.exports = onChat;

metagpt/mineflayer_env/mineflayer/lib/observation/onError.js

@@ -0,0 +1,22 @@
+const Observation = require("./base.js").Observation;
+
+class onError extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "onError";
+        this.obs = null;
+        bot.on("error", (err) => {
+            // Save entity status to local variable
+            this.obs = err;
+            this.bot.event(this.name);
+        });
+    }
+
+    observe() {
+        const result = this.obs;
+        this.obs = null;
+        return result;
+    }
+}
+
+module.exports = onError;

metagpt/mineflayer_env/mineflayer/lib/observation/onSave.js

@@ -0,0 +1,22 @@
+const Observation = require("./base.js").Observation;
+
+class onSave extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "onSave";
+        this.obs = null;
+        bot.on("save", (eventName) => {
+            // Save entity status to local variable
+            this.obs = eventName;
+            this.bot.event(this.name);
+        });
+    }
+
+    observe() {
+        const result = this.obs;
+        this.obs = null;
+        return result;
+    }
+}
+
+module.exports = onSave;

metagpt/mineflayer_env/mineflayer/lib/observation/status.js

@@ -0,0 +1,103 @@
+const Observation = require("./base.js").Observation;
+
+class Status extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "status";
+    }
+
+    observe() {
+        return {
+            health: this.bot.health,
+            food: this.bot.food,
+            saturation: this.bot.foodSaturation,
+            oxygen: this.bot.oxygenLevel,
+            position: this.bot.entity.position,
+            velocity: this.bot.entity.velocity,
+            yaw: this.bot.entity.yaw,
+            pitch: this.bot.entity.pitch,
+            onGround: this.bot.entity.onGround,
+            equipment: this.getEquipment(),
+            name: this.bot.entity.username,
+            timeSinceOnGround: this.bot.entity.timeSinceOnGround,
+            isInWater: this.bot.entity.isInWater,
+            isInLava: this.bot.entity.isInLava,
+            isInWeb: this.bot.entity.isInWeb,
+            isCollidedHorizontally: this.bot.entity.isCollidedHorizontally,
+            isCollidedVertically: this.bot.entity.isCollidedVertically,
+            biome: this.bot.blockAt(this.bot.entity.position)
+                ? this.bot.blockAt(this.bot.entity.position).biome.name
+                : "None",
+            entities: this.getEntities(),
+            timeOfDay: this.getTime(),
+            inventoryUsed: this.bot.inventoryUsed(),
+            elapsedTime: this.bot.globalTickCounter,
+        };
+    }
+
+    itemToObs(item) {
+        if (!item) return null;
+        return item.name;
+    }
+
+    getTime() {
+        const timeOfDay = this.bot.time.timeOfDay;
+        let time = "";
+        if (timeOfDay < 1000) {
+            time = "sunrise";
+        } else if (timeOfDay < 6000) {
+            time = "day";
+        } else if (timeOfDay < 12000) {
+            time = "noon";
+        } else if (timeOfDay < 13000) {
+            time = "sunset";
+        } else if (timeOfDay < 18000) {
+            time = "night";
+        } else if (timeOfDay < 22000) {
+            time = "midnight";
+        } else {
+            time = "sunrise";
+        }
+        return time;
+    }
+
+    // For each item in equipment, if it exists, return the name of the item
+    // otherwise return null
+    getEquipment() {
+        const slots = this.bot.inventory.slots;
+        const mainHand = this.bot.heldItem;
+        return slots
+            .slice(5, 9)
+            .concat(mainHand, slots[45])
+            .map(this.itemToObs);
+    }
+
+    getEntities() {
+        const entities = this.bot.entities;
+        if (!entities) return {};
+        // keep all monsters in one list, keep other mobs in another list
+        const mobs = {};
+        for (const id in entities) {
+            const entity = entities[id];
+            if (!entity.displayName) continue;
+            if (entity.name === "player" || entity.name === "item") continue;
+            if (entity.position.distanceTo(this.bot.entity.position) < 32) {
+                if (!mobs[entity.name]) {
+                    mobs[entity.name] = entity.position.distanceTo(
+                        this.bot.entity.position
+                    );
+                } else if (
+                    mobs[entity.name] >
+                    entity.position.distanceTo(this.bot.entity.position)
+                ) {
+                    mobs[entity.name] = entity.position.distanceTo(
+                        this.bot.entity.position
+                    );
+                }
+            }
+        }
+        return mobs;
+    }
+}
+
+module.exports = Status;

metagpt/mineflayer_env/mineflayer/lib/observation/voxels.js

@@ -0,0 +1,67 @@
+// Blocks = require("./blocks")
+const { Observation } = require("./base");
+
+class Voxels extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "voxels";
+    }
+
+    observe() {
+        return Array.from(getSurroundingBlocks(this.bot, 8, 2, 8));
+    }
+}
+
+class BlockRecords extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "blockRecords";
+        this.records = new Set();
+        this.tick = 0;
+        bot.on("physicsTick", () => {
+            this.tick++;
+            if (this.tick >= 100) {
+                const items = getInventoryItems(this.bot);
+                getSurroundingBlocks(this.bot, 8, 2, 8).forEach((block) => {
+                    if (!items.has(block)) this.records.add(block);
+                });
+                this.tick = 0;
+            }
+        });
+    }
+
+    observe() {
+        return Array.from(this.records);
+    }
+
+    reset() {
+        this.records = new Set();
+    }
+}
+
+function getSurroundingBlocks(bot, x_distance, y_distance, z_distance) {
+    const surroundingBlocks = new Set();
+
+    for (let x = -x_distance; x <= x_distance; x++) {
+        for (let y = -y_distance; y <= y_distance; y++) {
+            for (let z = -z_distance; z <= z_distance; z++) {
+                const block = bot.blockAt(bot.entity.position.offset(x, y, z));
+                if (block && block.type !== 0) {
+                    surroundingBlocks.add(block.name);
+                }
+            }
+        }
+    }
+    // console.log(surroundingBlocks);
+    return surroundingBlocks;
+}
+
+function getInventoryItems(bot) {
+    const items = new Set();
+    bot.inventory.items().forEach((item) => {
+        if (item) items.add(item.name);
+    });
+    return items;
+}
+
+module.exports = { Voxels, BlockRecords };

metagpt/mineflayer_env/mineflayer/lib/skillLoader.js

@@ -0,0 +1,79 @@
+function inject(bot) {
+    bot._sleep = bot.sleep;
+    bot.sleep = async (bedBlock) => {
+        await bot.waitForTicks(20);
+        await bot._sleep(bedBlock);
+        await bot.waitForTicks(135);
+    };
+
+    bot._fish = bot.fish;
+    bot.fish = async () => {
+        if (bot.heldItem?.name !== "fishing_rod") {
+            bot.chat("I'm not holding a fishing rod!");
+            return;
+        }
+        let timeout = null;
+        await Promise.race([
+            bot._fish(),
+            new Promise(
+                (resolve, reject) =>
+                    (timeout = setTimeout(() => {
+                        bot.activateItem();
+                        reject(
+                            new Error(
+                                "Finishing timeout, make sure you get to and look at a water block!"
+                            )
+                        );
+                    }, 60000))
+            ),
+        ]);
+        clearTimeout(timeout);
+        await bot.waitForTicks(20);
+    };
+
+    bot._consume = bot.consume;
+    bot.consume = async () => {
+        // action_count.activateItem++;
+        await bot._consume();
+        await bot.waitForTicks(20);
+    };
+
+    bot._useOn = bot.useOn;
+    bot.useOn = async (entity) => {
+        if (entity.position.distanceTo(bot.entity.position) > 6) {
+            bot.chat("Please goto a place near the entity first!");
+            return;
+        }
+        await bot._useOn(entity);
+        await bot.waitForTicks(20);
+    };
+
+    bot._activateBlock = bot.activateBlock;
+    bot.activateBlock = async (block) => {
+        if (block.position.distanceTo(bot.entity.position) > 6) {
+            bot.chat("Please goto a place near the block first!");
+            return;
+        }
+        // action_count.activateBlock++;
+        await bot._activateBlock(block);
+    };
+
+    bot._chat = bot.chat;
+    bot.chat = (message) => {
+        // action_count.chat++;
+        bot.emit("chatEvent", "bot", message);
+        bot._chat(message);
+    };
+
+    bot.inventoryUsed = () => {
+        return bot.inventory.slots.slice(9, 45).filter((item) => item !== null)
+            .length;
+    };
+
+    bot.save = function (eventName) {
+        bot.emit("save", eventName);
+    };
+}
+
+// export all control_primitives
+module.exports = { inject };

metagpt/mineflayer_env/mineflayer/lib/utils.js

@@ -0,0 +1,31 @@
+let gameTimeCounter = 0;
+let gameTimeList = [];
+const initCounter = (bot) => {
+    gameTimeList = [];
+    for (let i = 0; i < 13000; i += 1000) {
+        gameTimeList.push(i);
+    }
+    for (let i = 13000; i < 24000; i += 2000) {
+        gameTimeList.push(i);
+    }
+    const timeOfDay = bot.time.timeOfDay;
+    for (let i = 0; i < gameTimeList.length; i++) {
+        if (gameTimeList[i] > timeOfDay) {
+            gameTimeCounter = i - 1;
+            break;
+        }
+    }
+};
+
+const getNextTime = () => {
+    gameTimeCounter++;
+    if (gameTimeCounter >= gameTimeList.length) {
+        gameTimeCounter = 0;
+    }
+    return gameTimeList[gameTimeCounter];
+};
+
+module.exports = {
+    initCounter,
+    getNextTime,
+};

metagpt/mineflayer_environment.py

@@ -38,6 +38,9 @@ class MineflayerEnv:
 
         os.makedirs(f"{CKPT_DIR}/curriculum/vectordb", exist_ok=True)
         os.makedirs(f"{CKPT_DIR}/action", exist_ok=True)
+        os.makedirs(f"{CKPT_DIR}/skill/code", exist_ok=True)
+        os.makedirs(f"{CKPT_DIR}/skill/description", exist_ok=True)
+        os.makedirs(f"{CKPT_DIR}/skill/vectordb", exist_ok=True)
 
     def _set_warmup(self):
         warm_up = DEFAULT_WARMUP

metagpt/roles/minecraft/action_developer.py

@@ -182,10 +182,15 @@ class ActionDeveloper(Base):
         return len(self._rc.news)
 
     async def generate_action_code(self, human_msg, system_msg, *args, **kwargs):
-        code = await GenerateActionCode().run(human_msg, system_msg, *args, **kwargs)
+        code, program_name = await GenerateActionCode().run(
+            human_msg, system_msg, *args, **kwargs
+        )
         # logger.warning(type(code))
         # logger.info(f"Code is Here:{code}")
         self.perform_game_info_callback(code, self.game_memory.update_code)
+        self.perform_game_info_callback(
+            program_name, self.game_memory.update_program_name
+        )
         msg = Message(
             content=f"{code}",
             instruct_content="generate_action_code",

metagpt/roles/minecraft/critic_agent.py

@@ -50,8 +50,9 @@ class CriticReviewer(Base):
 
         for i, (event_type, event) in enumerate(events):
             if event_type == "onError":
-                print(f"\033[31mCritic Agent: Error occurs {event['onError']}\033[0m")
-                return None
+                logger.info(f"\033[31mCritic Agent: Error occurs {event['onError']}\033[0m")
+                # return None
+                return HumanMessage(content="")
 
         observation = ""
 
@@ -119,6 +120,7 @@ class CriticReviewer(Base):
         self.maintain_actions(todo)
         # 获取最新的游戏周边信息
         events = await self._obtain_events()
+        self.perform_game_info_callback(events, self.game_memory.update_event) # update chest_memory / chest observation
         context = self.game_memory.context
         task = self.game_memory.current_task
         chest_observation = self.game_memory.chest_observation

metagpt/roles/minecraft/curriculum_agent.py

@@ -346,7 +346,7 @@ class CurriculumDesigner(Base):
                 updated_failed_tasks.remove(task)
 
         self.game_memory.completed_tasks = updated_completed_tasks
-        self.failed_tasks = updated_failed_tasks
+        self.game_memory.failed_tasks = updated_failed_tasks
 
         # dump to json
         with open(f"{CKPT_DIR}/curriculum/completed_tasks.json", "w") as f:

metagpt/roles/minecraft/skill_manager.py

@@ -6,55 +6,107 @@ from metagpt.logs import logger
 from metagpt.roles.minecraft.minecraft_base import Minecraft as Base
 from metagpt.roles.minecraft.minecraft_base import agent_registry
 from metagpt.schema import Message, HumanMessage, SystemMessage
-from metagpt.actions.minecraft.manage_skills import GenerateSkillDescription, RetrieveSkills, AddNewSkills
+from metagpt.actions.minecraft.manage_skills import (
+    GenerateSkillDescription,
+    RetrieveSkills,
+    AddNewSkills,
+)
 from metagpt.actions.minecraft.review_task import VerifyTask
 from metagpt.actions.minecraft.design_curriculumn import DesignCurriculum
-
+from metagpt.utils.minecraft import load_prompt
 
 
 @agent_registry.register("skill_manager")
 class SkillManager(Base):
     def __init__(
-            self,
-            name: str = "John",
-            profile: str = "Skills Management Specialist",
-            goal: str = "To oversee and optimize the acquisition, development, and utilization of skills within the organization, ensuring workforce competence and efficiency.",
-            constraints: str = "Resource allocation, training budgets, and alignment with organizational goals.",
+        self,
+        name: str = "John",
+        profile: str = "Skills Management Specialist",
+        goal: str = "To oversee and optimize the acquisition, development, and utilization of skills within the organization, ensuring workforce competence and efficiency.",
+        constraints: str = "Resource allocation, training budgets, and alignment with organizational goals.",
     ) -> None:
         super().__init__(name, profile, goal, constraints)
+
         # Initialize actions specific to the SkillManager role
         self._init_actions([RetrieveSkills, GenerateSkillDescription]) #AddNewSkills])#先去掉add
         
         # Set events or actions the SkillManager should watch or be aware of
-        self._watch([DesignCurriculum, VerifyTask, RetrieveSkills, GenerateSkillDescription])
-    
-    async def retrieve_skills(self, human_msg, system_msg, *args, **kwargs):
-        skills = await RetrieveSkills().run(human_msg)
-        logger.info(
-            f"\033[33mRender Action Agent system message with {len(skills)} skills\033[0m"
+        self._watch(
+            [DesignCurriculum, VerifyTask, RetrieveSkills, GenerateSkillDescription]
         )
-        return Message(content=f"{skills}", instruct_content="retrieve_skills", role=self.profile,
-                       send_to=agent_registry.entries["action_developer"]()._setting.name)
-    
+
+    def encapsule_message(self, program_code, program_name, *args, **kwargs):
+        human_msg = self.render_system_message(load_prompt("skill"))
+        system_msg = self.render_human_message(
+            program_code + "\n\n" + f"The main function is `{program_name}`."
+        )
+        return {"system_msg": [system_msg.content], "human_msg": human_msg.content}
+
+    async def retrieve_skills(self, query, *args, **kwargs):
+        skills = await RetrieveSkills().run(query)
+        logger.info(f"Render Action Agent system message with {len(skills)} skills")
+        return Message(content=f"{skills}", instruct_content="retrieve_skills", 
+                       role=self.profile, send_to=agent_registry.entries["action_developer"]()._setting.name)
+        # return Message(
+        #     content=f"{skills}", instruct_content="retrieve_skills", role=self.profile
+        # )  # Unit test only
+
     async def generate_skill_descp(self, human_msg, system_msg, *args, **kwargs):
-        desp = await GenerateSkillDescription().run(human_msg)
-        return Message(content=f"{desp}", instruct_content="generate_skill_descp", role=self.profile)
-    
-    async def handle_add_new_skills(self, human_msg, system_msg, *args, **kwargs):
-        new_skills = await AddNewSkills().run(human_msg)
-        return Message(content=f"", instruct_content="generate_skill_descp", role=self.profile)
-    
+        program_name = self.game_memory.program_name
+        desp = await GenerateSkillDescription().run(program_name, human_msg, system_msg)
+        self.perform_game_info_callback(desp, self.game_memory.update_skill_desp)
+        return Message(
+            content=f"{desp}",
+            instruct_content="generate_skill_descp",
+            role=self.profile,
+        )
+
+    async def handle_add_new_skills(
+        self, task, program_name, program_code, skills, *args, **kwargs
+    ):
+        skill_desp = self.game_memory.skill_desp
+        new_skills_info = await AddNewSkills().run(
+            task, program_name, program_code, skills, skill_desp
+        )
+        self.perform_game_info_callback(new_skills_info, self.game_memory.append_skill)
+        return Message(
+            content=f"{new_skills_info}",
+            instruct_content="handle_add_new_skills",
+            role=self.profile,
+        )
+
     async def _act(self) -> Message:
         todo = self._rc.todo
         logger.debug(f"Todo is {todo}")
         self.maintain_actions(todo)    
         # 获取最新的游戏周边信息
         context = self.game_memory.context
-        
-        msg = self._rc.memory.get(k=1)[0]
-        
-        message = self.encapsule_message(context)
-        
+        task = self.game_memory.current_task
+        event_summary = self.game_memory.event_summary
+        code = self.game_memory.code
+        program_code = code["program_code"]
+        program_name = self.game_memory.program_name
+        skills = self.game_memory.skills
+
+        # TODO: mv to PlayerAction
+        RetrieveSkills.set_skills(skills)
+        AddNewSkills.set_skills(skills)
+
+        # msg = self._rc.memory.get(k=1)[0]
+
+        retrieve_skills_message_step1 = {"query": context}
+
+        retrieve_skills_message_step2 = {"query": context + "\n\n" + event_summary}
+
+        generate_skill_message = self.encapsule_message(program_code, program_name)
+
+        add_new_skills_message = {
+            "task": task,
+            "program_name": program_name,
+            "program_code": program_code,
+            "skills": skills,
+        }
+
         handler_map = {
             DesignCurriculum: self.retrieve_skills,
             RetrieveSkills: self.retrieve_skills,
@@ -63,10 +115,18 @@ class SkillManager(Base):
         }
         handler = handler_map.get(type(todo))
         if handler:
-            msg = await handler(**message)
+            if type(todo) == "DesignCurriculum":
+                msg = await handler(**retrieve_skills_message_step1)
+            elif type(todo) == "RetrieveSkills":
+                msg = await handler(**retrieve_skills_message_step2)
+            elif type(todo) == "GenerateSkillDescription":
+                msg = await handler(**generate_skill_message)
+            else:
+                msg = await handler(**add_new_skills_message)
+
             msg.cause_by = type(todo)
             msg.round_id = self.round_id
             self._publish_message(msg)
             return msg
-        
+
         raise ValueError(f"Unknown todo type: {type(todo)}")