update base env and android_env

metagpt/const.py

@@ -129,3 +129,45 @@ IGNORED_MESSAGE_ID = "0"
 GENERALIZATION = "Generalize"
 COMPOSITION = "Composite"
 AGGREGATION = "Aggregate"
+
+# For Android Assistant Agent
+ADB_EXEC_FAIL = "FAILED"
+
+# For Mincraft Game Agent
+MC_CKPT_DIR = METAGPT_ROOT / "data/mincraft/ckpt"
+MC_LOG_DIR = METAGPT_ROOT / "logs"
+MC_DEFAULT_WARMUP = {
+    "context": 15,
+    "biome": 10,
+    "time": 15,
+    "nearby_blocks": 0,
+    "other_blocks": 10,
+    "nearby_entities": 5,
+    "health": 15,
+    "hunger": 15,
+    "position": 0,
+    "equipment": 0,
+    "inventory": 0,
+    "optional_inventory_items": 7,
+    "chests": 0,
+    "completed_tasks": 0,
+    "failed_tasks": 0,
+}
+MC_CURRICULUM_OB = [
+    "context",
+    "biome",
+    "time",
+    "nearby_blocks",
+    "other_blocks",
+    "nearby_entities",
+    "health",
+    "hunger",
+    "position",
+    "equipment",
+    "inventory",
+    "chests",
+    "completed_tasks",
+    "failed_tasks",
+]
+MC_CORE_INVENTORY_ITEMS = r".*_log|.*_planks|stick|crafting_table|furnace"
+r"|cobblestone|dirt|coal|.*_pickaxe|.*_sword|.*_axe",  # curriculum_agent: only show these items in inventory before optional_inventory_items reached in warm up

metagpt/environment/__init__.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :
+
+from metagpt.environment.base_env import Environment
+from metagpt.environment.android_env.android_env import AndroidEnv
+from metagpt.environment.mincraft_env.mincraft_env import MincraftExtEnv
+from metagpt.environment.werewolf_env.werewolf_env import WerewolfEnv
+from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv
+from metagpt.environment.software_env.software_env import SoftwareEnv
+
+
+__all__ = ["AndroidEnv", "MincraftExtEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"]

metagpt/environment/android_env/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

metagpt/environment/android_env/android_env.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : MG Android Env
+
+from pydantic import Field
+
+from metagpt.environment.android_env.android_ext_env import AndroidExtEnv
+from metagpt.environment.base_env import Environment
+
+
+class AndroidEnv(Environment, AndroidExtEnv):
+    rows: int = Field(default=0, description="rows of a grid on the screenshot")
+    cols: int = Field(default=0, description="cols of a grid on the screenshot")

metagpt/environment/android_env/android_ext_env.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : The Android external environment to integrate with Android apps
+
+import subprocess
+from pathlib import Path
+from typing import Any, Optional
+
+from pydantic import Field
+
+from metagpt.const import ADB_EXEC_FAIL
+from metagpt.environment.base_env import ExtEnv, mark_as_readable, mark_as_writeable
+
+
+class AndroidExtEnv(ExtEnv):
+    device_id: Optional[str] = Field(default=None)
+    screenshot_dir: Optional[Path] = Field(default=None)
+    xml_dir: Optional[Path] = Field(default=None)
+    width: int = Field(default=720, description="device screen width")
+    height: int = Field(default=1080, description="device screen height")
+
+    def __init__(self, **data: Any):
+        super().__init__(**data)
+        if data.get("device_id"):
+            (width, height) = self.device_shape
+            self.width = data.get("width", width)
+            self.height = data.get("height", height)
+
+    @property
+    def adb_prefix_si(self):
+        """adb cmd prefix with `device_id` and `shell input`"""
+        return f"adb -s {self.device_id} shell input "
+
+    @property
+    def adb_prefix_shell(self):
+        """adb cmd prefix with `device_id` and `shell`"""
+        return f"adb -s {self.device_id} shell "
+
+    @property
+    def adb_prefix(self):
+        """adb cmd prefix with `device_id`"""
+        return f"adb -s {self.device_id} "
+
+    def execute_adb_with_cmd(self, adb_cmd: str) -> str:
+        res = subprocess.run(adb_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        exec_res = ADB_EXEC_FAIL
+        if not res.returncode:
+            exec_res = res.stdout.strip()
+        return exec_res
+
+    @property
+    def device_shape(self) -> tuple[int, int]:
+        adb_cmd = f"{self.adb_prefix_shell} wm size"
+        shape = (0, 0)
+        shape_res = self.execute_adb_with_cmd(adb_cmd)
+        if shape_res != ADB_EXEC_FAIL:
+            shape = tuple(map(int, shape_res.split(": ")[1].split("x")))
+        return shape
+
+    def list_devices(self):
+        adb_cmd = "adb devices"
+        res = self.execute_adb_with_cmd(adb_cmd)
+        devices = []
+        if res != ADB_EXEC_FAIL:
+            devices = res.split("\n")[1:]
+            devices = [device.split()[0] for device in devices]
+        return devices
+
+    @mark_as_readable
+    def get_screenshot(self, ss_name: str, local_save_dir: Path) -> Path:
+        """
+        ss_name: screenshot file name
+        local_save_dir: local dir to store image from virtual machine
+        """
+        assert self.screenshot_dir
+        ss_remote_path = Path(self.screenshot_dir).joinpath(f"{ss_name}.png")
+        ss_cmd = f"{self.adb_prefix_shell} screencap -p {ss_remote_path}"
+        ss_res = self.execute_adb_with_cmd(ss_cmd)
+
+        res = ADB_EXEC_FAIL
+        if ss_res != ADB_EXEC_FAIL:
+            ss_local_path = Path(local_save_dir).joinpath(f"{ss_name}.png")
+            pull_cmd = f"{self.adb_prefix} pull {ss_remote_path} {ss_local_path}"
+            pull_res = self.execute_adb_with_cmd(pull_cmd)
+            if pull_res != ADB_EXEC_FAIL:
+                res = ss_local_path
+        return Path(res)
+
+    @mark_as_readable
+    def get_xml(self, xml_name: str, local_save_dir: Path) -> Path:
+        xml_remote_path = Path(self.xml_dir).joinpath(f"{xml_name}.xml")
+        dump_cmd = f"{self.adb_prefix_shell} uiautomator dump {xml_remote_path}"
+        xml_res = self.execute_adb_with_cmd(dump_cmd)
+
+        res = ADB_EXEC_FAIL
+        if xml_res != ADB_EXEC_FAIL:
+            xml_local_path = Path(local_save_dir).joinpath(f"{xml_name}.xml")
+            pull_cmd = f"{self.adb_prefix} pull {xml_remote_path} {xml_local_path}"
+            pull_res = self.execute_adb_with_cmd(pull_cmd)
+            if pull_res != ADB_EXEC_FAIL:
+                res = xml_local_path
+        return Path(res)
+
+    @mark_as_writeable
+    def system_back(self) -> str:
+        adb_cmd = f"{self.adb_prefix_si} keyevent KEYCODE_BACK"
+        back_res = self.execute_adb_with_cmd(adb_cmd)
+        return back_res
+
+    @mark_as_writeable
+    def system_tap(self, x: int, y: int) -> str:
+        adb_cmd = f"{self.adb_prefix_si} tap {x} {y}"
+        tap_res = self.execute_adb_with_cmd(adb_cmd)
+        return tap_res
+
+    @mark_as_writeable
+    def user_input(self, input_txt: str) -> str:
+        input_txt = input_txt.replace(" ", "%s").replace("'", "")
+        adb_cmd = f"{self.adb_prefix_si} text {input_txt}"
+        input_res = self.execute_adb_with_cmd(adb_cmd)
+        return input_res
+
+    @mark_as_writeable
+    def user_longpress(self, x: int, y: int, duration: int = 500) -> str:
+        adb_cmd = f"{self.adb_prefix_si} swipe {x} {y} {x} {y} {duration}"
+        press_res = self.execute_adb_with_cmd(adb_cmd)
+        return press_res
+
+    @mark_as_writeable
+    def user_swipe(self, x: int, y: int, orient: str = "up", dist: str = "medium", if_quick: bool = False) -> str:
+        dist_unit = int(self.width / 10)
+        if dist == "long":
+            dist_unit *= 3
+        elif dist == "medium":
+            dist_unit *= 2
+
+        if orient == "up":
+            offset = 0, -2 * dist_unit
+        elif orient == "down":
+            offset = 0, 2 * dist_unit
+        elif orient == "left":
+            offset = -1 * dist_unit, 0
+        elif orient == "right":
+            offset = dist_unit, 0
+        else:
+            return ADB_EXEC_FAIL
+
+        duration = 100 if if_quick else 400
+        adb_cmd = f"{self.adb_prefix_si} swipe {x} {y} {x + offset[0]} {y + offset[1]} {duration}"
+        swipe_res = self.execute_adb_with_cmd(adb_cmd)
+        return swipe_res
+
+    @mark_as_writeable
+    def user_swipe_to(self, start: tuple[int, int], end: tuple[int, int], duration: int = 400):
+        adb_cmd = f"{self.adb_prefix_si} swipe {start[0]} {start[1]} {end[0]} {end[1]} {duration}"
+        swipe_res = self.execute_adb_with_cmd(adb_cmd)
+        return swipe_res

metagpt/environment/api/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

metagpt/environment/api/env_api.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :  the environment api store
+
+from typing import Callable
+
+from pydantic import BaseModel, Field
+
+
+class EnvAPIAbstract(BaseModel):
+    """api/interface summary description"""
+
+    api_name: str = Field(default="", description="the api function name or id")
+    args: set = Field(default={}, description="the api function `args` params")
+    kwargs: dict = Field(default=dict(), description="the api function `kwargs` params")
+
+
+class EnvAPIRegistry(BaseModel):
+    """the registry to store environment w&r api/interface"""
+
+    registry: dict[str, Callable] = Field(default=dict(), exclude=True)
+
+    def get(self, api_name: str):
+        return self.registry.get(api_name)
+
+    def __getitem__(self, api_name: str) -> Callable:
+        return self.get(api_name)
+
+    def __setitem__(self, api_name: str, func: Callable):
+        self.registry[api_name] = func
+
+    def __len__(self):
+        return len(self.registry)
+
+
+class WriteAPIRegistry(EnvAPIRegistry):
+    """just as a explicit class name"""
+
+    pass
+
+
+class ReadAPIRegistry(EnvAPIRegistry):
+    """just as a explicit class name"""
+
+    pass

metagpt/environment/base_env.py

@@ -1,29 +1,91 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/11 22:12
-@Author  : alexanderwu
-@File    : environment.py
-@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.2 of RFC 116:
-    1. Remove the functionality of `Environment` class as a public message buffer.
-    2. Standardize the message forwarding behavior of the `Environment` class.
-    3. Add the `is_idle` property.
-@Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing
-    functionality is to be consolidated into the `Environment` class.
-"""
+# @Desc   : base env of executing environment
+
+from enum import Enum
+from typing import Iterable, Optional, Set, Union
 import asyncio
-from typing import Iterable, Set
 
 from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, model_validator
 
 from metagpt.context import Context
+from metagpt.environment.api.env_api import (
+    EnvAPIAbstract,
+    ReadAPIRegistry,
+    WriteAPIRegistry,
+)
 from metagpt.logs import logger
 from metagpt.roles.role import Role
 from metagpt.schema import Message
-from metagpt.utils.common import is_send_to
+from metagpt.utils.common import is_send_to, is_coroutine_func
 
 
-class Environment(BaseModel):
+class EnvType(Enum):
+    ANDROID = "Android"
+    GYM = "Gym"
+    WEREWOLF = "Werewolf"
+    MINCRAFT = "Mincraft"
+    STANFORDTOWN = "StanfordTown"
+
+
+env_write_api_registry = WriteAPIRegistry()
+env_read_api_registry = ReadAPIRegistry()
+
+
+def mark_as_readable(func):
+    """mark functionn as a readable one in ExtEnv, it observes something from ExtEnv"""
+    env_read_api_registry[func.__name__] = func
+    return func
+
+
+def mark_as_writeable(func):
+    """mark functionn as a writeable one in ExtEnv, it does something to ExtEnv"""
+    env_write_api_registry[func.__name__] = func
+    return func
+
+
+class ExtEnv(BaseModel):
+    """External Env to intergate actual game environment"""
+
+    def _check_api_exist(self, rw_api: Optional[str] = None):
+        if not rw_api:
+            raise ValueError(f"{rw_api} not exists")
+
+    async def observe(self, env_action: Union[str, EnvAPIAbstract]):
+        """get observation from particular api of ExtEnv"""
+        if isinstance(env_action, str):
+            read_api = env_read_api_registry.get(api_name=env_action)
+            self._check_api_exist(read_api)
+            if is_coroutine_func(read_api):
+                res = await read_api(self)
+            else:
+                res = read_api(self)
+        elif isinstance(env_action, EnvAPIAbstract):
+            read_api = env_read_api_registry.get(api_name=env_action.api_name)
+            self._check_api_exist(read_api)
+            if is_coroutine_func(read_api):
+                res = await read_api(self, *env_action.args, **env_action.kwargs)
+            else:
+                res = read_api(self, *env_action.args, **env_action.kwargs)
+        return res
+
+    async def step(self, env_action: Union[str, Message, EnvAPIAbstract, list[EnvAPIAbstract]]):
+        """execute through particular api of ExtEnv"""
+        res = None
+        if isinstance(env_action, Message):
+            self.publish_message(env_action)
+        elif isinstance(env_action, EnvAPIAbstract):
+            write_api = env_write_api_registry.get(env_action.api_name)
+            self._check_api_exist(write_api)
+            if is_coroutine_func(write_api):
+                res = await write_api(self, *env_action.args, **env_action.kwargs)
+            else:
+                res = write_api(self, *env_action.args, **env_action.kwargs)
+
+        return res
+
+
+class Environment(ExtEnv):
     """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到
     Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles
     """

metagpt/logs.py

@@ -15,14 +15,15 @@ from loguru import logger as _logger
 from metagpt.const import METAGPT_ROOT
 
 
-def define_log_level(print_level="INFO", logfile_level="DEBUG"):
+def define_log_level(print_level="INFO", logfile_level="DEBUG", name: str = None):
     """Adjust the log level to above level"""
     current_date = datetime.now()
     formatted_date = current_date.strftime("%Y%m%d")
+    log_name = f"{name}_{formatted_date}" if name else formatted_date  # name a log with prefix name
 
     _logger.remove()
     _logger.add(sys.stderr, level=print_level)
-    _logger.add(METAGPT_ROOT / f"logs/{formatted_date}.txt", level=logfile_level)
+    _logger.add(METAGPT_ROOT / f"logs/{log_name}.txt", level=logfile_level)
     return _logger
 
 

metagpt/utils/common.py

@@ -26,7 +26,7 @@ import traceback
 import typing
 from io import BytesIO
 from pathlib import Path
-from typing import Any, List, Tuple, Union
+from typing import Any, List, Tuple, Union, Callable
 
 import aiofiles
 import loguru
@@ -603,6 +603,25 @@ def list_files(root: str | Path) -> List[Path]:
     return files
 
 
+def is_coroutine_func(func: Callable) -> bool:
+    return inspect.iscoroutinefunction(func)
+
+
+def load_mc_skills_code(skill_names: list[str] = None, skills_dir: Path = None) -> list[str]:
+    """load mincraft skill from js files"""
+    if not skills_dir:
+        skills_dir = Path(__file__).parent.absolute()
+    if skill_names is None:
+        skill_names = [
+            skill[:-3] for skill in os.listdir(f"{skills_dir}") if skill.endswith(".js")
+        ]
+    skills = [
+        skills_dir.joinpath(f"{skill_name}.js").read_text()
+        for skill_name in skill_names
+    ]
+    return skills
+
+
 def encode_image(image_path_or_pil: Union[Path, Image], encoding: str = "utf-8") -> str:
     """encode image from file or PIL.Image into base64"""
     if isinstance(image_path_or_pil, Image):

tests/metagpt/environment/android_env/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

tests/metagpt/environment/android_env/test_android_ext_env.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : the unittest of AndroidExtEnv
+
+from pathlib import Path
+
+from metagpt.const import ADB_EXEC_FAIL
+from metagpt.environment.android_env.android_ext_env import AndroidExtEnv
+
+
+def mock_device_shape(self, adb_cmd: str) -> str:
+    return "shape: 720x1080"
+
+
+def mock_device_shape_invalid(self, adb_cmd: str) -> str:
+    return ADB_EXEC_FAIL
+
+
+def mock_list_devices(self, adb_cmd: str) -> str:
+    return "devices\nemulator-5554"
+
+
+def mock_get_screenshot(self, adb_cmd: str) -> str:
+    return "screenshot_xxxx-xx-xx"
+
+
+def mock_get_xml(self, adb_cmd: str) -> str:
+    return "xml_xxxx-xx-xx"
+
+
+def mock_write_read_operation(self, adb_cmd: str) -> str:
+    return "OK"
+
+
+def test_android_ext_env(mocker):
+    device_id = "emulator-5554"
+    mocker.patch(
+        "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_device_shape
+    )
+
+    ext_env = AndroidExtEnv(device_id=device_id, screenshot_dir="/data2/", xml_dir="/data2/")
+    assert ext_env.adb_prefix == f"adb -s {device_id} "
+    assert ext_env.adb_prefix_shell == f"adb -s {device_id} shell "
+    assert ext_env.adb_prefix_si == f"adb -s {device_id} shell input "
+
+    assert ext_env.device_shape == (720, 1080)
+
+    mocker.patch(
+        "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_device_shape_invalid
+    )
+    assert ext_env.device_shape == (0, 0)
+
+    mocker.patch(
+        "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_list_devices
+    )
+    assert ext_env.list_devices() == [device_id]
+
+    mocker.patch(
+        "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_get_screenshot
+    )
+    assert ext_env.get_screenshot("screenshot_xxxx-xx-xx", "/data/") == Path("/data/screenshot_xxxx-xx-xx.png")
+
+    mocker.patch("metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_get_xml)
+    assert ext_env.get_xml("xml_xxxx-xx-xx", "/data/") == Path("/data/xml_xxxx-xx-xx.xml")
+
+    mocker.patch(
+        "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_write_read_operation
+    )
+    res = "OK"
+    assert ext_env.system_back() == res
+    assert ext_env.system_tap(10, 10) == res
+    assert ext_env.user_input("test_input") == res
+    assert ext_env.user_longpress(10, 10) == res
+    assert ext_env.user_swipe(10, 10) == res
+    assert ext_env.user_swipe_to((10, 10), (20, 20)) == res

tests/metagpt/environment/api/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

tests/metagpt/environment/api/test_env_api.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :
+
+from metagpt.environment.api.env_api import EnvAPIRegistry
+
+
+def test_env_api_registry():
+    def test_func():
+        pass
+
+    env_api_registry = EnvAPIRegistry()
+    env_api_registry["test"] = test_func
+
+    env_api_registry.get("test") == test_func

tests/metagpt/environment/test_base_env.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : the unittest of ExtEnv&Env
+
+import pytest
+
+from metagpt.environment.api.env_api import EnvAPIAbstract
+from metagpt.environment.base_env import (
+    Environment,
+    mark_as_readable,
+    mark_as_writeable,
+    env_read_api_registry,
+    env_write_api_registry
+)
+
+
+class ForTestEnv(Environment):
+    value: int = 0
+
+    @mark_as_readable
+    def read_api_no_parms(self):
+        return self.value
+
+    @mark_as_readable
+    def read_api(self, a: int, b: int):
+        return a + b
+
+    @mark_as_writeable
+    def write_api(self, a: int, b: int):
+        self.value = a + b
+
+    @mark_as_writeable
+    async def async_read_api(self, a: int, b: int):
+        return a + b
+
+
+@pytest.mark.asyncio
+async def test_ext_env():
+    env = ForTestEnv()
+    assert len(env_read_api_registry) > 0
+    assert len(env_write_api_registry) > 0
+
+    _ = await env.step(EnvAPIAbstract(api_name="write_api", kwargs={"a": 5, "b": 10}))
+    assert env.value == 15
+
+    with pytest.raises(ValueError):
+        await env.observe("not_exist_api")
+
+    assert await env.observe("read_api_no_parms") == 15
+    assert await env.observe(EnvAPIAbstract(api_name="read_api", kwargs={"a": 5, "b": 5})) == 10