diff --git a/metagpt/const.py b/metagpt/const.py index a1c650ce3..41a98356c 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -129,3 +129,45 @@ IGNORED_MESSAGE_ID = "0" GENERALIZATION = "Generalize" COMPOSITION = "Composite" AGGREGATION = "Aggregate" + +# For Android Assistant Agent +ADB_EXEC_FAIL = "FAILED" + +# For Mincraft Game Agent +MC_CKPT_DIR = METAGPT_ROOT / "data/mincraft/ckpt" +MC_LOG_DIR = METAGPT_ROOT / "logs" +MC_DEFAULT_WARMUP = { + "context": 15, + "biome": 10, + "time": 15, + "nearby_blocks": 0, + "other_blocks": 10, + "nearby_entities": 5, + "health": 15, + "hunger": 15, + "position": 0, + "equipment": 0, + "inventory": 0, + "optional_inventory_items": 7, + "chests": 0, + "completed_tasks": 0, + "failed_tasks": 0, +} +MC_CURRICULUM_OB = [ + "context", + "biome", + "time", + "nearby_blocks", + "other_blocks", + "nearby_entities", + "health", + "hunger", + "position", + "equipment", + "inventory", + "chests", + "completed_tasks", + "failed_tasks", +] +MC_CORE_INVENTORY_ITEMS = r".*_log|.*_planks|stick|crafting_table|furnace" +r"|cobblestone|dirt|coal|.*_pickaxe|.*_sword|.*_axe", # curriculum_agent: only show these items in inventory before optional_inventory_items reached in warm up diff --git a/metagpt/environment/__init__.py b/metagpt/environment/__init__.py new file mode 100644 index 000000000..692672fa7 --- /dev/null +++ b/metagpt/environment/__init__.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : + +from metagpt.environment.base_env import Environment +from metagpt.environment.android_env.android_env import AndroidEnv +from metagpt.environment.mincraft_env.mincraft_env import MincraftExtEnv +from metagpt.environment.werewolf_env.werewolf_env import WerewolfEnv +from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv +from metagpt.environment.software_env.software_env import SoftwareEnv + + +__all__ = ["AndroidEnv", "MincraftExtEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"] diff --git a/metagpt/environment/android_env/__init__.py b/metagpt/environment/android_env/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/metagpt/environment/android_env/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/metagpt/environment/android_env/android_env.py b/metagpt/environment/android_env/android_env.py new file mode 100644 index 000000000..c27e20541 --- /dev/null +++ b/metagpt/environment/android_env/android_env.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : MG Android Env + +from pydantic import Field + +from metagpt.environment.android_env.android_ext_env import AndroidExtEnv +from metagpt.environment.base_env import Environment + + +class AndroidEnv(Environment, AndroidExtEnv): + rows: int = Field(default=0, description="rows of a grid on the screenshot") + cols: int = Field(default=0, description="cols of a grid on the screenshot") diff --git a/metagpt/environment/android_env/android_ext_env.py b/metagpt/environment/android_env/android_ext_env.py new file mode 100644 index 000000000..7467d394c --- /dev/null +++ b/metagpt/environment/android_env/android_ext_env.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : The Android external environment to integrate with Android apps + +import subprocess +from pathlib import Path +from typing import Any, Optional + +from pydantic import Field + +from metagpt.const import ADB_EXEC_FAIL +from metagpt.environment.base_env import ExtEnv, mark_as_readable, mark_as_writeable + + +class AndroidExtEnv(ExtEnv): + device_id: Optional[str] = Field(default=None) + screenshot_dir: Optional[Path] = Field(default=None) + xml_dir: Optional[Path] = Field(default=None) + width: int = Field(default=720, description="device screen width") + height: int = Field(default=1080, description="device screen height") + + def __init__(self, **data: Any): + super().__init__(**data) + if data.get("device_id"): + (width, height) = self.device_shape + self.width = data.get("width", width) + self.height = data.get("height", height) + + @property + def adb_prefix_si(self): + """adb cmd prefix with `device_id` and `shell input`""" + return f"adb -s {self.device_id} shell input " + + @property + def adb_prefix_shell(self): + """adb cmd prefix with `device_id` and `shell`""" + return f"adb -s {self.device_id} shell " + + @property + def adb_prefix(self): + """adb cmd prefix with `device_id`""" + return f"adb -s {self.device_id} " + + def execute_adb_with_cmd(self, adb_cmd: str) -> str: + res = subprocess.run(adb_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + exec_res = ADB_EXEC_FAIL + if not res.returncode: + exec_res = res.stdout.strip() + return exec_res + + @property + def device_shape(self) -> tuple[int, int]: + adb_cmd = f"{self.adb_prefix_shell} wm size" + shape = (0, 0) + shape_res = self.execute_adb_with_cmd(adb_cmd) + if shape_res != ADB_EXEC_FAIL: + shape = tuple(map(int, shape_res.split(": ")[1].split("x"))) + return shape + + def list_devices(self): + adb_cmd = "adb devices" + res = self.execute_adb_with_cmd(adb_cmd) + devices = [] + if res != ADB_EXEC_FAIL: + devices = res.split("\n")[1:] + devices = [device.split()[0] for device in devices] + return devices + + @mark_as_readable + def get_screenshot(self, ss_name: str, local_save_dir: Path) -> Path: + """ + ss_name: screenshot file name + local_save_dir: local dir to store image from virtual machine + """ + assert self.screenshot_dir + ss_remote_path = Path(self.screenshot_dir).joinpath(f"{ss_name}.png") + ss_cmd = f"{self.adb_prefix_shell} screencap -p {ss_remote_path}" + ss_res = self.execute_adb_with_cmd(ss_cmd) + + res = ADB_EXEC_FAIL + if ss_res != ADB_EXEC_FAIL: + ss_local_path = Path(local_save_dir).joinpath(f"{ss_name}.png") + pull_cmd = f"{self.adb_prefix} pull {ss_remote_path} {ss_local_path}" + pull_res = self.execute_adb_with_cmd(pull_cmd) + if pull_res != ADB_EXEC_FAIL: + res = ss_local_path + return Path(res) + + @mark_as_readable + def get_xml(self, xml_name: str, local_save_dir: Path) -> Path: + xml_remote_path = Path(self.xml_dir).joinpath(f"{xml_name}.xml") + dump_cmd = f"{self.adb_prefix_shell} uiautomator dump {xml_remote_path}" + xml_res = self.execute_adb_with_cmd(dump_cmd) + + res = ADB_EXEC_FAIL + if xml_res != ADB_EXEC_FAIL: + xml_local_path = Path(local_save_dir).joinpath(f"{xml_name}.xml") + pull_cmd = f"{self.adb_prefix} pull {xml_remote_path} {xml_local_path}" + pull_res = self.execute_adb_with_cmd(pull_cmd) + if pull_res != ADB_EXEC_FAIL: + res = xml_local_path + return Path(res) + + @mark_as_writeable + def system_back(self) -> str: + adb_cmd = f"{self.adb_prefix_si} keyevent KEYCODE_BACK" + back_res = self.execute_adb_with_cmd(adb_cmd) + return back_res + + @mark_as_writeable + def system_tap(self, x: int, y: int) -> str: + adb_cmd = f"{self.adb_prefix_si} tap {x} {y}" + tap_res = self.execute_adb_with_cmd(adb_cmd) + return tap_res + + @mark_as_writeable + def user_input(self, input_txt: str) -> str: + input_txt = input_txt.replace(" ", "%s").replace("'", "") + adb_cmd = f"{self.adb_prefix_si} text {input_txt}" + input_res = self.execute_adb_with_cmd(adb_cmd) + return input_res + + @mark_as_writeable + def user_longpress(self, x: int, y: int, duration: int = 500) -> str: + adb_cmd = f"{self.adb_prefix_si} swipe {x} {y} {x} {y} {duration}" + press_res = self.execute_adb_with_cmd(adb_cmd) + return press_res + + @mark_as_writeable + def user_swipe(self, x: int, y: int, orient: str = "up", dist: str = "medium", if_quick: bool = False) -> str: + dist_unit = int(self.width / 10) + if dist == "long": + dist_unit *= 3 + elif dist == "medium": + dist_unit *= 2 + + if orient == "up": + offset = 0, -2 * dist_unit + elif orient == "down": + offset = 0, 2 * dist_unit + elif orient == "left": + offset = -1 * dist_unit, 0 + elif orient == "right": + offset = dist_unit, 0 + else: + return ADB_EXEC_FAIL + + duration = 100 if if_quick else 400 + adb_cmd = f"{self.adb_prefix_si} swipe {x} {y} {x + offset[0]} {y + offset[1]} {duration}" + swipe_res = self.execute_adb_with_cmd(adb_cmd) + return swipe_res + + @mark_as_writeable + def user_swipe_to(self, start: tuple[int, int], end: tuple[int, int], duration: int = 400): + adb_cmd = f"{self.adb_prefix_si} swipe {start[0]} {start[1]} {end[0]} {end[1]} {duration}" + swipe_res = self.execute_adb_with_cmd(adb_cmd) + return swipe_res diff --git a/metagpt/environment/api/__init__.py b/metagpt/environment/api/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/metagpt/environment/api/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/metagpt/environment/api/env_api.py b/metagpt/environment/api/env_api.py new file mode 100644 index 000000000..6469e5b4c --- /dev/null +++ b/metagpt/environment/api/env_api.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : the environment api store + +from typing import Callable + +from pydantic import BaseModel, Field + + +class EnvAPIAbstract(BaseModel): + """api/interface summary description""" + + api_name: str = Field(default="", description="the api function name or id") + args: set = Field(default={}, description="the api function `args` params") + kwargs: dict = Field(default=dict(), description="the api function `kwargs` params") + + +class EnvAPIRegistry(BaseModel): + """the registry to store environment w&r api/interface""" + + registry: dict[str, Callable] = Field(default=dict(), exclude=True) + + def get(self, api_name: str): + return self.registry.get(api_name) + + def __getitem__(self, api_name: str) -> Callable: + return self.get(api_name) + + def __setitem__(self, api_name: str, func: Callable): + self.registry[api_name] = func + + def __len__(self): + return len(self.registry) + + +class WriteAPIRegistry(EnvAPIRegistry): + """just as a explicit class name""" + + pass + + +class ReadAPIRegistry(EnvAPIRegistry): + """just as a explicit class name""" + + pass diff --git a/metagpt/environment.py b/metagpt/environment/base_env.py similarity index 61% rename from metagpt/environment.py rename to metagpt/environment/base_env.py index 5a2dd339b..01582d8d8 100644 --- a/metagpt/environment.py +++ b/metagpt/environment/base_env.py @@ -1,29 +1,91 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" -@Time : 2023/5/11 22:12 -@Author : alexanderwu -@File : environment.py -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.2 of RFC 116: - 1. Remove the functionality of `Environment` class as a public message buffer. - 2. Standardize the message forwarding behavior of the `Environment` class. - 3. Add the `is_idle` property. -@Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing - functionality is to be consolidated into the `Environment` class. -""" +# @Desc : base env of executing environment + +from enum import Enum +from typing import Iterable, Optional, Set, Union import asyncio -from typing import Iterable, Set from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, model_validator from metagpt.context import Context +from metagpt.environment.api.env_api import ( + EnvAPIAbstract, + ReadAPIRegistry, + WriteAPIRegistry, +) from metagpt.logs import logger from metagpt.roles.role import Role from metagpt.schema import Message -from metagpt.utils.common import is_send_to +from metagpt.utils.common import is_send_to, is_coroutine_func -class Environment(BaseModel): +class EnvType(Enum): + ANDROID = "Android" + GYM = "Gym" + WEREWOLF = "Werewolf" + MINCRAFT = "Mincraft" + STANFORDTOWN = "StanfordTown" + + +env_write_api_registry = WriteAPIRegistry() +env_read_api_registry = ReadAPIRegistry() + + +def mark_as_readable(func): + """mark functionn as a readable one in ExtEnv, it observes something from ExtEnv""" + env_read_api_registry[func.__name__] = func + return func + + +def mark_as_writeable(func): + """mark functionn as a writeable one in ExtEnv, it does something to ExtEnv""" + env_write_api_registry[func.__name__] = func + return func + + +class ExtEnv(BaseModel): + """External Env to intergate actual game environment""" + + def _check_api_exist(self, rw_api: Optional[str] = None): + if not rw_api: + raise ValueError(f"{rw_api} not exists") + + async def observe(self, env_action: Union[str, EnvAPIAbstract]): + """get observation from particular api of ExtEnv""" + if isinstance(env_action, str): + read_api = env_read_api_registry.get(api_name=env_action) + self._check_api_exist(read_api) + if is_coroutine_func(read_api): + res = await read_api(self) + else: + res = read_api(self) + elif isinstance(env_action, EnvAPIAbstract): + read_api = env_read_api_registry.get(api_name=env_action.api_name) + self._check_api_exist(read_api) + if is_coroutine_func(read_api): + res = await read_api(self, *env_action.args, **env_action.kwargs) + else: + res = read_api(self, *env_action.args, **env_action.kwargs) + return res + + async def step(self, env_action: Union[str, Message, EnvAPIAbstract, list[EnvAPIAbstract]]): + """execute through particular api of ExtEnv""" + res = None + if isinstance(env_action, Message): + self.publish_message(env_action) + elif isinstance(env_action, EnvAPIAbstract): + write_api = env_write_api_registry.get(env_action.api_name) + self._check_api_exist(write_api) + if is_coroutine_func(write_api): + res = await write_api(self, *env_action.args, **env_action.kwargs) + else: + res = write_api(self, *env_action.args, **env_action.kwargs) + + return res + + +class Environment(ExtEnv): """环境,承载一批角色,角色可以向环境发布消息,可以被其他角色观察到 Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles """ diff --git a/metagpt/logs.py b/metagpt/logs.py index fb0fdd553..90bac21aa 100644 --- a/metagpt/logs.py +++ b/metagpt/logs.py @@ -15,14 +15,15 @@ from loguru import logger as _logger from metagpt.const import METAGPT_ROOT -def define_log_level(print_level="INFO", logfile_level="DEBUG"): +def define_log_level(print_level="INFO", logfile_level="DEBUG", name: str = None): """Adjust the log level to above level""" current_date = datetime.now() formatted_date = current_date.strftime("%Y%m%d") + log_name = f"{name}_{formatted_date}" if name else formatted_date # name a log with prefix name _logger.remove() _logger.add(sys.stderr, level=print_level) - _logger.add(METAGPT_ROOT / f"logs/{formatted_date}.txt", level=logfile_level) + _logger.add(METAGPT_ROOT / f"logs/{log_name}.txt", level=logfile_level) return _logger diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 93921e983..4e7e25531 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -26,7 +26,7 @@ import traceback import typing from io import BytesIO from pathlib import Path -from typing import Any, List, Tuple, Union +from typing import Any, List, Tuple, Union, Callable import aiofiles import loguru @@ -603,6 +603,25 @@ def list_files(root: str | Path) -> List[Path]: return files +def is_coroutine_func(func: Callable) -> bool: + return inspect.iscoroutinefunction(func) + + +def load_mc_skills_code(skill_names: list[str] = None, skills_dir: Path = None) -> list[str]: + """load mincraft skill from js files""" + if not skills_dir: + skills_dir = Path(__file__).parent.absolute() + if skill_names is None: + skill_names = [ + skill[:-3] for skill in os.listdir(f"{skills_dir}") if skill.endswith(".js") + ] + skills = [ + skills_dir.joinpath(f"{skill_name}.js").read_text() + for skill_name in skill_names + ] + return skills + + def encode_image(image_path_or_pil: Union[Path, Image], encoding: str = "utf-8") -> str: """encode image from file or PIL.Image into base64""" if isinstance(image_path_or_pil, Image): diff --git a/tests/metagpt/environment/android_env/__init__.py b/tests/metagpt/environment/android_env/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/tests/metagpt/environment/android_env/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/tests/metagpt/environment/android_env/test_android_ext_env.py b/tests/metagpt/environment/android_env/test_android_ext_env.py new file mode 100644 index 000000000..955210868 --- /dev/null +++ b/tests/metagpt/environment/android_env/test_android_ext_env.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : the unittest of AndroidExtEnv + +from pathlib import Path + +from metagpt.const import ADB_EXEC_FAIL +from metagpt.environment.android_env.android_ext_env import AndroidExtEnv + + +def mock_device_shape(self, adb_cmd: str) -> str: + return "shape: 720x1080" + + +def mock_device_shape_invalid(self, adb_cmd: str) -> str: + return ADB_EXEC_FAIL + + +def mock_list_devices(self, adb_cmd: str) -> str: + return "devices\nemulator-5554" + + +def mock_get_screenshot(self, adb_cmd: str) -> str: + return "screenshot_xxxx-xx-xx" + + +def mock_get_xml(self, adb_cmd: str) -> str: + return "xml_xxxx-xx-xx" + + +def mock_write_read_operation(self, adb_cmd: str) -> str: + return "OK" + + +def test_android_ext_env(mocker): + device_id = "emulator-5554" + mocker.patch( + "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_device_shape + ) + + ext_env = AndroidExtEnv(device_id=device_id, screenshot_dir="/data2/", xml_dir="/data2/") + assert ext_env.adb_prefix == f"adb -s {device_id} " + assert ext_env.adb_prefix_shell == f"adb -s {device_id} shell " + assert ext_env.adb_prefix_si == f"adb -s {device_id} shell input " + + assert ext_env.device_shape == (720, 1080) + + mocker.patch( + "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_device_shape_invalid + ) + assert ext_env.device_shape == (0, 0) + + mocker.patch( + "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_list_devices + ) + assert ext_env.list_devices() == [device_id] + + mocker.patch( + "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_get_screenshot + ) + assert ext_env.get_screenshot("screenshot_xxxx-xx-xx", "/data/") == Path("/data/screenshot_xxxx-xx-xx.png") + + mocker.patch("metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_get_xml) + assert ext_env.get_xml("xml_xxxx-xx-xx", "/data/") == Path("/data/xml_xxxx-xx-xx.xml") + + mocker.patch( + "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_write_read_operation + ) + res = "OK" + assert ext_env.system_back() == res + assert ext_env.system_tap(10, 10) == res + assert ext_env.user_input("test_input") == res + assert ext_env.user_longpress(10, 10) == res + assert ext_env.user_swipe(10, 10) == res + assert ext_env.user_swipe_to((10, 10), (20, 20)) == res diff --git a/tests/metagpt/environment/api/__init__.py b/tests/metagpt/environment/api/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/tests/metagpt/environment/api/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/tests/metagpt/environment/api/test_env_api.py b/tests/metagpt/environment/api/test_env_api.py new file mode 100644 index 000000000..53f98c0d3 --- /dev/null +++ b/tests/metagpt/environment/api/test_env_api.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : + +from metagpt.environment.api.env_api import EnvAPIRegistry + + +def test_env_api_registry(): + def test_func(): + pass + + env_api_registry = EnvAPIRegistry() + env_api_registry["test"] = test_func + + env_api_registry.get("test") == test_func diff --git a/tests/metagpt/environment/test_base_env.py b/tests/metagpt/environment/test_base_env.py new file mode 100644 index 000000000..7404f9bf6 --- /dev/null +++ b/tests/metagpt/environment/test_base_env.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : the unittest of ExtEnv&Env + +import pytest + +from metagpt.environment.api.env_api import EnvAPIAbstract +from metagpt.environment.base_env import ( + Environment, + mark_as_readable, + mark_as_writeable, + env_read_api_registry, + env_write_api_registry +) + + +class ForTestEnv(Environment): + value: int = 0 + + @mark_as_readable + def read_api_no_parms(self): + return self.value + + @mark_as_readable + def read_api(self, a: int, b: int): + return a + b + + @mark_as_writeable + def write_api(self, a: int, b: int): + self.value = a + b + + @mark_as_writeable + async def async_read_api(self, a: int, b: int): + return a + b + + +@pytest.mark.asyncio +async def test_ext_env(): + env = ForTestEnv() + assert len(env_read_api_registry) > 0 + assert len(env_write_api_registry) > 0 + + _ = await env.step(EnvAPIAbstract(api_name="write_api", kwargs={"a": 5, "b": 10})) + assert env.value == 15 + + with pytest.raises(ValueError): + await env.observe("not_exist_api") + + assert await env.observe("read_api_no_parms") == 15 + assert await env.observe(EnvAPIAbstract(api_name="read_api", kwargs={"a": 5, "b": 5})) == 10