diff --git a/examples/andriod_assistant/actions/screenshot_parse.py b/examples/andriod_assistant/actions/screenshot_parse.py index 1dbd45b5e..ef9454979 100644 --- a/examples/andriod_assistant/actions/screenshot_parse.py +++ b/examples/andriod_assistant/actions/screenshot_parse.py @@ -104,7 +104,7 @@ class ScreenshotParse(Action): return elif isinstance(op_param, SwipeOp): x, y = elem_bbox_to_xy(elem_list[op_param.area - 1].bbox) - res = env.step(EnvAPIAbstract("user_swipe", kwargs={"x": x, "y": y, "dir": op_param.swipe_orient, "dist": op_param.dist})) + res = env.step(EnvAPIAbstract("user_swipe", kwargs={"x": x, "y": y, "orient": op_param.swipe_orient, "dist": op_param.dist})) if res == ADB_EXEC_FAIL: # TODO return diff --git a/metagpt/environment/android_env/android_env.py b/metagpt/environment/android_env/android_env.py index 87b49750d..6c83be530 100644 --- a/metagpt/environment/android_env/android_env.py +++ b/metagpt/environment/android_env/android_env.py @@ -2,8 +2,11 @@ # -*- coding: utf-8 -*- # @Desc : MG Android Env +from pydantic import Field + from metagpt.environment.android_env.android_ext_env import AndroidExtEnv class AndroidEnv(AndroidExtEnv): - pass + rows: int = Field(default=0, description="rows of a grid on the screenshot") + cols: int = Field(default=0, description="cols of a grid on the screenshot") diff --git a/metagpt/environment/android_env/android_ext_env.py b/metagpt/environment/android_env/android_ext_env.py index f5e62eb5e..7467d394c 100644 --- a/metagpt/environment/android_env/android_ext_env.py +++ b/metagpt/environment/android_env/android_ext_env.py @@ -67,39 +67,39 @@ class AndroidExtEnv(ExtEnv): return devices @mark_as_readable - def get_screenshot(self, ss_name: str, local_save_dir: Path) -> str: + def get_screenshot(self, ss_name: str, local_save_dir: Path) -> Path: """ ss_name: screenshot file name local_save_dir: local dir to store image from virtual machine """ assert self.screenshot_dir - ss_remote_path = str(Path(self.screenshot_dir).joinpath(f"{ss_name}.png")) + ss_remote_path = Path(self.screenshot_dir).joinpath(f"{ss_name}.png") ss_cmd = f"{self.adb_prefix_shell} screencap -p {ss_remote_path}" ss_res = self.execute_adb_with_cmd(ss_cmd) res = ADB_EXEC_FAIL if ss_res != ADB_EXEC_FAIL: - ss_local_path = str(Path(local_save_dir).joinpath(f"{ss_name}.png")) + ss_local_path = Path(local_save_dir).joinpath(f"{ss_name}.png") pull_cmd = f"{self.adb_prefix} pull {ss_remote_path} {ss_local_path}" pull_res = self.execute_adb_with_cmd(pull_cmd) if pull_res != ADB_EXEC_FAIL: res = ss_local_path - return res + return Path(res) @mark_as_readable - def get_xml(self, xml_name: str, local_save_dir: Path) -> str: - xml_remote_path = str(Path(self.xml_dir).joinpath(f"{xml_name}.xml")) + def get_xml(self, xml_name: str, local_save_dir: Path) -> Path: + xml_remote_path = Path(self.xml_dir).joinpath(f"{xml_name}.xml") dump_cmd = f"{self.adb_prefix_shell} uiautomator dump {xml_remote_path}" xml_res = self.execute_adb_with_cmd(dump_cmd) res = ADB_EXEC_FAIL if xml_res != ADB_EXEC_FAIL: - xml_local_path = str(Path(local_save_dir).joinpath(f"{xml_name}.xml")) + xml_local_path = Path(local_save_dir).joinpath(f"{xml_name}.xml") pull_cmd = f"{self.adb_prefix} pull {xml_remote_path} {xml_local_path}" pull_res = self.execute_adb_with_cmd(pull_cmd) if pull_res != ADB_EXEC_FAIL: res = xml_local_path - return res + return Path(res) @mark_as_writeable def system_back(self) -> str: @@ -127,20 +127,20 @@ class AndroidExtEnv(ExtEnv): return press_res @mark_as_writeable - def user_swipe(self, x: int, y: int, dir: str = "up", dist: str = "medium", if_quick: bool = False) -> str: + def user_swipe(self, x: int, y: int, orient: str = "up", dist: str = "medium", if_quick: bool = False) -> str: dist_unit = int(self.width / 10) if dist == "long": dist_unit *= 3 elif dist == "medium": dist_unit *= 2 - if dir == "up": + if orient == "up": offset = 0, -2 * dist_unit - elif dir == "down": + elif orient == "down": offset = 0, 2 * dist_unit - elif dir == "left": + elif orient == "left": offset = -1 * dist_unit, 0 - elif dir == "right": + elif orient == "right": offset = dist_unit, 0 else: return ADB_EXEC_FAIL