From c9588647e8827f1c7b3e7b085a8f76d431b7e7b3 Mon Sep 17 00:00:00 2001 From: didi Date: Wed, 28 Feb 2024 18:06:11 +0800 Subject: [PATCH] Update Role Test Current Bugs 1. Unable to perform multiple rounds 2. Problems with self-learning phase --- .../actions/manual_record.py | 6 ++-- .../actions/screenshot_parse.py | 4 +-- examples/andriod_assistant/run_assistant.py | 28 ++++++++++++++++--- metagpt/team.py | 3 +- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/examples/andriod_assistant/actions/manual_record.py b/examples/andriod_assistant/actions/manual_record.py index abcda3c8e..ef9796b55 100644 --- a/examples/andriod_assistant/actions/manual_record.py +++ b/examples/andriod_assistant/actions/manual_record.py @@ -127,7 +127,7 @@ class ManualRecord(Action): user_input = input() tl, br = elem_list[int(user_input) - 1].bbox x, y = (tl[0] + br[0]) // 2, (tl[1] + br[1]) // 2 - ret = env.step(EnvAPIAbstract(api_name="system_tap", kwargs={"x": x, "y": y})) + ret = await env.step(EnvAPIAbstract(api_name="system_tap", kwargs={"x": x, "y": y})) if ret == ADB_EXEC_FAIL: return AndroidActionOutput(action_state=RunState.FAIL) record_file.write(f"tap({int(user_input)}):::{elem_list[int(user_input) - 1].uid}\n") @@ -155,7 +155,7 @@ class ManualRecord(Action): user_input = input() tl, br = elem_list[int(user_input) - 1].bbox x, y = (tl[0] + br[0]) // 2, (tl[1] + br[1]) // 2 - ret = env.step(EnvAPIAbstract(api_name="user_longpress", kwargs={"x": x, "y": y})) + ret = await env.step(EnvAPIAbstract(api_name="user_longpress", kwargs={"x": x, "y": y})) if ret == ADB_EXEC_FAIL: return AndroidActionOutput(action_state=RunState.FAIL) record_file.write(f"long_press({int(user_input)}):::{elem_list[int(user_input) - 1].uid}\n") @@ -179,7 +179,7 @@ class ManualRecord(Action): user_input = input() tl, br = elem_list[int(user_input) - 1].bbox x, y = (tl[0] + br[0]) // 2, (tl[1] + br[1]) // 2 - ret = env.step(EnvAPIAbstract(api_name="user_swipe", kwargs={"x": x, "y": y, "orient": swipe_dir})) + ret = await env.step(EnvAPIAbstract(api_name="user_swipe", kwargs={"x": x, "y": y, "orient": swipe_dir})) if ret == ADB_EXEC_FAIL: return AndroidActionOutput(action_state=RunState.FAIL) record_file.write(f"swipe({int(user_input)}:sep:{swipe_dir}):::{elem_list[int(user_input) - 1].uid}\n") diff --git a/examples/andriod_assistant/actions/screenshot_parse.py b/examples/andriod_assistant/actions/screenshot_parse.py index c2bd16863..38db933ea 100644 --- a/examples/andriod_assistant/actions/screenshot_parse.py +++ b/examples/andriod_assistant/actions/screenshot_parse.py @@ -190,8 +190,8 @@ class ScreenshotParse(Action): if res == ADB_EXEC_FAIL: return AndroidActionOutput(action_state=RunState.FAIL) elif isinstance(op_param, SwipeGridOp): - start_x, start_y = area_to_xy(op_param.start_area, op_param.start_subarea, width, height, rows, cols) - end_x, end_y = area_to_xy(op_param.end_area, op_param.end_subarea, width, height, rows, cols) + start_x, start_y = area_to_xy(op_param.start_area, op_param.start_subarea, env.width, env.height, env.rows, env.cols) + end_x, end_y = area_to_xy(op_param.end_area, op_param.end_subarea, env.width, env.height, env.rows, env.cols) res = await env.step( EnvAPIAbstract(api_name="user_swipe_to", kwargs={"start": (start_x, start_y), "end": (end_x, end_y)})) if res == ADB_EXEC_FAIL: diff --git a/examples/andriod_assistant/run_assistant.py b/examples/andriod_assistant/run_assistant.py index 187a8032b..e1c4b0362 100644 --- a/examples/andriod_assistant/run_assistant.py +++ b/examples/andriod_assistant/run_assistant.py @@ -3,7 +3,7 @@ # @Desc : the entry of android assistant including learning and acting stage import asyncio - +from pathlib import Path import typer from examples.andriod_assistant.roles.android_assistant import AndroidAssistant @@ -36,6 +36,10 @@ def startup( default="/sdcard", help="The path to store xml files for determining UI elements localtion. Make sure it exists.", ), + device_id : str = typer.Option( + default="emulator-5554", + help="The Android device_id" + ), ): config.set_other( { @@ -46,17 +50,33 @@ def startup( "min_dist": min_dist, "android_screenshot_dir": android_screenshot_dir, "android_xml_dir": android_xml_dir, + "device_id":device_id } ) - team = Team(env=AndroidEnv()) + team = Team(env=AndroidEnv( + device_id=device_id, + xml_dir=Path(android_xml_dir), + screenshot_dir=Path(android_screenshot_dir), + )) + team.hire([AndroidAssistant()]) team.invest(investment) team.run_project(idea=task_desc) - asyncio.run(team.run(n_round=n_round)) if __name__ == "__main__": app() -# Command python run_assistant.py "Create a contact in Contacts App named zjy with a phone number +86 18831933368" \ No newline at end of file +# Command python run_assistant.py "Create a contact in Contacts App named zjy with a phone number +86 18831933368" + +# python run_assistant.py "Create a contact in Contacts App named zjy with a phone number +86 18831933368" --mode "auto" --app-name "Contacts"examples\andriod_assistant> + +# TODO +# 0. How to set Round ? +# 1. Manual Record & Parse Record Success +# 2. Self Learn Fail +# local variable 'action' referenced before assignment +# 3. Act +# 3.1 TODO Act with Manual Docs +# 3.2 TDOO Act with Auto Docs diff --git a/metagpt/team.py b/metagpt/team.py index 35f987b57..2cc5d659c 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -76,6 +76,7 @@ class Team(BaseModel): def hire(self, roles: list[Role]): """Hire roles to cooperate""" + only_role = roles[0] self.env.add_roles(roles) @property @@ -133,4 +134,4 @@ class Team(BaseModel): await self.env.run() self.env.archive(auto_archive) - return self.env.history + return self.env.history \ No newline at end of file