From a40e5dcd7ab650cfbea155b819b84950f060377f Mon Sep 17 00:00:00 2001 From: didi Date: Wed, 28 Feb 2024 18:06:11 +0800 Subject: [PATCH] Update Role Test Current Bugs 1. Unable to perform multiple rounds 2. Problems with self-learning phase --- .../actions/manual_record.py | 6 ++-- .../actions/screenshot_parse.py | 4 +-- examples/andriod_assistant/run_assistant.py | 28 ++++++++++++++++--- metagpt/team.py | 20 +++++++++---- 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/examples/andriod_assistant/actions/manual_record.py b/examples/andriod_assistant/actions/manual_record.py index abcda3c8e..ef9796b55 100644 --- a/examples/andriod_assistant/actions/manual_record.py +++ b/examples/andriod_assistant/actions/manual_record.py @@ -127,7 +127,7 @@ class ManualRecord(Action): user_input = input() tl, br = elem_list[int(user_input) - 1].bbox x, y = (tl[0] + br[0]) // 2, (tl[1] + br[1]) // 2 - ret = env.step(EnvAPIAbstract(api_name="system_tap", kwargs={"x": x, "y": y})) + ret = await env.step(EnvAPIAbstract(api_name="system_tap", kwargs={"x": x, "y": y})) if ret == ADB_EXEC_FAIL: return AndroidActionOutput(action_state=RunState.FAIL) record_file.write(f"tap({int(user_input)}):::{elem_list[int(user_input) - 1].uid}\n") @@ -155,7 +155,7 @@ class ManualRecord(Action): user_input = input() tl, br = elem_list[int(user_input) - 1].bbox x, y = (tl[0] + br[0]) // 2, (tl[1] + br[1]) // 2 - ret = env.step(EnvAPIAbstract(api_name="user_longpress", kwargs={"x": x, "y": y})) + ret = await env.step(EnvAPIAbstract(api_name="user_longpress", kwargs={"x": x, "y": y})) if ret == ADB_EXEC_FAIL: return AndroidActionOutput(action_state=RunState.FAIL) record_file.write(f"long_press({int(user_input)}):::{elem_list[int(user_input) - 1].uid}\n") @@ -179,7 +179,7 @@ class ManualRecord(Action): user_input = input() tl, br = elem_list[int(user_input) - 1].bbox x, y = (tl[0] + br[0]) // 2, (tl[1] + br[1]) // 2 - ret = env.step(EnvAPIAbstract(api_name="user_swipe", kwargs={"x": x, "y": y, "orient": swipe_dir})) + ret = await env.step(EnvAPIAbstract(api_name="user_swipe", kwargs={"x": x, "y": y, "orient": swipe_dir})) if ret == ADB_EXEC_FAIL: return AndroidActionOutput(action_state=RunState.FAIL) record_file.write(f"swipe({int(user_input)}:sep:{swipe_dir}):::{elem_list[int(user_input) - 1].uid}\n") diff --git a/examples/andriod_assistant/actions/screenshot_parse.py b/examples/andriod_assistant/actions/screenshot_parse.py index c2bd16863..38db933ea 100644 --- a/examples/andriod_assistant/actions/screenshot_parse.py +++ b/examples/andriod_assistant/actions/screenshot_parse.py @@ -190,8 +190,8 @@ class ScreenshotParse(Action): if res == ADB_EXEC_FAIL: return AndroidActionOutput(action_state=RunState.FAIL) elif isinstance(op_param, SwipeGridOp): - start_x, start_y = area_to_xy(op_param.start_area, op_param.start_subarea, width, height, rows, cols) - end_x, end_y = area_to_xy(op_param.end_area, op_param.end_subarea, width, height, rows, cols) + start_x, start_y = area_to_xy(op_param.start_area, op_param.start_subarea, env.width, env.height, env.rows, env.cols) + end_x, end_y = area_to_xy(op_param.end_area, op_param.end_subarea, env.width, env.height, env.rows, env.cols) res = await env.step( EnvAPIAbstract(api_name="user_swipe_to", kwargs={"start": (start_x, start_y), "end": (end_x, end_y)})) if res == ADB_EXEC_FAIL: diff --git a/examples/andriod_assistant/run_assistant.py b/examples/andriod_assistant/run_assistant.py index 187a8032b..e1c4b0362 100644 --- a/examples/andriod_assistant/run_assistant.py +++ b/examples/andriod_assistant/run_assistant.py @@ -3,7 +3,7 @@ # @Desc : the entry of android assistant including learning and acting stage import asyncio - +from pathlib import Path import typer from examples.andriod_assistant.roles.android_assistant import AndroidAssistant @@ -36,6 +36,10 @@ def startup( default="/sdcard", help="The path to store xml files for determining UI elements localtion. Make sure it exists.", ), + device_id : str = typer.Option( + default="emulator-5554", + help="The Android device_id" + ), ): config.set_other( { @@ -46,17 +50,33 @@ def startup( "min_dist": min_dist, "android_screenshot_dir": android_screenshot_dir, "android_xml_dir": android_xml_dir, + "device_id":device_id } ) - team = Team(env=AndroidEnv()) + team = Team(env=AndroidEnv( + device_id=device_id, + xml_dir=Path(android_xml_dir), + screenshot_dir=Path(android_screenshot_dir), + )) + team.hire([AndroidAssistant()]) team.invest(investment) team.run_project(idea=task_desc) - asyncio.run(team.run(n_round=n_round)) if __name__ == "__main__": app() -# Command python run_assistant.py "Create a contact in Contacts App named zjy with a phone number +86 18831933368" \ No newline at end of file +# Command python run_assistant.py "Create a contact in Contacts App named zjy with a phone number +86 18831933368" + +# python run_assistant.py "Create a contact in Contacts App named zjy with a phone number +86 18831933368" --mode "auto" --app-name "Contacts"examples\andriod_assistant> + +# TODO +# 0. How to set Round ? +# 1. Manual Record & Parse Record Success +# 2. Self Learn Fail +# local variable 'action' referenced before assignment +# 3. Act +# 3.1 TODO Act with Manual Docs +# 3.2 TDOO Act with Auto Docs diff --git a/metagpt/team.py b/metagpt/team.py index aec72970b..2cc5d659c 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -10,12 +10,13 @@ import warnings from pathlib import Path -from typing import Any +from typing import Any, Optional from pydantic import BaseModel, ConfigDict, Field from metagpt.actions import UserRequirement from metagpt.const import MESSAGE_ROUTE_TO_ALL, SERDESER_PATH +from metagpt.context import Context from metagpt.environment import Environment from metagpt.logs import logger from metagpt.roles import Role @@ -36,12 +37,17 @@ class Team(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) - env: Environment = Field(default_factory=Environment) + env: Optional[Environment] = None investment: float = Field(default=10.0) idea: str = Field(default="") - def __init__(self, **data: Any): + def __init__(self, context: Context = None, **data: Any): super(Team, self).__init__(**data) + ctx = context or Context() + if not self.env: + self.env = Environment(context=ctx) + else: + self.env.context = ctx # The `env` object is allocated by deserialization if "roles" in data: self.hire(data["roles"]) if "env_desc" in data: @@ -54,7 +60,7 @@ class Team(BaseModel): write_json_file(team_info_path, self.model_dump()) @classmethod - def deserialize(cls, stg_path: Path) -> "Team": + def deserialize(cls, stg_path: Path, context: Context = None) -> "Team": """stg_path = ./storage/team""" # recover team_info team_info_path = stg_path.joinpath("team.json") @@ -64,11 +70,13 @@ class Team(BaseModel): ) team_info: dict = read_json_file(team_info_path) - team = Team(**team_info) + ctx = context or Context() + team = Team(**team_info, context=ctx) return team def hire(self, roles: list[Role]): """Hire roles to cooperate""" + only_role = roles[0] self.env.add_roles(roles) @property @@ -126,4 +134,4 @@ class Team(BaseModel): await self.env.run() self.env.archive(auto_archive) - return self.env.history + return self.env.history \ No newline at end of file