diff --git a/examples/andriod_assistant/actions/manual_record.py b/examples/andriod_assistant/actions/manual_record.py index 4214a88b7..007ac36ba 100644 --- a/examples/andriod_assistant/actions/manual_record.py +++ b/examples/andriod_assistant/actions/manual_record.py @@ -82,8 +82,7 @@ class ManualRecord(Action): user_input = "xxx" logger.info( "Choose one of the following actions you want to perform on the current screen:\n" - "tap, text, long_press, swipe, stop", - "blue", + "tap, text, long_press, swipe, stop" ) while ( @@ -93,15 +92,13 @@ class ManualRecord(Action): and user_input.lower() != ActionOp.SWIPE.value and user_input.lower() != ActionOp.STOP.value ): - user_input = input() + user_input = input("user_input: ") if user_input.lower() == ActionOp.TAP.value: - logger.info( - f"Which element do you want to tap? Choose a numeric tag from 1 to {len(elem_list)}:", "blue" - ) + logger.info(f"Which element do you want to tap? Choose a numeric tag from 1 to {len(elem_list)}:") user_input = "xxx" while not user_input.isnumeric() or int(user_input) > len(elem_list) or int(user_input) < 1: - user_input = input() + user_input = input("user_input: ") tl, br = elem_list[int(user_input) - 1].bbox x, y = (tl[0] + br[0]) // 2, (tl[1] + br[1]) // 2 action = EnvAction(action_type=EnvActionType.SYSTEM_TAP, coord=(x, y)) @@ -109,25 +106,24 @@ class ManualRecord(Action): elif user_input.lower() == ActionOp.TEXT.value: logger.info( f"Which element do you want to input the text string? Choose a numeric tag from 1 to " - f"{len(elem_list)}:", - "blue", + f"{len(elem_list)}:" ) input_area = "xxx" while not input_area.isnumeric() or int(input_area) > len(elem_list) or int(input_area) < 1: - input_area = input() - logger.info("Enter your input text below:", "blue") + input_area = input("user_input: ") + logger.info("Enter your input text below:") user_input = "" while not user_input: - user_input = input() + user_input = input("user_input: ") action = EnvAction(action_type=EnvActionType.USER_INPUT, input_txt=user_input) log_str = f"text({input_area}:sep:'{user_input}'):::{elem_list[int(input_area) - 1].uid}\n" elif user_input.lower() == ActionOp.LONG_PRESS.value: logger.info( - f"Which element do you want to long press? Choose a numeric tag from 1 to {len(elem_list)}:", "blue" + f"Which element do you want to long press? Choose a numeric tag from 1 to {len(elem_list)}:" ) user_input = "xxx" while not user_input.isnumeric() or int(user_input) > len(elem_list) or int(user_input) < 1: - user_input = input() + user_input = input("user_input: ") tl, br = elem_list[int(user_input) - 1].bbox x, y = (tl[0] + br[0]) // 2, (tl[1] + br[1]) // 2 action = EnvAction(action_type=EnvActionType.USER_LONGPRESS, coord=(x, y)) @@ -135,8 +131,7 @@ class ManualRecord(Action): elif user_input.lower() == ActionOp.SWIPE.value: logger.info( "What is the direction of your swipe? Choose one from the following options:\n" - "up, down, left, right", - "blue", + "up, down, left, right" ) user_input = "" while ( @@ -145,11 +140,11 @@ class ManualRecord(Action): and user_input != SwipeOp.LEFT.value and user_input != SwipeOp.RIGHT.value ): - user_input = input() + user_input = input("user_input: ") swipe_dir = user_input logger.info(f"Which element do you want to swipe? Choose a numeric tag from 1 to {len(elem_list)}:") while not user_input.isnumeric() or int(user_input) > len(elem_list) or int(user_input) < 1: - user_input = input() + user_input = input("user_input: ") tl, br = elem_list[int(user_input) - 1].bbox x, y = (tl[0] + br[0]) // 2, (tl[1] + br[1]) // 2 diff --git a/examples/andriod_assistant/roles/android_assistant.py b/examples/andriod_assistant/roles/android_assistant.py index 33b521208..9e071a88f 100644 --- a/examples/andriod_assistant/roles/android_assistant.py +++ b/examples/andriod_assistant/roles/android_assistant.py @@ -47,25 +47,30 @@ class AndroidAssistant(Role): """Firstly, we decide the state with user config, further, we can do it automatically, like if it's new app, run the learn first and then do the act stage or learn it during the action. """ - if config.get_other("stage") == "learn" and config.get_other("mode") == "manual": + stage = config.get_other("stage") + mode = config.get_other("mode") + if stage == "learn" and mode == "manual": # choose ManualRecord and then run ParseRecord # Remember, only run each action only one time, no need to run n_round. self.set_actions([ManualRecord, ParseRecord]) self.task_dir = data_dir.joinpath(app_name, f"manual_learn_{cur_datetime}") self.docs_dir = data_dir.joinpath(app_name, "manual_docs") - elif config.get_other("stage") == "learn" and config.get_other("mode") == "auto": + elif stage == "learn" and mode == "auto": # choose SelfLearnAndReflect to run self.set_actions([SelfLearnAndReflect]) self.task_dir = data_dir.joinpath(app_name, f"auto_learn_{cur_datetime}") self.docs_dir = data_dir.joinpath(app_name, "auto_docs") - elif config.get_other("stage") == "act": + elif stage == "act": # choose ScreenshotParse to run self.set_actions([ScreenshotParse]) self.task_dir = data_dir.joinpath(app_name, f"act_{cur_datetime}") - if config.get_other("mode") == "manual": + if mode == "manual": self.docs_dir = data_dir.joinpath(app_name, "manual_docs") else: self.docs_dir = data_dir.joinpath(app_name, "auto_docs") + else: + raise ValueError(f"invalid stage: {stage}, mode: {mode}") + self._check_dir() self._set_react_mode(RoleReactMode.BY_ORDER) @@ -82,10 +87,11 @@ class AndroidAssistant(Role): async def _observe(self, ignore_memory=True) -> int: """ignore old memory to make it run multi rounds inside a role""" - newest_msg = self.rc.memory.get(k=1)[0] - if RunState.SUCCESS not in newest_msg.content: + newest_msgs = self.rc.memory.get(k=1) + newest_msg = newest_msgs[0] if newest_msgs else None + if newest_msg and (RunState.SUCCESS not in newest_msg.content): ignore_memory = False - logger.error("Latest action_state is FINISH or FAIL, won't react in remainder rounds", "red") + logger.error("Latest action_state is FINISH or FAIL, won't react in remainder rounds") return await super()._observe(ignore_memory) async def _act(self) -> Message: diff --git a/examples/andriod_assistant/utils/utils.py b/examples/andriod_assistant/utils/utils.py index c715de968..8cd5b6114 100644 --- a/examples/andriod_assistant/utils/utils.py +++ b/examples/andriod_assistant/utils/utils.py @@ -218,7 +218,6 @@ def draw_grid(img_path: Path, output_path: Path) -> tuple[int, int]: def area_to_xy(area: int, subarea: str, width: int, height: int, rows: int, cols: int) -> tuple[int, int]: area -= 1 - logger.info(f"{cols}") row, col = area // cols, area % cols x_0, y_0 = col * (width // cols), row * (height // rows) if subarea == "top-left":