Update test for action node & Modify extenv (self reflection)

examples/andriod_assistant/actions/self_learn_and_reflect.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+# !/usr/bin/env python
 # -*- coding: utf-8 -*-
 # @Desc   : LIKE scripts/self_explorer.py in stage=learn & mode=auto self_explore_task stage
 
@@ -58,21 +58,21 @@ class SelfLearnAndReflect(Action):
     ui_area: int = -1
 
     async def run(
-        self, round_count: int, task_desc: str, last_act: str, task_dir: Path, docs_dir: Path, env: AndroidEnv
+            self, round_count: int, task_desc: str, last_act: str, task_dir: Path, docs_dir: Path, env: AndroidEnv
     ) -> AndroidActionOutput:
-        resp = self.run_self_learn(round_count, task_desc, last_act, task_dir, env)
-        resp = self.run_reflect(round_count, task_desc, last_act, task_dir, docs_dir, env)
+        resp = await self.run_self_learn(round_count, task_desc, last_act, task_dir, env)
+        resp = await self.run_reflect(round_count, task_desc, last_act, task_dir, docs_dir, env)
         return resp
 
     async def run_self_learn(
-        self, round_count: int, task_desc: str, last_act: str, task_dir: Path, env: AndroidEnv
+            self, round_count: int, task_desc: str, last_act: str, task_dir: Path, env: AndroidEnv
     ) -> AndroidActionOutput:
-        screenshot_path: Path = env.step(
+        screenshot_path: Path = env.observe(
             EnvAPIAbstract(
                 api_name="get_screenshot", kwargs={"ss_name": f"{round_count}_before", "local_save_dir": task_dir}
             )
         )
-        xml_path: Path = env.step(
+        xml_path: Path = env.observe(
             EnvAPIAbstract(api_name="get_xml", kwargs={"xml_name": f"{round_count}", "local_save_dir": task_dir})
         )
         if not screenshot_path.exists() or not xml_path.exists():
@@ -80,6 +80,7 @@ class SelfLearnAndReflect(Action):
 
         clickable_list = []
         focusable_list = []
+        # TODO Tuple Bug
         traverse_xml_tree(xml_path, clickable_list, "clickable", True)
         traverse_xml_tree(xml_path, focusable_list, "focusable", True)
         elem_list = []
@@ -155,9 +156,9 @@ class SelfLearnAndReflect(Action):
         return AndroidActionOutput()
 
     async def run_reflect(
-        self, round_count: int, task_desc: str, last_act: str, task_dir: Path, docs_dir: Path, env: AndroidEnv
+            self, round_count: int, task_desc: str, last_act: str, task_dir: Path, docs_dir: Path, env: AndroidEnv
     ) -> AndroidActionOutput:
-        screenshot_path: Path = env.step(
+        screenshot_path: Path = env.observe(
             EnvAPIAbstract(
                 api_name="get_screenshot", kwargs={"ss_name": f"{round_count}_after", "local_save_dir": task_dir}
             )

examples/andriod_assistant/test_for_an.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : test on android emulator
+import asyncio
+import time
+from pathlib import Path
+from actions.manual_record import ManualRecord
+from actions.parse_record import ParseRecord
+from actions.self_learn_and_reflect import SelfLearnAndReflect
+from metagpt.environment.android_env.android_env import AndroidEnv
+
+TASK_PATH = Path("apps/Contacts")
+DOC_PATH = TASK_PATH.joinpath("docs")
+DEMO_NAME = str(time.time())
+# TODO Test for Self Learning、
+test_env_self_learn_android = AndroidEnv(
+    device_id="emulator-5554",
+    xml_dir=Path("/sdcard"),
+    screenshot_dir=Path("/sdcard/Pictures/Screenshots"),
+)
+test_self_learning = SelfLearnAndReflect()
+
+# TODO Test for Manual Learning
+test_env_manual_learn_android = AndroidEnv(
+    device_id="emulator-5554",
+    xml_dir=Path("/sdcard"),
+    screenshot_dir=Path("/sdcard/Pictures/Screenshots"),
+)
+test_manual_record = ManualRecord()
+test_manual_parse = ParseRecord()
+
+# 虚拟机效果实现
+# 不同 Action Node 结果符合预期(Action Node)
+
+if __name__ == "__main__":
+    loop = asyncio.get_event_loop()
+    test_action_list = [
+        test_self_learning.run(
+            round_count=20,
+            task_desc="Create a contact in Contacts App named zjy with a phone number +86 18831933368 ",
+            last_act="",
+            task_dir=TASK_PATH,
+            docs_dir=DOC_PATH,
+            env=test_env_self_learn_android
+        ),
+        # test_manual_record.run(
+        #     demo_name=DEMO_NAME,
+        #     task_dir=TASK_PATH,
+        #     env=test_env_manual_learn_android
+        # ),
+        # test_manual_parse.run(
+        #     app_name="Contacts",
+        #     demo_name=DEMO_NAME,
+        #     task_dir=TASK_PATH,
+        #     docs_dir=DOC_PATH,
+        #     env=test_env_manual_learn_android
+        # )
+    ]
+    loop.run_until_complete(asyncio.gather(*test_action_list))
+    loop.close()
+    print("Finish")