Merge pull request #1135 from better629/feat_android

Feat android assistant agent

tests/metagpt/environment/android_env/test_android_ext_env.py

@@ -16,8 +16,8 @@ def mock_device_shape_invalid(self, adb_cmd: str) -> str:
     return ADB_EXEC_FAIL
 
 
-def mock_list_devices(self, adb_cmd: str) -> str:
-    return "devices\nemulator-5554"
+def mock_list_devices(self) -> str:
+    return ["emulator-5554"]
 
 
 def mock_get_screenshot(self, adb_cmd: str) -> str:
@@ -35,6 +35,7 @@ def mock_write_read_operation(self, adb_cmd: str) -> str:
 def test_android_ext_env(mocker):
     device_id = "emulator-5554"
     mocker.patch("metagpt.environment.android.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_device_shape)
+    mocker.patch("metagpt.environment.android.android_ext_env.AndroidExtEnv.list_devices", mock_list_devices)
 
     ext_env = AndroidExtEnv(device_id=device_id, screenshot_dir="/data2/", xml_dir="/data2/")
     assert ext_env.adb_prefix == f"adb -s {device_id} "
@@ -48,7 +49,6 @@ def test_android_ext_env(mocker):
     )
     assert ext_env.device_shape == (0, 0)
 
-    mocker.patch("metagpt.environment.android.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_list_devices)
     assert ext_env.list_devices() == [device_id]
 
     mocker.patch("metagpt.environment.android.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_get_screenshot)

tests/metagpt/environment/test_base_env.py

@@ -64,7 +64,7 @@ async def test_ext_env():
     _ = await env.write_thru_api(EnvAPIAbstract(api_name="write_api", kwargs={"a": 5, "b": 10}))
     assert env.value == 15
 
-    with pytest.raises(ValueError):
+    with pytest.raises(KeyError):
         await env.read_from_api("not_exist_api")
 
     assert await env.read_from_api("read_api_no_param") == 15

tests/metagpt/ext/android_assistant/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

tests/metagpt/ext/android_assistant/test_an.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : test on android emulator action. After Modify Role Test, this script is discarded.
+
+import asyncio
+import time
+from pathlib import Path
+
+from metagpt.const import TEST_DATA_PATH
+from metagpt.environment.android.android_env import AndroidEnv
+from metagpt.ext.android_assistant.actions.manual_record import ManualRecord
+from metagpt.ext.android_assistant.actions.parse_record import ParseRecord
+from metagpt.ext.android_assistant.actions.screenshot_parse import ScreenshotParse
+from metagpt.ext.android_assistant.actions.self_learn_and_reflect import (
+    SelfLearnAndReflect,
+)
+
+TASK_PATH = TEST_DATA_PATH.joinpath("andriod_assistant/unitest_Contacts")
+TASK_PATH.mkdir(parents=True, exist_ok=True)
+DEMO_NAME = str(time.time())
+SELF_EXPLORE_DOC_PATH = TASK_PATH.joinpath("auto_docs")
+PARSE_RECORD_DOC_PATH = TASK_PATH.joinpath("demo_docs")
+
+device_id = "emulator-5554"
+xml_dir = Path("/sdcard")
+screenshot_dir = Path("/sdcard/Pictures/Screenshots")
+
+test_env_self_learn_android = AndroidEnv(
+    device_id=device_id,
+    xml_dir=xml_dir,
+    screenshot_dir=screenshot_dir,
+)
+test_self_learning = SelfLearnAndReflect()
+
+test_env_manual_learn_android = AndroidEnv(
+    device_id=device_id,
+    xml_dir=xml_dir,
+    screenshot_dir=screenshot_dir,
+)
+test_manual_record = ManualRecord()
+test_manual_parse = ParseRecord()
+
+test_env_screenshot_parse_android = AndroidEnv(
+    device_id=device_id,
+    xml_dir=xml_dir,
+    screenshot_dir=screenshot_dir,
+)
+test_screenshot_parse = ScreenshotParse()
+
+
+if __name__ == "__main__":
+    loop = asyncio.get_event_loop()
+
+    test_action_list = [
+        test_self_learning.run(
+            round_count=20,
+            task_desc="Create a contact in Contacts App named zjy with a phone number +86 18831933368 ",
+            last_act="",
+            task_dir=TASK_PATH / "demos" / f"self_learning_{DEMO_NAME}",
+            docs_dir=SELF_EXPLORE_DOC_PATH,
+            env=test_env_self_learn_android,
+        ),
+        test_manual_record.run(
+            task_dir=TASK_PATH / "demos" / f"manual_record_{DEMO_NAME}",
+            task_desc="Create a contact in Contacts App named zjy with a phone number +86 18831933368 ",
+            env=test_env_manual_learn_android,
+        ),
+        test_manual_parse.run(
+            task_dir=TASK_PATH / "demos" / f"manual_record_{DEMO_NAME}",  # 修要修改
+            docs_dir=PARSE_RECORD_DOC_PATH,  # 需要修改
+            env=test_env_manual_learn_android,
+        ),
+        test_screenshot_parse.run(
+            round_count=20,
+            task_desc="Create a contact in Contacts App named zjy with a phone number +86 18831933368 ",
+            last_act="",
+            task_dir=TASK_PATH / f"act_{DEMO_NAME}",
+            docs_dir=PARSE_RECORD_DOC_PATH,
+            env=test_env_screenshot_parse_android,
+            grid_on=False,
+        ),
+    ]
+
+    loop.run_until_complete(asyncio.gather(*test_action_list))
+    loop.close()

tests/metagpt/ext/android_assistant/test_parse_record.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : test case (imgs from appagent's)
+
+import asyncio
+
+from metagpt.actions.action import Action
+from metagpt.const import TEST_DATA_PATH
+from metagpt.ext.android_assistant.actions.parse_record import ParseRecord
+
+TASK_PATH = TEST_DATA_PATH.joinpath("andriod_assistant/demo_Contacts")
+TEST_BEFORE_PATH = TASK_PATH.joinpath("labeled_screenshots/0_labeled.png")
+TEST_AFTER_PATH = TASK_PATH.joinpath("labeled_screenshots/1_labeled.png")
+RECORD_PATH = TASK_PATH.joinpath("record.txt")
+TASK_DESC_PATH = TASK_PATH.joinpath("task_desc.txt")
+DOCS_DIR = TASK_PATH.joinpath("storage")
+
+test_action = Action(name="test")
+
+
+async def manual_learn_test():
+    parse_record = ParseRecord()
+    await parse_record.run(app_name="demo_Contacts", task_dir=TASK_PATH, docs_dir=DOCS_DIR)
+
+
+if __name__ == "__main__":
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(manual_learn_test())
+    loop.close()