mv android_assistant folder and fix conflicts

examples/android_assistant/run_assistant.py

@@ -7,9 +7,9 @@ from pathlib import Path
 
 import typer
 
-from examples.android_assistant.roles.android_assistant import AndroidAssistant
 from metagpt.config2 import config
-from metagpt.environment.android_env.android_env import AndroidEnv
+from metagpt.environment.android.android_env import AndroidEnv
+from metagpt.ext.android_assistant.roles.android_assistant import AndroidAssistant
 from metagpt.team import Team
 
 app = typer.Typer(add_completion=False, pretty_exceptions_show_locals=False)
@@ -61,7 +61,7 @@ def startup(
         )
     )
 
-    team.hire([AndroidAssistant()])
+    team.hire([AndroidAssistant(output_root_dir=Path(__file__))])
     team.invest(investment)
     team.run_project(idea=task_desc)
     asyncio.run(team.run(n_round=n_round))

examples/android_assistant/utils/const.py

@@ -1,8 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# @Desc   :
-
-from pathlib import Path
-
-ROOT_PATH = Path(__file__).parent.parent.absolute()
-ADB_EXEC_FAIL = "FAILED"

metagpt/ext/android_assistant/actions/manual_record.py

@@ -6,26 +6,26 @@ from pathlib import Path
 
 import cv2
 
-from examples.android_assistant.utils.const import ADB_EXEC_FAIL
-from examples.android_assistant.utils.schema import (
+from metagpt.actions.action import Action
+from metagpt.config2 import config
+from metagpt.environment.android.android_env import AndroidEnv
+from metagpt.environment.android.const import ADB_EXEC_FAIL
+from metagpt.environment.android.env_space import (
+    EnvAction,
+    EnvActionType,
+    EnvObsParams,
+    EnvObsType,
+)
+from metagpt.ext.android_assistant.utils.schema import (
     ActionOp,
     AndroidActionOutput,
     RunState,
     SwipeOp,
 )
-from examples.android_assistant.utils.utils import (
+from metagpt.ext.android_assistant.utils.utils import (
     draw_bbox_multi,
     elem_list_from_xml_tree,
 )
-from metagpt.actions.action import Action
-from metagpt.config2 import config
-from metagpt.environment.android_env.android_env import AndroidEnv
-from metagpt.environment.android_env.env_space import (
-    EnvAction,
-    EnvActionType,
-    EnvObsParams,
-    EnvObsType,
-)
 from metagpt.logs import logger
 
 

metagpt/ext/android_assistant/actions/parse_record.py

@@ -7,23 +7,23 @@ import ast
 import re
 from pathlib import Path
 
-from examples.android_assistant.actions.parse_record_an import RECORD_PARSE_NODE
-from examples.android_assistant.prompts.operation_prompt import (
+from metagpt.actions.action import Action
+from metagpt.config2 import config
+from metagpt.ext.android_assistant.actions.parse_record_an import RECORD_PARSE_NODE
+from metagpt.ext.android_assistant.prompts.operation_prompt import (
     long_press_doc_template,
     refine_doc_suffix,
     swipe_doc_template,
     tap_doc_template,
     text_doc_template,
 )
-from examples.android_assistant.utils.schema import (
+from metagpt.ext.android_assistant.utils.schema import (
     ActionOp,
     AndroidActionOutput,
     RecordLogItem,
     RunState,
     SwipeOp,
 )
-from metagpt.actions.action import Action
-from metagpt.config2 import config
 from metagpt.logs import logger
 from metagpt.utils.common import encode_image
 

metagpt/ext/android_assistant/actions/screenshot_parse.py

@@ -5,13 +5,24 @@
 import ast
 from pathlib import Path
 
-from examples.android_assistant.actions.screenshot_parse_an import SCREENSHOT_PARSE_NODE
-from examples.android_assistant.prompts.assistant_prompt import (
+from metagpt.actions.action import Action
+from metagpt.config2 import config
+from metagpt.environment.android.android_env import AndroidEnv
+from metagpt.environment.android.const import ADB_EXEC_FAIL
+from metagpt.environment.android.env_space import (
+    EnvAction,
+    EnvActionType,
+    EnvObsParams,
+    EnvObsType,
+)
+from metagpt.ext.android_assistant.actions.screenshot_parse_an import (
+    SCREENSHOT_PARSE_NODE,
+)
+from metagpt.ext.android_assistant.prompts.assistant_prompt import (
     screenshot_parse_template,
     screenshot_parse_with_grid_template,
 )
-from examples.android_assistant.utils.const import ADB_EXEC_FAIL
-from examples.android_assistant.utils.schema import (
+from metagpt.ext.android_assistant.utils.schema import (
     AndroidActionOutput,
     AndroidElement,
     GridOp,
@@ -25,7 +36,7 @@ from examples.android_assistant.utils.schema import (
     TapOp,
     TextOp,
 )
-from examples.android_assistant.utils.utils import (
+from metagpt.ext.android_assistant.utils.utils import (
     area_to_xy,
     draw_bbox_multi,
     draw_grid,
@@ -33,15 +44,6 @@ from examples.android_assistant.utils.utils import (
     screenshot_parse_extract,
     traverse_xml_tree,
 )
-from metagpt.actions.action import Action
-from metagpt.config2 import config
-from metagpt.environment.android_env.android_env import AndroidEnv
-from metagpt.environment.android_env.env_space import (
-    EnvAction,
-    EnvActionType,
-    EnvObsParams,
-    EnvObsType,
-)
 from metagpt.logs import logger
 from metagpt.utils.common import encode_image
 

metagpt/ext/android_assistant/actions/self_learn_and_reflect.py

@@ -5,18 +5,29 @@
 import ast
 from pathlib import Path
 
-from examples.android_assistant.actions.screenshot_parse_an import SCREENSHOT_PARSE_NODE
-from examples.android_assistant.actions.self_learn_reflect_an import (
+from metagpt.actions.action import Action
+from metagpt.config2 import config
+from metagpt.environment.android.android_env import AndroidEnv
+from metagpt.environment.android.const import ADB_EXEC_FAIL
+from metagpt.environment.android.env_space import (
+    EnvAction,
+    EnvActionType,
+    EnvObsParams,
+    EnvObsType,
+)
+from metagpt.ext.android_assistant.actions.screenshot_parse_an import (
+    SCREENSHOT_PARSE_NODE,
+)
+from metagpt.ext.android_assistant.actions.self_learn_reflect_an import (
     SELF_LEARN_REFLECT_NODE,
 )
-from examples.android_assistant.prompts.assistant_prompt import (
+from metagpt.ext.android_assistant.prompts.assistant_prompt import (
     screenshot_parse_self_explore_reflect_template as reflect_template,
 )
-from examples.android_assistant.prompts.assistant_prompt import (
+from metagpt.ext.android_assistant.prompts.assistant_prompt import (
     screenshot_parse_self_explore_template,
 )
-from examples.android_assistant.utils.const import ADB_EXEC_FAIL
-from examples.android_assistant.utils.schema import (
+from metagpt.ext.android_assistant.utils.schema import (
     ActionOp,
     AndroidActionOutput,
     AndroidElement,
@@ -31,22 +42,13 @@ from examples.android_assistant.utils.schema import (
     TapOp,
     TextOp,
 )
-from examples.android_assistant.utils.utils import (
+from metagpt.ext.android_assistant.utils.utils import (
     draw_bbox_multi,
     elem_bbox_to_xy,
     elem_list_from_xml_tree,
     reflect_parse_extarct,
     screenshot_parse_extract,
 )
-from metagpt.actions.action import Action
-from metagpt.config2 import config
-from metagpt.environment.android_env.android_env import AndroidEnv
-from metagpt.environment.android_env.env_space import (
-    EnvAction,
-    EnvActionType,
-    EnvObsParams,
-    EnvObsType,
-)
 from metagpt.logs import logger
 from metagpt.utils.common import encode_image
 

metagpt/ext/android_assistant/roles/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

metagpt/ext/android_assistant/roles/android_assistant.py

@@ -8,16 +8,16 @@ from typing import Optional
 
 from pydantic import Field
 
-from examples.android_assistant.actions.manual_record import ManualRecord
-from examples.android_assistant.actions.parse_record import ParseRecord
-from examples.android_assistant.actions.screenshot_parse import ScreenshotParse
-from examples.android_assistant.actions.self_learn_and_reflect import (
-    SelfLearnAndReflect,
-)
-from examples.android_assistant.utils.const import ROOT_PATH
-from examples.android_assistant.utils.schema import AndroidActionOutput, RunState
 from metagpt.actions.add_requirement import UserRequirement
 from metagpt.config2 import config
+from metagpt.const import EXAMPLE_PATH
+from metagpt.ext.android_assistant.actions.manual_record import ManualRecord
+from metagpt.ext.android_assistant.actions.parse_record import ParseRecord
+from metagpt.ext.android_assistant.actions.screenshot_parse import ScreenshotParse
+from metagpt.ext.android_assistant.actions.self_learn_and_reflect import (
+    SelfLearnAndReflect,
+)
+from metagpt.ext.android_assistant.utils.schema import AndroidActionOutput, RunState
 from metagpt.logs import logger
 from metagpt.roles.role import Role, RoleReactMode
 from metagpt.schema import Message
@@ -31,6 +31,7 @@ class AndroidAssistant(Role):
     task_desc: str = ""
     round_count: int = 0
     last_act: str = ""
+    output_root_dir: Optional[Path] = Field(default=None)
     task_dir: Optional[Path] = Field(default=None)
     docs_dir: Optional[Path] = Field(default=None)
     grid_on: bool = Field(default=False)
@@ -41,7 +42,7 @@ class AndroidAssistant(Role):
         self._watch([UserRequirement, AndroidActionOutput])
         self.task_desc = config.get_other("task_desc", "Just explore any app in this phone!")
         app_name = config.get_other("app_name", "demo")
-        data_dir = ROOT_PATH.joinpath("output")
+        data_dir = self.output_root_dir.absolute() or EXAMPLE_PATH.joinpath("android_assistant/output")
         cur_datetime = datetime.fromtimestamp(int(time.time())).strftime("%Y-%m-%d_%H-%M-%S")
 
         """Firstly, we decide the state with user config, further, we can do it automatically, like if it's new app,

metagpt/ext/android_assistant/utils/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

metagpt/ext/android_assistant/utils/utils.py

@@ -10,7 +10,7 @@ from xml.etree.ElementTree import Element, iterparse
 import cv2
 import pyshine as ps
 
-from examples.android_assistant.utils.schema import (
+from metagpt.ext.android_assistant.utils.schema import (
     ActionOp,
     AndroidElement,
     BaseGridOpParam,

metagpt/provider/openai_api.py

@@ -109,15 +109,9 @@ class OpenAILLM(BaseLLM):
         kwargs = {
             "messages": messages,
             "max_tokens": self._get_max_tokens(messages),
-<<<<<<< HEAD
             # "n": 1,  # Some services do not provide this parameter, such as mistral
             # "stop": None,  # default it's None and gpt4-v can't have this one
             "temperature": self.config.temperature,
-=======
-            "n": 1,
-            # "stop": None,  # default it's None and gpt4-v can't have this one
-            "temperature": 0.3,
->>>>>>> 9cbc3466 (add gpt4-v)
             "model": self.model,
             "timeout": self.get_timeout(timeout),
         }

metagpt/utils/common.py

@@ -25,17 +25,9 @@ import re
 import sys
 import traceback
 from io import BytesIO
-import typing
-<<<<<<< HEAD
-import base64
 from pathlib import Path
 from typing import Any, Callable, List, Literal, Tuple, Union
 from urllib.parse import quote, unquote
-=======
-from io import BytesIO
-from pathlib import Path
-from typing import Any, Callable, List, Tuple, Union
->>>>>>> 138bb6e6 (FIx Format and Some bugs in android_assistant.py)
 
 import aiofiles
 import chardet
@@ -371,19 +363,6 @@ def parse_recipient(text):
     return ""
 
 
-<<<<<<< HEAD
-=======
-def create_func_call_config(func_schema: dict) -> dict:
-    """Create new function call config"""
-    tools = [{"type": "function", "function": func_schema}]
-    tool_choice = {"type": "function", "function": {"name": func_schema["name"]}}
-    return {
-        "tools": tools,
-        "tool_choice": tool_choice,
-    }
-
-
->>>>>>> 138bb6e6 (FIx Format and Some bugs in android_assistant.py)
 def remove_comments(code_str: str) -> str:
     """Remove comments from code."""
     pattern = r"(\".*?\"|\'.*?\')|(\#.*?$)"
@@ -600,32 +579,6 @@ def write_json_file(json_file: str, data: list, encoding: str = None, indent: in
 
     with open(json_file, "w", encoding=encoding) as fout:
         json.dump(data, fout, ensure_ascii=False, indent=indent, default=to_jsonable_python)
-<<<<<<< HEAD
-
-
-def read_csv_to_list(curr_file: str, header=False, strip_trail=True):
-    """
-    Reads in a csv file to a list of list. If header is True, it returns a
-    tuple with (header row, all rows)
-    ARGS:
-      curr_file: path to the current csv file.
-    RETURNS:
-      List of list where the component lists are the rows of the file.
-    """
-    logger.debug(f"start read csv: {curr_file}")
-    analysis_list = []
-    with open(curr_file) as f_analysis_file:
-        data_reader = csv.reader(f_analysis_file, delimiter=",")
-        for count, row in enumerate(data_reader):
-            if strip_trail:
-                row = [i.strip() for i in row]
-            analysis_list += [row]
-    if not header:
-        return analysis_list
-    else:
-        return analysis_list[0], analysis_list[1:]
-=======
->>>>>>> 138bb6e6 (FIx Format and Some bugs in android_assistant.py)
 
 
 def read_csv_to_list(curr_file: str, header=False, strip_trail=True):
@@ -821,11 +774,7 @@ def is_coroutine_func(func: Callable) -> bool:
 
 
 def load_mc_skills_code(skill_names: list[str] = None, skills_dir: Path = None) -> list[str]:
-<<<<<<< HEAD
     """load minecraft skill from js files"""
-=======
-    """load mincraft skill from js files"""
->>>>>>> 138bb6e6 (FIx Format and Some bugs in android_assistant.py)
     if not skills_dir:
         skills_dir = Path(__file__).parent.absolute()
     if skill_names is None:
@@ -860,7 +809,6 @@ def decode_image(img_url_or_b64: str) -> Image:
         img_data = BytesIO(base64.b64decode(b64_data))
         img = Image.open(img_data)
     return img
-<<<<<<< HEAD
 
 
 def log_and_reraise(retry_state: RetryCallState):
@@ -890,5 +838,3 @@ def get_markdown_codeblock_type(filename: str) -> str:
         "application/sql": "sql",
     }
     return mappings.get(mime_type, "text")
-=======
->>>>>>> 138bb6e6 (FIx Format and Some bugs in android_assistant.py)

setup.py

@@ -39,6 +39,7 @@ extras_require = {
         "llama-index-vector-stores-elasticsearch==0.1.6",
         "llama-index-vector-stores-chroma==0.1.6",
     ],
+    "android_assistant": ["pyshine==0.0.9", "opencv-python==4.6.0.66"],
 }
 
 extras_require["test"] = [

tests/metagpt/actions/test_action_node.py

@@ -9,11 +9,7 @@ from pathlib import Path
 from typing import List, Tuple
 
 import pytest
-<<<<<<< HEAD
 from pydantic import BaseModel, Field, ValidationError
-=======
-from pydantic import ValidationError
->>>>>>> 7f8ae1f2 (update gpt4-v)
 
 from metagpt.actions import Action
 from metagpt.actions.action_node import ActionNode, ReviewMode, ReviseMode

tests/metagpt/ext/android_assistant/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

tests/metagpt/ext/android_assistant/test_an.py

@@ -6,16 +6,17 @@ import asyncio
 import time
 from pathlib import Path
 
-from examples.android_assistant.actions.manual_record import ManualRecord
-from examples.android_assistant.actions.parse_record import ParseRecord
-from examples.android_assistant.actions.screenshot_parse import ScreenshotParse
-from examples.android_assistant.actions.self_learn_and_reflect import (
+from metagpt.const import TEST_DATA_PATH
+from metagpt.environment.android.android_env import AndroidEnv
+from metagpt.ext.android_assistant.actions.manual_record import ManualRecord
+from metagpt.ext.android_assistant.actions.parse_record import ParseRecord
+from metagpt.ext.android_assistant.actions.screenshot_parse import ScreenshotParse
+from metagpt.ext.android_assistant.actions.self_learn_and_reflect import (
     SelfLearnAndReflect,
 )
-from examples.android_assistant.utils.const import ROOT_PATH
-from metagpt.environment.android_env.android_env import AndroidEnv
 
-TASK_PATH = ROOT_PATH.joinpath("unitest_Contacts")
+TASK_PATH = TEST_DATA_PATH.joinpath("andriod_assistant/unitest_Contacts")
+TASK_PATH.mkdir(parents=True, exist_ok=True)
 DEMO_NAME = str(time.time())
 SELF_EXPLORE_DOC_PATH = TASK_PATH.joinpath("auto_docs")
 PARSE_RECORD_DOC_PATH = TASK_PATH.joinpath("demo_docs")

tests/metagpt/ext/android_assistant/test_parse_record.py

@@ -4,11 +4,11 @@
 
 import asyncio
 
-from examples.android_assistant.actions.parse_record import ParseRecord
-from examples.android_assistant.utils.const import ROOT_PATH
 from metagpt.actions.action import Action
+from metagpt.const import TEST_DATA_PATH
+from metagpt.ext.android_assistant.actions.parse_record import ParseRecord
 
-TASK_PATH = ROOT_PATH.parent.joinpath("data/demo_Contacts")
+TASK_PATH = TEST_DATA_PATH.joinpath("andriod_assistant/demo_Contacts")
 TEST_BEFORE_PATH = TASK_PATH.joinpath("labeled_screenshots/0_labeled.png")
 TEST_AFTER_PATH = TASK_PATH.joinpath("labeled_screenshots/1_labeled.png")
 RECORD_PATH = TASK_PATH.joinpath("record.txt")