Merge branch 'feature/toollib/shell' of https://github.com/iorisa/MetaGPT into feature/toollib/merge

examples/di/custom_tool.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/3/22 10:54
+@Author  : alexanderwu
+@File    : custom_tool.py
+"""
+
+from metagpt.roles.di.data_interpreter import DataInterpreter
+from metagpt.tools.tool_registry import register_tool
+
+
+@register_tool()
+def magic_function(arg1: str, arg2: int) -> dict:
+    """
+    The magic function that does something.
+
+    Args:
+        arg1 (str): ...
+        arg2 (int): ...
+
+    Returns:
+        dict: ...
+    """
+    return {"arg1": arg1 * 3, "arg2": arg2 * 5}
+
+
+async def main():
+    di = DataInterpreter(tools=["magic_function"])
+    await di.run("Just call the magic function with arg1 'A' and arg2 2. Tell me the result.")
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

metagpt/environment/README.md

@@ -34,5 +34,5 @@ # do a `tap` action on the screen
 ## TODO
 - add android app operation assistant under `examples/android_assistant`
 - migrate roles/actions of werewolf game from old version into current version
-- migrate roles/actions of mincraft game from old version into current version
+- migrate roles/actions of minecraft game from old version into current version
 - migrate roles/actions of stanford_town game from old version into current version

metagpt/environment/__init__.py

@@ -4,10 +4,9 @@
 
 from metagpt.environment.base_env import Environment
 from metagpt.environment.android_env.android_env import AndroidEnv
-from metagpt.environment.mincraft_env.mincraft_env import MincraftExtEnv
 from metagpt.environment.werewolf_env.werewolf_env import WerewolfEnv
 from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv
 from metagpt.environment.software_env.software_env import SoftwareEnv
 
 
-__all__ = ["AndroidEnv", "MincraftExtEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"]
+__all__ = ["AndroidEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"]

metagpt/environment/base_env.py

@@ -26,7 +26,7 @@ class EnvType(Enum):
     ANDROID = "Android"
     GYM = "Gym"
     WEREWOLF = "Werewolf"
-    MINCRAFT = "Mincraft"
+    MINECRAFT = "Minecraft"
     STANFORDTOWN = "StanfordTown"
 
 

metagpt/environment/minecraft_env/const.py

@@ -4,8 +4,8 @@
 
 from metagpt.const import METAGPT_ROOT
 
-# For Mincraft Game Agent
-MC_CKPT_DIR = METAGPT_ROOT / "data/mincraft/ckpt"
+# For Minecraft Game Agent
+MC_CKPT_DIR = METAGPT_ROOT / "data/minecraft/ckpt"
 MC_LOG_DIR = METAGPT_ROOT / "logs"
 MC_DEFAULT_WARMUP = {
     "context": 15,

metagpt/environment/minecraft_env/minecraft_env.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# @Desc   : MG Mincraft Env
+# @Desc   : MG Minecraft Env
 #           refs to `voyager voyager.py`
 
 import json
@@ -12,15 +12,15 @@ from pydantic import ConfigDict, Field
 
 from metagpt.config2 import config as CONFIG
 from metagpt.environment.base_env import Environment
-from metagpt.environment.mincraft_env.const import MC_CKPT_DIR
-from metagpt.environment.mincraft_env.mincraft_ext_env import MincraftExtEnv
+from metagpt.environment.minecraft_env.const import MC_CKPT_DIR
+from metagpt.environment.minecraft_env.minecraft_ext_env import MinecraftExtEnv
 from metagpt.logs import logger
 from metagpt.rag.vector_stores.chroma import ChromaVectorStore
 from metagpt.utils.common import load_mc_skills_code, read_json_file, write_json_file
 
 
-class MincraftEnv(Environment, MincraftExtEnv):
-    """MincraftEnv, including shared memory of cache and information between roles"""
+class MinecraftEnv(Environment, MinecraftExtEnv):
+    """MinecraftEnv, including shared memory of cache and information between roles"""
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 

metagpt/environment/minecraft_env/minecraft_ext_env.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# @Desc   : The Mincraft external environment to integrate with Mincraft game
+# @Desc   : The Minecraft external environment to integrate with Minecraft game
 #           refs to `voyager bridge.py`
 
 import json
@@ -11,18 +11,18 @@ import requests
 from pydantic import ConfigDict, Field, model_validator
 
 from metagpt.environment.base_env import ExtEnv, mark_as_writeable
-from metagpt.environment.mincraft_env.const import (
+from metagpt.environment.minecraft_env.const import (
     MC_CKPT_DIR,
     MC_CORE_INVENTORY_ITEMS,
     MC_CURRICULUM_OB,
     MC_DEFAULT_WARMUP,
     METAGPT_ROOT,
 )
-from metagpt.environment.mincraft_env.process_monitor import SubprocessMonitor
+from metagpt.environment.minecraft_env.process_monitor import SubprocessMonitor
 from metagpt.logs import logger
 
 
-class MincraftExtEnv(ExtEnv):
+class MinecraftExtEnv(ExtEnv):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     mc_port: Optional[int] = Field(default=None)
@@ -48,7 +48,7 @@ class MincraftExtEnv(ExtEnv):
             self.mineflayer = SubprocessMonitor(
                 commands=[
                     "node",
-                    METAGPT_ROOT.joinpath("metagpt", "environment", "mincraft_env", "mineflayer", "index.js"),
+                    METAGPT_ROOT.joinpath("metagpt", "environment", "minecraft_env", "mineflayer", "index.js"),
                     str(self.server_port),
                 ],
                 name="mineflayer",

metagpt/roles/__init__.py

@@ -14,7 +14,6 @@ from metagpt.roles.engineer import Engineer
 from metagpt.roles.qa_engineer import QaEngineer
 from metagpt.roles.searcher import Searcher
 from metagpt.roles.sales import Sales
-from metagpt.roles.customer_service import CustomerService
 
 
 __all__ = [
@@ -26,5 +25,4 @@ __all__ = [
     "QaEngineer",
     "Searcher",
     "Sales",
-    "CustomerService",
 ]

metagpt/tools/libs/shell.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+from typing import Dict, List, Tuple, Union
+
+from metagpt.tools.tool_registry import register_tool
+
+
+@register_tool(tags=["shell"])
+async def execute(
+    command: Union[List[str], str], cwd: str | Path = None, env: Dict = None, timeout: int = 600
+) -> Tuple[str, str]:
+    """
+    Execute a command asynchronously and return its standard output and standard error.
+
+    Args:
+        command (Union[List[str], str]): The command to execute and its arguments. It can be provided either as a list
+            of strings or as a single string.
+        cwd (str | Path, optional): The current working directory for the command. Defaults to None.
+        env (Dict, optional): Environment variables to set for the command. Defaults to None.
+        timeout (int, optional): Timeout for the command execution in seconds. Defaults to 600.
+
+    Returns:
+        Tuple[str, str]: A tuple containing the standard output and standard error of the executed command, both as
+         strings.
+
+    Raises:
+        ValueError: If the command times out, this error is raised. The error message contains both standard output and
+         standard error of the timed-out process.
+
+    Example:
+        >>> # command is a list
+        >>> stdout, stderr = await execute(command=["ls", "-l"], cwd="/home/user", env={"PATH": "/usr/bin"})
+        >>> print(stdout)
+        total 8
+        -rw-r--r-- 1 user user    0 Mar 22 10:00 file1.txt
+        -rw-r--r-- 1 user user    0 Mar 22 10:00 file2.txt
+        ...
+
+        >>> # command is a string of shell script
+        >>> stdout, stderr = await execute(command="ls -l", cwd="/home/user", env={"PATH": "/usr/bin"})
+        >>> print(stdout)
+        total 8
+        -rw-r--r-- 1 user user    0 Mar 22 10:00 file1.txt
+        -rw-r--r-- 1 user user    0 Mar 22 10:00 file2.txt
+        ...
+    """
+    cwd = str(cwd) if cwd else None
+    shell = True if isinstance(command, str) else False
+    process = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, shell=shell)
+    try:
+        # Wait for the process to complete, with a timeout
+        stdout, stderr = process.communicate(timeout=timeout)
+        return stdout.decode("utf-8"), stderr.decode("utf-8")
+    except subprocess.TimeoutExpired:
+        process.kill()  # Kill the process if it times out
+        stdout, stderr = process.communicate()
+        raise ValueError(f"{stdout.decode('utf-8')}\n{stderr.decode('utf-8')}")

metagpt/utils/common.py

@@ -774,7 +774,7 @@ def is_coroutine_func(func: Callable) -> bool:
 
 
 def load_mc_skills_code(skill_names: list[str] = None, skills_dir: Path = None) -> list[str]:
-    """load mincraft skill from js files"""
+    """load minecraft skill from js files"""
     if not skills_dir:
         skills_dir = Path(__file__).parent.absolute()
     if skill_names is None:

requirements.txt

@@ -10,14 +10,6 @@ typer==0.9.0
 # godot==0.1.1
 # google_api_python_client==2.93.0  # Used by search_engine.py
 lancedb==0.4.0
-llama-index-core==0.10.15
-llama-index-embeddings-azure-openai==0.1.6
-llama-index-embeddings-openai==0.1.5
-llama-index-llms-azure-openai==0.1.4
-llama-index-readers-file==0.1.4
-llama-index-retrievers-bm25==0.1.3
-llama-index-vector-stores-faiss==0.1.1
-chromadb==0.4.23
 loguru==0.6.0
 meilisearch==0.21.0
 numpy==1.24.3

setup.py

@@ -28,6 +28,16 @@ extras_require = {
     "search-google": ["google-api-python-client==2.94.0"],
     "search-ddg": ["duckduckgo-search~=4.1.1"],
     "ocr": ["paddlepaddle==2.4.2", "paddleocr>=2.0.1", "tabulate==0.9.0"],
+    "rag": [
+        "llama-index-core==0.10.15",
+        "llama-index-embeddings-azure-openai==0.1.6",
+        "llama-index-embeddings-openai==0.1.5",
+        "llama-index-llms-azure-openai==0.1.4",
+        "llama-index-readers-file==0.1.4",
+        "llama-index-retrievers-bm25==0.1.3",
+        "llama-index-vector-stores-faiss==0.1.1",
+        "chromadb==0.4.23",
+    ],
 }
 
 extras_require["test"] = [
@@ -42,7 +52,6 @@ extras_require["test"] = [
     "connexion[uvicorn]~=3.0.5",
     "azure-cognitiveservices-speech~=1.31.0",
     "aioboto3~=11.3.0",
-    "chromadb==0.4.23",
     "gradio==3.0.0",
     "grpcio-status==1.48.2",
     "pylint==3.0.3",

tests/metagpt/environment/mincraft_env/test_mincraft_ext_env.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# @Desc   : the unittest of MincraftExtEnv
-
-
-from metagpt.environment.mincraft_env.const import MC_CKPT_DIR
-from metagpt.environment.mincraft_env.mincraft_ext_env import MincraftExtEnv
-
-
-def test_mincraft_ext_env():
-    ext_env = MincraftExtEnv()
-    assert ext_env.server, f"{ext_env.server_host}:{ext_env.server_port}"
-    assert MC_CKPT_DIR.joinpath("skill/code").exists()
-    assert ext_env.warm_up.get("optional_inventory_items") == 7

tests/metagpt/environment/minecraft_env/test_minecraft_ext_env.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : the unittest of MinecraftExtEnv
+
+
+from metagpt.environment.minecraft_env.const import MC_CKPT_DIR
+from metagpt.environment.minecraft_env.minecraft_ext_env import MinecraftExtEnv
+
+
+def test_minecraft_ext_env():
+    ext_env = MinecraftExtEnv()
+    assert ext_env.server, f"{ext_env.server_host}:{ext_env.server_port}"
+    assert MC_CKPT_DIR.joinpath("skill/code").exists()
+    assert ext_env.warm_up.get("optional_inventory_items") == 7

tests/metagpt/tools/libs/test_shell.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import pytest
+
+from metagpt.tools.libs.shell import execute
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ["command", "expect_stdout", "expect_stderr"],
+    [
+        (["file", f"{__file__}"], "Python script text executable, ASCII text", ""),
+        (f"file {__file__}", "Python script text executable, ASCII text", ""),
+    ],
+)
+async def test_shell(command, expect_stdout, expect_stderr):
+    stdout, stderr = await execute(command)
+    assert expect_stdout in stdout
+    assert stderr == expect_stderr
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-s"])