Merge branch 'dev' into v0.5-release

config/config.yaml

@@ -23,7 +23,7 @@ RPM: 10
 #SPARK_URL : "ws://spark-api.xf-yun.com/v2.1/chat"
 
 #### if Anthropic
-#Anthropic_API_KEY: "YOUR_API_KEY"
+#ANTHROPIC_API_KEY: "YOUR_API_KEY"
 
 #### if AZURE, check https://github.com/openai/openai-cookbook/blob/main/examples/azure/chat.ipynb
 #### You can use ENGINE or DEPLOYMENT mode

metagpt/actions/action.py

@@ -15,8 +15,7 @@ from metagpt.actions.action_output import ActionOutput
 from metagpt.llm import LLM
 from metagpt.logs import logger
 from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess
-from metagpt.utils.common import OutputParser
-from metagpt.utils.utils import general_after_log
+from metagpt.utils.common import OutputParser, general_after_log
 
 
 class Action(ABC):

metagpt/actions/action_node.py

@@ -43,7 +43,7 @@ Fill in the above nodes based on the format example.
 """
 
 
-def dict_to_markdown(d, prefix="-", postfix="\n"):
+def dict_to_markdown(d, prefix="###", postfix="\n"):
     markdown_str = ""
     for key, value in d.items():
         markdown_str += f"{prefix} {key}: {value}{postfix}"
@@ -52,6 +52,7 @@ def dict_to_markdown(d, prefix="-", postfix="\n"):
 
 class ActionNode:
     """ActionNode is a tree of nodes."""
+
     mode: str
 
     # Action Context
@@ -70,8 +71,15 @@ class ActionNode:
     content: str
     instruct_content: BaseModel
 
-    def __init__(self, key: str, expected_type: Type, instruction: str, example: str, content: str = "",
-                 children: dict[str, "ActionNode"] = None):
+    def __init__(
+        self,
+        key: str,
+        expected_type: Type,
+        instruction: str,
+        example: str,
+        content: str = "",
+        children: dict[str, "ActionNode"] = None,
+    ):
         self.key = key
         self.expected_type = expected_type
         self.instruction = instruction

metagpt/actions/project_management_an.py

@@ -44,7 +44,7 @@ FULL_API_SPEC = ActionNode(
     key="Full API spec",
     expected_type=str,
     instruction="Describe all APIs using OpenAPI 3.0 spec that may be used by both frontend and backend. If front-end "
-                "and back-end communication is not required, leave it blank.",
+    "and back-end communication is not required, leave it blank.",
     example="openapi: 3.0.0 ...",
 )
 

metagpt/actions/run_code.py

@@ -16,13 +16,13 @@
             class.
 """
 import subprocess
-import traceback
 from typing import Tuple
 
 from metagpt.actions.action import Action
 from metagpt.config import CONFIG
 from metagpt.logs import logger
 from metagpt.schema import RunCodeResult
+from metagpt.utils.exceptions import handle_exception
 
 PROMPT_TEMPLATE = """
 Role: You are a senior development and qa engineer, your role is summarize the code running result.
@@ -78,15 +78,12 @@ class RunCode(Action):
         super().__init__(name, context, llm)
 
     @classmethod
+    @handle_exception
     async def run_text(cls, code) -> Tuple[str, str]:
-        try:
-            # We will document_store the result in this dictionary
-            namespace = {}
-            exec(code, namespace)
-            return namespace.get("result", ""), ""
-        except Exception:
-            # If there is an error in the code, return the error message
-            return "", traceback.format_exc()
+        # We will document_store the result in this dictionary
+        namespace = {}
+        exec(code, namespace)
+        return namespace.get("result", ""), ""
 
     @classmethod
     async def run_script(cls, working_directory, additional_python_paths=[], command=[]) -> Tuple[str, str]:
@@ -145,18 +142,17 @@ class RunCode(Action):
         rsp = await self._aask(prompt)
         return RunCodeResult(summary=rsp, stdout=outs, stderr=errs)
 
+    @staticmethod
+    @handle_exception(exception_type=subprocess.CalledProcessError)
+    def _install_via_subprocess(cmd, check, cwd, env):
+        return subprocess.run(cmd, check=check, cwd=cwd, env=env)
+
     @staticmethod
     def _install_dependencies(working_directory, env):
         install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"]
         logger.info(" ".join(install_command))
-        try:
-            subprocess.run(install_command, check=True, cwd=working_directory, env=env)
-        except subprocess.CalledProcessError as e:
-            logger.warning(f"{e}")
+        RunCode._install_via_subprocess(install_command, check=True, cwd=working_directory, env=env)
 
         install_pytest_command = ["python", "-m", "pip", "install", "pytest"]
         logger.info(" ".join(install_pytest_command))
-        try:
-            subprocess.run(install_pytest_command, check=True, cwd=working_directory, env=env)
-        except subprocess.CalledProcessError as e:
-            logger.warning(f"{e}")
+        RunCode._install_via_subprocess(install_pytest_command, check=True, cwd=working_directory, env=env)

metagpt/actions/write_code_review.py

@@ -154,11 +154,15 @@ class WriteCodeReview(Action):
                 code=iterative_code,
                 filename=self.context.code_doc.filename,
             )
-            cr_prompt = EXAMPLE_AND_INSTRUCTION.format(format_example=format_example, )
+            cr_prompt = EXAMPLE_AND_INSTRUCTION.format(
+                format_example=format_example,
+            )
             logger.info(
                 f"Code review and rewrite {self.context.code_doc.filename}: {i+1}/{k} | {len(iterative_code)=}, {len(self.context.code_doc.content)=}"
             )
-            result, rewrited_code = await self.write_code_review_and_rewrite(context_prompt, cr_prompt, self.context.code_doc.filename)
+            result, rewrited_code = await self.write_code_review_and_rewrite(
+                context_prompt, cr_prompt, self.context.code_doc.filename
+            )
             if "LBTM" in result:
                 iterative_code = rewrited_code
             elif "LGTM" in result:

metagpt/config.py

@@ -8,6 +8,7 @@ Provide configuration, singleton
 """
 import os
 from copy import deepcopy
+from enum import Enum
 from pathlib import Path
 from typing import Any
 
@@ -31,6 +32,15 @@ class NotConfiguredException(Exception):
         super().__init__(self.message)
 
 
+class LLMProviderEnum(Enum):
+    OPENAI = "openai"
+    ANTHROPIC = "anthropic"
+    SPARK = "spark"
+    ZHIPUAI = "zhipuai"
+    FIREWORKS = "fireworks"
+    OPEN_LLM = "open_llm"
+
+
 class Config(metaclass=Singleton):
     """
     Regular usage method:
@@ -45,38 +55,58 @@ class Config(metaclass=Singleton):
     default_yaml_file = METAGPT_ROOT / "config/config.yaml"
 
     def __init__(self, yaml_file=default_yaml_file):
+
         golbal_options = OPTIONS.get()
+        # cli paras
+        self.project_path = ""
+        self.project_name = ""
+        self.inc = False
+        self.reqa_file = ""
+        self.max_auto_summarize_code = 0
+
         self._init_with_config_files_and_env(yaml_file)
-        logger.debug("Config loading done.")
         self._update()
         golbal_options.update(OPTIONS.get())
+        logger.debug("Config loading done.")
+        logger.info(f"OpenAI API Model: {self.openai_api_model}")
+
+    def get_default_llm_provider_enum(self):
+        if self._is_valid_llm_key(self.openai_api_key):
+            llm = LLMProviderEnum.OPENAI
+        elif self._is_valid_llm_key(self.anthropic_api_key):
+            llm = LLMProviderEnum.ANTHROPIC
+        elif self._is_valid_llm_key(self.zhipuai_api_key):
+            llm = LLMProviderEnum.ZHIPUAI
+        elif self._is_valid_llm_key(self.fireworks_api_key):
+            llm = LLMProviderEnum.FIREWORKS
+        elif self.open_llm_api_base:
+            llm = LLMProviderEnum.OPEN_LLM
+        else:
+            raise NotConfiguredException("You should config a LLM configuration first")
+        return llm
+
+    @staticmethod
+    def _is_valid_llm_key(k) -> bool:
+        return k and k != "YOUR_API_KEY"
 
     def _update(self):
         # logger.info("Config loading done.")
         self.global_proxy = self._get("GLOBAL_PROXY")
+
         self.openai_api_key = self._get("OPENAI_API_KEY")
-        self.anthropic_api_key = self._get("Anthropic_API_KEY")
+        self.anthropic_api_key = self._get("ANTHROPIC_API_KEY")
         self.zhipuai_api_key = self._get("ZHIPUAI_API_KEY")
         self.open_llm_api_base = self._get("OPEN_LLM_API_BASE")
         self.open_llm_api_model = self._get("OPEN_LLM_API_MODEL")
         self.fireworks_api_key = self._get("FIREWORKS_API_KEY")
-        if (
-            (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key)
-            and (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key)
-            and (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key)
-            and (not self.open_llm_api_base)
-            and (not self.fireworks_api_key or "YOUR_API_KEY" == self.fireworks_api_key)
-        ):
-            raise NotConfiguredException(
-                "Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first "
-                "or FIREWORKS_API_KEY or OPEN_LLM_API_BASE"
-            )
+        _ = self.get_default_llm_provider_enum()
+
         self.openai_api_base = self._get("OPENAI_API_BASE")
         self.openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy
         self.openai_api_type = self._get("OPENAI_API_TYPE")
         self.openai_api_version = self._get("OPENAI_API_VERSION")
         self.openai_api_rpm = self._get("RPM", 3)
-        self.openai_api_model = self._get("OPENAI_API_MODEL", "gpt-4")
+        self.openai_api_model = self._get("OPENAI_API_MODEL", "gpt-4-1106-preview")
         self.max_tokens_rsp = self._get("MAX_TOKENS", 2048)
         self.deployment_name = self._get("DEPLOYMENT_NAME")
         self.deployment_id = self._get("DEPLOYMENT_ID")
@@ -90,7 +120,7 @@ class Config(metaclass=Singleton):
         self.fireworks_api_base = self._get("FIREWORKS_API_BASE")
         self.fireworks_api_model = self._get("FIREWORKS_API_MODEL")
 
-        self.claude_api_key = self._get("Anthropic_API_KEY")
+        self.claude_api_key = self._get("ANTHROPIC_API_KEY")
         self.serpapi_api_key = self._get("SERPAPI_API_KEY")
         self.serper_api_key = self._get("SERPER_API_KEY")
         self.google_api_key = self._get("GOOGLE_API_KEY")
@@ -120,6 +150,19 @@ class Config(metaclass=Singleton):
         self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT))
         self._ensure_workspace_exists()
 
+    def update_via_cli(self, project_path, project_name, inc, reqa_file, max_auto_summarize_code):
+        """update config via cli"""
+
+        # Use in the PrepareDocuments action according to Section 2.2.3.5.1 of RFC 135.
+        if project_path:
+            inc = True
+            project_name = project_name or Path(project_path).name
+        self.project_path = project_path
+        self.project_name = project_name
+        self.inc = inc
+        self.reqa_file = reqa_file
+        self.max_auto_summarize_code = max_auto_summarize_code
+
     def _ensure_workspace_exists(self):
         self.workspace_path.mkdir(parents=True, exist_ok=True)
         logger.debug(f"WORKSPACE_PATH set to {self.workspace_path}")
@@ -142,8 +185,8 @@ class Config(metaclass=Singleton):
 
     @staticmethod
     def _get(*args, **kwargs):
-        m = OPTIONS.get()
-        return m.get(*args, **kwargs)
+        i = OPTIONS.get()
+        return i.get(*args, **kwargs)
 
     def get(self, key, *args, **kwargs):
         """Search for a value in config/key.yaml, config/config.yaml, and env; raise an error if not found"""
@@ -156,8 +199,8 @@ class Config(metaclass=Singleton):
         OPTIONS.get()[name] = value
 
     def __getattr__(self, name: str) -> Any:
-        m = OPTIONS.get()
-        return m.get(name)
+        i = OPTIONS.get()
+        return i.get(name)
 
     def set_context(self, options: dict):
         """Update current config"""
@@ -176,8 +219,8 @@ class Config(metaclass=Singleton):
     def new_environ(self):
         """Return a new os.environ object"""
         env = os.environ.copy()
-        m = self.options
-        env.update({k: v for k, v in m.items() if isinstance(v, str)})
+        i = self.options
+        env.update({k: v for k, v in i.items() if isinstance(v, str)})
         return env
 
 

metagpt/inspect_module.py

@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/28 14:54
-@Author  : alexanderwu
-@File    : inspect_module.py
-"""
-
-import inspect
-
-import metagpt  # replace with your module
-
-
-def print_classes_and_functions(module):
-    """FIXME: NOT WORK.."""
-    for name, obj in inspect.getmembers(module):
-        if inspect.isclass(obj):
-            print(f"Class: {name}")
-        elif inspect.isfunction(obj):
-            print(f"Function: {name}")
-        else:
-            print(name)
-
-    print(dir(module))
-
-
-if __name__ == "__main__":
-    print_classes_and_functions(metagpt)

metagpt/llm.py

@@ -8,12 +8,8 @@
 
 from metagpt.config import CONFIG
 from metagpt.provider.base_gpt_api import BaseGPTAPI
-from metagpt.provider.fireworks_api import FireWorksGPTAPI
 from metagpt.provider.human_provider import HumanProvider
-from metagpt.provider.open_llm_api import OpenLLMGPTAPI
-from metagpt.provider.openai_api import OpenAIGPTAPI
-from metagpt.provider.spark_api import SparkAPI
-from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI
+from metagpt.provider.llm_provider_registry import LLMProviderRegistry
 
 _ = HumanProvider()  # Avoid pre-commit error
 
@@ -21,17 +17,4 @@ _ = HumanProvider()  # Avoid pre-commit error
 def LLM() -> BaseGPTAPI:
     """initialize different LLM instance according to the key field existence"""
     # TODO a little trick, can use registry to initialize LLM instance further
-    if CONFIG.openai_api_key:
-        llm = OpenAIGPTAPI()
-    elif CONFIG.spark_api_key:
-        llm = SparkAPI()
-    elif CONFIG.zhipuai_api_key:
-        llm = ZhiPuAIGPTAPI()
-    elif CONFIG.open_llm_api_base:
-        llm = OpenLLMGPTAPI()
-    elif CONFIG.fireworks_api_key:
-        llm = FireWorksGPTAPI()
-    else:
-        raise RuntimeError("You should config a LLM configuration first")
-
-    return llm
+    return LLMProviderRegistry.get_provider(CONFIG.get_default_llm_provider_enum())

metagpt/provider/anthropic_api.py

@@ -14,7 +14,7 @@ from metagpt.config import CONFIG
 
 class Claude2:
     def ask(self, prompt):
-        client = Anthropic(api_key=CONFIG.claude_api_key)
+        client = Anthropic(api_key=CONFIG.anthropic_api_key)
 
         res = client.completions.create(
             model="claude-2",
@@ -24,7 +24,7 @@ class Claude2:
         return res.completion
 
     async def aask(self, prompt):
-        client = Anthropic(api_key=CONFIG.claude_api_key)
+        client = Anthropic(api_key=CONFIG.anthropic_api_key)
 
         res = client.completions.create(
             model="claude-2",

metagpt/provider/fireworks_api.py

@@ -4,10 +4,12 @@
 
 import openai
 
-from metagpt.config import CONFIG
+from metagpt.config import CONFIG, LLMProviderEnum
+from metagpt.provider.llm_provider_registry import register_provider
 from metagpt.provider.openai_api import CostManager, OpenAIGPTAPI, RateLimiter
 
 
+@register_provider(LLMProviderEnum.FIREWORKS)
 class FireWorksGPTAPI(OpenAIGPTAPI):
     def __init__(self):
         self.__init_fireworks(CONFIG)

metagpt/provider/llm_provider_registry.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/12/19 17:26
+@Author  : alexanderwu
+@File    : llm_provider_registry.py
+"""
+from metagpt.config import LLMProviderEnum
+
+
+class LLMProviderRegistry:
+    def __init__(self):
+        self.providers = {}
+
+    def register(self, key, provider_cls):
+        self.providers[key] = provider_cls
+
+    def get_provider(self, enum: LLMProviderEnum):
+        """get provider instance according to the enum"""
+        return self.providers[enum]()
+
+
+# Registry instance
+LLM_REGISTRY = LLMProviderRegistry()
+
+
+def register_provider(key):
+    """register provider to registry"""
+
+    def decorator(cls):
+        LLM_REGISTRY.register(key, cls)
+        return cls
+
+    return decorator

metagpt/provider/open_llm_api.py

@@ -4,8 +4,9 @@
 
 import openai
 
-from metagpt.config import CONFIG
+from metagpt.config import CONFIG, LLMProviderEnum
 from metagpt.logs import logger
+from metagpt.provider.llm_provider_registry import register_provider
 from metagpt.provider.openai_api import CostManager, OpenAIGPTAPI, RateLimiter
 
 
@@ -31,6 +32,7 @@ class OpenLLMCostManager(CostManager):
         CONFIG.total_cost = self.total_cost
 
 
+@register_provider(LLMProviderEnum.OPEN_LLM)
 class OpenLLMGPTAPI(OpenAIGPTAPI):
     def __init__(self):
         self.__init_openllm(CONFIG)

metagpt/provider/openai_api.py

@@ -18,10 +18,11 @@ from tenacity import (
     wait_random_exponential,
 )
 
-from metagpt.config import CONFIG
+from metagpt.config import CONFIG, LLMProviderEnum
 from metagpt.logs import logger
 from metagpt.provider.base_gpt_api import BaseGPTAPI
 from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA, GENERAL_TOOL_CHOICE
+from metagpt.provider.llm_provider_registry import register_provider
 from metagpt.schema import Message
 from metagpt.utils.singleton import Singleton
 from metagpt.utils.token_counter import (
@@ -137,6 +138,7 @@ See FAQ 5.8
     raise retry_state.outcome.exception()
 
 
+@register_provider(LLMProviderEnum.OPENAI)
 class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
     """
     Check https://platform.openai.com/examples for examples
@@ -329,7 +331,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
                 usage["completion_tokens"] = completion_tokens
                 return usage
             except Exception as e:
-                logger.error("usage calculation failed!", e)
+                logger.error(f"{self.model} usage calculation failed!", e)
+                return {}
         else:
             return usage
 
@@ -360,7 +363,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
         return results
 
     def _update_costs(self, usage: dict):
-        if CONFIG.calc_usage:
+        if CONFIG.calc_usage and usage:
             try:
                 prompt_tokens = int(usage["prompt_tokens"])
                 completion_tokens = int(usage["completion_tokens"])

metagpt/provider/spark_api.py

@@ -19,11 +19,13 @@ from wsgiref.handlers import format_date_time
 
 import websocket  # 使用websocket_client
 
-from metagpt.config import CONFIG
+from metagpt.config import CONFIG, LLMProviderEnum
 from metagpt.logs import logger
 from metagpt.provider.base_gpt_api import BaseGPTAPI
+from metagpt.provider.llm_provider_registry import register_provider
 
 
+@register_provider(LLMProviderEnum.SPARK)
 class SparkAPI(BaseGPTAPI):
     def __init__(self):
         logger.warning("当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。")

metagpt/provider/zhipuai_api.py

@@ -16,9 +16,10 @@ from tenacity import (
     wait_random_exponential,
 )
 
-from metagpt.config import CONFIG
+from metagpt.config import CONFIG, LLMProviderEnum
 from metagpt.logs import logger
 from metagpt.provider.base_gpt_api import BaseGPTAPI
+from metagpt.provider.llm_provider_registry import register_provider
 from metagpt.provider.openai_api import CostManager, log_and_reraise
 from metagpt.provider.zhipuai.zhipu_model_api import ZhiPuModelAPI
 
@@ -30,6 +31,7 @@ class ZhiPuEvent(Enum):
     FINISH = "finish"
 
 
+@register_provider(LLMProviderEnum.ZHIPUAI)
 class ZhiPuAIGPTAPI(BaseGPTAPI):
     """
     Refs to `https://open.bigmodel.cn/dev/api#chatglm_turbo`

metagpt/repo_parser.py

@@ -15,17 +15,17 @@ from pydantic import BaseModel, Field
 
 from metagpt.config import CONFIG
 from metagpt.logs import logger
+from metagpt.utils.exceptions import handle_exception
 
 
 class RepoParser(BaseModel):
     base_directory: Path = Field(default=None)
 
-    def parse_file(self, file_path):
+    @classmethod
+    @handle_exception(exception_type=Exception, default_return=[])
+    def _parse_file(cls, file_path: Path) -> list:
         """Parse a Python file in the repository."""
-        try:
-            return ast.parse(file_path.read_text()).body
-        except:
-            return []
+        return ast.parse(file_path.read_text()).body
 
     def extract_class_and_function_info(self, tree, file_path):
         """Extract class, function, and global variable information from the AST."""
@@ -52,7 +52,7 @@ class RepoParser(BaseModel):
         files_classes = []
         directory = self.base_directory
         for path in directory.rglob("*.py"):
-            tree = self.parse_file(path)
+            tree = self._parse_file(path)
             file_info = self.extract_class_and_function_info(tree, path)
             files_classes.append(file_info)
 
@@ -90,5 +90,10 @@ def main():
     logger.info(pformat(symbols))
 
 
+def error():
+    """raise Exception and logs it"""
+    RepoParser._parse_file(Path("test.py"))
+
+
 if __name__ == "__main__":
     main()

metagpt/roles/architect.py

@@ -28,7 +28,7 @@ class Architect(Role):
         profile: str = "Architect",
         goal: str = "design a concise, usable, complete software system",
         constraints: str = "make sure the architecture is simple enough and use appropriate open source libraries."
-                           "Use same language as user requirement"
+        "Use same language as user requirement",
     ) -> None:
         """Initializes the Architect with given attributes."""
         super().__init__(name, profile, goal, constraints)

metagpt/roles/engineer.py

@@ -73,7 +73,7 @@ class Engineer(Role):
         profile: str = "Engineer",
         goal: str = "write elegant, readable, extensible, efficient code",
         constraints: str = "the code should conform to standards like google-style and be modular and maintainable. "
-                           "Use same language as user requirement",
+        "Use same language as user requirement",
         n_borg: int = 1,
         use_code_review: bool = False,
     ) -> None:

metagpt/roles/project_manager.py

@@ -26,7 +26,7 @@ class ProjectManager(Role):
         name: str = "Eve",
         profile: str = "Project Manager",
         goal: str = "break down tasks according to PRD/technical design, generate a task list, and analyze task "
-            "dependencies to start with the prerequisite modules",
+        "dependencies to start with the prerequisite modules",
         constraints: str = "use same language as user requirement",
     ) -> None:
         """

metagpt/schema.py

@@ -21,7 +21,7 @@ import uuid
 from asyncio import Queue, QueueEmpty, wait_for
 from json import JSONDecodeError
 from pathlib import Path
-from typing import Dict, List, Optional, Set, TypedDict
+from typing import Dict, List, Optional, Set, Type, TypedDict, TypeVar
 
 from pydantic import BaseModel, Field
 
@@ -36,6 +36,7 @@ from metagpt.const import (
 )
 from metagpt.logs import logger
 from metagpt.utils.common import any_to_str, any_to_str_set
+from metagpt.utils.exceptions import handle_exception
 
 
 class RawMessage(TypedDict):
@@ -160,14 +161,11 @@ class Message(BaseModel):
         return self.json(exclude_none=True)
 
     @staticmethod
+    @handle_exception(exception_type=JSONDecodeError, default_return=None)
     def load(val):
         """Convert the json string to object."""
-        try:
-            d = json.loads(val)
-            return Message(**d)
-        except JSONDecodeError as err:
-            logger.error(f"parse json failed: {val}, error:{err}")
-        return None
+        i = json.loads(val)
+        return Message(**i)
 
 
 class UserMessage(Message):
@@ -249,50 +247,46 @@ class MessageQueue:
         return json.dumps(lst)
 
     @staticmethod
-    def load(self, v) -> "MessageQueue":
+    def load(data) -> "MessageQueue":
         """Convert the json string to the `MessageQueue` object."""
-        q = MessageQueue()
+        queue = MessageQueue()
         try:
-            lst = json.loads(v)
+            lst = json.loads(data)
             for i in lst:
                 msg = Message(**i)
-                q.push(msg)
+                queue.push(msg)
         except JSONDecodeError as e:
-            logger.warning(f"JSON load failed: {v}, error:{e}")
+            logger.warning(f"JSON load failed: {data}, error:{e}")
 
-        return q
+        return queue
 
 
-class CodingContext(BaseModel):
+# 定义一个泛型类型变量
+T = TypeVar("T", bound="BaseModel")
+
+
+class BaseContext(BaseModel):
+    @classmethod
+    @handle_exception
+    def loads(cls: Type[T], val: str) -> Optional[T]:
+        i = json.loads(val)
+        return cls(**i)
+
+
+class CodingContext(BaseContext):
     filename: str
     design_doc: Optional[Document]
     task_doc: Optional[Document]
     code_doc: Optional[Document]
 
-    @staticmethod
-    def loads(val: str) -> CodingContext | None:
-        try:
-            m = json.loads(val)
-            return CodingContext(**m)
-        except Exception:
-            return None
 
-
-class TestingContext(BaseModel):
+class TestingContext(BaseContext):
     filename: str
     code_doc: Document
     test_doc: Optional[Document]
 
-    @staticmethod
-    def loads(val: str) -> TestingContext | None:
-        try:
-            m = json.loads(val)
-            return TestingContext(**m)
-        except Exception:
-            return None
 
-
-class RunCodeContext(BaseModel):
+class RunCodeContext(BaseContext):
     mode: str = "script"
     code: Optional[str]
     code_filename: str = ""
@@ -304,28 +298,12 @@ class RunCodeContext(BaseModel):
     output_filename: Optional[str]
     output: Optional[str]
 
-    @staticmethod
-    def loads(val: str) -> RunCodeContext | None:
-        try:
-            m = json.loads(val)
-            return RunCodeContext(**m)
-        except Exception:
-            return None
 
-
-class RunCodeResult(BaseModel):
+class RunCodeResult(BaseContext):
     summary: str
     stdout: str
     stderr: str
 
-    @staticmethod
-    def loads(val: str) -> RunCodeResult | None:
-        try:
-            m = json.loads(val)
-            return RunCodeResult(**m)
-        except Exception:
-            return None
-
 
 class CodeSummarizeContext(BaseModel):
     design_filename: str = ""
@@ -349,5 +327,5 @@ class CodeSummarizeContext(BaseModel):
         return hash((self.design_filename, self.task_filename))
 
 
-class BugFixContext(BaseModel):
+class BugFixContext(BaseContext):
     filename: str = ""

metagpt/startup.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import asyncio
-from pathlib import Path
 
 import typer
 
@@ -27,7 +26,8 @@ def startup(
     reqa_file: str = typer.Option(default="", help="Specify the source file name for rewriting the quality test code."),
     max_auto_summarize_code: int = typer.Option(
         default=0,
-        help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. This parameter is used for debugging the workflow.",
+        help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating "
+        "unlimited. This parameter is used for debugging the workflow.",
     ),
 ):
     """Run a startup. Be a boss."""
@@ -40,15 +40,7 @@ def startup(
     )
     from metagpt.team import Team
 
-    # Use in the PrepareDocuments action according to Section 2.2.3.5.1 of RFC 135.
-    CONFIG.project_path = project_path
-    if project_path:
-        inc = True
-        project_name = project_name or Path(project_path).name
-    CONFIG.project_name = project_name
-    CONFIG.inc = inc
-    CONFIG.reqa_file = reqa_file
-    CONFIG.max_auto_summarize_code = max_auto_summarize_code
+    CONFIG.update_via_cli(project_path, project_name, inc, reqa_file, max_auto_summarize_code)
 
     company = Team()
     company.hire(

metagpt/team.py

@@ -22,8 +22,8 @@ from metagpt.utils.common import NoMoneyException
 
 class Team(BaseModel):
     """
-    Team: Possesses one or more roles (agents), SOP (Standard Operating Procedures), and a platform for instant messaging,
-    dedicated to perform any multi-agent activity, such as collaboratively writing executable code.
+    Team: Possesses one or more roles (agents), SOP (Standard Operating Procedures), and a env for instant messaging,
+    dedicated to env any multi-agent activity, such as collaboratively writing executable code.
     """
 
     env: Environment = Field(default_factory=Environment)

metagpt/tools/search_engine_meilisearch.py

@@ -11,6 +11,8 @@ from typing import List
 import meilisearch
 from meilisearch.index import Index
 
+from metagpt.utils.exceptions import handle_exception
+
 
 class DataSource:
     def __init__(self, name: str, url: str):
@@ -34,11 +36,7 @@ class MeilisearchEngine:
         index.add_documents(documents)
         self.set_index(index)
 
+    @handle_exception(exception_type=Exception, default_return=[])
     def search(self, query):
-        try:
-            search_results = self._index.search(query)
-            return search_results["hits"]
-        except Exception as e:
-            # Handle MeiliSearch API errors
-            print(f"MeiliSearch API error: {e}")
-            return []
+        search_results = self._index.search(query)
+        return search_results["hits"]

metagpt/utils/common.py

@@ -17,10 +17,16 @@ import inspect
 import os
 import platform
 import re
+import typing
 from typing import List, Tuple, Union
 
+import aiofiles
+import loguru
+from tenacity import RetryCallState, _utils
+
 from metagpt.const import MESSAGE_ROUTE_TO_ALL
 from metagpt.logs import logger
+from metagpt.utils.exceptions import handle_exception
 
 
 def check_cmd_exists(command) -> int:
@@ -291,9 +297,6 @@ class NoMoneyException(Exception):
 def print_members(module, indent=0):
     """
     https://stackoverflow.com/questions/1796180/how-can-i-get-a-list-of-all-classes-within-current-module-in-python
-    :param module:
-    :param indent:
-    :return:
     """
     prefix = " " * indent
     for name, obj in inspect.getmembers(module):
@@ -311,6 +314,7 @@ def print_members(module, indent=0):
 
 
 def parse_recipient(text):
+    # FIXME: use ActionNode instead.
     pattern = r"## Send To:\s*([A-Za-z]+)\s*?"  # hard code for now
     recipient = re.search(pattern, text)
     if recipient:
@@ -327,18 +331,12 @@ def get_class_name(cls) -> str:
     return f"{cls.__module__}.{cls.__name__}"
 
 
-def get_object_name(obj) -> str:
-    """Return class name of the object"""
-    cls = type(obj)
-    return f"{cls.__module__}.{cls.__name__}"
-
-
-def any_to_str(val) -> str:
+def any_to_str(val: str | typing.Callable) -> str:
     """Return the class name or the class name of the object, or 'val' if it's a string type."""
     if isinstance(val, str):
         return val
     if not callable(val):
-        return get_object_name(val)
+        return get_class_name(type(val))
 
     return get_class_name(val)
 
@@ -346,20 +344,68 @@ def any_to_str(val) -> str:
 def any_to_str_set(val) -> set:
     """Convert any type to string set."""
     res = set()
-    if isinstance(val, dict) or isinstance(val, list) or isinstance(val, set) or isinstance(val, tuple):
+
+    # Check if the value is iterable, but not a string (since strings are technically iterable)
+    if isinstance(val, (dict, list, set, tuple)):
+        # Special handling for dictionaries to iterate over values
+        if isinstance(val, dict):
+            val = val.values()
+
         for i in val:
             res.add(any_to_str(i))
     else:
         res.add(any_to_str(val))
+
     return res
 
 
-def is_subscribed(message, tags):
+def is_subscribed(message: "Message", tags: set):
     """Return whether it's consumer"""
     if MESSAGE_ROUTE_TO_ALL in message.send_to:
         return True
 
-    for t in tags:
-        if t in message.send_to:
+    for i in tags:
+        if i in message.send_to:
             return True
     return False
+
+
+def general_after_log(i: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]:
+    """
+    Generates a logging function to be used after a call is retried.
+
+    This generated function logs an error message with the outcome of the retried function call. It includes
+    the name of the function, the time taken for the call in seconds (formatted according to `sec_format`),
+    the number of attempts made, and the exception raised, if any.
+
+    :param i: A Logger instance from the loguru library used to log the error message.
+    :param sec_format: A string format specifier for how to format the number of seconds since the start of the call.
+                       Defaults to three decimal places.
+    :return: A callable that accepts a RetryCallState object and returns None. This callable logs the details
+             of the retried call.
+    """
+
+    def log_it(retry_state: "RetryCallState") -> None:
+        # If the function name is not known, default to "<unknown>"
+        if retry_state.fn is None:
+            fn_name = "<unknown>"
+        else:
+            # Retrieve the callable's name using a utility function
+            fn_name = _utils.get_callback_name(retry_state.fn)
+
+        # Log an error message with the function name, time since start, attempt number, and the exception
+        i.error(
+            f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), "
+            f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. "
+            f"exp: {retry_state.outcome.exception()}"
+        )
+
+    return log_it
+
+
+@handle_exception
+async def aread(file_path: str) -> str:
+    """Read file asynchronously."""
+    async with aiofiles.open(str(file_path), mode="r") as reader:
+        content = await reader.read()
+    return content

metagpt/utils/custom_decoder.py

@@ -25,7 +25,7 @@ def py_make_scanner(context):
         except IndexError:
             raise StopIteration(idx) from None
 
-        if nextchar == '"' or nextchar == "'":
+        if nextchar in ("'", '"'):
             if idx + 2 < len(string) and string[idx + 1] == nextchar and string[idx + 2] == nextchar:
                 # Handle the case where the next two characters are the same as nextchar
                 return parse_string(string, idx + 3, strict, delimiter=nextchar * 3)  # triple quote

metagpt/utils/dependency_file.py

@@ -15,7 +15,8 @@ from typing import Set
 import aiofiles
 
 from metagpt.config import CONFIG
-from metagpt.logs import logger
+from metagpt.utils.common import aread
+from metagpt.utils.exceptions import handle_exception
 
 
 class DependencyFile:
@@ -36,21 +37,14 @@ class DependencyFile:
         """Load dependencies from the file asynchronously."""
         if not self._filename.exists():
             return
-        try:
-            async with aiofiles.open(str(self._filename), mode="r") as reader:
-                data = await reader.read()
-            self._dependencies = json.loads(data)
-        except Exception as e:
-            logger.error(f"Failed to load {str(self._filename)}, error:{e}")
+        self._dependencies = json.loads(await aread(self._filename))
 
+    @handle_exception
     async def save(self):
         """Save dependencies to the file asynchronously."""
-        try:
-            data = json.dumps(self._dependencies)
-            async with aiofiles.open(str(self._filename), mode="w") as writer:
-                await writer.write(data)
-        except Exception as e:
-            logger.error(f"Failed to save {str(self._filename)}, error:{e}")
+        data = json.dumps(self._dependencies)
+        async with aiofiles.open(str(self._filename), mode="w") as writer:
+            await writer.write(data)
 
     async def update(self, filename: Path | str, dependencies: Set[Path | str], persist=True):
         """Update dependencies for a file asynchronously.

metagpt/utils/exceptions.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/12/19 14:46
+@Author  : alexanderwu
+@File    : exceptions.py
+"""
+
+
+import asyncio
+import functools
+import traceback
+from typing import Any, Callable, Tuple, Type, TypeVar, Union
+
+from metagpt.logs import logger
+
+ReturnType = TypeVar("ReturnType")
+
+
+def handle_exception(
+    _func: Callable[..., ReturnType] = None,
+    *,
+    exception_type: Union[Type[Exception], Tuple[Type[Exception], ...]] = Exception,
+    default_return: Any = None,
+) -> Callable[..., ReturnType]:
+    """handle exception, return default value"""
+
+    def decorator(func: Callable[..., ReturnType]) -> Callable[..., ReturnType]:
+        @functools.wraps(func)
+        async def async_wrapper(*args: Any, **kwargs: Any) -> ReturnType:
+            try:
+                return await func(*args, **kwargs)
+            except exception_type as e:
+                logger.opt(depth=1).error(
+                    f"Calling {func.__name__} with args: {args}, kwargs: {kwargs} failed: {e}, "
+                    f"stack: {traceback.format_exc()}"
+                )
+                return default_return
+
+        @functools.wraps(func)
+        def sync_wrapper(*args: Any, **kwargs: Any) -> ReturnType:
+            try:
+                return func(*args, **kwargs)
+            except exception_type as e:
+                logger.opt(depth=1).error(
+                    f"Calling {func.__name__} with args: {args}, kwargs: {kwargs} failed: {e}, "
+                    f"stack: {traceback.format_exc()}"
+                )
+                return default_return
+
+        if asyncio.iscoroutinefunction(func):
+            return async_wrapper
+        else:
+            return sync_wrapper
+
+    if _func is None:
+        return decorator
+    else:
+        return decorator(_func)

metagpt/utils/file.py

@@ -11,6 +11,7 @@ from pathlib import Path
 import aiofiles
 
 from metagpt.logs import logger
+from metagpt.utils.exceptions import handle_exception
 
 
 class File:
@@ -19,6 +20,7 @@ class File:
     CHUNK_SIZE = 64 * 1024
 
     @classmethod
+    @handle_exception
     async def write(cls, root_path: Path, filename: str, content: bytes) -> Path:
         """Write the file content to the local specified path.
 
@@ -33,18 +35,15 @@ class File:
         Raises:
             Exception: If an unexpected error occurs during the file writing process.
         """
-        try:
-            root_path.mkdir(parents=True, exist_ok=True)
-            full_path = root_path / filename
-            async with aiofiles.open(full_path, mode="wb") as writer:
-                await writer.write(content)
-                logger.debug(f"Successfully write file: {full_path}")
-                return full_path
-        except Exception as e:
-            logger.error(f"Error writing file: {e}")
-            raise e
+        root_path.mkdir(parents=True, exist_ok=True)
+        full_path = root_path / filename
+        async with aiofiles.open(full_path, mode="wb") as writer:
+            await writer.write(content)
+            logger.debug(f"Successfully write file: {full_path}")
+            return full_path
 
     @classmethod
+    @handle_exception
     async def read(cls, file_path: Path, chunk_size: int = None) -> bytes:
         """Partitioning read the file content from the local specified path.
 
@@ -58,18 +57,14 @@ class File:
         Raises:
             Exception: If an unexpected error occurs during the file reading process.
         """
-        try:
-            chunk_size = chunk_size or cls.CHUNK_SIZE
-            async with aiofiles.open(file_path, mode="rb") as reader:
-                chunks = list()
-                while True:
-                    chunk = await reader.read(chunk_size)
-                    if not chunk:
-                        break
-                    chunks.append(chunk)
-                content = b"".join(chunks)
-                logger.debug(f"Successfully read file, the path of file: {file_path}")
-                return content
-        except Exception as e:
-            logger.error(f"Error reading file: {e}")
-            raise e
+        chunk_size = chunk_size or cls.CHUNK_SIZE
+        async with aiofiles.open(file_path, mode="rb") as reader:
+            chunks = list()
+            while True:
+                chunk = await reader.read(chunk_size)
+                if not chunk:
+                    break
+                chunks.append(chunk)
+            content = b"".join(chunks)
+            logger.debug(f"Successfully read file, the path of file: {file_path}")
+            return content

metagpt/utils/file_repository.py

@@ -19,6 +19,7 @@ import aiofiles
 from metagpt.config import CONFIG
 from metagpt.logs import logger
 from metagpt.schema import Document
+from metagpt.utils.common import aread
 from metagpt.utils.json_to_markdown import json_to_markdown
 
 
@@ -97,15 +98,7 @@ class FileRepository:
         path_name = self.workdir / filename
         if not path_name.exists():
             return None
-        try:
-            async with aiofiles.open(str(path_name), mode="r") as reader:
-                doc.content = await reader.read()
-        except FileNotFoundError as e:
-            logger.info(f"open {str(path_name)} failed:{e}")
-            return None
-        except Exception as e:
-            logger.info(f"open {str(path_name)} failed:{e}")
-            return None
+        doc.content = await aread(path_name)
         return doc
 
     async def get_all(self) -> List[Document]:

metagpt/utils/token_counter.py

@@ -56,6 +56,7 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"):
     if model in {
         "gpt-3.5-turbo-0613",
         "gpt-3.5-turbo-16k-0613",
+        "gpt-3.5-turbo-16k",
         "gpt-3.5-turbo-1106",
         "gpt-4-0314",
         "gpt-4-32k-0314",
@@ -63,7 +64,7 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"):
         "gpt-4-32k-0613",
         "gpt-4-1106-preview",
     }:
-        tokens_per_message = 3
+        tokens_per_message = 3  # # every reply is primed with <|start|>assistant<|message|>
         tokens_per_name = 1
     elif model == "gpt-3.5-turbo-0301":
         tokens_per_message = 4  # every message follows <|start|>{role/name}\n{content}<|end|>\n

metagpt/utils/utils.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# @Desc   :
-
-import typing
-
-from tenacity import _utils
-
-
-def general_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]:
-    def log_it(retry_state: "RetryCallState") -> None:
-        if retry_state.fn is None:
-            fn_name = "<unknown>"
-        else:
-            fn_name = _utils.get_callback_name(retry_state.fn)
-        logger.error(
-            f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), "
-            f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. "
-            f"exp: {retry_state.outcome.exception()}"
-        )
-
-    return log_it

requirements-ocr.txt

@@ -1,4 +0,0 @@
-paddlepaddle==2.4.2
-paddleocr>=2.0.1
-tabulate==0.9.0
--r requirements.txt

setup.py

@@ -31,14 +31,14 @@ with open(path.join(here, "requirements.txt"), encoding="utf-8") as f:
 setup(
     name="metagpt",
     version="0.5.2",
-    description="The Multi-Role Meta Programming Framework",
+    description="The Multi-Agent Framework",
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/geekan/MetaGPT",
     author="Alexander Wu",
     author_email="alexanderwu@deepwisdom.ai",
     license="MIT",
-    keywords="metagpt multi-role multi-agent programming gpt llm metaprogramming",
+    keywords="metagpt multi-agent multi-role programming gpt llm metaprogramming",
     packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]),
     python_requires=">=3.9",
     install_requires=requirements,
@@ -48,6 +48,7 @@ setup(
         "search-google": ["google-api-python-client==2.94.0"],
         "search-ddg": ["duckduckgo-search==3.8.5"],
         "pyppeteer": ["pyppeteer>=1.0.2"],
+        "ocr": ["paddlepaddle==2.4.2", "paddleocr>=2.0.1", "tabulate==0.9.0"],
     },
     cmdclass={
         "install_mermaid": InstallMermaidCLI,

tests/metagpt/test_role.py

@@ -18,7 +18,7 @@ from metagpt.actions import Action, ActionOutput, UserRequirement
 from metagpt.environment import Environment
 from metagpt.roles import Role
 from metagpt.schema import Message
-from metagpt.utils.common import any_to_str, get_class_name
+from metagpt.utils.common import any_to_str
 
 
 class MockAction(Action):
@@ -88,13 +88,13 @@ async def test_react():
 @pytest.mark.asyncio
 async def test_msg_to():
     m = Message(content="a", send_to=["a", MockRole, Message])
-    assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)})
+    assert m.send_to == {"a", any_to_str(MockRole), any_to_str(Message)}
 
     m = Message(content="a", cause_by=MockAction, send_to={"a", MockRole, Message})
-    assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)})
+    assert m.send_to == {"a", any_to_str(MockRole), any_to_str(Message)}
 
     m = Message(content="a", send_to=("a", MockRole, Message))
-    assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)})
+    assert m.send_to == {"a", any_to_str(MockRole), any_to_str(Message)}
 
 
 if __name__ == "__main__":

tests/metagpt/test_schema.py

@@ -13,7 +13,7 @@ import pytest
 
 from metagpt.actions import Action
 from metagpt.schema import AIMessage, Message, SystemMessage, UserMessage
-from metagpt.utils.common import get_class_name
+from metagpt.utils.common import any_to_str
 
 
 @pytest.mark.asyncio
@@ -54,9 +54,9 @@ def test_message():
     m.cause_by = "Message"
     assert m.cause_by == "Message"
     m.cause_by = Action
-    assert m.cause_by == get_class_name(Action)
+    assert m.cause_by == any_to_str(Action)
     m.cause_by = Action()
-    assert m.cause_by == get_class_name(Action)
+    assert m.cause_by == any_to_str(Action)
     m.content = "b"
     assert m.content == "b"
 
@@ -67,7 +67,7 @@ def test_routes():
     m.send_to = "b"
     assert m.send_to == {"b"}
     m.send_to = {"e", Action}
-    assert m.send_to == {"e", get_class_name(Action)}
+    assert m.send_to == {"e", any_to_str(Action)}
 
 
 if __name__ == "__main__":