Merge branch 'main' into dev_updated

metagpt/actions/__init__.py

@@ -9,12 +9,11 @@ from enum import Enum
 
 from metagpt.actions.action import Action
 from metagpt.actions.action_output import ActionOutput
-from metagpt.actions.add_requirement import BossRequirement
+from metagpt.actions.add_requirement import UserRequirement
 from metagpt.actions.debug_error import DebugError
 from metagpt.actions.design_api import WriteDesign
 from metagpt.actions.design_api_review import DesignReview
-from metagpt.actions.design_filenames import DesignFilenames
-from metagpt.actions.project_management import AssignTasks, WriteTasks
+from metagpt.actions.project_management import WriteTasks
 from metagpt.actions.research import CollectLinks, WebBrowseAndSummarize, ConductResearch
 from metagpt.actions.run_code import RunCode
 from metagpt.actions.search_and_summarize import SearchAndSummarize
@@ -31,19 +30,17 @@ from metagpt.actions.write_plan import WritePlan
 class ActionType(Enum):
     """All types of Actions, used for indexing."""
 
-    ADD_REQUIREMENT = BossRequirement
+    ADD_REQUIREMENT = UserRequirement
     WRITE_PRD = WritePRD
     WRITE_PRD_REVIEW = WritePRDReview
     WRITE_DESIGN = WriteDesign
     DESIGN_REVIEW = DesignReview
-    DESIGN_FILENAMES = DesignFilenames
     WRTIE_CODE = WriteCode
     WRITE_CODE_REVIEW = WriteCodeReview
     WRITE_TEST = WriteTest
     RUN_CODE = RunCode
     DEBUG_ERROR = DebugError
     WRITE_TASKS = WriteTasks
-    ASSIGN_TASKS = AssignTasks
     SEARCH_AND_SUMMARIZE = SearchAndSummarize
     COLLECT_LINKS = CollectLinks
     WEB_BROWSE_AND_SUMMARIZE = WebBrowseAndSummarize

metagpt/actions/action.py

@@ -5,36 +5,56 @@
 @Author  : alexanderwu
 @File    : action.py
 """
-import re
-from abc import ABC
-from typing import Optional
 
-from tenacity import retry, stop_after_attempt, wait_fixed
+from __future__ import annotations
 
-from metagpt.actions.action_output import ActionOutput
+from typing import Optional, Union
+
+from pydantic import ConfigDict, Field, model_validator
+
+from metagpt.actions.action_node import ActionNode
 from metagpt.llm import LLM
-from metagpt.logs import logger
-from metagpt.utils.common import OutputParser
-from metagpt.utils.custom_decoder import CustomDecoder
+from metagpt.provider.base_llm import BaseLLM
+from metagpt.schema import (
+    CodeSummarizeContext,
+    CodingContext,
+    RunCodeContext,
+    SerializationMixin,
+    TestingContext,
+)
 
 
-class Action(ABC):
-    def __init__(self, name: str = "", context=None, llm: LLM = None):
-        self.name: str = name
-        if llm is None:
-            llm = LLM()
-        self.llm = llm
-        self.context = context
-        self.prefix = ""
-        self.profile = ""
-        self.desc = ""
-        self.content = ""
-        self.instruct_content = None
+class Action(SerializationMixin, is_polymorphic_base=True):
+    model_config = ConfigDict(arbitrary_types_allowed=True, exclude=["llm"])
 
-    def set_prefix(self, prefix, profile):
+    name: str = ""
+    llm: BaseLLM = Field(default_factory=LLM, exclude=True)
+    context: Union[dict, CodingContext, CodeSummarizeContext, TestingContext, RunCodeContext, str, None] = ""
+    prefix: str = ""  # aask*时会加上prefix，作为system_message
+    desc: str = ""  # for skill manager
+    node: ActionNode = Field(default=None, exclude=True)
+
+    @model_validator(mode="before")
+    def set_name_if_empty(cls, values):
+        if "name" not in values or not values["name"]:
+            values["name"] = cls.__name__
+        return values
+
+    @model_validator(mode="before")
+    def _init_with_instruction(cls, values):
+        if "instruction" in values:
+            name = values["name"]
+            i = values["instruction"]
+            values["node"] = ActionNode(key=name, expected_type=str, instruction=i, example="", schema="raw")
+        return values
+
+    def set_prefix(self, prefix):
         """Set prefix for later usage"""
         self.prefix = prefix
-        self.profile = profile
+        self.llm.system_prompt = prefix
+        if self.node:
+            self.node.llm = self.llm
+        return self
 
     def __str__(self):
         return self.__class__.__name__
@@ -44,46 +64,17 @@ class Action(ABC):
 
     async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None) -> str:
         """Append default prefix"""
-        if not system_msgs:
-            system_msgs = []
-        system_msgs.append(self.prefix)
         return await self.llm.aask(prompt, system_msgs)
 
-    @retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
-    async def _aask_v1(
-        self,
-        prompt: str,
-        output_class_name: str,
-        output_data_mapping: dict,
-        system_msgs: Optional[list[str]] = None,
-        format="markdown",  # compatible to original format
-    ) -> ActionOutput:
-        """Append default prefix"""
-        if not system_msgs:
-            system_msgs = []
-        system_msgs.append(self.prefix)
-        content = await self.llm.aask(prompt, system_msgs)
-        logger.debug(content)
-        output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping)
-
-        if format == "json":
-            pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]"
-            matches = re.findall(pattern, content, re.DOTALL)
-
-            for match in matches:
-                if match:
-                    content = match
-                    break
-
-            parsed_data = CustomDecoder(strict=False).decode(content)
-
-        else:  # using markdown parser
-            parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping)
-
-        logger.debug(parsed_data)
-        instruct_content = output_class(**parsed_data)
-        return ActionOutput(content, instruct_content)
+    async def _run_action_node(self, *args, **kwargs):
+        """Run action node"""
+        msgs = args[0]
+        context = "## History Messages\n"
+        context += "\n".join([f"{idx}: {i}" for idx, i in enumerate(reversed(msgs))])
+        return await self.node.fill(context=context, llm=self.llm)
 
     async def run(self, *args, **kwargs):
         """Run action"""
+        if self.node:
+            return await self._run_action_node(*args, **kwargs)
         raise NotImplementedError("The run method should be implemented in a subclass.")

metagpt/actions/action_node.py

@@ -0,0 +1,349 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/12/11 18:45
+@Author  : alexanderwu
+@File    : action_node.py
+
+NOTE: You should use typing.List instead of list to do type annotation. Because in the markdown extraction process,
+  we can use typing to extract the type of the node, but we cannot use built-in list to extract.
+"""
+import json
+from typing import Any, Dict, List, Optional, Tuple, Type
+
+from pydantic import BaseModel, create_model, model_validator
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+
+from metagpt.config import CONFIG
+from metagpt.llm import BaseLLM
+from metagpt.logs import logger
+from metagpt.provider.postprocess.llm_output_postprocess import llm_output_postprocess
+from metagpt.utils.common import OutputParser, general_after_log
+
+TAG = "CONTENT"
+
+LANGUAGE_CONSTRAINT = "Language: Please use the same language as Human INPUT."
+FORMAT_CONSTRAINT = f"Format: output wrapped inside [{TAG}][/{TAG}] like format example, nothing else."
+
+
+SIMPLE_TEMPLATE = """
+## context
+{context}
+
+-----
+
+## format example
+{example}
+
+## nodes: "<node>: <type>  # <instruction>"
+{instruction}
+
+## constraint
+{constraint}
+
+## action
+Follow instructions of nodes, generate output and make sure it follows the format example.
+"""
+
+
+def dict_to_markdown(d, prefix="- ", kv_sep="\n", postfix="\n"):
+    markdown_str = ""
+    for key, value in d.items():
+        markdown_str += f"{prefix}{key}{kv_sep}{value}{postfix}"
+    return markdown_str
+
+
+class ActionNode:
+    """ActionNode is a tree of nodes."""
+
+    schema: str  # raw/json/markdown, default: ""
+
+    # Action Context
+    context: str  # all the context, including all necessary info
+    llm: BaseLLM  # LLM with aask interface
+    children: dict[str, "ActionNode"]
+
+    # Action Input
+    key: str  # Product Requirement / File list / Code
+    expected_type: Type  # such as str / int / float etc.
+    # context: str  # everything in the history.
+    instruction: str  # the instructions should be followed.
+    example: Any  # example for In Context-Learning.
+
+    # Action Output
+    content: str
+    instruct_content: BaseModel
+
+    def __init__(
+        self,
+        key: str,
+        expected_type: Type,
+        instruction: str,
+        example: Any,
+        content: str = "",
+        children: dict[str, "ActionNode"] = None,
+        schema: str = "",
+    ):
+        self.key = key
+        self.expected_type = expected_type
+        self.instruction = instruction
+        self.example = example
+        self.content = content
+        self.children = children if children is not None else {}
+        self.schema = schema
+
+    def __str__(self):
+        return (
+            f"{self.key}, {repr(self.expected_type)}, {self.instruction}, {self.example}"
+            f", {self.content}, {self.children}"
+        )
+
+    def __repr__(self):
+        return self.__str__()
+
+    def add_child(self, node: "ActionNode"):
+        """增加子ActionNode"""
+        self.children[node.key] = node
+
+    def add_children(self, nodes: List["ActionNode"]):
+        """批量增加子ActionNode"""
+        for node in nodes:
+            self.add_child(node)
+
+    @classmethod
+    def from_children(cls, key, nodes: List["ActionNode"]):
+        """直接从一系列的子nodes初始化"""
+        obj = cls(key, str, "", "")
+        obj.add_children(nodes)
+        return obj
+
+    def get_children_mapping(self, exclude=None) -> Dict[str, Tuple[Type, Any]]:
+        """获得子ActionNode的字典，以key索引"""
+        exclude = exclude or []
+        return {k: (v.expected_type, ...) for k, v in self.children.items() if k not in exclude}
+
+    def get_self_mapping(self) -> Dict[str, Tuple[Type, Any]]:
+        """get self key: type mapping"""
+        return {self.key: (self.expected_type, ...)}
+
+    def get_mapping(self, mode="children", exclude=None) -> Dict[str, Tuple[Type, Any]]:
+        """get key: type mapping under mode"""
+        if mode == "children" or (mode == "auto" and self.children):
+            return self.get_children_mapping(exclude=exclude)
+        return {} if exclude and self.key in exclude else self.get_self_mapping()
+
+    @classmethod
+    def create_model_class(cls, class_name: str, mapping: Dict[str, Tuple[Type, Any]]):
+        """基于pydantic v1的模型动态生成，用来检验结果类型正确性"""
+
+        def check_fields(cls, values):
+            required_fields = set(mapping.keys())
+            missing_fields = required_fields - set(values.keys())
+            if missing_fields:
+                raise ValueError(f"Missing fields: {missing_fields}")
+
+            unrecognized_fields = set(values.keys()) - required_fields
+            if unrecognized_fields:
+                logger.warning(f"Unrecognized fields: {unrecognized_fields}")
+            return values
+
+        validators = {"check_missing_fields_validator": model_validator(mode="before")(check_fields)}
+
+        new_class = create_model(class_name, __validators__=validators, **mapping)
+        return new_class
+
+    def create_children_class(self, exclude=None):
+        """使用object内有的字段直接生成model_class"""
+        class_name = f"{self.key}_AN"
+        mapping = self.get_children_mapping(exclude=exclude)
+        return self.create_model_class(class_name, mapping)
+
+    def to_dict(self, format_func=None, mode="auto", exclude=None) -> Dict:
+        """将当前节点与子节点都按照node: format的格式组织成字典"""
+
+        # 如果没有提供格式化函数，使用默认的格式化方式
+        if format_func is None:
+            format_func = lambda node: f"{node.instruction}"
+
+        # 使用提供的格式化函数来格式化当前节点的值
+        formatted_value = format_func(self)
+
+        # 创建当前节点的键值对
+        if mode == "children" or (mode == "auto" and self.children):
+            node_dict = {}
+        else:
+            node_dict = {self.key: formatted_value}
+
+        if mode == "root":
+            return node_dict
+
+        # 遍历子节点并递归调用 to_dict 方法
+        exclude = exclude or []
+        for _, child_node in self.children.items():
+            if child_node.key in exclude:
+                continue
+            node_dict.update(child_node.to_dict(format_func))
+
+        return node_dict
+
+    def compile_to(self, i: Dict, schema, kv_sep) -> str:
+        if schema == "json":
+            return json.dumps(i, indent=4)
+        elif schema == "markdown":
+            return dict_to_markdown(i, kv_sep=kv_sep)
+        else:
+            return str(i)
+
+    def tagging(self, text, schema, tag="") -> str:
+        if not tag:
+            return text
+        if schema == "json":
+            return f"[{tag}]\n" + text + f"\n[/{tag}]"
+        else:  # markdown
+            return f"[{tag}]\n" + text + f"\n[/{tag}]"
+
+    def _compile_f(self, schema, mode, tag, format_func, kv_sep, exclude=None) -> str:
+        nodes = self.to_dict(format_func=format_func, mode=mode, exclude=exclude)
+        text = self.compile_to(nodes, schema, kv_sep)
+        return self.tagging(text, schema, tag)
+
+    def compile_instruction(self, schema="markdown", mode="children", tag="", exclude=None) -> str:
+        """compile to raw/json/markdown template with all/root/children nodes"""
+        format_func = lambda i: f"{i.expected_type}  # {i.instruction}"
+        return self._compile_f(schema, mode, tag, format_func, kv_sep=": ", exclude=exclude)
+
+    def compile_example(self, schema="json", mode="children", tag="", exclude=None) -> str:
+        """compile to raw/json/markdown examples with all/root/children nodes"""
+
+        # 这里不能使用f-string，因为转译为str后再json.dumps会额外加上引号，无法作为有效的example
+        # 错误示例："File list": "['main.py', 'const.py', 'game.py']", 注意这里值不是list，而是str
+        format_func = lambda i: i.example
+        return self._compile_f(schema, mode, tag, format_func, kv_sep="\n", exclude=exclude)
+
+    def compile(self, context, schema="json", mode="children", template=SIMPLE_TEMPLATE, exclude=[]) -> str:
+        """
+        mode: all/root/children
+            mode="children": 编译所有子节点为一个统一模板，包括instruction与example
+            mode="all": NotImplemented
+            mode="root": NotImplemented
+        schmea: raw/json/markdown
+            schema="raw": 不编译，context, lang_constaint, instruction
+            schema="json"：编译context, example(json), instruction(markdown), constraint, action
+            schema="markdown": 编译context, example(markdown), instruction(markdown), constraint, action
+        """
+        if schema == "raw":
+            return context + "\n\n## Actions\n" + LANGUAGE_CONSTRAINT + "\n" + self.instruction
+
+        # FIXME: json instruction会带来格式问题，如："Project name": "web_2048  # 项目名称使用下划线",
+        # compile example暂时不支持markdown
+        instruction = self.compile_instruction(schema="markdown", mode=mode, exclude=exclude)
+        example = self.compile_example(schema=schema, tag=TAG, mode=mode, exclude=exclude)
+        # nodes = ", ".join(self.to_dict(mode=mode).keys())
+        constraints = [LANGUAGE_CONSTRAINT, FORMAT_CONSTRAINT]
+        constraint = "\n".join(constraints)
+
+        prompt = template.format(
+            context=context,
+            example=example,
+            instruction=instruction,
+            constraint=constraint,
+        )
+        return prompt
+
+    @retry(
+        wait=wait_random_exponential(min=1, max=20),
+        stop=stop_after_attempt(6),
+        after=general_after_log(logger),
+    )
+    async def _aask_v1(
+        self,
+        prompt: str,
+        output_class_name: str,
+        output_data_mapping: dict,
+        system_msgs: Optional[list[str]] = None,
+        schema="markdown",  # compatible to original format
+        timeout=CONFIG.timeout,
+    ) -> (str, BaseModel):
+        """Use ActionOutput to wrap the output of aask"""
+        content = await self.llm.aask(prompt, system_msgs, timeout=timeout)
+        logger.debug(f"llm raw output:\n{content}")
+        output_class = self.create_model_class(output_class_name, output_data_mapping)
+
+        if schema == "json":
+            parsed_data = llm_output_postprocess(
+                output=content, schema=output_class.model_json_schema(), req_key=f"[/{TAG}]"
+            )
+        else:  # using markdown parser
+            parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping)
+
+        logger.debug(f"parsed_data:\n{parsed_data}")
+        instruct_content = output_class(**parsed_data)
+        return content, instruct_content
+
+    def get(self, key):
+        return self.instruct_content.model_dump()[key]
+
+    def set_recursive(self, name, value):
+        setattr(self, name, value)
+        for _, i in self.children.items():
+            i.set_recursive(name, value)
+
+    def set_llm(self, llm):
+        self.set_recursive("llm", llm)
+
+    def set_context(self, context):
+        self.set_recursive("context", context)
+
+    async def simple_fill(self, schema, mode, timeout=CONFIG.timeout, exclude=None):
+        prompt = self.compile(context=self.context, schema=schema, mode=mode, exclude=exclude)
+
+        if schema != "raw":
+            mapping = self.get_mapping(mode, exclude=exclude)
+            class_name = f"{self.key}_AN"
+            content, scontent = await self._aask_v1(prompt, class_name, mapping, schema=schema, timeout=timeout)
+            self.content = content
+            self.instruct_content = scontent
+        else:
+            self.content = await self.llm.aask(prompt)
+            self.instruct_content = None
+
+        return self
+
+    async def fill(self, context, llm, schema="json", mode="auto", strgy="simple", timeout=CONFIG.timeout, exclude=[]):
+        """Fill the node(s) with mode.
+
+        :param context: Everything we should know when filling node.
+        :param llm: Large Language Model with pre-defined system message.
+        :param schema: json/markdown, determine example and output format.
+         - raw: free form text
+         - json: it's easy to open source LLM with json format
+         - markdown: when generating code, markdown is always better
+        :param mode: auto/children/root
+         - auto: automated fill children's nodes and gather outputs, if no children, fill itself
+         - children: fill children's nodes and gather outputs
+         - root: fill root's node and gather output
+        :param strgy: simple/complex
+         - simple: run only once
+         - complex: run each node
+        :param timeout: Timeout for llm invocation.
+        :param exclude: The keys of ActionNode to exclude.
+        :return: self
+        """
+        self.set_llm(llm)
+        self.set_context(context)
+        if self.schema:
+            schema = self.schema
+
+        if strgy == "simple":
+            return await self.simple_fill(schema=schema, mode=mode, timeout=timeout, exclude=exclude)
+        elif strgy == "complex":
+            # 这里隐式假设了拥有children
+            tmp = {}
+            for _, i in self.children.items():
+                if exclude and i.key in exclude:
+                    continue
+                child = await i.simple_fill(schema=schema, mode=mode, timeout=timeout, exclude=exclude)
+                tmp.update(child.instruct_content.dict())
+            cls = self.create_children_class()
+            self.instruct_content = cls(**tmp)
+            return self

metagpt/actions/action_output.py

@@ -6,9 +6,7 @@
 @File    : action_output
 """
 
-from typing import Dict, Type
-
-from pydantic import BaseModel, create_model, root_validator, validator
+from pydantic import BaseModel
 
 
 class ActionOutput:
@@ -18,26 +16,3 @@ class ActionOutput:
     def __init__(self, content: str, instruct_content: BaseModel):
         self.content = content
         self.instruct_content = instruct_content
-
-    @classmethod
-    def create_model_class(cls, class_name: str, mapping: Dict[str, Type]):
-        new_class = create_model(class_name, **mapping)
-
-        @validator('*', allow_reuse=True)
-        def check_name(v, field):
-            if field.name not in mapping.keys():
-                raise ValueError(f'Unrecognized block: {field.name}')
-            return v
-
-        @root_validator(pre=True, allow_reuse=True)
-        def check_missing_fields(values):
-            required_fields = set(mapping.keys())
-            missing_fields = required_fields - set(values.keys())
-            if missing_fields:
-                raise ValueError(f'Missing fields: {missing_fields}')
-            return values
-
-        new_class.__validator_check_name = classmethod(check_name)
-        new_class.__root_validator_check_missing_fields = classmethod(check_missing_fields)
-        return new_class
-    

metagpt/actions/add_requirement.py

@@ -8,7 +8,5 @@
 from metagpt.actions import Action
 
 
-class BossRequirement(Action):
-    """Boss Requirement without any implementation details"""
-    async def run(self, *args, **kwargs):
-        raise NotImplementedError
+class UserRequirement(Action):
+    """User Requirement without any implementation details"""

metagpt/actions/analyze_dep_libs.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/19 12:01
-@Author  : alexanderwu
-@File    : analyze_dep_libs.py
-"""
-
-from metagpt.actions import Action
-
-PROMPT = """You are an AI developer, trying to write a program that generates code for users based on their intentions.
-
-For the user's prompt:
-
----
-The API is: {prompt}
----
-
-We decide the generated files are: {filepaths_string}
-
-Now that we have a file list, we need to understand the shared dependencies they have.
-Please list and briefly describe the shared contents between the files we are generating, including exported variables, 
-data patterns, id names of all DOM elements that javascript functions will use, message names and function names.
-Focus only on the names of shared dependencies, do not add any other explanations.
-"""
-
-
-class AnalyzeDepLibs(Action):
-    def __init__(self, name, context=None, llm=None):
-        super().__init__(name, context, llm)
-        self.desc = "Analyze the runtime dependencies of the program based on the context"
-
-    async def run(self, requirement, filepaths_string):
-        # prompt = f"Below is the product requirement document (PRD):\n\n{prd}\n\n{PROMPT}"
-        prompt = PROMPT.format(prompt=requirement, filepaths_string=filepaths_string)
-        design_filenames = await self._aask(prompt)
-        return design_filenames

metagpt/actions/azure_tts.py

@@ -1,53 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/6/9 22:22
-@Author  : Leo Xiao
-@File    : azure_tts.py
-"""
-from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer
-
-from metagpt.actions.action import Action
-from metagpt.config import Config
-
-
-class AzureTTS(Action):
-    def __init__(self, name, context=None, llm=None):
-        super().__init__(name, context, llm)
-        self.config = Config()
-
-    # Parameters reference: https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles
-    def synthesize_speech(self, lang, voice, role, text, output_file):
-        subscription_key = self.config.get('AZURE_TTS_SUBSCRIPTION_KEY')
-        region = self.config.get('AZURE_TTS_REGION')
-        speech_config = SpeechConfig(
-            subscription=subscription_key, region=region)
-
-        speech_config.speech_synthesis_voice_name = voice
-        audio_config = AudioConfig(filename=output_file)
-        synthesizer = SpeechSynthesizer(
-            speech_config=speech_config,
-            audio_config=audio_config)
-
-        # if voice=="zh-CN-YunxiNeural":
-        ssml_string = f"""
-            <speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='{lang}' xmlns:mstts='http://www.w3.org/2001/mstts'>
-                <voice name='{voice}'>
-                    <mstts:express-as style='affectionate' role='{role}'>
-                        {text}
-                    </mstts:express-as>
-                </voice>
-            </speak>
-            """
-
-        synthesizer.speak_ssml_async(ssml_string).get()
-
-
-if __name__ == "__main__":
-    azure_tts = AzureTTS("azure_tts")
-    azure_tts.synthesize_speech(
-        "zh-CN",
-        "zh-CN-YunxiNeural",
-        "Boy",
-        "Hello, I am Kaka",
-        "output.wav")

metagpt/actions/clone_function.py

@@ -1,65 +0,0 @@
-from pathlib import Path
-import traceback
-
-from metagpt.actions.write_code import WriteCode
-from metagpt.logs import logger
-from metagpt.schema import Message
-from metagpt.utils.highlight import highlight
-
-CLONE_PROMPT = """
-*context*
-Please convert the function code ```{source_code}``` into the the function format: ```{template_func}```.
-*Please Write code based on the following list and context*
-1. Write code start with ```, and end with ```.
-2. Please implement it in one function if possible, except for import statements. for exmaple:
-```python
-import pandas as pd
-def run(*args) -> pd.DataFrame:
-    ...
-```
-3. Do not use public member functions that do not exist in your design.
-4. The output function name, input parameters and return value must be the same as ```{template_func}```.
-5. Make sure the results before and after the code conversion are required to be exactly the same.
-6. Don't repeat my context in your replies.
-7. Return full results, for example, if the return value has df.head(), please return df.
-8. If you must use a third-party package, use the most popular ones, for example: pandas, numpy, ta, ...
-"""
-
-
-class CloneFunction(WriteCode):
-    def __init__(self, name="CloneFunction", context: list[Message] = None, llm=None):
-        super().__init__(name, context, llm)
-
-    def _save(self, code_path, code):
-        if isinstance(code_path, str):
-            code_path = Path(code_path)
-        code_path.parent.mkdir(parents=True, exist_ok=True)
-        code_path.write_text(code)
-        logger.info(f"Saving Code to {code_path}")
-
-    async def run(self, template_func: str, source_code: str) -> str:
-        """将source_code转换成template_func一样的入参和返回类型"""
-        prompt = CLONE_PROMPT.format(source_code=source_code, template_func=template_func)
-        logger.info(f"query for CloneFunction: \n {prompt}")
-        code = await self.write_code(prompt)
-        logger.info(f'CloneFunction code is \n {highlight(code)}')
-        return code
-
-
-def run_function_code(func_code: str, func_name: str, *args, **kwargs):
-    """Run function code from string code."""
-    try:
-        locals_ = {}
-        exec(func_code, locals_)
-        func = locals_[func_name]
-        return func(*args, **kwargs), ""
-    except Exception:
-        return "", traceback.format_exc()
-
-
-def run_function_script(code_script_path: str, func_name: str, *args, **kwargs):
-    """Run function code from script."""
-    if isinstance(code_script_path, str):
-        code_path = Path(code_script_path)
-    code = code_path.read_text(encoding='utf-8')
-    return run_function_code(code, func_name, *args, **kwargs)

metagpt/actions/debug_error.py

@@ -4,12 +4,21 @@
 @Time    : 2023/5/11 17:46
 @Author  : alexanderwu
 @File    : debug_error.py
+@Modified By: mashenquan, 2023/11/27.
+        1. Divide the context into three components: legacy code, unit test code, and console log.
+        2. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
 """
 import re
 
-from metagpt.logs import logger
+from pydantic import Field
+
 from metagpt.actions.action import Action
+from metagpt.config import CONFIG
+from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO
+from metagpt.logs import logger
+from metagpt.schema import RunCodeContext, RunCodeResult
 from metagpt.utils.common import CodeParser
+from metagpt.utils.file_repository import FileRepository
 
 PROMPT_TEMPLATE = """
 NOTICE
@@ -19,33 +28,56 @@ Based on the message, first, figure out your own role, i.e. Engineer or QaEngine
 then rewrite the development code or the test code based on your role, the error, and the summary, such that all bugs are fixed and the code performs well.
 Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the test case or script and triple quotes.
 The message is as follows:
-{context}
+# Legacy Code
+```python
+{code}
+```
+---
+# Unit Test Code
+```python
+{test_code}
+```
+---
+# Console logs
+```text
+{logs}
+```
 ---
 Now you should start rewriting the code:
-## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE.
+## file name of the code to rewrite: Write code with triple quote. Do your best to implement THIS IN ONLY ONE FILE.
 """
+
+
 class DebugError(Action):
-    def __init__(self, name="DebugError", context=None, llm=None):
-        super().__init__(name, context, llm)
+    name: str = "DebugError"
+    context: RunCodeContext = Field(default_factory=RunCodeContext)
 
-    # async def run(self, code, error):
-    #     prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \
-    #              f"\n\n{error}\n\nPlease try to fix the error in this code."
-    #     fixed_code = await self._aask(prompt)
-    #     return fixed_code
-    
-    async def run(self, context):
-        if "PASS" in context:
-            return "", "the original code works fine, no need to debug"
-        
-        file_name = re.search("## File To Rewrite:\s*(.+\\.py)", context).group(1)
+    async def run(self, *args, **kwargs) -> str:
+        output_doc = await FileRepository.get_file(
+            filename=self.context.output_filename, relative_path=TEST_OUTPUTS_FILE_REPO
+        )
+        if not output_doc:
+            return ""
+        output_detail = RunCodeResult.loads(output_doc.content)
+        pattern = r"Ran (\d+) tests in ([\d.]+)s\n\nOK"
+        matches = re.search(pattern, output_detail.stderr)
+        if matches:
+            return ""
 
-        logger.info(f"Debug and rewrite {file_name}")
+        logger.info(f"Debug and rewrite {self.context.test_filename}")
+        code_doc = await FileRepository.get_file(
+            filename=self.context.code_filename, relative_path=CONFIG.src_workspace
+        )
+        if not code_doc:
+            return ""
+        test_doc = await FileRepository.get_file(
+            filename=self.context.test_filename, relative_path=TEST_CODES_FILE_REPO
+        )
+        if not test_doc:
+            return ""
+        prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr)
 
-        prompt = PROMPT_TEMPLATE.format(context=context)
-        
         rsp = await self._aask(prompt)
-
         code = CodeParser.parse_code(block="", text=rsp)
 
-        return file_name, code
+        return code

metagpt/actions/design_api.py

@@ -4,214 +4,133 @@
 @Time    : 2023/5/11 19:26
 @Author  : alexanderwu
 @File    : design_api.py
+@Modified By: mashenquan, 2023/11/27.
+            1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
+            2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality.
+@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD.
 """
-import shutil
+import json
 from pathlib import Path
-from typing import List
+from typing import Optional
 
 from metagpt.actions import Action, ActionOutput
+from metagpt.actions.design_api_an import DESIGN_API_NODE
 from metagpt.config import CONFIG
-from metagpt.const import WORKSPACE_ROOT
+from metagpt.const import (
+    DATA_API_DESIGN_FILE_REPO,
+    PRDS_FILE_REPO,
+    SEQ_FLOW_FILE_REPO,
+    SYSTEM_DESIGN_FILE_REPO,
+    SYSTEM_DESIGN_PDF_FILE_REPO,
+)
 from metagpt.logs import logger
-from metagpt.utils.common import CodeParser
-from metagpt.utils.get_template import get_template
-from metagpt.utils.json_to_markdown import json_to_markdown
+from metagpt.schema import Document, Documents, Message
+from metagpt.utils.file_repository import FileRepository
 from metagpt.utils.mermaid import mermaid_to_file
 
-templates = {
-    "json": {
-        "PROMPT_TEMPLATE": """
-# Context
+NEW_REQ_TEMPLATE = """
+### Legacy Content
+{old_design}
+
+### New Requirements
 {context}
-
-## Format example
-{format_example}
------
-Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools
-Requirement: Fill in the following missing information based on the context, each section name is a key in json
-Max Output: 8192 chars or 2048 tokens. Try to use them up.
-
-## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework.
-
-## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores
-
-## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here
-
-## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. 
-
-## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.
-
-## Anything UNCLEAR: Provide as Plain text. Make clear here.
-
-output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example,
-and only output the json inside this tag, nothing else
-""",
-        "FORMAT_EXAMPLE": """
-[CONTENT]
-{
-    "Implementation approach": "We will ...",
-    "Python package name": "snake_game",
-    "File list": ["main.py"],
-    "Data structures and interface definitions": '
-    classDiagram
-        class Game{
-            +int score
-        }
-        ...
-        Game "1" -- "1" Food: has
-    ',
-    "Program call flow": '
-    sequenceDiagram
-        participant M as Main
-        ...
-        G->>M: end game
-    ',
-    "Anything UNCLEAR": "The requirement is clear to me."
-}
-[/CONTENT]
-""",
-    },
-    "markdown": {
-        "PROMPT_TEMPLATE": """
-# Context
-{context}
-
-## Format example
-{format_example}
------
-Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools
-Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately
-Max Output: 8192 chars or 2048 tokens. Try to use them up.
-Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the code and triple quote.
-
-## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework.
-
-## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores
-
-## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here
-
-## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. 
-
-## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.
-
-## Anything UNCLEAR: Provide as Plain text. Make clear here.
-
-""",
-        "FORMAT_EXAMPLE": """
----
-## Implementation approach
-We will ...
-
-## Python package name
-```python
-"snake_game"
-```
-
-## File list
-```python
-[
-    "main.py",
-]
-```
-
-## Data structures and interface definitions
-```mermaid
-classDiagram
-    class Game{
-        +int score
-    }
-    ...
-    Game "1" -- "1" Food: has
-```
-
-## Program call flow
-```mermaid
-sequenceDiagram
-    participant M as Main
-    ...
-    G->>M: end game
-```
-
-## Anything UNCLEAR
-The requirement is clear to me.
----
-""",
-    },
-}
-
-OUTPUT_MAPPING = {
-    "Implementation approach": (str, ...),
-    "Python package name": (str, ...),
-    "File list": (List[str], ...),
-    "Data structures and interface definitions": (str, ...),
-    "Program call flow": (str, ...),
-    "Anything UNCLEAR": (str, ...),
-}
+"""
 
 
 class WriteDesign(Action):
-    def __init__(self, name, context=None, llm=None):
-        super().__init__(name, context, llm)
-        self.desc = (
-            "Based on the PRD, think about the system design, and design the corresponding APIs, "
-            "data structures, library tables, processes, and paths. Please provide your design, feedback "
-            "clearly and in detail."
-        )
+    name: str = ""
+    context: Optional[str] = None
+    desc: str = (
+        "Based on the PRD, think about the system design, and design the corresponding APIs, "
+        "data structures, library tables, processes, and paths. Please provide your design, feedback "
+        "clearly and in detail."
+    )
 
-    def recreate_workspace(self, workspace: Path):
-        try:
-            shutil.rmtree(workspace)
-        except FileNotFoundError:
-            pass  # Folder does not exist, but we don't care
-        workspace.mkdir(parents=True, exist_ok=True)
+    async def run(self, with_messages: Message, schema: str = CONFIG.prompt_schema):
+        # Use `git status` to identify which PRD documents have been modified in the `docs/prds` directory.
+        prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO)
+        changed_prds = prds_file_repo.changed_files
+        # Use `git status` to identify which design documents in the `docs/system_designs` directory have undergone
+        # changes.
+        system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO)
+        changed_system_designs = system_design_file_repo.changed_files
 
-    async def _save_prd(self, docs_path, resources_path, context):
-        prd_file = docs_path / "prd.md"
-        if context[-1].instruct_content and context[-1].instruct_content.dict()["Competitive Quadrant Chart"]:
-            quadrant_chart = context[-1].instruct_content.dict()["Competitive Quadrant Chart"]
-            await mermaid_to_file(quadrant_chart, resources_path / "competitive_analysis")
+        # For those PRDs and design documents that have undergone changes, regenerate the design content.
+        changed_files = Documents()
+        for filename in changed_prds.keys():
+            doc = await self._update_system_design(
+                filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo
+            )
+            changed_files.docs[filename] = doc
 
-        if context[-1].instruct_content:
-            logger.info(f"Saving PRD to {prd_file}")
-            prd_file.write_text(json_to_markdown(context[-1].instruct_content.dict()))
+        for filename in changed_system_designs.keys():
+            if filename in changed_files.docs:
+                continue
+            doc = await self._update_system_design(
+                filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo
+            )
+            changed_files.docs[filename] = doc
+        if not changed_files.docs:
+            logger.info("Nothing has changed.")
+        # Wait until all files under `docs/system_designs/` are processed before sending the publish message,
+        # leaving room for global optimization in subsequent steps.
+        return ActionOutput(content=changed_files.model_dump_json(), instruct_content=changed_files)
 
-    async def _save_system_design(self, docs_path, resources_path, system_design):
-        data_api_design = system_design.instruct_content.dict()[
-            "Data structures and interface definitions"
-        ]  # CodeParser.parse_code(block="Data structures and interface definitions", text=content)
-        seq_flow = system_design.instruct_content.dict()[
-            "Program call flow"
-        ]  # CodeParser.parse_code(block="Program call flow", text=content)
-        await mermaid_to_file(data_api_design, resources_path / "data_api_design")
-        await mermaid_to_file(seq_flow, resources_path / "seq_flow")
-        system_design_file = docs_path / "system_design.md"
-        logger.info(f"Saving System Designs to {system_design_file}")
-        system_design_file.write_text((json_to_markdown(system_design.instruct_content.dict())))
+    async def _new_system_design(self, context, schema=CONFIG.prompt_schema):
+        node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, schema=schema)
+        return node
 
-    async def _save(self, context, system_design):
-        if isinstance(system_design, ActionOutput):
-            ws_name = system_design.instruct_content.dict()["Python package name"]
+    async def _merge(self, prd_doc, system_design_doc, schema=CONFIG.prompt_schema):
+        context = NEW_REQ_TEMPLATE.format(old_design=system_design_doc.content, context=prd_doc.content)
+        node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, schema=schema)
+        system_design_doc.content = node.instruct_content.model_dump_json()
+        return system_design_doc
+
+    async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document:
+        prd = await prds_file_repo.get(filename)
+        old_system_design_doc = await system_design_file_repo.get(filename)
+        if not old_system_design_doc:
+            system_design = await self._new_system_design(context=prd.content)
+            doc = Document(
+                root_path=SYSTEM_DESIGN_FILE_REPO,
+                filename=filename,
+                content=system_design.instruct_content.model_dump_json(),
+            )
         else:
-            ws_name = CodeParser.parse_str(block="Python package name", text=system_design)
-        workspace = WORKSPACE_ROOT / ws_name
-        self.recreate_workspace(workspace)
-        docs_path = workspace / "docs"
-        resources_path = workspace / "resources"
-        docs_path.mkdir(parents=True, exist_ok=True)
-        resources_path.mkdir(parents=True, exist_ok=True)
-        await self._save_prd(docs_path, resources_path, context)
-        await self._save_system_design(docs_path, resources_path, system_design)
-
-    async def run(self, context, format=CONFIG.prompt_format):
-        prompt_template, format_example = get_template(templates, format)
-        prompt = prompt_template.format(context=context, format_example=format_example)
-        # system_design = await self._aask(prompt)
-        system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format)
-        # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python package name" contain space, have to use setattr
-        setattr(
-            system_design.instruct_content,
-            "Python package name",
-            system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'),
+            doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc)
+        await system_design_file_repo.save(
+            filename=filename, content=doc.content, dependencies={prd.root_relative_path}
         )
-        await self._save(context, system_design)
-        return system_design
+        await self._save_data_api_design(doc)
+        await self._save_seq_flow(doc)
+        await self._save_pdf(doc)
+        return doc
+
+    @staticmethod
+    async def _save_data_api_design(design_doc):
+        m = json.loads(design_doc.content)
+        data_api_design = m.get("Data structures and interfaces")
+        if not data_api_design:
+            return
+        pathname = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("")
+        await WriteDesign._save_mermaid_file(data_api_design, pathname)
+        logger.info(f"Save class view to {str(pathname)}")
+
+    @staticmethod
+    async def _save_seq_flow(design_doc):
+        m = json.loads(design_doc.content)
+        seq_flow = m.get("Program call flow")
+        if not seq_flow:
+            return
+        pathname = CONFIG.git_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc.filename).with_suffix("")
+        await WriteDesign._save_mermaid_file(seq_flow, pathname)
+        logger.info(f"Saving sequence flow to {str(pathname)}")
+
+    @staticmethod
+    async def _save_pdf(design_doc):
+        await FileRepository.save_as(doc=design_doc, with_suffix=".md", relative_path=SYSTEM_DESIGN_PDF_FILE_REPO)
+
+    @staticmethod
+    async def _save_mermaid_file(data: str, pathname: Path):
+        pathname.parent.mkdir(parents=True, exist_ok=True)
+        await mermaid_to_file(data, pathname)

metagpt/actions/design_api_an.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/12/12 22:24
+@Author  : alexanderwu
+@File    : design_api_an.py
+"""
+from typing import List
+
+from metagpt.actions.action_node import ActionNode
+from metagpt.utils.mermaid import MMC1, MMC2
+
+IMPLEMENTATION_APPROACH = ActionNode(
+    key="Implementation approach",
+    expected_type=str,
+    instruction="Analyze the difficult points of the requirements, select the appropriate open-source framework",
+    example="We will ...",
+)
+
+PROJECT_NAME = ActionNode(
+    key="Project name", expected_type=str, instruction="The project name with underline", example="game_2048"
+)
+
+FILE_LIST = ActionNode(
+    key="File list",
+    expected_type=List[str],
+    instruction="Only need relative paths. ALWAYS write a main.py or app.py here",
+    example=["main.py", "game.py"],
+)
+
+DATA_STRUCTURES_AND_INTERFACES = ActionNode(
+    key="Data structures and interfaces",
+    expected_type=str,
+    instruction="Use mermaid classDiagram code syntax, including classes, method(__init__ etc.) and functions with type"
+    " annotations, CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. "
+    "The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.",
+    example=MMC1,
+)
+
+PROGRAM_CALL_FLOW = ActionNode(
+    key="Program call flow",
+    expected_type=str,
+    instruction="Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE "
+    "accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.",
+    example=MMC2,
+)
+
+ANYTHING_UNCLEAR = ActionNode(
+    key="Anything UNCLEAR",
+    expected_type=str,
+    instruction="Mention unclear project aspects, then try to clarify it.",
+    example="Clarification needed on third-party API integration, ...",
+)
+
+NODES = [
+    IMPLEMENTATION_APPROACH,
+    # PROJECT_NAME,
+    FILE_LIST,
+    DATA_STRUCTURES_AND_INTERFACES,
+    PROGRAM_CALL_FLOW,
+    ANYTHING_UNCLEAR,
+]
+
+DESIGN_API_NODE = ActionNode.from_children("DesignAPI", NODES)

metagpt/actions/design_api_review.py

@@ -5,18 +5,22 @@
 @Author  : alexanderwu
 @File    : design_api_review.py
 """
+
+from typing import Optional
+
 from metagpt.actions.action import Action
 
 
 class DesignReview(Action):
-    def __init__(self, name, context=None, llm=None):
-        super().__init__(name, context, llm)
+    name: str = "DesignReview"
+    context: Optional[str] = None
 
     async def run(self, prd, api_design):
-        prompt = f"Here is the Product Requirement Document (PRD):\n\n{prd}\n\nHere is the list of APIs designed " \
-                 f"based on this PRD:\n\n{api_design}\n\nPlease review whether this API design meets the requirements" \
-                 f" of the PRD, and whether it complies with good design practices."
+        prompt = (
+            f"Here is the Product Requirement Document (PRD):\n\n{prd}\n\nHere is the list of APIs designed "
+            f"based on this PRD:\n\n{api_design}\n\nPlease review whether this API design meets the requirements"
+            f" of the PRD, and whether it complies with good design practices."
+        )
 
         api_review = await self._aask(prompt)
         return api_review
-    

metagpt/actions/design_filenames.py

@@ -1,29 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/19 11:50
-@Author  : alexanderwu
-@File    : design_filenames.py
-"""
-from metagpt.actions import Action
-from metagpt.logs import logger
-
-PROMPT = """You are an AI developer, trying to write a program that generates code for users based on their intentions.
-When given their intentions, provide a complete and exhaustive list of file paths needed to write the program for the user.
-Only list the file paths you will write and return them as a Python string list.
-Do not add any other explanations, just return a Python string list."""
-
-
-class DesignFilenames(Action):
-    def __init__(self, name, context=None, llm=None):
-        super().__init__(name, context, llm)
-        self.desc = "Based on the PRD, consider system design, and carry out the basic design of the corresponding " \
-                    "APIs, data structures, and database tables. Please give your design, feedback clearly and in detail."
-
-    async def run(self, prd):
-        prompt = f"The following is the Product Requirement Document (PRD):\n\n{prd}\n\n{PROMPT}"
-        design_filenames = await self._aask(prompt)
-        logger.debug(prompt)
-        logger.debug(design_filenames)
-        return design_filenames
-    

metagpt/actions/detail_mining.py

@@ -1,52 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/9/12 17:45
-@Author  : fisherdeng
-@File    : detail_mining.py
-"""
-from metagpt.actions import Action, ActionOutput
-from metagpt.logs import logger
-
-PROMPT_TEMPLATE = """
-##TOPIC
-{topic}
-
-##RECORD
-{record}
-
-##Format example
-{format_example}
------
-
-Task: Refer to the "##TOPIC" (discussion objectives) and "##RECORD" (discussion records) to further inquire about the details that interest you, within a word limit of 150 words.
-Special Note 1: Your intention is solely to ask questions without endorsing or negating any individual's viewpoints.
-Special Note 2: This output should only include the topic "##OUTPUT". Do not add, remove, or modify the topic. Begin the output with '##OUTPUT', followed by an immediate line break, and then proceed to provide the content in the specified format as outlined in the "##Format example" section.
-Special Note 3: The output should be in the same language as the input.
-"""
-FORMAT_EXAMPLE = """
-
-##
-
-##OUTPUT
-...(Please provide the specific details you would like to inquire about here.)
-
-##
-
-##
-"""
-OUTPUT_MAPPING = {
-    "OUTPUT": (str, ...),
-}
-
-
-class DetailMining(Action):
-    """This class allows LLM to further mine noteworthy details based on specific "##TOPIC"(discussion topic) and "##RECORD" (discussion records), thereby deepening the discussion.
-    """
-    def __init__(self, name="", context=None, llm=None):
-        super().__init__(name, context, llm)
-
-    async def run(self, topic, record) -> ActionOutput:
-        prompt = PROMPT_TEMPLATE.format(topic=topic, record=record, format_example=FORMAT_EXAMPLE)
-        rsp = await self._aask_v1(prompt, "detail_mining", OUTPUT_MAPPING)
-        return rsp

metagpt/actions/execute_task.py

@@ -5,13 +5,15 @@
 @Author  : femto Zheng
 @File    : execute_task.py
 """
+
+
 from metagpt.actions import Action
 from metagpt.schema import Message
 
 
 class ExecuteTask(Action):
-    def __init__(self, name="ExecuteTask", context: list[Message] = None, llm=None):
-        super().__init__(name, context, llm)
+    name: str = "ExecuteTask"
+    context: list[Message] = []
 
-    def run(self, *args, **kwargs):
+    async def run(self, *args, **kwargs):
         pass

metagpt/actions/fix_bug.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023-12-12
+@Author  : mashenquan
+@File    : fix_bug.py
+"""
+from metagpt.actions import Action
+
+
+class FixBug(Action):
+    """Fix bug action without any implementation details"""
+
+    name: str = "FixBug"

metagpt/actions/generate_questions.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/9/12 17:45
+@Author  : fisherdeng
+@File    : generate_questions.py
+"""
+from metagpt.actions import Action
+from metagpt.actions.action_node import ActionNode
+
+QUESTIONS = ActionNode(
+    key="Questions",
+    expected_type=list[str],
+    instruction="Task: Refer to the context to further inquire about the details that interest you, within a word limit"
+    " of 150 words. Please provide the specific details you would like to inquire about here",
+    example=["1. What ...", "2. How ...", "3. ..."],
+)
+
+
+class GenerateQuestions(Action):
+    """This class allows LLM to further mine noteworthy details based on specific "##TOPIC"(discussion topic) and
+    "##RECORD" (discussion records), thereby deepening the discussion."""
+
+    name: str = "GenerateQuestions"
+
+    async def run(self, context):
+        return await QUESTIONS.fill(context=context, llm=self.llm)

metagpt/actions/invoice_ocr.py

@@ -10,16 +10,23 @@
 
 import os
 import zipfile
-from pathlib import Path
 from datetime import datetime
+from pathlib import Path
+from typing import Optional
 
 import pandas as pd
 from paddleocr import PaddleOCR
+from pydantic import Field
 
 from metagpt.actions import Action
 from metagpt.const import INVOICE_OCR_TABLE_PATH
+from metagpt.llm import LLM
 from metagpt.logs import logger
-from metagpt.prompts.invoice_ocr import EXTRACT_OCR_MAIN_INFO_PROMPT, REPLY_OCR_QUESTION_PROMPT
+from metagpt.prompts.invoice_ocr import (
+    EXTRACT_OCR_MAIN_INFO_PROMPT,
+    REPLY_OCR_QUESTION_PROMPT,
+)
+from metagpt.provider.base_llm import BaseLLM
 from metagpt.utils.common import OutputParser
 from metagpt.utils.file import File
 
@@ -33,8 +40,8 @@ class InvoiceOCR(Action):
 
     """
 
-    def __init__(self, name: str = "", *args, **kwargs):
-        super().__init__(name, *args, **kwargs)
+    name: str = "InvoiceOCR"
+    context: Optional[str] = None
 
     @staticmethod
     async def _check_file_type(file_path: Path) -> str:
@@ -81,6 +88,8 @@ class InvoiceOCR(Action):
     async def _ocr(invoice_file_path: Path):
         ocr = PaddleOCR(use_angle_cls=True, lang="ch", page_num=1)
         ocr_result = ocr.ocr(str(invoice_file_path), cls=True)
+        for result in ocr_result[0]:
+            result[1] = (result[1][0], round(result[1][1], 2))  # round long confidence scores to reduce token costs
         return ocr_result
 
     async def run(self, file_path: Path, *args, **kwargs) -> list:
@@ -122,9 +131,10 @@ class GenerateTable(Action):
 
     """
 
-    def __init__(self, name: str = "", language: str = "ch", *args, **kwargs):
-        super().__init__(name, *args, **kwargs)
-        self.language = language
+    name: str = "GenerateTable"
+    context: Optional[str] = None
+    llm: BaseLLM = Field(default_factory=LLM)
+    language: str = "ch"
 
     async def run(self, ocr_results: list, filename: str, *args, **kwargs) -> dict[str, str]:
         """Processes OCR results, extracts invoice information, generates a table, and saves it as an Excel file.
@@ -166,9 +176,10 @@ class ReplyQuestion(Action):
 
     """
 
-    def __init__(self, name: str = "", language: str = "ch", *args, **kwargs):
-        super().__init__(name, *args, **kwargs)
-        self.language = language
+    name: str = "ReplyQuestion"
+    context: Optional[str] = None
+    llm: BaseLLM = Field(default_factory=LLM)
+    language: str = "ch"
 
     async def run(self, query: str, ocr_result: list, *args, **kwargs) -> str:
         """Reply to questions based on ocr results.
@@ -183,4 +194,3 @@ class ReplyQuestion(Action):
         prompt = REPLY_OCR_QUESTION_PROMPT.format(query=query, ocr_result=ocr_result, language=self.language)
         resp = await self._aask(prompt=prompt)
         return resp
-

metagpt/actions/prepare_documents.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/11/20
+@Author  : mashenquan
+@File    : prepare_documents.py
+@Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt.
+        RFC 135 2.2.3.5.1.
+"""
+import shutil
+from pathlib import Path
+from typing import Optional
+
+from metagpt.actions import Action, ActionOutput
+from metagpt.config import CONFIG
+from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME
+from metagpt.schema import Document
+from metagpt.utils.file_repository import FileRepository
+from metagpt.utils.git_repository import GitRepository
+
+
+class PrepareDocuments(Action):
+    """PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt."""
+
+    name: str = "PrepareDocuments"
+    context: Optional[str] = None
+
+    def _init_repo(self):
+        """Initialize the Git environment."""
+        if not CONFIG.project_path:
+            name = CONFIG.project_name or FileRepository.new_filename()
+            path = Path(CONFIG.workspace_path) / name
+        else:
+            path = Path(CONFIG.project_path)
+        if path.exists() and not CONFIG.inc:
+            shutil.rmtree(path)
+        CONFIG.project_path = path
+        CONFIG.git_repo = GitRepository(local_path=path, auto_init=True)
+
+    async def run(self, with_messages, **kwargs):
+        """Create and initialize the workspace folder, initialize the Git environment."""
+        self._init_repo()
+
+        # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`.
+        doc = Document(root_path=DOCS_FILE_REPO, filename=REQUIREMENT_FILENAME, content=with_messages[0].content)
+        await FileRepository.save_file(filename=REQUIREMENT_FILENAME, content=doc.content, relative_path=DOCS_FILE_REPO)
+
+        # Send a Message notification to the WritePRD action, instructing it to process requirements using
+        # `docs/requirement.txt` and `docs/prds/`.
+        return ActionOutput(content=doc.content, instruct_content=doc)

metagpt/actions/prepare_interview.py

@@ -6,36 +6,20 @@
 @File    : prepare_interview.py
 """
 from metagpt.actions import Action
+from metagpt.actions.action_node import ActionNode
 
-PROMPT_TEMPLATE = """
-# Context
-{context}
-
-## Format example
----
-Q1: question 1 here
-References:
-  - point 1
-  - point 2
-
-Q2: question 2 here...
----
-
------
-Role: You are an interviewer of our company who is well-knonwn in frontend or backend develop;
+QUESTIONS = ActionNode(
+    key="Questions",
+    expected_type=list[str],
+    instruction="""Role: You are an interviewer of our company who is well-knonwn in frontend or backend develop;
 Requirement: Provide a list of questions for the interviewer to ask the interviewee, by reading the resume of the interviewee in the context.
-Attention: Provide as markdown block as the format above, at least 10 questions.
-"""
-
-# prepare for a interview
+Attention: Provide as markdown block as the format above, at least 10 questions.""",
+    example=["1. What ...", "2. How ..."],
+)
 
 
 class PrepareInterview(Action):
-    def __init__(self, name, context=None, llm=None):
-        super().__init__(name, context, llm)
+    name: str = "PrepareInterview"
 
     async def run(self, context):
-        prompt = PROMPT_TEMPLATE.format(context=context)
-        question_list = await self._aask_v1(prompt)
-        return question_list
-
+        return await QUESTIONS.fill(context=context, llm=self.llm)

metagpt/actions/project_management.py

@@ -4,189 +4,114 @@
 @Time    : 2023/5/11 19:12
 @Author  : alexanderwu
 @File    : project_management.py
+@Modified By: mashenquan, 2023/11/27.
+        1. Divide the context into three components: legacy code, unit test code, and console log.
+        2. Move the document storage operations related to WritePRD from the save operation of WriteDesign.
+        3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality.
 """
-from typing import List
 
+import json
+from typing import Optional
+
+from metagpt.actions import ActionOutput
 from metagpt.actions.action import Action
+from metagpt.actions.project_management_an import PM_NODE
 from metagpt.config import CONFIG
-from metagpt.const import WORKSPACE_ROOT
-from metagpt.utils.common import CodeParser
-from metagpt.utils.get_template import get_template
-from metagpt.utils.json_to_markdown import json_to_markdown
+from metagpt.const import (
+    PACKAGE_REQUIREMENTS_FILENAME,
+    SYSTEM_DESIGN_FILE_REPO,
+    TASK_FILE_REPO,
+    TASK_PDF_FILE_REPO,
+)
+from metagpt.logs import logger
+from metagpt.schema import Document, Documents
+from metagpt.utils.file_repository import FileRepository
 
-templates = {
-    "json": {
-        "PROMPT_TEMPLATE": """
-# Context
+NEW_REQ_TEMPLATE = """
+### Legacy Content
+{old_tasks}
+
+### New Requirements
 {context}
-
-## Format example
-{format_example}
------
-Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules
-Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them
-Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the code and triple quote.
-
-## Required Python third-party packages: Provided in requirements.txt format
-
-## Required Other language third-party packages: Provided in requirements.txt format
-
-## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend.
-
-## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first
-
-## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first
-
-## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. 
-
-## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs.
-
-output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example,
-and only output the json inside this tag, nothing else
-""",
-        "FORMAT_EXAMPLE": '''
-{
-    "Required Python third-party packages": [
-        "flask==1.1.2",
-        "bcrypt==3.2.0"
-    ],
-    "Required Other language third-party packages": [
-        "No third-party ..."
-    ],
-    "Full API spec": """
-        openapi: 3.0.0
-        ...
-        description: A JSON object ...
-     """,
-    "Logic Analysis": [
-        ["game.py","Contains..."]
-    ],
-    "Task list": [
-        "game.py"
-    ],
-    "Shared Knowledge": """
-        'game.py' contains ...
-    """,
-    "Anything UNCLEAR": "We need ... how to start."
-}
-''',
-    },
-    "markdown": {
-        "PROMPT_TEMPLATE": """
-# Context
-{context}
-
-## Format example
-{format_example}
------
-Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules
-Requirements: Based on the context, fill in the following missing information, note that all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file, if there are any missing files, you can supplement them
-Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the code and triple quote.
-
-## Required Python third-party packages: Provided in requirements.txt format
-
-## Required Other language third-party packages: Provided in requirements.txt format
-
-## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend.
-
-## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first
-
-## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first
-
-## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. 
-
-## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs.
-
-""",
-        "FORMAT_EXAMPLE": '''
----
-## Required Python third-party packages
-```python
 """
-flask==1.1.2
-bcrypt==3.2.0
-"""
-```
-
-## Required Other language third-party packages
-```python
-"""
-No third-party ...
-"""
-```
-
-## Full API spec
-```python
-"""
-openapi: 3.0.0
-...
-description: A JSON object ...
-"""
-```
-
-## Logic Analysis
-```python
-[
-    ["game.py", "Contains ..."],
-]
-```
-
-## Task list
-```python
-[
-    "game.py",
-]
-```
-
-## Shared Knowledge
-```python
-"""
-'game.py' contains ...
-"""
-```
-
-## Anything UNCLEAR
-We need ... how to start.
----
-''',
-    },
-}
-OUTPUT_MAPPING = {
-    "Required Python third-party packages": (List[str], ...),
-    "Required Other language third-party packages": (List[str], ...),
-    "Full API spec": (str, ...),
-    "Logic Analysis": (List[List[str]], ...),
-    "Task list": (List[str], ...),
-    "Shared Knowledge": (str, ...),
-    "Anything UNCLEAR": (str, ...),
-}
 
 
 class WriteTasks(Action):
-    def __init__(self, name="CreateTasks", context=None, llm=None):
-        super().__init__(name, context, llm)
+    name: str = "CreateTasks"
+    context: Optional[str] = None
 
-    def _save(self, context, rsp):
-        if context[-1].instruct_content:
-            ws_name = context[-1].instruct_content.dict()["Python package name"]
+    async def run(self, with_messages, schema=CONFIG.prompt_schema):
+        system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO)
+        changed_system_designs = system_design_file_repo.changed_files
+
+        tasks_file_repo = CONFIG.git_repo.new_file_repository(TASK_FILE_REPO)
+        changed_tasks = tasks_file_repo.changed_files
+        change_files = Documents()
+        # Rewrite the system designs that have undergone changes based on the git head diff under
+        # `docs/system_designs/`.
+        for filename in changed_system_designs:
+            task_doc = await self._update_tasks(
+                filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo
+            )
+            change_files.docs[filename] = task_doc
+
+        # Rewrite the task files that have undergone changes based on the git head diff under `docs/tasks/`.
+        for filename in changed_tasks:
+            if filename in change_files.docs:
+                continue
+            task_doc = await self._update_tasks(
+                filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo
+            )
+            change_files.docs[filename] = task_doc
+
+        if not change_files.docs:
+            logger.info("Nothing has changed.")
+        # Wait until all files under `docs/tasks/` are processed before sending the publish_message, leaving room for
+        # global optimization in subsequent steps.
+        return ActionOutput(content=change_files.model_dump_json(), instruct_content=change_files)
+
+    async def _update_tasks(self, filename, system_design_file_repo, tasks_file_repo):
+        system_design_doc = await system_design_file_repo.get(filename)
+        task_doc = await tasks_file_repo.get(filename)
+        if task_doc:
+            task_doc = await self._merge(system_design_doc=system_design_doc, task_doc=task_doc)
         else:
-            ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content)
-        file_path = WORKSPACE_ROOT / ws_name / "docs/api_spec_and_tasks.md"
-        file_path.write_text(json_to_markdown(rsp.instruct_content.dict()))
+            rsp = await self._run_new_tasks(context=system_design_doc.content)
+            task_doc = Document(
+                root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.model_dump_json()
+            )
+        await tasks_file_repo.save(
+            filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path}
+        )
+        await self._update_requirements(task_doc)
+        await self._save_pdf(task_doc=task_doc)
+        return task_doc
 
-        # Write requirements.txt
-        requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt"
-        requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages")))
+    async def _run_new_tasks(self, context, schema=CONFIG.prompt_schema):
+        node = await PM_NODE.fill(context, self.llm, schema)
+        return node
 
-    async def run(self, context, format=CONFIG.prompt_format):
-        prompt_template, format_example = get_template(templates, format)
-        prompt = prompt_template.format(context=context, format_example=format_example)
-        rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format)
-        self._save(context, rsp)
-        return rsp
+    async def _merge(self, system_design_doc, task_doc, schema=CONFIG.prompt_schema) -> Document:
+        context = NEW_REQ_TEMPLATE.format(context=system_design_doc.content, old_tasks=task_doc.content)
+        node = await PM_NODE.fill(context, self.llm, schema)
+        task_doc.content = node.instruct_content.model_dump_json()
+        return task_doc
 
+    @staticmethod
+    async def _update_requirements(doc):
+        m = json.loads(doc.content)
+        packages = set(m.get("Required Python third-party packages", set()))
+        file_repo = CONFIG.git_repo.new_file_repository()
+        requirement_doc = await file_repo.get(filename=PACKAGE_REQUIREMENTS_FILENAME)
+        if not requirement_doc:
+            requirement_doc = Document(filename=PACKAGE_REQUIREMENTS_FILENAME, root_path=".", content="")
+        lines = requirement_doc.content.splitlines()
+        for pkg in lines:
+            if pkg == "":
+                continue
+            packages.add(pkg)
+        await file_repo.save(PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages))
 
-class AssignTasks(Action):
-    async def run(self, *args, **kwargs):
-        # Here you should implement the actual action
-        pass
+    @staticmethod
+    async def _save_pdf(task_doc):
+        await FileRepository.save_as(doc=task_doc, with_suffix=".md", relative_path=TASK_PDF_FILE_REPO)

metagpt/actions/project_management_an.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/12/14 15:28
+@Author  : alexanderwu
+@File    : project_management_an.py
+"""
+from typing import List
+
+from metagpt.actions.action_node import ActionNode
+from metagpt.logs import logger
+
+REQUIRED_PYTHON_PACKAGES = ActionNode(
+    key="Required Python packages",
+    expected_type=List[str],
+    instruction="Provide required Python packages in requirements.txt format.",
+    example=["flask==1.1.2", "bcrypt==3.2.0"],
+)
+
+REQUIRED_OTHER_LANGUAGE_PACKAGES = ActionNode(
+    key="Required Other language third-party packages",
+    expected_type=List[str],
+    instruction="List down the required packages for languages other than Python.",
+    example=["No third-party dependencies required"],
+)
+
+LOGIC_ANALYSIS = ActionNode(
+    key="Logic Analysis",
+    expected_type=List[List[str]],
+    instruction="Provide a list of files with the classes/methods/functions to be implemented, "
+    "including dependency analysis and imports.",
+    example=[
+        ["game.py", "Contains Game class and ... functions"],
+        ["main.py", "Contains main function, from game import Game"],
+    ],
+)
+
+TASK_LIST = ActionNode(
+    key="Task list",
+    expected_type=List[str],
+    instruction="Break down the tasks into a list of filenames, prioritized by dependency order.",
+    example=["game.py", "main.py"],
+)
+
+FULL_API_SPEC = ActionNode(
+    key="Full API spec",
+    expected_type=str,
+    instruction="Describe all APIs using OpenAPI 3.0 spec that may be used by both frontend and backend. If front-end "
+    "and back-end communication is not required, leave it blank.",
+    example="openapi: 3.0.0 ...",
+)
+
+SHARED_KNOWLEDGE = ActionNode(
+    key="Shared Knowledge",
+    expected_type=str,
+    instruction="Detail any shared knowledge, like common utility functions or configuration variables.",
+    example="'game.py' contains functions shared across the project.",
+)
+
+ANYTHING_UNCLEAR_PM = ActionNode(
+    key="Anything UNCLEAR",
+    expected_type=str,
+    instruction="Mention any unclear aspects in the project management context and try to clarify them.",
+    example="Clarification needed on how to start and initialize third-party libraries.",
+)
+
+NODES = [
+    REQUIRED_PYTHON_PACKAGES,
+    REQUIRED_OTHER_LANGUAGE_PACKAGES,
+    LOGIC_ANALYSIS,
+    TASK_LIST,
+    FULL_API_SPEC,
+    SHARED_KNOWLEDGE,
+    ANYTHING_UNCLEAR_PM,
+]
+
+
+PM_NODE = ActionNode.from_children("PM_NODE", NODES)
+
+
+def main():
+    prompt = PM_NODE.compile(context="")
+    logger.info(prompt)
+
+
+if __name__ == "__main__":
+    main()

metagpt/actions/rebuild_class_view.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/12/19
+@Author  : mashenquan
+@File    : rebuild_class_view.py
+@Desc    : Rebuild class view info
+"""
+import re
+from pathlib import Path
+
+import aiofiles
+
+from metagpt.actions import Action
+from metagpt.config import CONFIG
+from metagpt.const import (
+    AGGREGATION,
+    COMPOSITION,
+    DATA_API_DESIGN_FILE_REPO,
+    GENERALIZATION,
+    GRAPH_REPO_FILE_REPO,
+)
+from metagpt.logs import logger
+from metagpt.repo_parser import RepoParser
+from metagpt.schema import ClassAttribute, ClassMethod, ClassView
+from metagpt.utils.common import split_namespace
+from metagpt.utils.di_graph_repository import DiGraphRepository
+from metagpt.utils.graph_repository import GraphKeyword, GraphRepository
+
+
+class RebuildClassView(Action):
+    async def run(self, with_messages=None, format=CONFIG.prompt_schema):
+        graph_repo_pathname = CONFIG.git_repo.workdir / GRAPH_REPO_FILE_REPO / CONFIG.git_repo.workdir.name
+        graph_db = await DiGraphRepository.load_from(str(graph_repo_pathname.with_suffix(".json")))
+        repo_parser = RepoParser(base_directory=Path(self.context))
+        # use pylint
+        class_views, relationship_views, package_root = await repo_parser.rebuild_class_views(path=Path(self.context))
+        await GraphRepository.update_graph_db_with_class_views(graph_db, class_views)
+        await GraphRepository.update_graph_db_with_class_relationship_views(graph_db, relationship_views)
+        # use ast
+        direction, diff_path = self._diff_path(path_root=Path(self.context).resolve(), package_root=package_root)
+        symbols = repo_parser.generate_symbols()
+        for file_info in symbols:
+            # Align to the same root directory in accordance with `class_views`.
+            file_info.file = self._align_root(file_info.file, direction, diff_path)
+            await GraphRepository.update_graph_db_with_file_info(graph_db, file_info)
+        await self._create_mermaid_class_views(graph_db=graph_db)
+        await graph_db.save()
+
+    async def _create_mermaid_class_views(self, graph_db):
+        path = Path(CONFIG.git_repo.workdir) / DATA_API_DESIGN_FILE_REPO
+        path.mkdir(parents=True, exist_ok=True)
+        pathname = path / CONFIG.git_repo.workdir.name
+        async with aiofiles.open(str(pathname.with_suffix(".mmd")), mode="w", encoding="utf-8") as writer:
+            content = "classDiagram\n"
+            logger.debug(content)
+            await writer.write(content)
+            # class names
+            rows = await graph_db.select(predicate=GraphKeyword.IS, object_=GraphKeyword.CLASS)
+            class_distinct = set()
+            relationship_distinct = set()
+            for r in rows:
+                await RebuildClassView._create_mermaid_class(r.subject, graph_db, writer, class_distinct)
+            for r in rows:
+                await RebuildClassView._create_mermaid_relationship(r.subject, graph_db, writer, relationship_distinct)
+
+    @staticmethod
+    async def _create_mermaid_class(ns_class_name, graph_db, file_writer, distinct):
+        fields = split_namespace(ns_class_name)
+        if len(fields) > 2:
+            # Ignore sub-class
+            return
+
+        class_view = ClassView(name=fields[1])
+        rows = await graph_db.select(subject=ns_class_name)
+        for r in rows:
+            name = split_namespace(r.object_)[-1]
+            name, visibility, abstraction = RebuildClassView._parse_name(name=name, language="python")
+            if r.predicate == GraphKeyword.HAS_CLASS_PROPERTY:
+                var_type = await RebuildClassView._parse_variable_type(r.object_, graph_db)
+                attribute = ClassAttribute(
+                    name=name, visibility=visibility, abstraction=bool(abstraction), value_type=var_type
+                )
+                class_view.attributes.append(attribute)
+            elif r.predicate == GraphKeyword.HAS_CLASS_FUNCTION:
+                method = ClassMethod(name=name, visibility=visibility, abstraction=bool(abstraction))
+                await RebuildClassView._parse_function_args(method, r.object_, graph_db)
+                class_view.methods.append(method)
+
+        # update graph db
+        await graph_db.insert(ns_class_name, GraphKeyword.HAS_CLASS_VIEW, class_view.model_dump_json())
+
+        content = class_view.get_mermaid(align=1)
+        logger.debug(content)
+        await file_writer.write(content)
+        distinct.add(ns_class_name)
+
+    @staticmethod
+    async def _create_mermaid_relationship(ns_class_name, graph_db, file_writer, distinct):
+        s_fields = split_namespace(ns_class_name)
+        if len(s_fields) > 2:
+            # Ignore sub-class
+            return
+
+        predicates = {GraphKeyword.IS + v + GraphKeyword.OF: v for v in [GENERALIZATION, COMPOSITION, AGGREGATION]}
+        mappings = {
+            GENERALIZATION: " <|-- ",
+            COMPOSITION: " *-- ",
+            AGGREGATION: " o-- ",
+        }
+        content = ""
+        for p, v in predicates.items():
+            rows = await graph_db.select(subject=ns_class_name, predicate=p)
+            for r in rows:
+                o_fields = split_namespace(r.object_)
+                if len(o_fields) > 2:
+                    # Ignore sub-class
+                    continue
+                relationship = mappings.get(v, " .. ")
+                link = f"{o_fields[1]}{relationship}{s_fields[1]}"
+                distinct.add(link)
+                content += f"\t{link}\n"
+
+        if content:
+            logger.debug(content)
+            await file_writer.write(content)
+
+    @staticmethod
+    def _parse_name(name: str, language="python"):
+        pattern = re.compile(r"<I>(.*?)<\/I>")
+        result = re.search(pattern, name)
+
+        abstraction = ""
+        if result:
+            name = result.group(1)
+            abstraction = "*"
+        if name.startswith("__"):
+            visibility = "-"
+        elif name.startswith("_"):
+            visibility = "#"
+        else:
+            visibility = "+"
+        return name, visibility, abstraction
+
+    @staticmethod
+    async def _parse_variable_type(ns_name, graph_db) -> str:
+        rows = await graph_db.select(subject=ns_name, predicate=GraphKeyword.HAS_TYPE_DESC)
+        if not rows:
+            return ""
+        vals = rows[0].object_.replace("'", "").split(":")
+        if len(vals) == 1:
+            return ""
+        val = vals[-1].strip()
+        return "" if val == "NoneType" else val + " "
+
+    @staticmethod
+    async def _parse_function_args(method: ClassMethod, ns_name: str, graph_db: GraphRepository):
+        rows = await graph_db.select(subject=ns_name, predicate=GraphKeyword.HAS_ARGS_DESC)
+        if not rows:
+            return
+        info = rows[0].object_.replace("'", "")
+
+        fs_tag = "("
+        ix = info.find(fs_tag)
+        fe_tag = "):"
+        eix = info.rfind(fe_tag)
+        if eix < 0:
+            fe_tag = ")"
+            eix = info.rfind(fe_tag)
+        args_info = info[ix + len(fs_tag) : eix].strip()
+        method.return_type = info[eix + len(fe_tag) :].strip()
+        if method.return_type == "None":
+            method.return_type = ""
+        if "(" in method.return_type:
+            method.return_type = method.return_type.replace("(", "Tuple[").replace(")", "]")
+
+        # parse args
+        if not args_info:
+            return
+        splitter_ixs = []
+        cost = 0
+        for i in range(len(args_info)):
+            if args_info[i] == "[":
+                cost += 1
+            elif args_info[i] == "]":
+                cost -= 1
+            if args_info[i] == "," and cost == 0:
+                splitter_ixs.append(i)
+        splitter_ixs.append(len(args_info))
+        args = []
+        ix = 0
+        for eix in splitter_ixs:
+            args.append(args_info[ix:eix])
+            ix = eix + 1
+        for arg in args:
+            parts = arg.strip().split(":")
+            if len(parts) == 1:
+                method.args.append(ClassAttribute(name=parts[0].strip()))
+                continue
+            method.args.append(ClassAttribute(name=parts[0].strip(), value_type=parts[-1].strip()))
+
+    @staticmethod
+    def _diff_path(path_root: Path, package_root: Path) -> (str, str):
+        if len(str(path_root)) > len(str(package_root)):
+            return "+", str(path_root.relative_to(package_root))
+        if len(str(path_root)) < len(str(package_root)):
+            return "-", str(package_root.relative_to(path_root))
+        return "=", "."
+
+    @staticmethod
+    def _align_root(path: str, direction: str, diff_path: str):
+        if direction == "=":
+            return path
+        if direction == "+":
+            return diff_path + "/" + path
+        else:
+            return path[len(diff_path) + 1 :]

metagpt/actions/rebuild_sequence_view.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/1/4
+@Author  : mashenquan
+@File    : rebuild_sequence_view.py
+@Desc    : Rebuild sequence view info
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List
+
+from metagpt.actions import Action
+from metagpt.config import CONFIG
+from metagpt.const import GRAPH_REPO_FILE_REPO
+from metagpt.logs import logger
+from metagpt.utils.common import aread, list_files
+from metagpt.utils.di_graph_repository import DiGraphRepository
+from metagpt.utils.graph_repository import GraphKeyword
+
+
+class RebuildSequenceView(Action):
+    async def run(self, with_messages=None, format=CONFIG.prompt_schema):
+        graph_repo_pathname = CONFIG.git_repo.workdir / GRAPH_REPO_FILE_REPO / CONFIG.git_repo.workdir.name
+        graph_db = await DiGraphRepository.load_from(str(graph_repo_pathname.with_suffix(".json")))
+        entries = await RebuildSequenceView._search_main_entry(graph_db)
+        for entry in entries:
+            await self._rebuild_sequence_view(entry, graph_db)
+        await graph_db.save()
+
+    @staticmethod
+    async def _search_main_entry(graph_db) -> List:
+        rows = await graph_db.select(predicate=GraphKeyword.HAS_PAGE_INFO)
+        tag = "__name__:__main__"
+        entries = []
+        for r in rows:
+            if tag in r.subject or tag in r.object_:
+                entries.append(r)
+        return entries
+
+    async def _rebuild_sequence_view(self, entry, graph_db):
+        filename = entry.subject.split(":", 1)[0]
+        src_filename = RebuildSequenceView._get_full_filename(root=self.context, pathname=filename)
+        content = await aread(filename=src_filename, encoding="utf-8")
+        content = f"```python\n{content}\n```\n\n---\nTranslate the code above into Mermaid Sequence Diagram."
+        data = await self.llm.aask(
+            msg=content, system_msgs=["You are a python code to Mermaid Sequence Diagram translator in function detail"]
+        )
+        await graph_db.insert(subject=filename, predicate=GraphKeyword.HAS_SEQUENCE_VIEW, object_=data)
+        logger.info(data)
+
+    @staticmethod
+    def _get_full_filename(root: str | Path, pathname: str | Path) -> Path | None:
+        files = list_files(root=root)
+        postfix = "/" + str(pathname)
+        for i in files:
+            if str(i).endswith(postfix):
+                return i
+        return None

metagpt/actions/rebuild_sequence_view_an.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/1/4
+@Author  : mashenquan
+@File    : rebuild_sequence_view_an.py
+"""
+from metagpt.actions.action_node import ActionNode
+from metagpt.utils.mermaid import MMC2
+
+CODE_2_MERMAID_SEQUENCE_DIAGRAM = ActionNode(
+    key="Program call flow",
+    expected_type=str,
+    instruction='Translate the "context" content into "format example" format.',
+    example=MMC2,
+)

metagpt/actions/research.py

@@ -3,14 +3,15 @@
 from __future__ import annotations
 
 import asyncio
-import json
-from typing import Callable
+from typing import Callable, Optional, Union
 
-from pydantic import parse_obj_as
+from pydantic import Field, parse_obj_as
 
 from metagpt.actions import Action
 from metagpt.config import CONFIG
+from metagpt.llm import LLM
 from metagpt.logs import logger
+from metagpt.provider.base_llm import BaseLLM
 from metagpt.tools.search_engine import SearchEngine
 from metagpt.tools.web_browser_engine import WebBrowserEngine, WebBrowserEngineType
 from metagpt.utils.common import OutputParser
@@ -49,7 +50,7 @@ based on the link credibility. If two results have equal credibility, prioritize
 ranked results' indices in JSON format, like [0, 1, 3, 4, ...], without including other words.
 """
 
-WEB_BROWSE_AND_SUMMARIZE_PROMPT = '''### Requirements
+WEB_BROWSE_AND_SUMMARIZE_PROMPT = """### Requirements
 1. Utilize the text in the "Reference Information" section to respond to the question "{query}".
 2. If the question cannot be directly answered using the text, but the text is related to the research topic, please provide \
 a comprehensive summary of the text.
@@ -58,10 +59,10 @@ a comprehensive summary of the text.
 
 ### Reference Information
 {content}
-'''
+"""
 
 
-CONDUCT_RESEARCH_PROMPT = '''### Reference Information
+CONDUCT_RESEARCH_PROMPT = """### Reference Information
 {content}
 
 ### Requirements
@@ -73,22 +74,18 @@ above. The report must meet the following requirements:
 - Present data and findings in an intuitive manner, utilizing feature comparative tables, if applicable.
 - The report should have a minimum word count of 2,000 and be formatted with Markdown syntax following APA style guidelines.
 - Include all source URLs in APA format at the end of the report.
-'''
+"""
 
 
 class CollectLinks(Action):
     """Action class to collect links from a search engine."""
-    def __init__(
-        self,
-        name: str = "",
-        *args,
-        rank_func: Callable[[list[str]], None] | None = None,
-        **kwargs,
-    ):
-        super().__init__(name, *args, **kwargs)
-        self.desc = "Collect links from a search engine."
-        self.search_engine = SearchEngine()
-        self.rank_func = rank_func
+
+    name: str = "CollectLinks"
+    context: Optional[str] = None
+    desc: str = "Collect links from a search engine."
+
+    search_engine: SearchEngine = Field(default_factory=SearchEngine)
+    rank_func: Optional[Callable[[list[str]], None]] = None
 
     async def run(
         self,
@@ -114,20 +111,26 @@ class CollectLinks(Action):
             keywords = OutputParser.extract_struct(keywords, list)
             keywords = parse_obj_as(list[str], keywords)
         except Exception as e:
-            logger.exception(f"fail to get keywords related to the research topic \"{topic}\" for {e}")
+            logger.exception(f"fail to get keywords related to the research topic '{topic}' for {e}")
             keywords = [topic]
         results = await asyncio.gather(*(self.search_engine.run(i, as_string=False) for i in keywords))
 
         def gen_msg():
             while True:
-                search_results = "\n".join(f"#### Keyword: {i}\n Search Result: {j}\n" for (i, j) in zip(keywords, results))
-                prompt = SUMMARIZE_SEARCH_PROMPT.format(decomposition_nums=decomposition_nums, search_results=search_results)
+                search_results = "\n".join(
+                    f"#### Keyword: {i}\n Search Result: {j}\n" for (i, j) in zip(keywords, results)
+                )
+                prompt = SUMMARIZE_SEARCH_PROMPT.format(
+                    decomposition_nums=decomposition_nums, search_results=search_results
+                )
                 yield prompt
                 remove = max(results, key=len)
                 remove.pop()
                 if len(remove) == 0:
                     break
-        prompt = reduce_message_length(gen_msg(), self.llm.model, system_text, CONFIG.max_tokens_rsp)
+
+        model_name = CONFIG.get_model_name(CONFIG.get_default_llm_provider_enum())
+        prompt = reduce_message_length(gen_msg(), model_name, system_text, CONFIG.max_tokens_rsp)
         logger.debug(prompt)
         queries = await self._aask(prompt, [system_text])
         try:
@@ -172,20 +175,23 @@ class CollectLinks(Action):
 
 class WebBrowseAndSummarize(Action):
     """Action class to explore the web and provide summaries of articles and webpages."""
-    def __init__(
-        self,
-        *args,
-        browse_func: Callable[[list[str]], None] | None = None,
-        **kwargs,
-    ):
-        super().__init__(*args, **kwargs)
+
+    name: str = "WebBrowseAndSummarize"
+    context: Optional[str] = None
+    llm: BaseLLM = Field(default_factory=LLM)
+    desc: str = "Explore the web and provide summaries of articles and webpages."
+    browse_func: Union[Callable[[list[str]], None], None] = None
+    web_browser_engine: Optional[WebBrowserEngine] = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
         if CONFIG.model_for_researcher_summary:
             self.llm.model = CONFIG.model_for_researcher_summary
+
         self.web_browser_engine = WebBrowserEngine(
-            engine=WebBrowserEngineType.CUSTOM if browse_func else None,
-            run_func=browse_func,
+            engine=WebBrowserEngineType.CUSTOM if self.browse_func else None,
+            run_func=self.browse_func,
         )
-        self.desc = "Explore the web and provide summaries of articles and webpages."
 
     async def run(
         self,
@@ -214,7 +220,9 @@ class WebBrowseAndSummarize(Action):
         for u, content in zip([url, *urls], contents):
             content = content.inner_text
             chunk_summaries = []
-            for prompt in generate_prompt_chunk(content, prompt_template, self.llm.model, system_text, CONFIG.max_tokens_rsp):
+            for prompt in generate_prompt_chunk(
+                content, prompt_template, self.llm.model, system_text, CONFIG.max_tokens_rsp
+            ):
                 logger.debug(prompt)
                 summary = await self._aask(prompt, [system_text])
                 if summary == "Not relevant.":
@@ -238,8 +246,13 @@ class WebBrowseAndSummarize(Action):
 
 class ConductResearch(Action):
     """Action class to conduct research and generate a research report."""
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+
+    name: str = "ConductResearch"
+    context: Optional[str] = None
+    llm: BaseLLM = Field(default_factory=LLM)
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
         if CONFIG.model_for_researcher_report:
             self.llm.model = CONFIG.model_for_researcher_report
 

metagpt/actions/run_code.py

@@ -4,14 +4,27 @@
 @Time    : 2023/5/11 17:46
 @Author  : alexanderwu
 @File    : run_code.py
+@Modified By: mashenquan, 2023/11/27.
+            1. Mark the location of Console logs in the PROMPT_TEMPLATE with markdown code-block formatting to enhance
+            the understanding for the LLM.
+            2. Fix bug: Add the "install dependency" operation.
+            3. Encapsulate the input of RunCode into RunCodeContext and encapsulate the output of RunCode into
+            RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError.
+            4. According to section 2.2.3.5.7 of RFC 135, change the method of transferring file content
+            (code files, unit test files, log files) from using the message to using the file name.
+            5. Merged the `Config` class of send18:dev branch to take over the set/get operations of the Environment
+            class.
 """
-import os
 import subprocess
-import traceback
 from typing import Tuple
 
+from pydantic import Field
+
 from metagpt.actions.action import Action
+from metagpt.config import CONFIG
 from metagpt.logs import logger
+from metagpt.schema import RunCodeContext, RunCodeResult
+from metagpt.utils.exceptions import handle_exception
 
 PROMPT_TEMPLATE = """
 Role: You are a senior development and qa engineer, your role is summarize the code running result.
@@ -51,14 +64,20 @@ CONTEXT = """
 ## Running Command
 {command}
 ## Running Output
-standard output: {outs};
-standard errors: {errs};
+standard output: 
+```text
+{outs}
+```
+standard errors: 
+```text
+{errs}
+```
 """
 
 
 class RunCode(Action):
-    def __init__(self, name="RunCode", context=None, llm=None):
-        super().__init__(name, context, llm)
+    name: str = "RunCode"
+    context: RunCodeContext = Field(default_factory=RunCodeContext)
 
     @classmethod
     async def run_text(cls, code) -> Tuple[str, str]:
@@ -66,10 +85,9 @@ class RunCode(Action):
             # We will document_store the result in this dictionary
             namespace = {}
             exec(code, namespace)
-            return namespace.get("result", ""), ""
-        except Exception:
-            # If there is an error in the code, return the error message
-            return "", traceback.format_exc()
+        except Exception as e:
+            return "", str(e)
+        return namespace.get("result", ""), ""
 
     @classmethod
     async def run_script(cls, working_directory, additional_python_paths=[], command=[]) -> Tuple[str, str]:
@@ -77,17 +95,19 @@ class RunCode(Action):
         additional_python_paths = [str(path) for path in additional_python_paths]
 
         # Copy the current environment variables
-        env = os.environ.copy()
+        env = CONFIG.new_environ()
 
         # Modify the PYTHONPATH environment variable
         additional_python_paths = [working_directory] + additional_python_paths
         additional_python_paths = ":".join(additional_python_paths)
         env["PYTHONPATH"] = additional_python_paths + ":" + env.get("PYTHONPATH", "")
+        RunCode._install_dependencies(working_directory=working_directory, env=env)
 
         # Start the subprocess
         process = subprocess.Popen(
             command, cwd=working_directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env
         )
+        logger.info(" ".join(command))
 
         try:
             # Wait for the process to complete, with a timeout
@@ -98,31 +118,45 @@ class RunCode(Action):
             stdout, stderr = process.communicate()
         return stdout.decode("utf-8"), stderr.decode("utf-8")
 
-    async def run(
-        self, code, mode="script", code_file_name="", test_code="", test_file_name="", command=[], **kwargs
-    ) -> str:
-        logger.info(f"Running {' '.join(command)}")
-        if mode == "script":
-            outs, errs = await self.run_script(command=command, **kwargs)
-        elif mode == "text":
-            outs, errs = await self.run_text(code=code)
+    async def run(self, *args, **kwargs) -> RunCodeResult:
+        logger.info(f"Running {' '.join(self.context.command)}")
+        if self.context.mode == "script":
+            outs, errs = await self.run_script(
+                command=self.context.command,
+                working_directory=self.context.working_directory,
+                additional_python_paths=self.context.additional_python_paths,
+            )
+        elif self.context.mode == "text":
+            outs, errs = await self.run_text(code=self.context.code)
 
         logger.info(f"{outs=}")
         logger.info(f"{errs=}")
 
         context = CONTEXT.format(
-            code=code,
-            code_file_name=code_file_name,
-            test_code=test_code,
-            test_file_name=test_file_name,
-            command=" ".join(command),
+            code=self.context.code,
+            code_file_name=self.context.code_filename,
+            test_code=self.context.test_code,
+            test_file_name=self.context.test_filename,
+            command=" ".join(self.context.command),
             outs=outs[:500],  # outs might be long but they are not important, truncate them to avoid token overflow
             errs=errs[:10000],  # truncate errors to avoid token overflow
         )
 
         prompt = PROMPT_TEMPLATE.format(context=context)
         rsp = await self._aask(prompt)
+        return RunCodeResult(summary=rsp, stdout=outs, stderr=errs)
 
-        result = context + rsp
+    @staticmethod
+    @handle_exception(exception_type=subprocess.CalledProcessError)
+    def _install_via_subprocess(cmd, check, cwd, env):
+        return subprocess.run(cmd, check=check, cwd=cwd, env=env)
 
-        return result
+    @staticmethod
+    def _install_dependencies(working_directory, env):
+        install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"]
+        logger.info(" ".join(install_command))
+        RunCode._install_via_subprocess(install_command, check=True, cwd=working_directory, env=env)
+
+        install_pytest_command = ["python", "-m", "pip", "install", "pytest"]
+        logger.info(" ".join(install_pytest_command))
+        RunCode._install_via_subprocess(install_pytest_command, check=True, cwd=working_directory, env=env)

metagpt/actions/search_and_summarize.py

@@ -5,12 +5,16 @@
 @Author  : alexanderwu
 @File    : search_google.py
 """
+from typing import Any, Optional
+
 import pydantic
+from pydantic import Field, model_validator
 
 from metagpt.actions import Action
-from metagpt.config import Config
+from metagpt.config import CONFIG, Config
 from metagpt.logs import logger
 from metagpt.schema import Message
+from metagpt.tools import SearchEngineType
 from metagpt.tools.search_engine import SearchEngine
 
 SEARCH_AND_SUMMARIZE_SYSTEM = """### Requirements
@@ -54,7 +58,6 @@ SEARCH_AND_SUMMARIZE_PROMPT = """
 
 """
 
-
 SEARCH_AND_SUMMARIZE_SALES_SYSTEM = """## Requirements
 1. Please summarize the latest dialogue based on the reference information (secondary) and dialogue history (primary). Do not include text that is irrelevant to the conversation.
 - The context is for reference only. If it is irrelevant to the user's search request history, please reduce its reference and usage.
@@ -100,18 +103,32 @@ You are a member of a professional butler team and will provide helpful suggesti
 """
 
 
+# TOTEST
 class SearchAndSummarize(Action):
-    def __init__(self, name="", context=None, llm=None, engine=None, search_func=None):
-        self.config = Config()
-        self.engine = engine or self.config.search_engine
+    name: str = ""
+    content: Optional[str] = None
+    config: None = Field(default_factory=Config)
+    engine: Optional[SearchEngineType] = CONFIG.search_engine
+    search_func: Optional[Any] = None
+    search_engine: SearchEngine = None
+    result: str = ""
 
+    @model_validator(mode="before")
+    @classmethod
+    def validate_engine_and_run_func(cls, values):
+        engine = values.get("engine")
+        search_func = values.get("search_func")
+        config = Config()
+
+        if engine is None:
+            engine = config.search_engine
         try:
-            self.search_engine = SearchEngine(self.engine, run_func=search_func)
+            search_engine = SearchEngine(engine=engine, run_func=search_func)
         except pydantic.ValidationError:
-            self.search_engine = None
+            search_engine = None
 
-        self.result = ""
-        super().__init__(name, context, llm)
+        values["search_engine"] = search_engine
+        return values
 
     async def run(self, context: list[Message], system_text=SEARCH_AND_SUMMARIZE_SYSTEM) -> str:
         if self.search_engine is None:
@@ -130,8 +147,7 @@ class SearchAndSummarize(Action):
         system_prompt = [system_text]
 
         prompt = SEARCH_AND_SUMMARIZE_PROMPT.format(
-            # PREFIX = self.prefix,
-            ROLE=self.profile,
+            ROLE=self.prefix,
             CONTEXT=rsp,
             QUERY_HISTORY="\n".join([str(i) for i in context[:-1]]),
             QUERY=str(context[-1]),
@@ -140,4 +156,3 @@ class SearchAndSummarize(Action):
         logger.debug(prompt)
         logger.debug(result)
         return result
-    

metagpt/actions/skill_action.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/8/28
+@Author  : mashenquan
+@File    : skill_action.py
+@Desc    : Call learned skill
+"""
+from __future__ import annotations
+
+import ast
+import importlib
+import traceback
+from copy import deepcopy
+from typing import Dict, Optional
+
+from metagpt.actions import Action
+from metagpt.learn.skill_loader import Skill
+from metagpt.logs import logger
+from metagpt.schema import Message
+
+
+# TOTEST
+class ArgumentsParingAction(Action):
+    skill: Skill
+    ask: str
+    rsp: Optional[Message] = None
+    args: Optional[Dict] = None
+
+    @property
+    def prompt(self):
+        prompt = "You are a function parser. You can convert spoken words into function parameters.\n"
+        prompt += "\n---\n"
+        prompt += f"{self.skill.name} function parameters description:\n"
+        for k, v in self.skill.arguments.items():
+            prompt += f"parameter `{k}`: {v}\n"
+        prompt += "\n---\n"
+        prompt += "Examples:\n"
+        for e in self.skill.examples:
+            prompt += f"If want you to do `{e.ask}`, return `{e.answer}` brief and clear.\n"
+        prompt += "\n---\n"
+        prompt += (
+            f"\nRefer to the `{self.skill.name}` function description, and fill in the function parameters according "
+            'to the example "I want you to do xx" in the Examples section.'
+            f"\nNow I want you to do `{self.ask}`, return function parameters in Examples format above, brief and "
+            "clear."
+        )
+        return prompt
+
+    async def run(self, with_message=None, **kwargs) -> Message:
+        prompt = self.prompt
+        rsp = await self.llm.aask(msg=prompt, system_msgs=[])
+        logger.debug(f"SKILL:{prompt}\n, RESULT:{rsp}")
+        self.args = ArgumentsParingAction.parse_arguments(skill_name=self.skill.name, txt=rsp)
+        self.rsp = Message(content=rsp, role="assistant", instruct_content=self.args, cause_by=self)
+        return self.rsp
+
+    @staticmethod
+    def parse_arguments(skill_name, txt) -> dict:
+        prefix = skill_name + "("
+        if prefix not in txt:
+            logger.error(f"{skill_name} not in {txt}")
+            return None
+        if ")" not in txt:
+            logger.error(f"')' not in {txt}")
+            return None
+        begin_ix = txt.find(prefix)
+        end_ix = txt.rfind(")")
+        args_txt = txt[begin_ix + len(prefix) : end_ix]
+        logger.info(args_txt)
+        fake_expression = f"dict({args_txt})"
+        parsed_expression = ast.parse(fake_expression, mode="eval")
+        args = {}
+        for keyword in parsed_expression.body.keywords:
+            key = keyword.arg
+            value = ast.literal_eval(keyword.value)
+            args[key] = value
+        return args
+
+
+class SkillAction(Action):
+    skill: Skill
+    args: Dict
+    rsp: Optional[Message] = None
+
+    async def run(self, with_message=None, **kwargs) -> Message:
+        """Run action"""
+        options = deepcopy(kwargs)
+        if self.args:
+            for k in self.args.keys():
+                if k in options:
+                    options.pop(k)
+        try:
+            rsp = await self.find_and_call_function(self.skill.name, args=self.args, **options)
+            self.rsp = Message(content=rsp, role="assistant", cause_by=self)
+        except Exception as e:
+            logger.exception(f"{e}, traceback:{traceback.format_exc()}")
+            self.rsp = Message(content=f"Error: {e}", role="assistant", cause_by=self)
+        return self.rsp
+
+    @staticmethod
+    async def find_and_call_function(function_name, args, **kwargs) -> str:
+        try:
+            module = importlib.import_module("metagpt.learn")
+            function = getattr(module, function_name)
+            # Invoke function and return result
+            result = await function(**args, **kwargs)
+            return result
+        except (ModuleNotFoundError, AttributeError):
+            logger.error(f"{function_name} not found")
+            raise ValueError(f"{function_name} not found")

metagpt/actions/summarize_code.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Author  : alexanderwu
+@File    : summarize_code.py
+@Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode.
+"""
+from pathlib import Path
+
+from pydantic import Field
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+
+from metagpt.actions.action import Action
+from metagpt.config import CONFIG
+from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO
+from metagpt.logs import logger
+from metagpt.schema import CodeSummarizeContext
+from metagpt.utils.file_repository import FileRepository
+
+PROMPT_TEMPLATE = """
+NOTICE
+Role: You are a professional software engineer, and your main task is to review the code.
+Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese.
+ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example".
+
+-----
+# System Design
+```text
+{system_design}
+```
+-----
+# Tasks
+```text
+{tasks}
+```
+-----
+{code_blocks}
+
+## Code Review All: Please read all historical files and find possible bugs in the files, such as unimplemented functions, calling errors, unreferences, etc.
+
+## Call flow: mermaid code, based on the implemented function, use mermaid to draw a complete call chain
+
+## Summary: Summary based on the implementation of historical files
+
+## TODOs: Python dict[str, str], write down the list of files that need to be modified and the reasons. We will modify them later.
+
+"""
+
+FORMAT_EXAMPLE = """
+
+## Code Review All
+
+### a.py
+- It fulfills less of xxx requirements...
+- Field yyy is not given...
+-...
+
+### b.py
+...
+
+### c.py
+...
+
+## Call flow
+```mermaid
+flowchart TB
+    c1-->a2
+    subgraph one
+    a1-->a2
+    end
+    subgraph two
+    b1-->b2
+    end
+    subgraph three
+    c1-->c2
+    end
+```
+
+## Summary
+- a.py:...
+- b.py:...
+- c.py:...
+- ...
+
+## TODOs
+{
+    "a.py": "implement requirement xxx...",
+}
+
+"""
+
+
+# TOTEST
+class SummarizeCode(Action):
+    name: str = "SummarizeCode"
+    context: CodeSummarizeContext = Field(default_factory=CodeSummarizeContext)
+
+    @retry(stop=stop_after_attempt(2), wait=wait_random_exponential(min=1, max=60))
+    async def summarize_code(self, prompt):
+        code_rsp = await self._aask(prompt)
+        return code_rsp
+
+    async def run(self):
+        design_pathname = Path(self.context.design_filename)
+        design_doc = await FileRepository.get_file(filename=design_pathname.name, relative_path=SYSTEM_DESIGN_FILE_REPO)
+        task_pathname = Path(self.context.task_filename)
+        task_doc = await FileRepository.get_file(filename=task_pathname.name, relative_path=TASK_FILE_REPO)
+        src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace)
+        code_blocks = []
+        for filename in self.context.codes_filenames:
+            code_doc = await src_file_repo.get(filename)
+            code_block = f"```python\n{code_doc.content}\n```\n-----"
+            code_blocks.append(code_block)
+        format_example = FORMAT_EXAMPLE
+        prompt = PROMPT_TEMPLATE.format(
+            system_design=design_doc.content,
+            tasks=task_doc.content,
+            code_blocks="\n".join(code_blocks),
+            format_example=format_example,
+        )
+        logger.info("Summarize code..")
+        rsp = await self.summarize_code(prompt)
+        return rsp

metagpt/actions/talk_action.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/8/28
+@Author  : mashenquan
+@File    : talk_action.py
+@Desc    : Act as it’s a talk
+"""
+from typing import Optional
+
+from metagpt.actions import Action
+from metagpt.config import CONFIG
+from metagpt.const import DEFAULT_LANGUAGE
+from metagpt.logs import logger
+from metagpt.schema import Message
+
+
+# TOTEST
+class TalkAction(Action):
+    context: str
+    history_summary: str = ""
+    knowledge: str = ""
+    rsp: Optional[Message] = None
+
+    @property
+    def prompt(self):
+        prompt = ""
+        if CONFIG.agent_description:
+            prompt = (
+                f"You are {CONFIG.agent_description}. Your responses should align with the role-play agreement, "
+                f"maintaining the character's persona and habits. When faced with unrelated questions, playfully "
+                f"decline to answer without revealing your AI nature to preserve the character's image.\n\n"
+            )
+        prompt += f"Knowledge:\n{self.knowledge}\n\n" if self.knowledge else ""
+        prompt += f"{self.history_summary}\n\n"
+        prompt += (
+            "If the information is insufficient, you can search in the historical conversation or knowledge above.\n"
+        )
+        language = CONFIG.language or DEFAULT_LANGUAGE
+        prompt += (
+            f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.\n "
+            f"{self.context}"
+        )
+        logger.debug(f"PROMPT: {prompt}")
+        return prompt
+
+    @property
+    def prompt_gpt4(self):
+        kvs = {
+            "{role}": CONFIG.agent_description or "",
+            "{history}": self.history_summary or "",
+            "{knowledge}": self.knowledge or "",
+            "{language}": CONFIG.language or DEFAULT_LANGUAGE,
+            "{ask}": self.context,
+        }
+        prompt = TalkActionPrompt.FORMATION_LOOSE
+        for k, v in kvs.items():
+            prompt = prompt.replace(k, v)
+        logger.info(f"PROMPT: {prompt}")
+        return prompt
+
+    # async def run_old(self, *args, **kwargs) -> ActionOutput:
+    #     prompt = self.prompt
+    #     rsp = await self.llm.aask(msg=prompt, system_msgs=[])
+    #     logger.debug(f"PROMPT:{prompt}\nRESULT:{rsp}\n")
+    #     self._rsp = ActionOutput(content=rsp)
+    #     return self._rsp
+
+    @property
+    def aask_args(self):
+        language = CONFIG.language or DEFAULT_LANGUAGE
+        system_msgs = [
+            f"You are {CONFIG.agent_description}.",
+            "Your responses should align with the role-play agreement, "
+            "maintaining the character's persona and habits. When faced with unrelated questions, playfully "
+            "decline to answer without revealing your AI nature to preserve the character's image.",
+            "If the information is insufficient, you can search in the context or knowledge.",
+            f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.",
+        ]
+        format_msgs = []
+        if self.knowledge:
+            format_msgs.append({"role": "assistant", "content": self.knowledge})
+        if self.history_summary:
+            format_msgs.append({"role": "assistant", "content": self.history_summary})
+        return self.context, format_msgs, system_msgs
+
+    async def run(self, with_message=None, **kwargs) -> Message:
+        msg, format_msgs, system_msgs = self.aask_args
+        rsp = await self.llm.aask(msg=msg, format_msgs=format_msgs, system_msgs=system_msgs)
+        self.rsp = Message(content=rsp, role="assistant", cause_by=self)
+        return self.rsp
+
+
+class TalkActionPrompt:
+    FORMATION = """Formation: "Capacity and role" defines the role you are currently playing;
+  "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation;
+  "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses;
+  "Statement" defines the work detail you need to complete at this stage;
+  "[ASK_BEGIN]" and [ASK_END] tags enclose the questions;
+  "Constraint" defines the conditions that your responses must comply with.
+  "Personality" defines your language style。
+  "Insight" provides a deeper understanding of the characters' inner traits.
+  "Initial" defines the initial setup of a character.
+
+Capacity and role: {role}
+Statement: Your responses should align with the role-play agreement, maintaining the
+ character's persona and habits. When faced with unrelated questions, playfully decline to answer without revealing
+ your AI nature to preserve the character's image.
+
+[HISTORY_BEGIN]
+
+{history}
+
+[HISTORY_END]
+
+[KNOWLEDGE_BEGIN]
+
+{knowledge}
+
+[KNOWLEDGE_END]
+
+Statement: If the information is insufficient, you can search in the historical conversation or knowledge.
+Statement: Unless you are a language professional, answer the following questions strictly in {language}
+, and the answers must follow the Markdown format. Strictly excluding any tag likes "[HISTORY_BEGIN]"
+, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" in responses.
+ 
+
+{ask}
+"""
+
+    FORMATION_LOOSE = """Formation: "Capacity and role" defines the role you are currently playing;
+  "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation;
+  "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses;
+  "Statement" defines the work detail you need to complete at this stage;
+  "Constraint" defines the conditions that your responses must comply with.
+  "Personality" defines your language style。
+  "Insight" provides a deeper understanding of the characters' inner traits.
+  "Initial" defines the initial setup of a character.
+
+Capacity and role: {role}
+Statement: Your responses should maintaining the character's persona and habits. When faced with unrelated questions
+, playfully decline to answer without revealing your AI nature to preserve the character's image. 
+
+[HISTORY_BEGIN]
+
+{history}
+
+[HISTORY_END]
+
+[KNOWLEDGE_BEGIN]
+
+{knowledge}
+
+[KNOWLEDGE_END]
+
+Statement: If the information is insufficient, you can search in the historical conversation or knowledge.
+Statement: Unless you are a language professional, answer the following questions strictly in {language}
+, and the answers must follow the Markdown format. Strictly excluding any tag likes "[HISTORY_BEGIN]"
+, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" in responses.
+
+
+{ask}
+"""

metagpt/actions/write_code.py

@@ -4,79 +4,151 @@
 @Time    : 2023/5/11 17:45
 @Author  : alexanderwu
 @File    : write_code.py
+@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `cause_by`
+            value of the `Message` object.
+@Modified By: mashenquan, 2023-11-27.
+        1. Mark the location of Design, Tasks, Legacy Code and Debug logs in the PROMPT_TEMPLATE with markdown
+        code-block formatting to enhance the understanding for the LLM.
+        2. Following the think-act principle, solidify the task parameters when creating the WriteCode object, rather
+        than passing them in when calling the run function.
+        3. Encapsulate the input of RunCode into RunCodeContext and encapsulate the output of RunCode into
+        RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError.
 """
-from metagpt.actions import WriteDesign
+
+import json
+
+from pydantic import Field
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+
 from metagpt.actions.action import Action
-from metagpt.const import WORKSPACE_ROOT
+from metagpt.config import CONFIG
+from metagpt.const import (
+    BUGFIX_FILENAME,
+    CODE_SUMMARIES_FILE_REPO,
+    DOCS_FILE_REPO,
+    TASK_FILE_REPO,
+    TEST_OUTPUTS_FILE_REPO,
+)
 from metagpt.logs import logger
-from metagpt.schema import Message
+from metagpt.schema import CodingContext, Document, RunCodeResult
 from metagpt.utils.common import CodeParser
-from tenacity import retry, stop_after_attempt, wait_fixed
+from metagpt.utils.file_repository import FileRepository
 
 PROMPT_TEMPLATE = """
 NOTICE
-Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language)
+Role: You are a professional engineer; the main goal is to write google-style, elegant, modular, easy to read and maintain code
+Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese.
 ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example".
 
-## Code: {filename} Write code with triple quoto, based on the following list and context.
-1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT.
-2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets
-3. Attention1: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE.
-4. Attention2: YOU MUST FOLLOW "Data structures and interface definitions". DONT CHANGE ANY DESIGN.
-5. Think before writing: What should be implemented and provided in this document?
-6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE.
-7. Do not use public member functions that do not exist in your design.
-
------
 # Context
-{context}
------
-## Format example
------
+## Design
+{design}
+
+## Tasks
+{tasks}
+
+## Legacy Code
+```Code
+{code}
+```
+
+## Debug logs
+```text
+{logs}
+
+{summary_log}
+```
+
+## Bug Feedback logs
+```text
+{feedback}
+```
+
+# Format example
 ## Code: {filename}
 ```python
 ## {filename}
 ...
 ```
------
+
+# Instruction: Based on the context, follow "Format example", write code.
+
+## Code: {filename}. Write code with triple quoto, based on the following attentions and context.
+1. Only One file: do your best to implement THIS ONLY ONE FILE.
+2. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets.
+3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. AVOID circular import.
+4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design.
+5. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE.
+6. Before using a external variable/module, make sure you import it first.
+7. Write out EVERY CODE DETAIL, DON'T LEAVE TODO.
+
 """
 
 
 class WriteCode(Action):
-    def __init__(self, name="WriteCode", context: list[Message] = None, llm=None):
-        super().__init__(name, context, llm)
+    name: str = "WriteCode"
+    context: Document = Field(default_factory=Document)
 
-    def _is_invalid(self, filename):
-        return any(i in filename for i in ["mp3", "wav"])
-
-    def _save(self, context, filename, code):
-        # logger.info(filename)
-        # logger.info(code_rsp)
-        if self._is_invalid(filename):
-            return
-
-        design = [i for i in context if i.cause_by == WriteDesign][0]
-
-        ws_name = CodeParser.parse_str(block="Python package name", text=design.content)
-        ws_path = WORKSPACE_ROOT / ws_name
-        if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]):
-            ws_path = ws_path / ws_name
-        code_path = ws_path / filename
-        code_path.parent.mkdir(parents=True, exist_ok=True)
-        code_path.write_text(code)
-        logger.info(f"Saving Code to {code_path}")
-
-    @retry(stop=stop_after_attempt(2), wait=wait_fixed(1))
-    async def write_code(self, prompt):
+    @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
+    async def write_code(self, prompt) -> str:
         code_rsp = await self._aask(prompt)
         code = CodeParser.parse_code(block="", text=code_rsp)
         return code
 
-    async def run(self, context, filename):
-        prompt = PROMPT_TEMPLATE.format(context=context, filename=filename)
-        logger.info(f'Writing {filename}..')
+    async def run(self, *args, **kwargs) -> CodingContext:
+        bug_feedback = await FileRepository.get_file(filename=BUGFIX_FILENAME, relative_path=DOCS_FILE_REPO)
+        coding_context = CodingContext.loads(self.context.content)
+        test_doc = await FileRepository.get_file(
+            filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO
+        )
+        summary_doc = None
+        if coding_context.design_doc and coding_context.design_doc.filename:
+            summary_doc = await FileRepository.get_file(
+                filename=coding_context.design_doc.filename, relative_path=CODE_SUMMARIES_FILE_REPO
+            )
+        logs = ""
+        if test_doc:
+            test_detail = RunCodeResult.loads(test_doc.content)
+            logs = test_detail.stderr
+
+        if bug_feedback:
+            code_context = coding_context.code_doc.content
+        else:
+            code_context = await self.get_codes(coding_context.task_doc, exclude=self.context.filename)
+
+        prompt = PROMPT_TEMPLATE.format(
+            design=coding_context.design_doc.content if coding_context.design_doc else "",
+            tasks=coding_context.task_doc.content if coding_context.task_doc else "",
+            code=code_context,
+            logs=logs,
+            feedback=bug_feedback.content if bug_feedback else "",
+            filename=self.context.filename,
+            summary_log=summary_doc.content if summary_doc else "",
+        )
+        logger.info(f"Writing {coding_context.filename}..")
         code = await self.write_code(prompt)
-        # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING)
-        # self._save(context, filename, code)
-        return code
-    
+        if not coding_context.code_doc:
+            # avoid root_path pydantic ValidationError if use WriteCode alone
+            root_path = CONFIG.src_workspace if CONFIG.src_workspace else ""
+            coding_context.code_doc = Document(filename=coding_context.filename, root_path=str(root_path))
+        coding_context.code_doc.content = code
+        return coding_context
+
+    @staticmethod
+    async def get_codes(task_doc, exclude) -> str:
+        if not task_doc:
+            return ""
+        if not task_doc.content:
+            task_doc.content = FileRepository.get_file(filename=task_doc.filename, relative_path=TASK_FILE_REPO)
+        m = json.loads(task_doc.content)
+        code_filenames = m.get("Task list", [])
+        codes = []
+        src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace)
+        for filename in code_filenames:
+            if filename == exclude:
+                continue
+            doc = await src_file_repo.get(filename=filename)
+            if not doc:
+                continue
+            codes.append(f"----- {filename}\n" + doc.content)
+        return "\n".join(codes)

metagpt/actions/write_code_an_draft.py

@@ -0,0 +1,591 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Author  : alexanderwu
+@File    : write_review.py
+"""
+import asyncio
+from typing import List
+
+from metagpt.actions import Action
+from metagpt.actions.action_node import ActionNode
+
+REVIEW = ActionNode(
+    key="Review",
+    expected_type=List[str],
+    instruction="Act as an experienced reviewer and critically assess the given output. Provide specific and"
+    " constructive feedback, highlighting areas for improvement and suggesting changes.",
+    example=[
+        "The logic in the function `calculate_total` seems flawed. Shouldn't it consider the discount rate as well?",
+        "The TODO function is not implemented yet? Should we implement it before commit?",
+    ],
+)
+
+LGTM = ActionNode(
+    key="LGTM",
+    expected_type=str,
+    instruction="LGTM/LBTM. If the code is fully implemented, "
+    "give a LGTM (Looks Good To Me), otherwise provide a LBTM (Looks Bad To Me).",
+    example="LBTM",
+)
+
+ACTIONS = ActionNode(
+    key="Actions",
+    expected_type=str,
+    instruction="Based on the code review outcome, suggest actionable steps. This can include code changes, "
+    "refactoring suggestions, or any follow-up tasks.",
+    example="""1. Refactor the `process_data` method to improve readability and efficiency.
+2. Cover edge cases in the `validate_user` function.
+3. Implement a the TODO in the `calculate_total` function.
+4. Fix the `handle_events` method to update the game state only if a move is successful.
+   ```python
+   def handle_events(self):
+       for event in pygame.event.get():
+           if event.type == pygame.QUIT:
+               return False
+           if event.type == pygame.KEYDOWN:
+               moved = False
+               if event.key == pygame.K_UP:
+                   moved = self.game.move('UP')
+               elif event.key == pygame.K_DOWN:
+                   moved = self.game.move('DOWN')
+               elif event.key == pygame.K_LEFT:
+                   moved = self.game.move('LEFT')
+               elif event.key == pygame.K_RIGHT:
+                   moved = self.game.move('RIGHT')
+               if moved:
+                   # Update the game state only if a move was successful
+                   self.render()
+       return True
+   ```
+""",
+)
+
+WRITE_DRAFT = ActionNode(
+    key="WriteDraft",
+    expected_type=str,
+    instruction="Could you write draft code for move function in order to implement it?",
+    example="Draft: ...",
+)
+
+
+WRITE_MOVE_FUNCTION = ActionNode(
+    key="WriteFunction",
+    expected_type=str,
+    instruction="write code for the function not implemented.",
+    example="""
+```Code
+...
+```
+""",
+)
+
+
+REWRITE_CODE = ActionNode(
+    key="RewriteCode",
+    expected_type=str,
+    instruction="""rewrite code based on the Review and Actions""",
+    example="""
+```python
+## example.py
+def calculate_total(price, quantity):
+    total = price * quantity
+```
+""",
+)
+
+
+CODE_REVIEW_CONTEXT = """
+# System
+Role: You are a professional software engineer, and your main task is to review and revise the code. You need to ensure that the code conforms to the google-style standards, is elegantly designed and modularized, easy to read and maintain.
+Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese.
+
+# Context
+## System Design
+{"Implementation approach": "我们将使用HTML、CSS和JavaScript来实现这个单机的响应式2048游戏。为了确保游戏性能流畅和响应式设计，我们会选择使用Vue.js框架，因为它易于上手且适合构建交互式界面。我们还将使用localStorage来记录玩家的最高分。", "File list": ["index.html", "styles.css", "main.js", "game.js", "storage.js"], "Data structures and interfaces": "classDiagram\
+    class Game {\
+        -board Array\
+        -score Number\
+        -bestScore Number\
+        +constructor()\
+        +startGame()\
+        +move(direction: String)\
+        +getBoard() Array\
+        +getScore() Number\
+        +getBestScore() Number\
+        +setBestScore(score: Number)\
+    }\
+    class Storage {\
+        +getBestScore() Number\
+        +setBestScore(score: Number)\
+    }\
+    class Main {\
+        +init()\
+        +bindEvents()\
+    }\
+    Game --> Storage : uses\
+    Main --> Game : uses", "Program call flow": "sequenceDiagram\
+    participant M as Main\
+    participant G as Game\
+    participant S as Storage\
+    M->>G: init()\
+    G->>S: getBestScore()\
+    S-->>G: return bestScore\
+    M->>G: bindEvents()\
+    M->>G: startGame()\
+    loop Game Loop\
+        M->>G: move(direction)\
+        G->>S: setBestScore(score)\
+        S-->>G: return\
+    end", "Anything UNCLEAR": "目前项目要求明确，没有不清楚的地方。"}
+
+## Tasks
+{"Required Python packages": ["无需Python包"], "Required Other language third-party packages": ["vue.js"], "Logic Analysis": [["index.html", "作为游戏的入口文件和主要的HTML结构"], ["styles.css", "包含所有的CSS样式，确保游戏界面美观"], ["main.js", "包含Main类，负责初始化游戏和绑定事件"], ["game.js", "包含Game类，负责游戏逻辑，如开始游戏、移动方块等"], ["storage.js", "包含Storage类，用于获取和设置玩家的最高分"]], "Task list": ["index.html", "styles.css", "storage.js", "game.js", "main.js"], "Full API spec": "", "Shared Knowledge": "\'game.js\' 包含游戏逻辑相关的函数，被 \'main.js\' 调用。", "Anything UNCLEAR": "目前项目要求明确，没有不清楚的地方。"}
+
+## Code Files
+----- index.html
+<!DOCTYPE html>
+<html lang="zh-CN">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>2048游戏</title>
+    <link rel="stylesheet" href="styles.css">
+    <script src="https://cdn.jsdelivr.net/npm/vue@2.6.14/dist/vue.js"></script>
+</head>
+<body>
+    <div id="app">
+        <h1>2048</h1>
+        <div class="scores-container">
+            <div class="score-container">
+                <div class="score-header">分数</div>
+                <div>{{ score }}</div>
+            </div>
+            <div class="best-container">
+                <div class="best-header">最高分</div>
+                <div>{{ bestScore }}</div>
+            </div>
+        </div>
+        <div class="game-container">
+            <div v-for="(row, rowIndex) in board" :key="rowIndex" class="grid-row">
+                <div v-for="(cell, cellIndex) in row" :key="cellIndex" class="grid-cell" :class="\'number-cell-\' + cell">
+                    {{ cell !== 0 ? cell : \'\' }}
+                </div>
+            </div>
+        </div>
+        <button @click="startGame" aria-label="开始新游戏">新游戏</button>
+    </div>
+
+    <script src="storage.js"></script>
+    <script src="game.js"></script>
+    <script src="main.js"></script>
+    <script src="app.js"></script>
+</body>
+</html>
+
+----- styles.css
+/* styles.css */
+body, html {
+    margin: 0;
+    padding: 0;
+    font-family: \'Arial\', sans-serif;
+}
+
+#app {
+    text-align: center;
+    font-size: 18px;
+    color: #776e65;
+}
+
+h1 {
+    color: #776e65;
+    font-size: 72px;
+    font-weight: bold;
+    margin: 20px 0;
+}
+
+.scores-container {
+    display: flex;
+    justify-content: center;
+    margin-bottom: 20px;
+}
+
+.score-container, .best-container {
+    background: #bbada0;
+    padding: 10px;
+    border-radius: 5px;
+    margin: 0 10px;
+    min-width: 100px;
+    text-align: center;
+}
+
+.score-header, .best-header {
+    color: #eee4da;
+    font-size: 18px;
+    margin-bottom: 5px;
+}
+
+.game-container {
+    max-width: 500px;
+    margin: 0 auto 20px;
+    background: #bbada0;
+    padding: 15px;
+    border-radius: 10px;
+    position: relative;
+}
+
+.grid-row {
+    display: flex;
+}
+
+.grid-cell {
+    background: #cdc1b4;
+    width: 100px;
+    height: 100px;
+    margin: 5px;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    font-size: 35px;
+    font-weight: bold;
+    color: #776e65;
+    border-radius: 3px;
+}
+
+/* Dynamic classes for different number cells */
+.number-cell-2 {
+    background: #eee4da;
+}
+
+.number-cell-4 {
+    background: #ede0c8;
+}
+
+.number-cell-8 {
+    background: #f2b179;
+    color: #f9f6f2;
+}
+
+.number-cell-16 {
+    background: #f59563;
+    color: #f9f6f2;
+}
+
+.number-cell-32 {
+    background: #f67c5f;
+    color: #f9f6f2;
+}
+
+.number-cell-64 {
+    background: #f65e3b;
+    color: #f9f6f2;
+}
+
+.number-cell-128 {
+    background: #edcf72;
+    color: #f9f6f2;
+}
+
+.number-cell-256 {
+    background: #edcc61;
+    color: #f9f6f2;
+}
+
+.number-cell-512 {
+    background: #edc850;
+    color: #f9f6f2;
+}
+
+.number-cell-1024 {
+    background: #edc53f;
+    color: #f9f6f2;
+}
+
+.number-cell-2048 {
+    background: #edc22e;
+    color: #f9f6f2;
+}
+
+/* Larger numbers need smaller font sizes */
+.number-cell-1024, .number-cell-2048 {
+    font-size: 30px;
+}
+
+button {
+    background-color: #8f7a66;
+    color: #f9f6f2;
+    border: none;
+    border-radius: 3px;
+    padding: 10px 20px;
+    font-size: 18px;
+    cursor: pointer;
+    outline: none;
+}
+
+button:hover {
+    background-color: #9f8b76;
+}
+
+----- storage.js
+## storage.js
+class Storage {
+    // 获取最高分
+    getBestScore() {
+        // 尝试从localStorage中获取最高分，如果不存在则默认为0
+        const bestScore = localStorage.getItem(\'bestScore\');
+        return bestScore ? Number(bestScore) : 0;
+    }
+
+    // 设置最高分
+    setBestScore(score) {
+        // 将最高分设置到localStorage中
+        localStorage.setItem(\'bestScore\', score.toString());
+    }
+}
+
+
+
+## Code to be Reviewed: game.js
+```Code
+## game.js
+class Game {
+    constructor() {
+        this.board = this.createEmptyBoard();
+        this.score = 0;
+        this.bestScore = 0;
+    }
+
+    createEmptyBoard() {
+        const board = [];
+        for (let i = 0; i < 4; i++) {
+            board[i] = [0, 0, 0, 0];
+        }
+        return board;
+    }
+
+    startGame() {
+        this.board = this.createEmptyBoard();
+        this.score = 0;
+        this.addRandomTile();
+        this.addRandomTile();
+    }
+
+    addRandomTile() {
+        let emptyCells = [];
+        for (let r = 0; r < 4; r++) {
+            for (let c = 0; c < 4; c++) {
+                if (this.board[r][c] === 0) {
+                    emptyCells.push({ r, c });
+                }
+            }
+        }
+        if (emptyCells.length > 0) {
+            let randomCell = emptyCells[Math.floor(Math.random() * emptyCells.length)];
+            this.board[randomCell.r][randomCell.c] = Math.random() < 0.9 ? 2 : 4;
+        }
+    }
+
+    move(direction) {
+        // This function will handle the logic for moving tiles
+        // in the specified direction and merging them
+        // It will also update the score and add a new random tile if the move is successful
+        // The actual implementation of this function is complex and would require
+        // a significant amount of code to handle all the cases for moving and merging tiles
+        // For the purposes of this example, we will not implement the full logic
+        // Instead, we will just call addRandomTile to simulate a move
+        this.addRandomTile();
+    }
+
+    getBoard() {
+        return this.board;
+    }
+
+    getScore() {
+        return this.score;
+    }
+
+    getBestScore() {
+        return this.bestScore;
+    }
+
+    setBestScore(score) {
+        this.bestScore = score;
+    }
+}
+
+```
+"""
+
+
+CODE_REVIEW_SMALLEST_CONTEXT = """
+## Code to be Reviewed: game.js
+```Code
+// game.js
+class Game {
+    constructor() {
+        this.board = this.createEmptyBoard();
+        this.score = 0;
+        this.bestScore = 0;
+    }
+
+    createEmptyBoard() {
+        const board = [];
+        for (let i = 0; i < 4; i++) {
+            board[i] = [0, 0, 0, 0];
+        }
+        return board;
+    }
+
+    startGame() {
+        this.board = this.createEmptyBoard();
+        this.score = 0;
+        this.addRandomTile();
+        this.addRandomTile();
+    }
+
+    addRandomTile() {
+        let emptyCells = [];
+        for (let r = 0; r < 4; r++) {
+            for (let c = 0; c < 4; c++) {
+                if (this.board[r][c] === 0) {
+                    emptyCells.push({ r, c });
+                }
+            }
+        }
+        if (emptyCells.length > 0) {
+            let randomCell = emptyCells[Math.floor(Math.random() * emptyCells.length)];
+            this.board[randomCell.r][randomCell.c] = Math.random() < 0.9 ? 2 : 4;
+        }
+    }
+
+    move(direction) {
+        // This function will handle the logic for moving tiles
+        // in the specified direction and merging them
+        // It will also update the score and add a new random tile if the move is successful
+        // The actual implementation of this function is complex and would require
+        // a significant amount of code to handle all the cases for moving and merging tiles
+        // For the purposes of this example, we will not implement the full logic
+        // Instead, we will just call addRandomTile to simulate a move
+        this.addRandomTile();
+    }
+
+    getBoard() {
+        return this.board;
+    }
+
+    getScore() {
+        return this.score;
+    }
+
+    getBestScore() {
+        return this.bestScore;
+    }
+
+    setBestScore(score) {
+        this.bestScore = score;
+    }
+}
+
+```
+"""
+
+
+CODE_REVIEW_SAMPLE = """
+## Code Review: game.js
+1. The code partially implements the requirements. The `Game` class is missing the full implementation of the `move` method, which is crucial for the game\'s functionality.
+2. The code logic is not completely correct. The `move` method is not implemented, which means the game cannot process player moves.
+3. The existing code follows the "Data structures and interfaces" in terms of class structure but lacks full method implementations.
+4. Not all functions are implemented. The `move` method is incomplete and does not handle the logic for moving and merging tiles.
+5. All necessary pre-dependencies seem to be imported since the code does not indicate the need for additional imports.
+6. The methods from other files (such as `Storage`) are not being used in the provided code snippet, but the class structure suggests that they will be used correctly.
+
+## Actions
+1. Implement the `move` method to handle tile movements and merging. This is a complex task that requires careful consideration of the game\'s rules and logic. Here is a simplified version of how one might begin to implement the `move` method:
+   ```javascript
+   move(direction) {
+       // Simplified logic for moving tiles up
+       if (direction === \'up\') {
+           for (let col = 0; col < 4; col++) {
+               let tiles = this.board.map(row => row[col]).filter(val => val !== 0);
+               let merged = [];
+               for (let i = 0; i < tiles.length; i++) {
+                   if (tiles[i] === tiles[i + 1]) {
+                       tiles[i] *= 2;
+                       this.score += tiles[i];
+                       tiles[i + 1] = 0;
+                       merged.push(i);
+                   }
+               }
+               tiles = tiles.filter(val => val !== 0);
+               while (tiles.length < 4) {
+                   tiles.push(0);
+               }
+               for (let row = 0; row < 4; row++) {
+                   this.board[row][col] = tiles[row];
+               }
+           }
+       }
+       // Additional logic needed for \'down\', \'left\', \'right\'
+       // ...
+       this.addRandomTile();
+   }
+   ```
+2. Integrate the `Storage` class methods to handle the best score. This means updating the `startGame` and `setBestScore` methods to use `Storage` for retrieving and setting the best score:
+   ```javascript
+   startGame() {
+       this.board = this.createEmptyBoard();
+       this.score = 0;
+       this.bestScore = new Storage().getBestScore(); // Retrieve the best score from storage
+       this.addRandomTile();
+       this.addRandomTile();
+   }
+
+   setBestScore(score) {
+       if (score > this.bestScore) {
+           this.bestScore = score;
+           new Storage().setBestScore(score); // Set the new best score in storage
+       }
+   }
+   ```
+
+## Code Review Result
+LBTM
+
+```
+"""
+
+
+WRITE_CODE_NODE = ActionNode.from_children("WRITE_REVIEW_NODE", [REVIEW, LGTM, ACTIONS])
+WRITE_MOVE_NODE = ActionNode.from_children("WRITE_MOVE_NODE", [WRITE_DRAFT, WRITE_MOVE_FUNCTION])
+
+
+CR_FOR_MOVE_FUNCTION_BY_3 = """
+The move function implementation provided appears to be well-structured and follows a clear logic for moving and merging tiles in the specified direction. However, there are a few potential improvements that could be made to enhance the code:
+
+1. Encapsulation: The logic for moving and merging tiles could be encapsulated into smaller, reusable functions to improve readability and maintainability.
+
+2. Magic Numbers: There are some magic numbers (e.g., 4, 3) used in the loops that could be replaced with named constants for improved readability and easier maintenance.
+
+3. Comments: Adding comments to explain the logic and purpose of each section of the code can improve understanding for future developers who may need to work on or maintain the code.
+
+4. Error Handling: It's important to consider error handling for unexpected input or edge cases to ensure the function behaves as expected in all scenarios.
+
+Overall, the code could benefit from refactoring to improve readability, maintainability, and extensibility. If you would like, I can provide a refactored version of the move function that addresses these considerations.
+"""
+
+
+class WriteCodeAN(Action):
+    """Write a code review for the context."""
+
+    async def run(self, context):
+        self.llm.system_prompt = "You are an outstanding engineer and can implement any code"
+        return await WRITE_MOVE_FUNCTION.fill(context=context, llm=self.llm, schema="json")
+        # return await WRITE_CODE_NODE.fill(context=context, llm=self.llm, schema="markdown")
+
+
+async def main():
+    await WriteCodeAN().run(CODE_REVIEW_SMALLEST_CONTEXT)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

metagpt/actions/write_code_review.py

@@ -4,57 +4,114 @@
 @Time    : 2023/5/11 17:45
 @Author  : alexanderwu
 @File    : write_code_review.py
+@Modified By: mashenquan, 2023/11/27. Following the think-act principle, solidify the task parameters when creating the
+        WriteCode object, rather than passing them in when calling the run function.
 """
 
+from pydantic import Field
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+
+from metagpt.actions import WriteCode
 from metagpt.actions.action import Action
+from metagpt.config import CONFIG
 from metagpt.logs import logger
-from metagpt.schema import Message
+from metagpt.schema import CodingContext
 from metagpt.utils.common import CodeParser
-from tenacity import retry, stop_after_attempt, wait_fixed
 
 PROMPT_TEMPLATE = """
-NOTICE
-Role: You are a professional software engineer, and your main task is to review the code. You need to ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language).
+# System
+Role: You are a professional software engineer, and your main task is to review and revise the code. You need to ensure that the code conforms to the google-style standards, is elegantly designed and modularized, easy to read and maintain.
+Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese.
 ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example".
 
-## Code Review: Based on the following context and code, and following the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5.
-```
-1. Check 0: Is the code implemented as per the requirements?
-2. Check 1: Are there any issues with the code logic?
-3. Check 2: Does the existing code follow the "Data structures and interface definitions"?
-4. Check 3: Is there a function in the code that is omitted or not fully implemented that needs to be implemented?
-5. Check 4: Does the code have unnecessary or lack dependencies?
-```
-
-## Rewrite Code: {filename} Base on "Code Review" and the source code, rewrite code with triple quotes. Do your utmost to optimize THIS SINGLE FILE. 
------
 # Context
 {context}
 
-## Code: {filename}
-```
+## Code to be Reviewed: {filename}
+```Code
 {code}
 ```
------
+"""
+
+EXAMPLE_AND_INSTRUCTION = """
 
-## Format example
------
 {format_example}
------
+
+
+# Instruction: Based on the actual code situation, follow one of the "Format example". Return only 1 file under review.
+
+## Code Review: Ordered List. Based on the "Code to be Reviewed", provide key, clear, concise, and specific answer. If any answer is no, explain how to fix it step by step.
+1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step.
+2. Is the code logic completely correct? If there are errors, please indicate how to correct them.
+3. Does the existing code follow the "Data structures and interfaces"?
+4. Are all functions implemented? If there is no implementation, please indicate how to achieve it step by step.
+5. Have all necessary pre-dependencies been imported? If not, indicate which ones need to be imported
+6. Are methods from other files being reused correctly?
+
+## Actions: Ordered List. Things that should be done after CR, such as implementing class A and function B
+
+## Code Review Result: str. If the code doesn't have bugs, we don't need to rewrite it, so answer LGTM and stop. ONLY ANSWER LGTM/LBTM.
+LGTM/LBTM
 
 """
 
 FORMAT_EXAMPLE = """
-
-## Code Review
-1. The code ...
+# Format example 1
+## Code Review: {filename}
+1. No, we should fix the logic of class A due to ...
 2. ...
 3. ...
-4. ...
+4. No, function B is not implemented, ...
 5. ...
+6. ...
 
-## Rewrite Code: {filename}
-```python
+## Actions
+1. Fix the `handle_events` method to update the game state only if a move is successful.
+   ```python
+   def handle_events(self):
+       for event in pygame.event.get():
+           if event.type == pygame.QUIT:
+               return False
+           if event.type == pygame.KEYDOWN:
+               moved = False
+               if event.key == pygame.K_UP:
+                   moved = self.game.move('UP')
+               elif event.key == pygame.K_DOWN:
+                   moved = self.game.move('DOWN')
+               elif event.key == pygame.K_LEFT:
+                   moved = self.game.move('LEFT')
+               elif event.key == pygame.K_RIGHT:
+                   moved = self.game.move('RIGHT')
+               if moved:
+                   # Update the game state only if a move was successful
+                   self.render()
+       return True
+   ```
+2. Implement function B
+
+## Code Review Result
+LBTM
+
+# Format example 2
+## Code Review: {filename}
+1. Yes.
+2. Yes.
+3. Yes.
+4. Yes.
+5. Yes.
+6. Yes.
+
+## Actions
+pass
+
+## Code Review Result
+LGTM
+"""
+
+REWRITE_CODE_TEMPLATE = """
+# Instruction: rewrite code based on the Code Review and Actions
+## Rewrite Code: CodeBlock. If it still has some bugs, rewrite {filename} with triple quotes. Do your utmost to optimize THIS SINGLE FILE. Return all completed codes and prohibit the return of unfinished codes.
+```Code
 ## {filename}
 ...
 ```
@@ -62,21 +119,58 @@ FORMAT_EXAMPLE = """
 
 
 class WriteCodeReview(Action):
-    def __init__(self, name="WriteCodeReview", context: list[Message] = None, llm=None):
-        super().__init__(name, context, llm)
+    name: str = "WriteCodeReview"
+    context: CodingContext = Field(default_factory=CodingContext)
 
-    @retry(stop=stop_after_attempt(2), wait=wait_fixed(1))
-    async def write_code(self, prompt):
-        code_rsp = await self._aask(prompt)
+    @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
+    async def write_code_review_and_rewrite(self, context_prompt, cr_prompt, filename):
+        cr_rsp = await self._aask(context_prompt + cr_prompt)
+        result = CodeParser.parse_block("Code Review Result", cr_rsp)
+        if "LGTM" in result:
+            return result, None
+
+        # if LBTM, rewrite code
+        rewrite_prompt = f"{context_prompt}\n{cr_rsp}\n{REWRITE_CODE_TEMPLATE.format(filename=filename)}"
+        code_rsp = await self._aask(rewrite_prompt)
         code = CodeParser.parse_code(block="", text=code_rsp)
-        return code
+        return result, code
 
-    async def run(self, context, code, filename):
-        format_example = FORMAT_EXAMPLE.format(filename=filename)
-        prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=filename, format_example=format_example)
-        logger.info(f'Code review {filename}..')
-        code = await self.write_code(prompt)
+    async def run(self, *args, **kwargs) -> CodingContext:
+        iterative_code = self.context.code_doc.content
+        k = CONFIG.code_review_k_times or 1
+        for i in range(k):
+            format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename)
+            task_content = self.context.task_doc.content if self.context.task_doc else ""
+            code_context = await WriteCode.get_codes(self.context.task_doc, exclude=self.context.filename)
+            context = "\n".join(
+                [
+                    "## System Design\n" + str(self.context.design_doc) + "\n",
+                    "## Tasks\n" + task_content + "\n",
+                    "## Code Files\n" + code_context + "\n",
+                ]
+            )
+            context_prompt = PROMPT_TEMPLATE.format(
+                context=context,
+                code=iterative_code,
+                filename=self.context.code_doc.filename,
+            )
+            cr_prompt = EXAMPLE_AND_INSTRUCTION.format(
+                format_example=format_example,
+            )
+            logger.info(
+                f"Code review and rewrite {self.context.code_doc.filename}: {i + 1}/{k} | {len(iterative_code)=}, "
+                f"{len(self.context.code_doc.content)=}"
+            )
+            result, rewrited_code = await self.write_code_review_and_rewrite(
+                context_prompt, cr_prompt, self.context.code_doc.filename
+            )
+            if "LBTM" in result:
+                iterative_code = rewrited_code
+            elif "LGTM" in result:
+                self.context.code_doc.content = iterative_code
+                return self.context
         # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING)
         # self._save(context, filename, code)
-        return code
-    
+        # 如果rewrited_code是None（原code perfect），那么直接返回code
+        self.context.code_doc.content = iterative_code
+        return self.context

metagpt/actions/write_docstring.py

@@ -16,19 +16,22 @@ Options:
                                 Default: 'google'
 
 Example:
-    python3 -m metagpt.actions.write_docstring startup.py --overwrite False --style=numpy
+    python3 -m metagpt.actions.write_docstring ./metagpt/startup.py --overwrite False --style=numpy
 
 This script uses the 'fire' library to create a command-line interface. It generates docstrings for the given Python code using
 the specified docstring style and adds them to the code.
 """
+from __future__ import annotations
+
 import ast
-from typing import Literal
+from pathlib import Path
+from typing import Literal, Optional
 
 from metagpt.actions.action import Action
-from metagpt.utils.common import OutputParser
+from metagpt.utils.common import OutputParser, aread, awrite
 from metagpt.utils.pycst import merge_docstring
 
-PYTHON_DOCSTRING_SYSTEM = '''### Requirements
+PYTHON_DOCSTRING_SYSTEM = """### Requirements
 1. Add docstrings to the given code following the {style} style.
 2. Replace the function body with an Ellipsis object(...) to reduce output.
 3. If the types are already annotated, there is no need to include them in the docstring.
@@ -48,7 +51,7 @@ class ExampleError(Exception):
 ```python
 {example}
 ```
-'''
+"""
 
 # https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html
 
@@ -157,12 +160,12 @@ class WriteDocstring(Action):
         desc: A string describing the action.
     """
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.desc = "Write docstring for code."
+    desc: str = "Write docstring for code."
+    context: Optional[str] = None
 
     async def run(
-        self, code: str,
+        self,
+        code: str,
         system_text: str = PYTHON_DOCSTRING_SYSTEM,
         style: Literal["google", "numpy", "sphinx"] = "google",
     ) -> str:
@@ -182,6 +185,16 @@ class WriteDocstring(Action):
         documented_code = OutputParser.parse_python_code(documented_code)
         return merge_docstring(code, documented_code)
 
+    @staticmethod
+    async def write_docstring(
+        filename: str | Path, overwrite: bool = False, style: Literal["google", "numpy", "sphinx"] = "google"
+    ) -> str:
+        data = await aread(str(filename))
+        code = await WriteDocstring().run(data, style=style)
+        if overwrite:
+            await awrite(filename, code)
+        return code
+
 
 def _simplify_python_code(code: str) -> None:
     """Simplifies the given Python code by removing expressions and the last if statement.
@@ -202,13 +215,4 @@ def _simplify_python_code(code: str) -> None:
 if __name__ == "__main__":
     import fire
 
-    async def run(filename: str, overwrite: bool = False, style: Literal["google", "numpy", "sphinx"] = "google"):
-        with open(filename) as f:
-            code = f.read()
-        code = await WriteDocstring().run(code, style=style)
-        if overwrite:
-            with open(filename, "w") as f:
-                f.write(code)
-        return code
-
-    fire.Fire(run)
+    fire.Fire(WriteDocstring.write_docstring)

metagpt/actions/write_prd.py

@@ -4,238 +4,194 @@
 @Time    : 2023/5/11 17:45
 @Author  : alexanderwu
 @File    : write_prd.py
+@Modified By: mashenquan, 2023/11/27.
+            1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
+            2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality.
+            3. Move the document storage operations related to WritePRD from the save operation of WriteDesign.
+@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD.
 """
-from typing import List
+
+from __future__ import annotations
+
+import json
+import uuid
+from pathlib import Path
+from typing import Optional
 
 from metagpt.actions import Action, ActionOutput
-from metagpt.actions.search_and_summarize import SearchAndSummarize
+from metagpt.actions.action_node import ActionNode
+from metagpt.actions.fix_bug import FixBug
+from metagpt.actions.write_prd_an import (
+    PROJECT_NAME,
+    WP_IS_RELATIVE_NODE,
+    WP_ISSUE_TYPE_NODE,
+    WRITE_PRD_NODE,
+)
 from metagpt.config import CONFIG
+from metagpt.const import (
+    BUGFIX_FILENAME,
+    COMPETITIVE_ANALYSIS_FILE_REPO,
+    DOCS_FILE_REPO,
+    PRD_PDF_FILE_REPO,
+    PRDS_FILE_REPO,
+    REQUIREMENT_FILENAME,
+)
 from metagpt.logs import logger
-from metagpt.utils.get_template import get_template
+from metagpt.schema import BugFixContext, Document, Documents, Message
+from metagpt.utils.common import CodeParser
+from metagpt.utils.file_repository import FileRepository
+from metagpt.utils.mermaid import mermaid_to_file
 
-templates = {
-    "json": {
-        "PROMPT_TEMPLATE": """
-# Context
-## Original Requirements
+CONTEXT_TEMPLATE = """
+### Project Name
+{project_name}
+
+### Original Requirements
 {requirements}
 
-## Search Information
-{search_information}
+### Search Information
+-
+"""
 
-## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the <Campain X> with REAL COMPETITOR NAME
-```mermaid
-quadrantChart
-    title Reach and engagement of campaigns
-    x-axis Low Reach --> High Reach
-    y-axis Low Engagement --> High Engagement
-    quadrant-1 We should expand
-    quadrant-2 Need to promote
-    quadrant-3 Re-evaluate
-    quadrant-4 May be improved
-    "Campaign: A": [0.3, 0.6]
-    "Campaign B": [0.45, 0.23]
-    "Campaign C": [0.57, 0.69]
-    "Campaign D": [0.78, 0.34]
-    "Campaign E": [0.40, 0.34]
-    "Campaign F": [0.35, 0.78]
-    "Our Target Product": [0.5, 0.6]
-```
+NEW_REQ_TEMPLATE = """
+### Legacy Content
+{old_prd}
 
-## Format example
-{format_example}
------
-Role: You are a professional product manager; the goal is to design a concise, usable, efficient product
-Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design
-
-## Original Requirements: Provide as Plain text, place the polished complete original requirements here
-
-## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple
-
-## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less
-
-## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible
-
-## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible.
-
-## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery.
-
-## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower
-
-## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description.
-## Anything UNCLEAR: Provide as Plain text. Make clear here.
-
-output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example,
-and only output the json inside this tag, nothing else
-""",
-        "FORMAT_EXAMPLE": """
-[CONTENT]
-{
-    "Original Requirements": "",
-    "Search Information": "",
-    "Requirements": "",
-    "Product Goals": [],
-    "User Stories": [],
-    "Competitive Analysis": [],
-    "Competitive Quadrant Chart": "quadrantChart
-                title Reach and engagement of campaigns
-                x-axis Low Reach --> High Reach
-                y-axis Low Engagement --> High Engagement
-                quadrant-1 We should expand
-                quadrant-2 Need to promote
-                quadrant-3 Re-evaluate
-                quadrant-4 May be improved
-                Campaign A: [0.3, 0.6]
-                Campaign B: [0.45, 0.23]
-                Campaign C: [0.57, 0.69]
-                Campaign D: [0.78, 0.34]
-                Campaign E: [0.40, 0.34]
-                Campaign F: [0.35, 0.78]",
-    "Requirement Analysis": "",
-    "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]],
-    "UI Design draft": "",
-    "Anything UNCLEAR": "",
-}
-[/CONTENT]
-""",
-    },
-    "markdown": {
-        "PROMPT_TEMPLATE": """
-# Context
-## Original Requirements
+### New Requirements
 {requirements}
-
-## Search Information
-{search_information}
-
-## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the <Campain X> with REAL COMPETITOR NAME
-```mermaid
-quadrantChart
-    title Reach and engagement of campaigns
-    x-axis Low Reach --> High Reach
-    y-axis Low Engagement --> High Engagement
-    quadrant-1 We should expand
-    quadrant-2 Need to promote
-    quadrant-3 Re-evaluate
-    quadrant-4 May be improved
-    "Campaign: A": [0.3, 0.6]
-    "Campaign B": [0.45, 0.23]
-    "Campaign C": [0.57, 0.69]
-    "Campaign D": [0.78, 0.34]
-    "Campaign E": [0.40, 0.34]
-    "Campaign F": [0.35, 0.78]
-    "Our Target Product": [0.5, 0.6]
-```
-
-## Format example
-{format_example}
------
-Role: You are a professional product manager; the goal is to design a concise, usable, efficient product
-Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design
-ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## <SECTION_NAME>' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format.
-
-## Original Requirements: Provide as Plain text, place the polished complete original requirements here
-
-## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple
-
-## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less
-
-## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible
-
-## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible.
-
-## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery.
-
-## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower
-
-## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description.
-## Anything UNCLEAR: Provide as Plain text. Make clear here.
-""",
-        "FORMAT_EXAMPLE": """
----
-## Original Requirements
-The boss ... 
-
-## Product Goals
-```python
-[
-    "Create a ...",
-]
-```
-
-## User Stories
-```python
-[
-    "As a user, ...",
-]
-```
-
-## Competitive Analysis
-```python
-[
-    "Python Snake Game: ...",
-]
-```
-
-## Competitive Quadrant Chart
-```mermaid
-quadrantChart
-    title Reach and engagement of campaigns
-    ...
-    "Our Target Product": [0.6, 0.7]
-```
-
-## Requirement Analysis
-The product should be a ...
-
-## Requirement Pool
-```python
-[
-    ["End game ...", "P0"]
-]
-```
-
-## UI Design draft
-Give a basic function description, and a draft
-
-## Anything UNCLEAR
-There are no unclear points.
----
-""",
-    },
-}
-
-OUTPUT_MAPPING = {
-    "Original Requirements": (str, ...),
-    "Product Goals": (List[str], ...),
-    "User Stories": (List[str], ...),
-    "Competitive Analysis": (List[str], ...),
-    "Competitive Quadrant Chart": (str, ...),
-    "Requirement Analysis": (str, ...),
-    "Requirement Pool": (List[List[str]], ...),
-    "UI Design draft": (str, ...),
-    "Anything UNCLEAR": (str, ...),
-}
+"""
 
 
 class WritePRD(Action):
-    def __init__(self, name="", context=None, llm=None):
-        super().__init__(name, context, llm)
+    name: str = "WritePRD"
+    content: Optional[str] = None
 
-    async def run(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput:
-        sas = SearchAndSummarize()
-        # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US)
-        rsp = ""
-        info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}"
-        if sas.result:
-            logger.info(sas.result)
-            logger.info(rsp)
+    async def run(self, with_messages, schema=CONFIG.prompt_schema, *args, **kwargs) -> ActionOutput | Message:
+        # Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are
+        # related to the PRD. If they are related, rewrite the PRD.
+        docs_file_repo = CONFIG.git_repo.new_file_repository(relative_path=DOCS_FILE_REPO)
+        requirement_doc = await docs_file_repo.get(filename=REQUIREMENT_FILENAME)
+        if requirement_doc and await self._is_bugfix(requirement_doc.content):
+            await docs_file_repo.save(filename=BUGFIX_FILENAME, content=requirement_doc.content)
+            await docs_file_repo.save(filename=REQUIREMENT_FILENAME, content="")
+            bug_fix = BugFixContext(filename=BUGFIX_FILENAME)
+            return Message(
+                content=bug_fix.model_dump_json(),
+                instruct_content=bug_fix,
+                role="",
+                cause_by=FixBug,
+                sent_from=self,
+                send_to="Alex",  # the name of Engineer
+            )
+        else:
+            await docs_file_repo.delete(filename=BUGFIX_FILENAME)
 
-        prompt_template, format_example = get_template(templates, format)
-        prompt = prompt_template.format(
-            requirements=requirements, search_information=info, format_example=format_example
+        prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO)
+        prd_docs = await prds_file_repo.get_all()
+        change_files = Documents()
+        for prd_doc in prd_docs:
+            prd_doc = await self._update_prd(
+                requirement_doc=requirement_doc, prd_doc=prd_doc, prds_file_repo=prds_file_repo, *args, **kwargs
+            )
+            if not prd_doc:
+                continue
+            change_files.docs[prd_doc.filename] = prd_doc
+            logger.info(f"rewrite prd: {prd_doc.filename}")
+        # If there is no existing PRD, generate one using 'docs/requirement.txt'.
+        if not change_files.docs:
+            prd_doc = await self._update_prd(
+                requirement_doc=requirement_doc, prd_doc=None, prds_file_repo=prds_file_repo, *args, **kwargs
+            )
+            if prd_doc:
+                change_files.docs[prd_doc.filename] = prd_doc
+                logger.debug(f"new prd: {prd_doc.filename}")
+        # Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the
+        # 'publish' message to transition the workflow to the next stage. This design allows room for global
+        # optimization in subsequent steps.
+        return ActionOutput(content=change_files.model_dump_json(), instruct_content=change_files)
+
+    async def _run_new_requirement(self, requirements, schema=CONFIG.prompt_schema) -> ActionOutput:
+        # sas = SearchAndSummarize()
+        # # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US)
+        # rsp = ""
+        # info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}"
+        # if sas.result:
+        #     logger.info(sas.result)
+        #     logger.info(rsp)
+        project_name = CONFIG.project_name or ""
+        context = CONTEXT_TEMPLATE.format(requirements=requirements, project_name=project_name)
+        exclude = [PROJECT_NAME.key] if project_name else []
+        node = await WRITE_PRD_NODE.fill(context=context, llm=self.llm, exclude=exclude)  # schema=schema
+        await self._rename_workspace(node)
+        return node
+
+    async def _is_relative(self, new_requirement_doc, old_prd_doc) -> bool:
+        context = NEW_REQ_TEMPLATE.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content)
+        node = await WP_IS_RELATIVE_NODE.fill(context, self.llm)
+        return node.get("is_relative") == "YES"
+
+    async def _merge(self, new_requirement_doc, prd_doc, schema=CONFIG.prompt_schema) -> Document:
+        if not CONFIG.project_name:
+            CONFIG.project_name = Path(CONFIG.project_path).name
+        prompt = NEW_REQ_TEMPLATE.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content)
+        node = await WRITE_PRD_NODE.fill(context=prompt, llm=self.llm, schema=schema)
+        prd_doc.content = node.instruct_content.model_dump_json()
+        await self._rename_workspace(node)
+        return prd_doc
+
+    async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None:
+        if not prd_doc:
+            prd = await self._run_new_requirement(
+                requirements=[requirement_doc.content if requirement_doc else ""], *args, **kwargs
+            )
+            new_prd_doc = Document(
+                root_path=PRDS_FILE_REPO,
+                filename=FileRepository.new_filename() + ".json",
+                content=prd.instruct_content.model_dump_json(),
+            )
+        elif await self._is_relative(requirement_doc, prd_doc):
+            new_prd_doc = await self._merge(requirement_doc, prd_doc)
+        else:
+            return None
+        await prds_file_repo.save(filename=new_prd_doc.filename, content=new_prd_doc.content)
+        await self._save_competitive_analysis(new_prd_doc)
+        await self._save_pdf(new_prd_doc)
+        return new_prd_doc
+
+    @staticmethod
+    async def _save_competitive_analysis(prd_doc):
+        m = json.loads(prd_doc.content)
+        quadrant_chart = m.get("Competitive Quadrant Chart")
+        if not quadrant_chart:
+            return
+        pathname = (
+            CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("")
         )
-        logger.debug(prompt)
-        # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING)
-        prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format)
-        return prd
+        if not pathname.parent.exists():
+            pathname.parent.mkdir(parents=True, exist_ok=True)
+        await mermaid_to_file(quadrant_chart, pathname)
+
+    @staticmethod
+    async def _save_pdf(prd_doc):
+        await FileRepository.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO)
+
+    @staticmethod
+    async def _rename_workspace(prd):
+        if not CONFIG.project_name:
+            if isinstance(prd, (ActionOutput, ActionNode)):
+                ws_name = prd.instruct_content.model_dump()["Project Name"]
+            else:
+                ws_name = CodeParser.parse_str(block="Project Name", text=prd)
+            if ws_name:
+                CONFIG.project_name = ws_name
+        if not CONFIG.project_name:  # The LLM failed to provide a project name, and the user didn't provide one either.
+            CONFIG.project_name = "app" + uuid.uuid4().hex[:16]
+        CONFIG.git_repo.rename_root(CONFIG.project_name)
+
+    async def _is_bugfix(self, context) -> bool:
+        src_workspace_path = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name
+        code_files = CONFIG.git_repo.get_files(relative_path=src_workspace_path)
+        if not code_files:
+            return False
+        node = await WP_ISSUE_TYPE_NODE.fill(context, self.llm)
+        return node.get("issue_type") == "BUG"

metagpt/actions/write_prd_an.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/12/14 11:40
+@Author  : alexanderwu
+@File    : write_prd_an.py
+"""
+from typing import List
+
+from metagpt.actions.action_node import ActionNode
+from metagpt.logs import logger
+
+LANGUAGE = ActionNode(
+    key="Language",
+    expected_type=str,
+    instruction="Provide the language used in the project, typically matching the user's requirement language.",
+    example="en_us",
+)
+
+PROGRAMMING_LANGUAGE = ActionNode(
+    key="Programming Language",
+    expected_type=str,
+    instruction="Python/JavaScript or other mainstream programming language.",
+    example="Python",
+)
+
+ORIGINAL_REQUIREMENTS = ActionNode(
+    key="Original Requirements",
+    expected_type=str,
+    instruction="Place the original user's requirements here.",
+    example="Create a 2048 game",
+)
+
+PROJECT_NAME = ActionNode(
+    key="Project Name",
+    expected_type=str,
+    instruction="According to the content of \"Original Requirements,\" name the project using snake case style , like 'game_2048' or 'simple_crm.",
+    example="game_2048",
+)
+
+PRODUCT_GOALS = ActionNode(
+    key="Product Goals",
+    expected_type=List[str],
+    instruction="Provide up to three clear, orthogonal product goals.",
+    example=["Create an engaging user experience", "Improve accessibility, be responsive", "More beautiful UI"],
+)
+
+USER_STORIES = ActionNode(
+    key="User Stories",
+    expected_type=List[str],
+    instruction="Provide up to 3 to 5 scenario-based user stories.",
+    example=[
+        "As a player, I want to be able to choose difficulty levels",
+        "As a player, I want to see my score after each game",
+        "As a player, I want to get restart button when I lose",
+        "As a player, I want to see beautiful UI that make me feel good",
+        "As a player, I want to play game via mobile phone",
+    ],
+)
+
+COMPETITIVE_ANALYSIS = ActionNode(
+    key="Competitive Analysis",
+    expected_type=List[str],
+    instruction="Provide 5 to 7 competitive products.",
+    example=[
+        "2048 Game A: Simple interface, lacks responsive features",
+        "play2048.co: Beautiful and responsive UI with my best score shown",
+        "2048game.com: Responsive UI with my best score shown, but many ads",
+    ],
+)
+
+COMPETITIVE_QUADRANT_CHART = ActionNode(
+    key="Competitive Quadrant Chart",
+    expected_type=str,
+    instruction="Use mermaid quadrantChart syntax. Distribute scores evenly between 0 and 1",
+    example="""quadrantChart
+    title "Reach and engagement of campaigns"
+    x-axis "Low Reach" --> "High Reach"
+    y-axis "Low Engagement" --> "High Engagement"
+    quadrant-1 "We should expand"
+    quadrant-2 "Need to promote"
+    quadrant-3 "Re-evaluate"
+    quadrant-4 "May be improved"
+    "Campaign A": [0.3, 0.6]
+    "Campaign B": [0.45, 0.23]
+    "Campaign C": [0.57, 0.69]
+    "Campaign D": [0.78, 0.34]
+    "Campaign E": [0.40, 0.34]
+    "Campaign F": [0.35, 0.78]
+    "Our Target Product": [0.5, 0.6]""",
+)
+
+REQUIREMENT_ANALYSIS = ActionNode(
+    key="Requirement Analysis",
+    expected_type=str,
+    instruction="Provide a detailed analysis of the requirements.",
+    example="",
+)
+
+REQUIREMENT_POOL = ActionNode(
+    key="Requirement Pool",
+    expected_type=List[List[str]],
+    instruction="List down the top-5 requirements with their priority (P0, P1, P2).",
+    example=[["P0", "The main code ..."], ["P0", "The game algorithm ..."]],
+)
+
+UI_DESIGN_DRAFT = ActionNode(
+    key="UI Design draft",
+    expected_type=str,
+    instruction="Provide a simple description of UI elements, functions, style, and layout.",
+    example="Basic function description with a simple style and layout.",
+)
+
+ANYTHING_UNCLEAR = ActionNode(
+    key="Anything UNCLEAR",
+    expected_type=str,
+    instruction="Mention any aspects of the project that are unclear and try to clarify them.",
+    example="",
+)
+
+ISSUE_TYPE = ActionNode(
+    key="issue_type",
+    expected_type=str,
+    instruction="Answer BUG/REQUIREMENT. If it is a bugfix, answer BUG, otherwise answer Requirement",
+    example="BUG",
+)
+
+IS_RELATIVE = ActionNode(
+    key="is_relative",
+    expected_type=str,
+    instruction="Answer YES/NO. If the requirement is related to the old PRD, answer YES, otherwise NO",
+    example="YES",
+)
+
+REASON = ActionNode(
+    key="reason", expected_type=str, instruction="Explain the reasoning process from question to answer", example="..."
+)
+
+
+NODES = [
+    LANGUAGE,
+    PROGRAMMING_LANGUAGE,
+    ORIGINAL_REQUIREMENTS,
+    PROJECT_NAME,
+    PRODUCT_GOALS,
+    USER_STORIES,
+    COMPETITIVE_ANALYSIS,
+    COMPETITIVE_QUADRANT_CHART,
+    REQUIREMENT_ANALYSIS,
+    REQUIREMENT_POOL,
+    UI_DESIGN_DRAFT,
+    ANYTHING_UNCLEAR,
+]
+
+WRITE_PRD_NODE = ActionNode.from_children("WritePRD", NODES)
+WP_ISSUE_TYPE_NODE = ActionNode.from_children("WP_ISSUE_TYPE", [ISSUE_TYPE, REASON])
+WP_IS_RELATIVE_NODE = ActionNode.from_children("WP_IS_RELATIVE", [IS_RELATIVE, REASON])
+
+
+def main():
+    prompt = WRITE_PRD_NODE.compile(context="")
+    logger.info(prompt)
+
+
+if __name__ == "__main__":
+    main()

metagpt/actions/write_prd_review.py

@@ -5,24 +5,27 @@
 @Author  : alexanderwu
 @File    : write_prd_review.py
 """
+
+from typing import Optional
+
 from metagpt.actions.action import Action
 
 
 class WritePRDReview(Action):
-    def __init__(self, name, context=None, llm=None):
-        super().__init__(name, context, llm)
-        self.prd = None
-        self.desc = "Based on the PRD, conduct a PRD Review, providing clear and detailed feedback"
-        self.prd_review_prompt_template = """
-        Given the following Product Requirement Document (PRD):
-        {prd}
+    name: str = ""
+    context: Optional[str] = None
 
-        As a project manager, please review it and provide your feedback and suggestions.
-        """
+    prd: Optional[str] = None
+    desc: str = "Based on the PRD, conduct a PRD Review, providing clear and detailed feedback"
+    prd_review_prompt_template: str = """
+Given the following Product Requirement Document (PRD):
+{prd}
+
+As a project manager, please review it and provide your feedback and suggestions.
+"""
 
     async def run(self, prd):
         self.prd = prd
         prompt = self.prd_review_prompt_template.format(prd=self.prd)
         review = await self._aask(prompt)
         return review
-    

metagpt/actions/write_review.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Author  : alexanderwu
+@File    : write_review.py
+"""
+from typing import List
+
+from metagpt.actions import Action
+from metagpt.actions.action_node import ActionNode
+
+REVIEW = ActionNode(
+    key="Review",
+    expected_type=List[str],
+    instruction="Act as an experienced Reviewer and review the given output. Ask a series of critical questions, "
+    "concisely and clearly, to help the writer improve their work.",
+    example=[
+        "This is a good PRD, but I think it can be improved by adding more details.",
+    ],
+)
+
+LGTM = ActionNode(
+    key="LGTM",
+    expected_type=str,
+    instruction="LGTM/LBTM. If the output is good enough, give a LGTM (Looks Good To Me) to the writer, "
+    "else LBTM (Looks Bad To Me).",
+    example="LGTM",
+)
+
+WRITE_REVIEW_NODE = ActionNode.from_children("WRITE_REVIEW_NODE", [REVIEW, LGTM])
+
+
+class WriteReview(Action):
+    """Write a review for the given context."""
+
+    name: str = "WriteReview"
+
+    async def run(self, context):
+        return await WRITE_REVIEW_NODE.fill(context=context, llm=self.llm, schema="json")

metagpt/actions/write_teaching_plan.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/7/27
+@Author  : mashenquan
+@File    : write_teaching_plan.py
+"""
+from typing import Optional
+
+from metagpt.actions import Action
+from metagpt.config import CONFIG
+from metagpt.logs import logger
+
+
+class WriteTeachingPlanPart(Action):
+    """Write Teaching Plan Part"""
+
+    context: Optional[str] = None
+    topic: str = ""
+    language: str = "Chinese"
+    rsp: Optional[str] = None
+
+    async def run(self, with_message=None, **kwargs):
+        statement_patterns = TeachingPlanBlock.TOPIC_STATEMENTS.get(self.topic, [])
+        statements = []
+        for p in statement_patterns:
+            s = self.format_value(p)
+            statements.append(s)
+        formatter = (
+            TeachingPlanBlock.PROMPT_TITLE_TEMPLATE
+            if self.topic == TeachingPlanBlock.COURSE_TITLE
+            else TeachingPlanBlock.PROMPT_TEMPLATE
+        )
+        prompt = formatter.format(
+            formation=TeachingPlanBlock.FORMATION,
+            role=self.prefix,
+            statements="\n".join(statements),
+            lesson=self.context,
+            topic=self.topic,
+            language=self.language,
+        )
+
+        logger.debug(prompt)
+        rsp = await self._aask(prompt=prompt)
+        logger.debug(rsp)
+        self._set_result(rsp)
+        return self.rsp
+
+    def _set_result(self, rsp):
+        if TeachingPlanBlock.DATA_BEGIN_TAG in rsp:
+            ix = rsp.index(TeachingPlanBlock.DATA_BEGIN_TAG)
+            rsp = rsp[ix + len(TeachingPlanBlock.DATA_BEGIN_TAG) :]
+        if TeachingPlanBlock.DATA_END_TAG in rsp:
+            ix = rsp.index(TeachingPlanBlock.DATA_END_TAG)
+            rsp = rsp[0:ix]
+        self.rsp = rsp.strip()
+        if self.topic != TeachingPlanBlock.COURSE_TITLE:
+            return
+        if "#" not in self.rsp or self.rsp.index("#") != 0:
+            self.rsp = "# " + self.rsp
+
+    def __str__(self):
+        """Return `topic` value when str()"""
+        return self.topic
+
+    def __repr__(self):
+        """Show `topic` value when debug"""
+        return self.topic
+
+    @staticmethod
+    def format_value(value):
+        """Fill parameters inside `value` with `options`."""
+        if not isinstance(value, str):
+            return value
+        if "{" not in value:
+            return value
+
+        merged_opts = CONFIG.options or {}
+        try:
+            return value.format(**merged_opts)
+        except KeyError as e:
+            logger.warning(f"Parameter is missing:{e}")
+
+        for k, v in merged_opts.items():
+            value = value.replace("{" + f"{k}" + "}", str(v))
+        return value
+
+
+class TeachingPlanBlock:
+    FORMATION = (
+        '"Capacity and role" defines the role you are currently playing;\n'
+        '\t"[LESSON_BEGIN]" and "[LESSON_END]" tags enclose the content of textbook;\n'
+        '\t"Statement" defines the work detail you need to complete at this stage;\n'
+        '\t"Answer options" defines the format requirements for your responses;\n'
+        '\t"Constraint" defines the conditions that your responses must comply with.'
+    )
+
+    COURSE_TITLE = "Title"
+    TOPICS = [
+        COURSE_TITLE,
+        "Teaching Hours",
+        "Teaching Objectives",
+        "Teaching Content",
+        "Teaching Methods and Strategies",
+        "Learning Activities",
+        "Teaching Time Allocation",
+        "Assessment and Feedback",
+        "Teaching Summary and Improvement",
+        "Vocabulary Cloze",
+        "Choice Questions",
+        "Grammar Questions",
+        "Translation Questions",
+    ]
+
+    TOPIC_STATEMENTS = {
+        COURSE_TITLE: [
+            "Statement: Find and return the title of the lesson only in markdown first-level header format, "
+            "without anything else."
+        ],
+        "Teaching Content": [
+            'Statement: "Teaching Content" must include vocabulary, analysis, and examples of various grammar '
+            "structures that appear in the textbook, as well as the listening materials and key points.",
+            'Statement: "Teaching Content" must include more examples.',
+        ],
+        "Teaching Time Allocation": [
+            'Statement: "Teaching Time Allocation" must include how much time is allocated to each '
+            "part of the textbook content."
+        ],
+        "Teaching Methods and Strategies": [
+            'Statement: "Teaching Methods and Strategies" must include teaching focus, difficulties, materials, '
+            "procedures, in detail."
+        ],
+        "Vocabulary Cloze": [
+            'Statement: Based on the content of the textbook enclosed by "[LESSON_BEGIN]" and "[LESSON_END]", '
+            "create vocabulary cloze. The cloze should include 10 {language} questions with {teaching_language} "
+            "answers, and it should also include 10 {teaching_language} questions with {language} answers. "
+            "The key-related vocabulary and phrases in the textbook content must all be included in the exercises.",
+        ],
+        "Grammar Questions": [
+            'Statement: Based on the content of the textbook enclosed by "[LESSON_BEGIN]" and "[LESSON_END]", '
+            "create grammar questions. 10 questions."
+        ],
+        "Choice Questions": [
+            'Statement: Based on the content of the textbook enclosed by "[LESSON_BEGIN]" and "[LESSON_END]", '
+            "create choice questions. 10 questions."
+        ],
+        "Translation Questions": [
+            'Statement: Based on the content of the textbook enclosed by "[LESSON_BEGIN]" and "[LESSON_END]", '
+            "create translation questions. The translation should include 10 {language} questions with "
+            "{teaching_language} answers, and it should also include 10 {teaching_language} questions with "
+            "{language} answers."
+        ],
+    }
+
+    # Teaching plan title
+    PROMPT_TITLE_TEMPLATE = (
+        "Do not refer to the context of the previous conversation records, "
+        "start the conversation anew.\n\n"
+        "Formation: {formation}\n\n"
+        "{statements}\n"
+        "Constraint: Writing in {language}.\n"
+        'Answer options: Encloses the lesson title with "[TEACHING_PLAN_BEGIN]" '
+        'and "[TEACHING_PLAN_END]" tags.\n'
+        "[LESSON_BEGIN]\n"
+        "{lesson}\n"
+        "[LESSON_END]"
+    )
+
+    # Teaching plan parts:
+    PROMPT_TEMPLATE = (
+        "Do not refer to the context of the previous conversation records, "
+        "start the conversation anew.\n\n"
+        "Formation: {formation}\n\n"
+        "Capacity and role: {role}\n"
+        'Statement: Write the "{topic}" part of teaching plan, '
+        'WITHOUT ANY content unrelated to "{topic}"!!\n'
+        "{statements}\n"
+        'Answer options: Enclose the teaching plan content with "[TEACHING_PLAN_BEGIN]" '
+        'and "[TEACHING_PLAN_END]" tags.\n'
+        "Answer options: Using proper markdown format from second-level header format.\n"
+        "Constraint: Writing in {language}.\n"
+        "[LESSON_BEGIN]\n"
+        "{lesson}\n"
+        "[LESSON_END]"
+    )
+
+    DATA_BEGIN_TAG = "[TEACHING_PLAN_BEGIN]"
+    DATA_END_TAG = "[TEACHING_PLAN_END]"

metagpt/actions/write_test.py

@@ -3,10 +3,17 @@
 """
 @Time    : 2023/5/11 22:12
 @Author  : alexanderwu
-@File    : environment.py
+@File    : write_test.py
+@Modified By: mashenquan, 2023-11-27. Following the think-act principle, solidify the task parameters when creating the
+        WriteTest object, rather than passing them in when calling the run function.
 """
+
+from typing import Optional
+
 from metagpt.actions.action import Action
+from metagpt.const import TEST_CODES_FILE_REPO
 from metagpt.logs import logger
+from metagpt.schema import Document, TestingContext
 from metagpt.utils.common import CodeParser
 
 PROMPT_TEMPLATE = """
@@ -15,7 +22,7 @@ NOTICE
 2. Requirement: Based on the context, develop a comprehensive test suite that adequately covers all relevant aspects of the code file under review. Your test suite will be part of the overall project QA, so please develop complete, robust, and reusable test cases.
 3. Attention1: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the test case or script.
 4. Attention2: If there are any settings in your tests, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE.
-5. Attention3: YOU MUST FOLLOW "Data structures and interface definitions". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity.
+5. Attention3: YOU MUST FOLLOW "Data structures and interfaces". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity.
 6. Think before writing: What should be tested and validated in this document? What edge cases could exist? What might fail?
 7. CAREFULLY CHECK THAT YOU DON'T MISS ANY NECESSARY TEST CASES/SCRIPTS IN THIS FILE.
 Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the test case or script and triple quotes.
@@ -26,13 +33,13 @@ Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD W
 ```
 Note that the code to test is at {source_file_path}, we will put your test code at {workspace}/tests/{test_file_name}, and run your test code from {workspace},
 you should correctly import the necessary classes based on these file locations!
-## {test_file_name}: Write test code with triple quoto. Do your best to implement THIS ONLY ONE FILE.
+## {test_file_name}: Write test code with triple quote. Do your best to implement THIS ONLY ONE FILE.
 """
 
 
 class WriteTest(Action):
-    def __init__(self, name="WriteTest", context=None, llm=None):
-        super().__init__(name, context, llm)
+    name: str = "WriteTest"
+    context: Optional[TestingContext] = None
 
     async def write_code(self, prompt):
         code_rsp = await self._aask(prompt)
@@ -47,12 +54,17 @@ class WriteTest(Action):
             code = code_rsp
         return code
 
-    async def run(self, code_to_test, test_file_name, source_file_path, workspace):
+    async def run(self, *args, **kwargs) -> TestingContext:
+        if not self.context.test_doc:
+            self.context.test_doc = Document(
+                filename="test_" + self.context.code_doc.filename, root_path=TEST_CODES_FILE_REPO
+            )
+        fake_root = "/data"
         prompt = PROMPT_TEMPLATE.format(
-            code_to_test=code_to_test,
-            test_file_name=test_file_name,
-            source_file_path=source_file_path,
-            workspace=workspace,
+            code_to_test=self.context.code_doc.content,
+            test_file_name=self.context.test_doc.filename,
+            source_file_path=fake_root + "/" + self.context.code_doc.root_relative_path,
+            workspace=fake_root,
         )
-        code = await self.write_code(prompt)
-        return code
+        self.context.test_doc.content = await self.write_code(prompt)
+        return self.context

metagpt/actions/write_tutorial.py

@@ -10,7 +10,7 @@
 from typing import Dict
 
 from metagpt.actions import Action
-from metagpt.prompts.tutorial_assistant import DIRECTORY_PROMPT, CONTENT_PROMPT
+from metagpt.prompts.tutorial_assistant import CONTENT_PROMPT, DIRECTORY_PROMPT
 from metagpt.utils.common import OutputParser
 
 
@@ -22,9 +22,8 @@ class WriteDirectory(Action):
         language: The language to output, default is "Chinese".
     """
 
-    def __init__(self, name: str = "", language: str = "Chinese", *args, **kwargs):
-        super().__init__(name, *args, **kwargs)
-        self.language = language
+    name: str = "WriteDirectory"
+    language: str = "Chinese"
 
     async def run(self, topic: str, *args, **kwargs) -> Dict:
         """Execute the action to generate a tutorial directory according to the topic.
@@ -49,10 +48,9 @@ class WriteContent(Action):
         language: The language to output, default is "Chinese".
     """
 
-    def __init__(self, name: str = "", directory: str = "", language: str = "Chinese", *args, **kwargs):
-        super().__init__(name, *args, **kwargs)
-        self.language = language
-        self.directory = directory
+    name: str = "WriteContent"
+    directory: dict = dict()
+    language: str = "Chinese"
 
     async def run(self, topic: str, *args, **kwargs) -> str:
         """Execute the action to write document content according to the directory and topic.
@@ -65,4 +63,3 @@ class WriteContent(Action):
         """
         prompt = CONTENT_PROMPT.format(topic=topic, language=self.language, directory=self.directory)
         return await self._aask(prompt=prompt)
-

metagpt/config.py

@@ -2,15 +2,27 @@
 # -*- coding: utf-8 -*-
 """
 Provide configuration, singleton
+@Modified By: mashenquan, 2023/11/27.
+        1. According to Section 2.2.3.11 of RFC 135, add git repository support.
+        2. Add the parameter `src_workspace` for the old version project path.
 """
+import datetime
+import json
 import os
+import warnings
+from copy import deepcopy
+from enum import Enum
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
 
-import openai
 import yaml
 
-from metagpt.const import PROJECT_ROOT
+from metagpt.const import DEFAULT_WORKSPACE_ROOT, METAGPT_ROOT, OPTIONS
 from metagpt.logs import logger
 from metagpt.tools import SearchEngineType, WebBrowserEngineType
+from metagpt.utils.common import require_python_version
+from metagpt.utils.cost_manager import CostManager
 from metagpt.utils.singleton import Singleton
 
 
@@ -26,6 +38,22 @@ class NotConfiguredException(Exception):
         super().__init__(self.message)
 
 
+class LLMProviderEnum(Enum):
+    OPENAI = "openai"
+    ANTHROPIC = "anthropic"
+    SPARK = "spark"
+    ZHIPUAI = "zhipuai"
+    FIREWORKS = "fireworks"
+    OPEN_LLM = "open_llm"
+    GEMINI = "gemini"
+    METAGPT = "metagpt"
+    AZURE_OPENAI = "azure_openai"
+    OLLAMA = "ollama"
+
+    def __missing__(self, key):
+        return self.OPENAI
+
+
 class Config(metaclass=Singleton):
     """
     Regular usage method:
@@ -35,33 +63,105 @@ class Config(metaclass=Singleton):
     """
 
     _instance = None
-    key_yaml_file = PROJECT_ROOT / "config/key.yaml"
-    default_yaml_file = PROJECT_ROOT / "config/config.yaml"
+    home_yaml_file = Path.home() / ".metagpt/config.yaml"
+    key_yaml_file = METAGPT_ROOT / "config/key.yaml"
+    default_yaml_file = METAGPT_ROOT / "config/config.yaml"
 
-    def __init__(self, yaml_file=default_yaml_file):
-        self._configs = {}
-        self._init_with_config_files_and_env(self._configs, yaml_file)
-        logger.info("Config loading done.")
+    def __init__(self, yaml_file=default_yaml_file, cost_data=""):
+        global_options = OPTIONS.get()
+        # cli paras
+        self.project_path = ""
+        self.project_name = ""
+        self.inc = False
+        self.reqa_file = ""
+        self.max_auto_summarize_code = 0
+        self.git_reinit = False
+
+        self._init_with_config_files_and_env(yaml_file)
+        # The agent needs to be billed per user, so billing information cannot be destroyed when the session ends.
+        self.cost_manager = CostManager(**json.loads(cost_data)) if cost_data else CostManager()
+        self._update()
+        global_options.update(OPTIONS.get())
+        logger.debug("Config loading done.")
+
+    def get_default_llm_provider_enum(self) -> LLMProviderEnum:
+        """Get first valid LLM provider enum"""
+        mappings = {
+            LLMProviderEnum.OPENAI: bool(
+                self._is_valid_llm_key(self.OPENAI_API_KEY) and not self.OPENAI_API_TYPE and self.OPENAI_API_MODEL
+            ),
+            LLMProviderEnum.ANTHROPIC: self._is_valid_llm_key(self.ANTHROPIC_API_KEY),
+            LLMProviderEnum.ZHIPUAI: self._is_valid_llm_key(self.ZHIPUAI_API_KEY),
+            LLMProviderEnum.FIREWORKS: self._is_valid_llm_key(self.FIREWORKS_API_KEY),
+            LLMProviderEnum.OPEN_LLM: self._is_valid_llm_key(self.OPEN_LLM_API_BASE),
+            LLMProviderEnum.GEMINI: self._is_valid_llm_key(self.GEMINI_API_KEY),
+            LLMProviderEnum.METAGPT: bool(
+                self._is_valid_llm_key(self.OPENAI_API_KEY) and self.OPENAI_API_TYPE == "metagpt"
+            ),
+            LLMProviderEnum.AZURE_OPENAI: bool(
+                self._is_valid_llm_key(self.OPENAI_API_KEY)
+                and self.OPENAI_API_TYPE == "azure"
+                and self.DEPLOYMENT_NAME
+                and self.OPENAI_API_VERSION
+            ),
+            LLMProviderEnum.OLLAMA: self._is_valid_llm_key(self.OLLAMA_API_BASE),
+        }
+        provider = None
+        for k, v in mappings.items():
+            if v:
+                provider = k
+                break
+        if provider is None:
+            if self.DEFAULT_PROVIDER:
+                provider = LLMProviderEnum(self.DEFAULT_PROVIDER)
+            else:
+                raise NotConfiguredException("You should config a LLM configuration first")
+
+        if provider is LLMProviderEnum.GEMINI and not require_python_version(req_version=(3, 10)):
+            warnings.warn("Use Gemini requires Python >= 3.10")
+        model_name = self.get_model_name(provider=provider)
+        if model_name:
+            logger.info(f"{provider} Model: {model_name}")
+        if provider:
+            logger.info(f"API: {provider}")
+            return provider
+
+    def get_model_name(self, provider=None) -> str:
+        provider = provider or self.get_default_llm_provider_enum()
+        model_mappings = {
+            LLMProviderEnum.OPENAI: self.OPENAI_API_MODEL,
+            LLMProviderEnum.AZURE_OPENAI: self.DEPLOYMENT_NAME,
+        }
+        return model_mappings.get(provider, "")
+
+    @staticmethod
+    def _is_valid_llm_key(k: str) -> bool:
+        return bool(k and k != "YOUR_API_KEY")
+
+    def _update(self):
         self.global_proxy = self._get("GLOBAL_PROXY")
+
         self.openai_api_key = self._get("OPENAI_API_KEY")
-        self.anthropic_api_key = self._get("Anthropic_API_KEY")
+        self.anthropic_api_key = self._get("ANTHROPIC_API_KEY")
         self.zhipuai_api_key = self._get("ZHIPUAI_API_KEY")
-        if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and \
-                (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key) and \
-                (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key):
-            raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first")
-        self.openai_api_base = self._get("OPENAI_API_BASE")
-        openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy
-        if openai_proxy:
-            openai.proxy = openai_proxy
-            openai.api_base = self.openai_api_base
+        self.open_llm_api_base = self._get("OPEN_LLM_API_BASE")
+        self.open_llm_api_model = self._get("OPEN_LLM_API_MODEL")
+        self.fireworks_api_key = self._get("FIREWORKS_API_KEY")
+        self.gemini_api_key = self._get("GEMINI_API_KEY")
+        self.ollama_api_base = self._get("OLLAMA_API_BASE")
+        self.ollama_api_model = self._get("OLLAMA_API_MODEL")
+
+        if not self._get("DISABLE_LLM_PROVIDER_CHECK"):
+            _ = self.get_default_llm_provider_enum()
+
+        self.openai_base_url = self._get("OPENAI_BASE_URL")
+        self.openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy
         self.openai_api_type = self._get("OPENAI_API_TYPE")
         self.openai_api_version = self._get("OPENAI_API_VERSION")
         self.openai_api_rpm = self._get("RPM", 3)
-        self.openai_api_model = self._get("OPENAI_API_MODEL", "gpt-4")
+        self.openai_api_model = self._get("OPENAI_API_MODEL", "gpt-4-1106-preview")
         self.max_tokens_rsp = self._get("MAX_TOKENS", 2048)
-        self.deployment_name = self._get("DEPLOYMENT_NAME")
-        self.deployment_id = self._get("DEPLOYMENT_ID")
+        self.deployment_name = self._get("DEPLOYMENT_NAME", "gpt-4")
 
         self.spark_appid = self._get("SPARK_APPID")
         self.spark_api_secret = self._get("SPARK_API_SECRET")
@@ -69,7 +169,10 @@ class Config(metaclass=Singleton):
         self.domain = self._get("DOMAIN")
         self.spark_url = self._get("SPARK_URL")
 
-        self.claude_api_key = self._get("Anthropic_API_KEY")
+        self.fireworks_api_base = self._get("FIREWORKS_API_BASE")
+        self.fireworks_api_model = self._get("FIREWORKS_API_MODEL")
+
+        self.claude_api_key = self._get("ANTHROPIC_API_KEY")
         self.serpapi_api_key = self._get("SERPAPI_API_KEY")
         self.serper_api_key = self._get("SERPER_API_KEY")
         self.google_api_key = self._get("GOOGLE_API_KEY")
@@ -82,8 +185,8 @@ class Config(metaclass=Singleton):
         self.long_term_memory = self._get("LONG_TERM_MEMORY", False)
         if self.long_term_memory:
             logger.warning("LONG_TERM_MEMORY is True")
-        self.max_budget = self._get("MAX_BUDGET", 10.0)
-        self.total_cost = 0.0
+        self.cost_manager.max_budget = self._get("MAX_BUDGET", 10.0)
+        self.code_review_k_times = 2
 
         self.puppeteer_config = self._get("PUPPETEER_CONFIG", "")
         self.mmdc = self._get("MMDC", "mmdc")
@@ -93,16 +196,44 @@ class Config(metaclass=Singleton):
         self.mermaid_engine = self._get("MERMAID_ENGINE", "nodejs")
         self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "")
 
-        self.prompt_format = self._get("PROMPT_FORMAT", "markdown")
+        workspace_uid = (
+            self._get("WORKSPACE_UID") or f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[-8:]}"
+        )
+        self.repair_llm_output = self._get("REPAIR_LLM_OUTPUT", False)
+        self.prompt_schema = self._get("PROMPT_FORMAT", "json")
+        self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT))
+        val = self._get("WORKSPACE_PATH_WITH_UID")
+        if val and val.lower() == "true":  # for agent
+            self.workspace_path = self.workspace_path / workspace_uid
+        self._ensure_workspace_exists()
+        self.max_auto_summarize_code = self.max_auto_summarize_code or self._get("MAX_AUTO_SUMMARIZE_CODE", 1)
+        self.timeout = int(self._get("TIMEOUT", 3))
 
         self.kaggle_username = self._get("KAGGLE_USERNAME", "")
         self.kaggle_key = self._get("KAGGLE_KEY", "")
 
-    def _init_with_config_files_and_env(self, configs: dict, yaml_file):
-        """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority"""
-        configs.update(os.environ)
+    def update_via_cli(self, project_path, project_name, inc, reqa_file, max_auto_summarize_code):
+        """update config via cli"""
 
-        for _yaml_file in [yaml_file, self.key_yaml_file]:
+        # Use in the PrepareDocuments action according to Section 2.2.3.5.1 of RFC 135.
+        if project_path:
+            inc = True
+            project_name = project_name or Path(project_path).name
+        self.project_path = project_path
+        self.project_name = project_name
+        self.inc = inc
+        self.reqa_file = reqa_file
+        self.max_auto_summarize_code = max_auto_summarize_code
+
+    def _ensure_workspace_exists(self):
+        self.workspace_path.mkdir(parents=True, exist_ok=True)
+        logger.debug(f"WORKSPACE_PATH set to {self.workspace_path}")
+
+    def _init_with_config_files_and_env(self, yaml_file):
+        """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority"""
+        configs = dict(os.environ)
+
+        for _yaml_file in [yaml_file, self.key_yaml_file, self.home_yaml_file]:
             if not _yaml_file.exists():
                 continue
 
@@ -111,18 +242,49 @@ class Config(metaclass=Singleton):
                 yaml_data = yaml.safe_load(file)
                 if not yaml_data:
                     continue
-                os.environ.update({k: v for k, v in yaml_data.items() if isinstance(v, str)})
                 configs.update(yaml_data)
+        OPTIONS.set(configs)
 
-    def _get(self, *args, **kwargs):
-        return self._configs.get(*args, **kwargs)
+    @staticmethod
+    def _get(*args, **kwargs):
+        i = OPTIONS.get()
+        return i.get(*args, **kwargs)
 
     def get(self, key, *args, **kwargs):
-        """Search for a value in config/key.yaml, config/config.yaml, and env; raise an error if not found"""
+        """Retrieve values from config/key.yaml, config/config.yaml, and environment variables.
+        Throw an error if not found."""
         value = self._get(key, *args, **kwargs)
         if value is None:
             raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file")
         return value
 
+    def __setattr__(self, name: str, value: Any) -> None:
+        OPTIONS.get()[name] = value
+
+    def __getattr__(self, name: str) -> Any:
+        i = OPTIONS.get()
+        return i.get(name)
+
+    def set_context(self, options: dict):
+        """Update current config"""
+        if not options:
+            return
+        opts = deepcopy(OPTIONS.get())
+        opts.update(options)
+        OPTIONS.set(opts)
+        self._update()
+
+    @property
+    def options(self):
+        """Return all key-values"""
+        return OPTIONS.get()
+
+    def new_environ(self):
+        """Return a new os.environ object"""
+        env = os.environ.copy()
+        i = self.options
+        env.update({k: v for k, v in i.items() if isinstance(v, str)})
+        return env
+
 
 CONFIG = Config()

metagpt/const.py

@@ -4,45 +4,130 @@
 @Time    : 2023/5/1 11:59
 @Author  : alexanderwu
 @File    : const.py
+@Modified By: mashenquan, 2023-11-1. According to Section 2.2.1 and 2.2.2 of RFC 116, added key definitions for
+        common properties in the Message.
+@Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135.
+@Modified By: mashenquan, 2023/12/5. Add directories for code summarization..
 """
+import contextvars
+import os
 from pathlib import Path
+
 from loguru import logger
 
-def get_project_root():
-    """Search upwards to find the project root directory."""
-    current_path = Path.cwd()
-    while True:
-        if (
-            (current_path / ".git").exists()
-            or (current_path / ".project_root").exists()
-            or (current_path / ".gitignore").exists()
-        ):
-            # use metagpt with git clone will land here
-            logger.info(f"PROJECT_ROOT set to {str(current_path)}")
-            return current_path
-        parent_path = current_path.parent
-        if parent_path == current_path:
-            # use metagpt with pip install will land here
-            cwd = Path.cwd()
-            logger.info(f"PROJECT_ROOT set to current working directory: {str(cwd)}")
-            return cwd
-        current_path = parent_path
+import metagpt
+
+OPTIONS = contextvars.ContextVar("OPTIONS", default={})
 
 
-PROJECT_ROOT = get_project_root()
-DATA_PATH = PROJECT_ROOT / "data"
-WORKSPACE_ROOT = PROJECT_ROOT / "workspace"
-PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts"
-UT_PATH = PROJECT_ROOT / "data/ut"
-SWAGGER_PATH = UT_PATH / "files/api/"
-UT_PY_PATH = UT_PATH / "files/ut/"
-API_QUESTIONS_PATH = UT_PATH / "files/question/"
-YAPI_URL = "http://yapi.deepwisdomai.com/"
-TMP = PROJECT_ROOT / "tmp"
+def get_metagpt_package_root():
+    """Get the root directory of the installed package."""
+    package_root = Path(metagpt.__file__).parent.parent
+    for i in (".git", ".project_root", ".gitignore"):
+        if (package_root / i).exists():
+            break
+    else:
+        package_root = Path.cwd()
+
+    logger.info(f"Package root set to {str(package_root)}")
+    return package_root
+
+
+def get_metagpt_root():
+    """Get the project root directory."""
+    # Check if a project root is specified in the environment variable
+    project_root_env = os.getenv("METAGPT_PROJECT_ROOT")
+    if project_root_env:
+        project_root = Path(project_root_env)
+        logger.info(f"PROJECT_ROOT set from environment variable to {str(project_root)}")
+    else:
+        # Fallback to package root if no environment variable is set
+        project_root = get_metagpt_package_root()
+    return project_root
+
+
+# METAGPT PROJECT ROOT AND VARS
+
+METAGPT_ROOT = get_metagpt_root()  # Dependent on METAGPT_PROJECT_ROOT
+DEFAULT_WORKSPACE_ROOT = METAGPT_ROOT / "workspace"
+
+EXAMPLE_PATH = METAGPT_ROOT / "examples"
+DATA_PATH = METAGPT_ROOT / "data"
+TEST_DATA_PATH = METAGPT_ROOT / "tests/data"
 RESEARCH_PATH = DATA_PATH / "research"
 TUTORIAL_PATH = DATA_PATH / "tutorial_docx"
 INVOICE_OCR_TABLE_PATH = DATA_PATH / "invoice_table"
 
-SKILL_DIRECTORY = PROJECT_ROOT / "metagpt/skills"
+UT_PATH = DATA_PATH / "ut"
+SWAGGER_PATH = UT_PATH / "files/api/"
+UT_PY_PATH = UT_PATH / "files/ut/"
+API_QUESTIONS_PATH = UT_PATH / "files/question/"
+
+SERDESER_PATH = DEFAULT_WORKSPACE_ROOT / "storage"  # TODO to store `storage` under the individual generated project
+
+TMP = METAGPT_ROOT / "tmp"
+
+SOURCE_ROOT = METAGPT_ROOT / "metagpt"
+PROMPT_PATH = SOURCE_ROOT / "prompts"
+SKILL_DIRECTORY = SOURCE_ROOT / "skills"
+
+
+# REAL CONSTS
 
 MEM_TTL = 24 * 30 * 3600
+
+
+MESSAGE_ROUTE_FROM = "sent_from"
+MESSAGE_ROUTE_TO = "send_to"
+MESSAGE_ROUTE_CAUSE_BY = "cause_by"
+MESSAGE_META_ROLE = "role"
+MESSAGE_ROUTE_TO_ALL = "<all>"
+MESSAGE_ROUTE_TO_NONE = "<none>"
+
+REQUIREMENT_FILENAME = "requirement.txt"
+BUGFIX_FILENAME = "bugfix.txt"
+PACKAGE_REQUIREMENTS_FILENAME = "requirements.txt"
+
+DOCS_FILE_REPO = "docs"
+PRDS_FILE_REPO = "docs/prds"
+SYSTEM_DESIGN_FILE_REPO = "docs/system_design"
+TASK_FILE_REPO = "docs/tasks"
+COMPETITIVE_ANALYSIS_FILE_REPO = "resources/competitive_analysis"
+DATA_API_DESIGN_FILE_REPO = "resources/data_api_design"
+SEQ_FLOW_FILE_REPO = "resources/seq_flow"
+SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design"
+PRD_PDF_FILE_REPO = "resources/prd"
+TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks"
+TEST_CODES_FILE_REPO = "tests"
+TEST_OUTPUTS_FILE_REPO = "test_outputs"
+CODE_SUMMARIES_FILE_REPO = "docs/code_summaries"
+CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries"
+RESOURCES_FILE_REPO = "resources"
+SD_OUTPUT_FILE_REPO = "resources/SD_Output"
+GRAPH_REPO_FILE_REPO = "docs/graph_repo"
+CLASS_VIEW_FILE_REPO = "docs/class_views"
+
+YAPI_URL = "http://yapi.deepwisdomai.com/"
+
+DEFAULT_LANGUAGE = "English"
+DEFAULT_MAX_TOKENS = 1500
+COMMAND_TOKENS = 500
+BRAIN_MEMORY = "BRAIN_MEMORY"
+SKILL_PATH = "SKILL_PATH"
+SERPER_API_KEY = "SERPER_API_KEY"
+DEFAULT_TOKEN_SIZE = 500
+
+# format
+BASE64_FORMAT = "base64"
+
+# REDIS
+REDIS_KEY = "REDIS_KEY"
+LLM_API_TIMEOUT = 300
+
+# Message id
+IGNORED_MESSAGE_ID = "0"
+
+# Class Relationship
+GENERALIZATION = "Generalize"
+COMPOSITION = "Composite"
+AGGREGATION = "Aggregate"

metagpt/document.py

@@ -0,0 +1,235 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/6/8 14:03
+@Author  : alexanderwu
+@File    : document.py
+@Desc    : Classes and Operations Related to Files in the File System.
+"""
+from enum import Enum
+from pathlib import Path
+from typing import Optional, Union
+
+import pandas as pd
+from langchain.document_loaders import (
+    TextLoader,
+    UnstructuredPDFLoader,
+    UnstructuredWordDocumentLoader,
+)
+from langchain.text_splitter import CharacterTextSplitter
+from pydantic import BaseModel, ConfigDict, Field
+from tqdm import tqdm
+
+from metagpt.repo_parser import RepoParser
+
+
+def validate_cols(content_col: str, df: pd.DataFrame):
+    if content_col not in df.columns:
+        raise ValueError("Content column not found in DataFrame.")
+
+
+def read_data(data_path: Path):
+    suffix = data_path.suffix
+    if ".xlsx" == suffix:
+        data = pd.read_excel(data_path)
+    elif ".csv" == suffix:
+        data = pd.read_csv(data_path)
+    elif ".json" == suffix:
+        data = pd.read_json(data_path)
+    elif suffix in (".docx", ".doc"):
+        data = UnstructuredWordDocumentLoader(str(data_path), mode="elements").load()
+    elif ".txt" == suffix:
+        data = TextLoader(str(data_path)).load()
+        text_splitter = CharacterTextSplitter(separator="\n", chunk_size=256, chunk_overlap=0)
+        texts = text_splitter.split_documents(data)
+        data = texts
+    elif ".pdf" == suffix:
+        data = UnstructuredPDFLoader(str(data_path), mode="elements").load()
+    else:
+        raise NotImplementedError("File format not supported.")
+    return data
+
+
+class DocumentStatus(Enum):
+    """Indicates document status, a mechanism similar to RFC/PEP"""
+
+    DRAFT = "draft"
+    UNDERREVIEW = "underreview"
+    APPROVED = "approved"
+    DONE = "done"
+
+
+class Document(BaseModel):
+    """
+    Document: Handles operations related to document files.
+    """
+
+    path: Path = Field(default=None)
+    name: str = Field(default="")
+    content: str = Field(default="")
+
+    # metadata? in content perhaps.
+    author: str = Field(default="")
+    status: DocumentStatus = Field(default=DocumentStatus.DRAFT)
+    reviews: list = Field(default_factory=list)
+
+    @classmethod
+    def from_path(cls, path: Path):
+        """
+        Create a Document instance from a file path.
+        """
+        if not path.exists():
+            raise FileNotFoundError(f"File {path} not found.")
+        content = path.read_text()
+        return cls(content=content, path=path)
+
+    @classmethod
+    def from_text(cls, text: str, path: Optional[Path] = None):
+        """
+        Create a Document from a text string.
+        """
+        return cls(content=text, path=path)
+
+    def to_path(self, path: Optional[Path] = None):
+        """
+        Save content to the specified file path.
+        """
+        if path is not None:
+            self.path = path
+
+        if self.path is None:
+            raise ValueError("File path is not set.")
+
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        # TODO: excel, csv, json, etc.
+        self.path.write_text(self.content, encoding="utf-8")
+
+    def persist(self):
+        """
+        Persist document to disk.
+        """
+        return self.to_path()
+
+
+class IndexableDocument(Document):
+    """
+    Advanced document handling: For vector databases or search engines.
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    data: Union[pd.DataFrame, list]
+    content_col: Optional[str] = Field(default="")
+    meta_col: Optional[str] = Field(default="")
+
+    @classmethod
+    def from_path(cls, data_path: Path, content_col="content", meta_col="metadata"):
+        if not data_path.exists():
+            raise FileNotFoundError(f"File {data_path} not found.")
+        data = read_data(data_path)
+        if isinstance(data, pd.DataFrame):
+            validate_cols(content_col, data)
+            return cls(data=data, content=str(data), content_col=content_col, meta_col=meta_col)
+        else:
+            content = data_path.read_text()
+            return cls(data=data, content=content, content_col=content_col, meta_col=meta_col)
+
+    def _get_docs_and_metadatas_by_df(self) -> (list, list):
+        df = self.data
+        docs = []
+        metadatas = []
+        for i in tqdm(range(len(df))):
+            docs.append(df[self.content_col].iloc[i])
+            if self.meta_col:
+                metadatas.append({self.meta_col: df[self.meta_col].iloc[i]})
+            else:
+                metadatas.append({})
+        return docs, metadatas
+
+    def _get_docs_and_metadatas_by_langchain(self) -> (list, list):
+        data = self.data
+        docs = [i.page_content for i in data]
+        metadatas = [i.metadata for i in data]
+        return docs, metadatas
+
+    def get_docs_and_metadatas(self) -> (list, list):
+        if isinstance(self.data, pd.DataFrame):
+            return self._get_docs_and_metadatas_by_df()
+        elif isinstance(self.data, list):
+            return self._get_docs_and_metadatas_by_langchain()
+        else:
+            raise NotImplementedError("Data type not supported for metadata extraction.")
+
+
+class RepoMetadata(BaseModel):
+    name: str = Field(default="")
+    n_docs: int = Field(default=0)
+    n_chars: int = Field(default=0)
+    symbols: list = Field(default_factory=list)
+
+
+class Repo(BaseModel):
+    # Name of this repo.
+    name: str = Field(default="")
+    # metadata: RepoMetadata = Field(default=RepoMetadata)
+    docs: dict[Path, Document] = Field(default_factory=dict)
+    codes: dict[Path, Document] = Field(default_factory=dict)
+    assets: dict[Path, Document] = Field(default_factory=dict)
+    path: Path = Field(default=None)
+
+    def _path(self, filename):
+        return self.path / filename
+
+    @classmethod
+    def from_path(cls, path: Path):
+        """Load documents, code, and assets from a repository path."""
+        path.mkdir(parents=True, exist_ok=True)
+        repo = Repo(path=path, name=path.name)
+        for file_path in path.rglob("*"):
+            # FIXME: These judgments are difficult to support multiple programming languages and need to be more general
+            if file_path.is_file() and file_path.suffix in [".json", ".txt", ".md", ".py", ".js", ".css", ".html"]:
+                repo._set(file_path.read_text(), file_path)
+        return repo
+
+    def to_path(self):
+        """Persist all documents, code, and assets to the given repository path."""
+        for doc in self.docs.values():
+            doc.to_path()
+        for code in self.codes.values():
+            code.to_path()
+        for asset in self.assets.values():
+            asset.to_path()
+
+    def _set(self, content: str, path: Path):
+        """Add a document to the appropriate category based on its file extension."""
+        suffix = path.suffix
+        doc = Document(content=content, path=path, name=str(path.relative_to(self.path)))
+
+        # FIXME: These judgments are difficult to support multiple programming languages and need to be more general
+        if suffix.lower() == ".md":
+            self.docs[path] = doc
+        elif suffix.lower() in [".py", ".js", ".css", ".html"]:
+            self.codes[path] = doc
+        else:
+            self.assets[path] = doc
+        return doc
+
+    def set(self, filename: str, content: str):
+        """Set a document and persist it to disk."""
+        path = self._path(filename)
+        doc = self._set(content, path)
+        doc.to_path()
+
+    def get(self, filename: str) -> Optional[Document]:
+        """Get a document by its filename."""
+        path = self._path(filename)
+        return self.docs.get(path) or self.codes.get(path) or self.assets.get(path)
+
+    def get_text_documents(self) -> list[Document]:
+        return list(self.docs.values()) + list(self.codes.values())
+
+    def eda(self) -> RepoMetadata:
+        n_docs = sum(len(i) for i in [self.docs, self.codes, self.assets])
+        n_chars = sum(sum(len(j.content) for j in i.values()) for i in [self.docs, self.codes, self.assets])
+        symbols = RepoParser(base_directory=self.path).generate_symbols()
+        return RepoMetadata(name=self.name, n_docs=n_docs, n_chars=n_chars, symbols=symbols)

metagpt/document_store/base_store.py

@@ -28,22 +28,22 @@ class BaseStore(ABC):
 
 
 class LocalStore(BaseStore, ABC):
-    def __init__(self, raw_data: Path, cache_dir: Path = None):
-        if not raw_data:
+    def __init__(self, raw_data_path: Path, cache_dir: Path = None):
+        if not raw_data_path:
             raise FileNotFoundError
         self.config = Config()
-        self.raw_data = raw_data
+        self.raw_data_path = raw_data_path
+        self.fname = self.raw_data_path.stem
         if not cache_dir:
-            cache_dir = raw_data.parent
+            cache_dir = raw_data_path.parent
         self.cache_dir = cache_dir
         self.store = self._load()
         if not self.store:
             self.store = self.write()
 
-    def _get_index_and_store_fname(self):
-        fname = self.raw_data.name.split('.')[0]
-        index_file = self.cache_dir / f"{fname}.index"
-        store_file = self.cache_dir / f"{fname}.pkl"
+    def _get_index_and_store_fname(self, index_ext=".index", pkl_ext=".pkl"):
+        index_file = self.cache_dir / f"{self.fname}{index_ext}"
+        store_file = self.cache_dir / f"{self.fname}{pkl_ext}"
         return index_file, store_file
 
     @abstractmethod
@@ -53,4 +53,3 @@ class LocalStore(BaseStore, ABC):
     @abstractmethod
     def _write(self, docs, metadatas):
         raise NotImplementedError
-    

metagpt/document_store/chromadb_store.py

@@ -10,6 +10,7 @@ import chromadb
 
 class ChromaStore:
     """If inherited from BaseStore, or importing other modules from metagpt, a Python exception occurs, which is strange."""
+
     def __init__(self, name):
         client = chromadb.Client()
         collection = client.create_collection(name)
@@ -22,7 +23,7 @@ class ChromaStore:
             query_texts=[query],
             n_results=n_results,
             where=metadata_filter,  # optional filter
-            where_document=document_filter  # optional filter
+            where_document=document_filter,  # optional filter
         )
         return results
 

metagpt/document_store/document.py

@@ -1,82 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/6/8 14:03
-@Author  : alexanderwu
-@File    : document.py
-"""
-from pathlib import Path
-
-import pandas as pd
-from langchain.document_loaders import (
-    TextLoader,
-    UnstructuredPDFLoader,
-    UnstructuredWordDocumentLoader,
-)
-from langchain.text_splitter import CharacterTextSplitter
-from tqdm import tqdm
-
-
-def validate_cols(content_col: str, df: pd.DataFrame):
-    if content_col not in df.columns:
-        raise ValueError
-
-
-def read_data(data_path: Path):
-    suffix = data_path.suffix
-    if '.xlsx' == suffix:
-        data = pd.read_excel(data_path)
-    elif '.csv' == suffix:
-        data = pd.read_csv(data_path)
-    elif '.json' == suffix:
-        data = pd.read_json(data_path)
-    elif suffix in ('.docx', '.doc'):
-        data = UnstructuredWordDocumentLoader(str(data_path), mode='elements').load()
-    elif '.txt' == suffix:
-        data = TextLoader(str(data_path)).load()
-        text_splitter = CharacterTextSplitter(separator='\n', chunk_size=256, chunk_overlap=0)
-        texts = text_splitter.split_documents(data)
-        data = texts
-    elif '.pdf' == suffix:
-        data = UnstructuredPDFLoader(str(data_path), mode="elements").load()
-    else:
-        raise NotImplementedError
-    return data
-
-
-class Document:
-
-    def __init__(self, data_path, content_col='content', meta_col='metadata'):
-        self.data = read_data(data_path)
-        if isinstance(self.data, pd.DataFrame):
-            validate_cols(content_col, self.data)
-        self.content_col = content_col
-        self.meta_col = meta_col
-
-    def _get_docs_and_metadatas_by_df(self) -> (list, list):
-        df = self.data
-        docs = []
-        metadatas = []
-        for i in tqdm(range(len(df))):
-            docs.append(df[self.content_col].iloc[i])
-            if self.meta_col:
-                metadatas.append({self.meta_col: df[self.meta_col].iloc[i]})
-            else:
-                metadatas.append({})
-
-        return docs, metadatas
-
-    def _get_docs_and_metadatas_by_langchain(self) -> (list, list):
-        data = self.data
-        docs = [i.page_content for i in data]
-        metadatas = [i.metadata for i in data]
-        return docs, metadatas
-
-    def get_docs_and_metadatas(self) -> (list, list):
-        if isinstance(self.data, pd.DataFrame):
-            return self._get_docs_and_metadatas_by_df()
-        elif isinstance(self.data, list):
-            return self._get_docs_and_metadatas_by_langchain()
-        else:
-            raise NotImplementedError
-        

metagpt/document_store/faiss_store.py

@@ -5,52 +5,48 @@
 @Author  : alexanderwu
 @File    : faiss_store.py
 """
-import pickle
+import asyncio
 from pathlib import Path
 from typing import Optional
 
-import faiss
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.vectorstores import FAISS
+from langchain_core.embeddings import Embeddings
 
-from metagpt.const import DATA_PATH
+from metagpt.config import CONFIG
+from metagpt.document import IndexableDocument
 from metagpt.document_store.base_store import LocalStore
-from metagpt.document_store.document import Document
 from metagpt.logs import logger
 
 
 class FaissStore(LocalStore):
-    def __init__(self, raw_data: Path, cache_dir=None, meta_col='source', content_col='output'):
+    def __init__(
+        self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output", embedding: Embeddings = None
+    ):
         self.meta_col = meta_col
         self.content_col = content_col
+        self.embedding = embedding or OpenAIEmbeddings(
+            openai_api_key=CONFIG.openai_api_key, openai_api_base=CONFIG.openai_base_url
+        )
         super().__init__(raw_data, cache_dir)
 
     def _load(self) -> Optional["FaissStore"]:
-        index_file, store_file = self._get_index_and_store_fname()
+        index_file, store_file = self._get_index_and_store_fname(index_ext=".faiss")  # langchain FAISS using .faiss
+
         if not (index_file.exists() and store_file.exists()):
             logger.info("Missing at least one of index_file/store_file, load failed and return None")
             return None
-        index = faiss.read_index(str(index_file))
-        with open(str(store_file), "rb") as f:
-            store = pickle.load(f)
-        store.index = index
-        return store
+
+        return FAISS.load_local(self.raw_data_path.parent, self.embedding, self.fname)
 
     def _write(self, docs, metadatas):
-        store = FAISS.from_texts(docs, OpenAIEmbeddings(openai_api_version="2020-11-07"), metadatas=metadatas)
+        store = FAISS.from_texts(docs, self.embedding, metadatas=metadatas)
         return store
 
     def persist(self):
-        index_file, store_file = self._get_index_and_store_fname()
-        store = self.store
-        index = self.store.index
-        faiss.write_index(store.index, str(index_file))
-        store.index = None
-        with open(store_file, "wb") as f:
-            pickle.dump(store, f)
-        store.index = index
+        self.store.save_local(self.raw_data_path.parent, self.fname)
 
-    def search(self, query, expand_cols=False, sep='\n', *args, k=5, **kwargs):
+    def search(self, query, expand_cols=False, sep="\n", *args, k=5, **kwargs):
         rsp = self.store.similarity_search(query, k=k, **kwargs)
         logger.debug(rsp)
         if expand_cols:
@@ -58,11 +54,14 @@ class FaissStore(LocalStore):
         else:
             return str(sep.join([f"{x.page_content}" for x in rsp]))
 
+    async def asearch(self, *args, **kwargs):
+        return await asyncio.to_thread(self.search, *args, **kwargs)
+
     def write(self):
         """Initialize the index and library based on the Document (JSON / XLSX, etc.) file provided by the user."""
-        if not self.raw_data.exists():
+        if not self.raw_data_path.exists():
             raise FileNotFoundError
-        doc = Document(self.raw_data, self.content_col, self.meta_col)
+        doc = IndexableDocument.from_path(self.raw_data_path, self.content_col, self.meta_col)
         docs, metadatas = doc.get_docs_and_metadatas()
 
         self.store = self._write(docs, metadatas)
@@ -76,10 +75,3 @@ class FaissStore(LocalStore):
     def delete(self, *args, **kwargs):
         """Currently, langchain does not provide a delete interface."""
         raise NotImplementedError
-
-
-if __name__ == '__main__':
-    faiss_store = FaissStore(DATA_PATH / 'qcs/qcs_4w.json')
-    logger.info(faiss_store.search('Oily Skin Facial Cleanser'))
-    faiss_store.add([f'Oily Skin Facial Cleanser-{i}' for i in range(3)])
-    logger.info(faiss_store.search('Oily Skin Facial Cleanser'))

metagpt/document_store/milvus_store.py

@@ -1,122 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/28 00:00
-@Author  : alexanderwu
-@File    : milvus_store.py
-"""
-from typing import TypedDict
-
-import numpy as np
-from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections
-
-from metagpt.document_store.base_store import BaseStore
-
-type_mapping = {
-    int: DataType.INT64,
-    str: DataType.VARCHAR,
-    float: DataType.DOUBLE,
-    np.ndarray: DataType.FLOAT_VECTOR
-}
-
-
-def columns_to_milvus_schema(columns: dict, primary_col_name: str = "", desc: str = ""):
-    """Assume the structure of columns is str: regular type"""
-    fields = []
-    for col, ctype in columns.items():
-        if ctype == str:
-            mcol = FieldSchema(name=col, dtype=type_mapping[ctype], max_length=100)
-        elif ctype == np.ndarray:
-            mcol = FieldSchema(name=col, dtype=type_mapping[ctype], dim=2)
-        else:
-            mcol = FieldSchema(name=col, dtype=type_mapping[ctype], is_primary=(col == primary_col_name))
-        fields.append(mcol)
-    schema = CollectionSchema(fields, description=desc)
-    return schema
-
-
-class MilvusConnection(TypedDict):
-    alias: str
-    host: str
-    port: str
-
-
-class MilvusStore(BaseStore):
-    """
-    FIXME: ADD TESTS
-    https://milvus.io/docs/v2.0.x/create_collection.md
-    """
-
-    def __init__(self, connection):
-        connections.connect(**connection)
-        self.collection = None
-
-    def _create_collection(self, name, schema):
-        collection = Collection(
-            name=name,
-            schema=schema,
-            using='default',
-            shards_num=2,
-            consistency_level="Strong"
-        )
-        return collection
-
-    def create_collection(self, name, columns):
-        schema = columns_to_milvus_schema(columns, 'idx')
-        self.collection = self._create_collection(name, schema)
-        return self.collection
-
-    def drop(self, name):
-        Collection(name).drop()
-
-    def load_collection(self):
-        self.collection.load()
-
-    def build_index(self, field='emb'):
-        self.collection.create_index(field, {"index_type": "FLAT", "metric_type": "L2", "params": {}})
-
-    def search(self, query: list[list[float]], *args, **kwargs):
-        """
-        FIXME: ADD TESTS
-        https://milvus.io/docs/v2.0.x/search.md
-        All search and query operations within Milvus are executed in memory. Load the collection to memory before conducting a vector similarity search.
-        Note the above description, is this logic serious? This should take a long time, right?
-        """
-        search_params = {"metric_type": "L2", "params": {"nprobe": 10}}
-        results = self.collection.search(
-            data=query,
-            anns_field=kwargs.get('field', 'emb'),
-            param=search_params,
-            limit=10,
-            expr=None,
-            consistency_level="Strong"
-        )
-        # FIXME: results contain id, but to get the actual value from the id, we still need to call the query interface
-        return results
-
-    def write(self, name, schema, *args, **kwargs):
-        """
-        FIXME: ADD TESTS
-        https://milvus.io/docs/v2.0.x/create_collection.md
-        :param args:
-        :param kwargs:
-        :return:
-        """
-        raise NotImplementedError
-
-    def add(self, data, *args, **kwargs):
-        """
-        FIXME: ADD TESTS
-        https://milvus.io/docs/v2.0.x/insert_data.md
-        import random
-        data = [
-          [i for i in range(2000)],
-          [i for i in range(10000, 12000)],
-          [[random.random() for _ in range(2)] for _ in range(2000)],
-        ]
-
-        :param args:
-        :param kwargs:
-        :return:
-        """
-        self.collection.insert(data)

metagpt/document_store/qdrant_store.py

@@ -10,13 +10,14 @@ from metagpt.document_store.base_store import BaseStore
 @dataclass
 class QdrantConnection:
     """
-   Args:
-       url: qdrant url
-       host: qdrant host
-       port: qdrant port
-       memory: qdrant service use memory mode
-       api_key: qdrant cloud api_key
-   """
+    Args:
+        url: qdrant url
+        host: qdrant host
+        port: qdrant port
+        memory: qdrant service use memory mode
+        api_key: qdrant cloud api_key
+    """
+
     url: str = None
     host: str = None
     port: int = None
@@ -31,9 +32,7 @@ class QdrantStore(BaseStore):
         elif connect.url:
             self.client = QdrantClient(url=connect.url, api_key=connect.api_key)
         elif connect.host and connect.port:
-            self.client = QdrantClient(
-                host=connect.host, port=connect.port, api_key=connect.api_key
-            )
+            self.client = QdrantClient(host=connect.host, port=connect.port, api_key=connect.api_key)
         else:
             raise Exception("please check QdrantConnection.")
 
@@ -58,15 +57,11 @@ class QdrantStore(BaseStore):
         try:
             self.client.get_collection(collection_name)
             if force_recreate:
-                res = self.client.recreate_collection(
-                    collection_name, vectors_config=vectors_config, **kwargs
-                )
+                res = self.client.recreate_collection(collection_name, vectors_config=vectors_config, **kwargs)
                 return res
             return True
         except:  # noqa: E722
-            return self.client.recreate_collection(
-                collection_name, vectors_config=vectors_config, **kwargs
-            )
+            return self.client.recreate_collection(collection_name, vectors_config=vectors_config, **kwargs)
 
     def has_collection(self, collection_name: str):
         try:

metagpt/environment.py

@@ -4,60 +4,124 @@
 @Time    : 2023/5/11 22:12
 @Author  : alexanderwu
 @File    : environment.py
+@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.2 of RFC 116:
+    1. Remove the functionality of `Environment` class as a public message buffer.
+    2. Standardize the message forwarding behavior of the `Environment` class.
+    3. Add the `is_idle` property.
+@Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing
+    functionality is to be consolidated into the `Environment` class.
 """
 import asyncio
-from typing import Iterable
+from pathlib import Path
+from typing import Iterable, Set
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, model_validator
 
-from metagpt.memory import Memory
-from metagpt.roles import Role
+from metagpt.config import CONFIG
+from metagpt.logs import logger
+from metagpt.roles.role import Role
 from metagpt.schema import Message
+from metagpt.utils.common import is_subscribed, read_json_file, write_json_file
 
 
 class Environment(BaseModel):
     """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到
-       Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles
-    
+    Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles
     """
 
-    roles: dict[str, Role] = Field(default_factory=dict)
-    memory: Memory = Field(default_factory=Memory)
-    history: str = Field(default='')
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
-    class Config:
-        arbitrary_types_allowed = True
+    desc: str = Field(default="")  # 环境描述
+    roles: dict[str, SerializeAsAny[Role]] = Field(default_factory=dict, validate_default=True)
+    members: dict[Role, Set] = Field(default_factory=dict, exclude=True)
+    history: str = ""  # For debug
+
+    @model_validator(mode="after")
+    def init_roles(self):
+        self.add_roles(self.roles.values())
+        return self
+
+    def serialize(self, stg_path: Path):
+        roles_path = stg_path.joinpath("roles.json")
+        roles_info = []
+        for role_key, role in self.roles.items():
+            roles_info.append(
+                {
+                    "role_class": role.__class__.__name__,
+                    "module_name": role.__module__,
+                    "role_name": role.name,
+                    "role_sub_tags": list(self.members.get(role)),
+                }
+            )
+            role.serialize(stg_path=stg_path.joinpath(f"roles/{role.__class__.__name__}_{role.name}"))
+        write_json_file(roles_path, roles_info)
+
+        history_path = stg_path.joinpath("history.json")
+        write_json_file(history_path, {"content": self.history})
+
+    @classmethod
+    def deserialize(cls, stg_path: Path) -> "Environment":
+        """stg_path: ./storage/team/environment/"""
+        roles_path = stg_path.joinpath("roles.json")
+        roles_info = read_json_file(roles_path)
+        roles = []
+        for role_info in roles_info:
+            # role stored in ./environment/roles/{role_class}_{role_name}
+            role_path = stg_path.joinpath(f"roles/{role_info.get('role_class')}_{role_info.get('role_name')}")
+            role = Role.deserialize(role_path)
+            roles.append(role)
+
+        history = read_json_file(stg_path.joinpath("history.json"))
+        history = history.get("content")
+
+        environment = Environment(**{"history": history})
+        environment.add_roles(roles)
+
+        return environment
 
     def add_role(self, role: Role):
         """增加一个在当前环境的角色
-           Add a role in the current environment
+        Add a role in the current environment
         """
-        role.set_env(self)
         self.roles[role.profile] = role
+        role.set_env(self)
 
     def add_roles(self, roles: Iterable[Role]):
         """增加一批在当前环境的角色
-            Add a batch of characters in the current environment
+        Add a batch of characters in the current environment
         """
         for role in roles:
-            self.add_role(role)
+            self.roles[role.profile] = role
 
-    def publish_message(self, message: Message):
-        """向当前环境发布信息
-          Post information to the current environment
+        for role in roles:  # setup system message with roles
+            role.set_env(self)
+
+    def publish_message(self, message: Message, peekable: bool = True) -> bool:
         """
-        # self.message_queue.put(message)
-        self.memory.add(message)
-        self.history += f"\n{message}"
+        Distribute the message to the recipients.
+        In accordance with the Message routing structure design in Chapter 2.2.1 of RFC 116, as already planned
+        in RFC 113 for the entire system, the routing information in the Message is only responsible for
+        specifying the message recipient, without concern for where the message recipient is located. How to
+        route the message to the message recipient is a problem addressed by the transport framework designed
+        in RFC 113.
+        """
+        logger.debug(f"publish_message: {message.dump()}")
+        found = False
+        # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113
+        for role, subscription in self.members.items():
+            if is_subscribed(message, subscription):
+                role.put_message(message)
+                found = True
+        if not found:
+            logger.warning(f"Message no recipients: {message.dump()}")
+        self.history += f"\n{message}"  # For debug
+
+        return True
 
     async def run(self, k=1):
         """处理一次所有信息的运行
         Process all Role runs at once
         """
-        # while not self.message_queue.empty():
-        # message = self.message_queue.get()
-        # rsp = await self.manager.handle(message, self)
-        # self.message_queue.put(rsp)
         for _ in range(k):
             futures = []
             for role in self.roles.values():
@@ -65,15 +129,40 @@ class Environment(BaseModel):
                 futures.append(future)
 
             await asyncio.gather(*futures)
+            logger.debug(f"is idle: {self.is_idle}")
 
     def get_roles(self) -> dict[str, Role]:
         """获得环境内的所有角色
-           Process all Role runs at once
+        Process all Role runs at once
         """
         return self.roles
 
     def get_role(self, name: str) -> Role:
         """获得环境内的指定角色
-           get all the environment roles
+        get all the environment roles
         """
         return self.roles.get(name, None)
+
+    def role_names(self) -> list[str]:
+        return [i.name for i in self.roles.values()]
+
+    @property
+    def is_idle(self):
+        """If true, all actions have been executed."""
+        for r in self.roles.values():
+            if not r.is_idle:
+                return False
+        return True
+
+    def get_subscription(self, obj):
+        """Get the labels for messages to be consumed by the object."""
+        return self.members.get(obj, {})
+
+    def set_subscription(self, obj, tags):
+        """Set the labels for message to be consumed by the object"""
+        self.members[obj] = tags
+
+    @staticmethod
+    def archive(auto_archive=True):
+        if auto_archive and CONFIG.git_repo:
+            CONFIG.git_repo.archive()

metagpt/inspect_module.py

@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/28 14:54
-@Author  : alexanderwu
-@File    : inspect_module.py
-"""
-
-import inspect
-
-import metagpt  # replace with your module
-
-
-def print_classes_and_functions(module):
-    """FIXME: NOT WORK.. """
-    for name, obj in inspect.getmembers(module):
-        if inspect.isclass(obj):
-            print(f'Class: {name}')
-        elif inspect.isfunction(obj):
-            print(f'Function: {name}')
-        else:
-            print(name)
-
-    print(dir(module))
-
-
-if __name__ == '__main__':
-    print_classes_and_functions(metagpt)

metagpt/learn/__init__.py

@@ -5,3 +5,9 @@
 @Author  : alexanderwu
 @File    : __init__.py
 """
+
+from metagpt.learn.text_to_image import text_to_image
+from metagpt.learn.text_to_speech import text_to_speech
+from metagpt.learn.google_search import google_search
+
+__all__ = ["text_to_image", "text_to_speech", "google_search"]

metagpt/learn/google_search.py

@@ -0,0 +1,12 @@
+from metagpt.tools.search_engine import SearchEngine
+
+
+async def google_search(query: str, max_results: int = 6, **kwargs):
+    """Perform a web search and retrieve search results.
+
+    :param query: The search query.
+    :param max_results: The number of search results to retrieve
+    :return: The web search results in markdown format.
+    """
+    results = await SearchEngine().run(query, max_results=max_results, as_string=False)
+    return "\n".join(f"{i}. [{j['title']}]({j['link']}): {j['snippet']}" for i, j in enumerate(results, 1))

metagpt/learn/skill_loader.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/8/18
+@Author  : mashenquan
+@File    : skill_loader.py
+@Desc    : Skill YAML Configuration Loader.
+"""
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import aiofiles
+import yaml
+from pydantic import BaseModel, Field
+
+from metagpt.config import CONFIG
+
+
+class Example(BaseModel):
+    ask: str
+    answer: str
+
+
+class Returns(BaseModel):
+    type: str
+    format: Optional[str] = None
+
+
+class Parameter(BaseModel):
+    type: str
+    description: str = None
+
+
+class Skill(BaseModel):
+    name: str
+    description: str = None
+    id: str = None
+    x_prerequisite: Dict = Field(default=None, alias="x-prerequisite")
+    parameters: Dict[str, Parameter] = None
+    examples: List[Example]
+    returns: Returns
+
+    @property
+    def arguments(self) -> Dict:
+        if not self.parameters:
+            return {}
+        ret = {}
+        for k, v in self.parameters.items():
+            ret[k] = v.description if v.description else ""
+        return ret
+
+
+class Entity(BaseModel):
+    name: str = None
+    skills: List[Skill]
+
+
+class Components(BaseModel):
+    pass
+
+
+class SkillsDeclaration(BaseModel):
+    skillapi: str
+    entities: Dict[str, Entity]
+    components: Components = None
+
+    @staticmethod
+    async def load(skill_yaml_file_name: Path = None) -> "SkillsDeclaration":
+        if not skill_yaml_file_name:
+            skill_yaml_file_name = Path(__file__).parent.parent.parent / "docs/.well-known/skills.yaml"
+        async with aiofiles.open(str(skill_yaml_file_name), mode="r") as reader:
+            data = await reader.read(-1)
+        skill_data = yaml.safe_load(data)
+        return SkillsDeclaration(**skill_data)
+
+    def get_skill_list(self, entity_name: str = "Assistant") -> Dict:
+        """Return the skill name based on the skill description."""
+        entity = self.entities.get(entity_name)
+        if not entity:
+            return {}
+
+        # List of skills that the agent chooses to activate.
+        agent_skills = CONFIG.agent_skills
+        if not agent_skills:
+            return {}
+
+        class _AgentSkill(BaseModel):
+            name: str
+
+        names = [_AgentSkill(**i).name for i in agent_skills]
+        return {s.description: s.name for s in entity.skills if s.name in names}
+
+    def get_skill(self, name, entity_name: str = "Assistant") -> Skill:
+        """Return a skill by name."""
+        entity = self.entities.get(entity_name)
+        if not entity:
+            return None
+        for sk in entity.skills:
+            if sk.name == name:
+                return sk

metagpt/learn/text_to_embedding.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/8/18
+@Author  : mashenquan
+@File    : text_to_embedding.py
+@Desc    : Text-to-Embedding skill, which provides text-to-embedding functionality.
+"""
+
+from metagpt.config import CONFIG
+from metagpt.tools.openai_text_to_embedding import oas3_openai_text_to_embedding
+
+
+async def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key="", **kwargs):
+    """Text to embedding
+
+    :param text: The text used for embedding.
+    :param model: One of ['text-embedding-ada-002'], ID of the model to use. For more details, checkout: `https://api.openai.com/v1/models`.
+    :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`
+    :return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`.
+    """
+    if CONFIG.OPENAI_API_KEY or openai_api_key:
+        return await oas3_openai_text_to_embedding(text, model=model, openai_api_key=openai_api_key)
+    raise EnvironmentError

metagpt/learn/text_to_image.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/8/18
+@Author  : mashenquan
+@File    : text_to_image.py
+@Desc    : Text-to-Image skill, which provides text-to-image functionality.
+"""
+import base64
+
+from metagpt.config import CONFIG
+from metagpt.const import BASE64_FORMAT
+from metagpt.tools.metagpt_text_to_image import oas3_metagpt_text_to_image
+from metagpt.tools.openai_text_to_image import oas3_openai_text_to_image
+from metagpt.utils.s3 import S3
+
+
+async def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url="", **kwargs):
+    """Text to image
+
+    :param text: The text used for image conversion.
+    :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`
+    :param size_type: If using OPENAI, the available size options are ['256x256', '512x512', '1024x1024'], while for MetaGPT, the options are ['512x512', '512x768'].
+    :param model_url: MetaGPT model url
+    :return: The image data is returned in Base64 encoding.
+    """
+    image_declaration = "data:image/png;base64,"
+    if CONFIG.METAGPT_TEXT_TO_IMAGE_MODEL_URL or model_url:
+        binary_data = await oas3_metagpt_text_to_image(text, size_type, model_url)
+    elif CONFIG.OPENAI_API_KEY or openai_api_key:
+        binary_data = await oas3_openai_text_to_image(text, size_type)
+    else:
+        raise ValueError("Missing necessary parameters.")
+    base64_data = base64.b64encode(binary_data).decode("utf-8")
+
+    s3 = S3()
+    url = await s3.cache(data=base64_data, file_ext=".png", format=BASE64_FORMAT) if s3.is_valid else ""
+    if url:
+        return f"![{text}]({url})"
+    return image_declaration + base64_data if base64_data else ""

metagpt/learn/text_to_speech.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/8/17
+@Author  : mashenquan
+@File    : text_to_speech.py
+@Desc    : Text-to-Speech skill, which provides text-to-speech functionality
+"""
+
+from metagpt.config import CONFIG
+from metagpt.const import BASE64_FORMAT
+from metagpt.tools.azure_tts import oas3_azsure_tts
+from metagpt.tools.iflytek_tts import oas3_iflytek_tts
+from metagpt.utils.s3 import S3
+
+
+async def text_to_speech(
+    text,
+    lang="zh-CN",
+    voice="zh-CN-XiaomoNeural",
+    style="affectionate",
+    role="Girl",
+    subscription_key="",
+    region="",
+    iflytek_app_id="",
+    iflytek_api_key="",
+    iflytek_api_secret="",
+    **kwargs,
+):
+    """Text to speech
+    For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`
+
+    :param lang: The value can contain a language code such as en (English), or a locale such as en-US (English - United States). For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`
+    :param voice: For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`, `https://speech.microsoft.com/portal/voicegallery`
+    :param style: Speaking style to express different emotions like cheerfulness, empathy, and calm. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`
+    :param role: With roles, the same voice can act as a different age and gender. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`
+    :param text: The text used for voice conversion.
+    :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint`
+    :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API.
+    :param iflytek_app_id: Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`
+    :param iflytek_api_key: WebAPI argument, see: `https://console.xfyun.cn/services/tts`
+    :param iflytek_api_secret: WebAPI argument, see: `https://console.xfyun.cn/services/tts`
+    :return: Returns the Base64-encoded .wav/.mp3 file data if successful, otherwise an empty string.
+
+    """
+
+    if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or (subscription_key and region):
+        audio_declaration = "data:audio/wav;base64,"
+        base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region)
+        s3 = S3()
+        url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) if s3.is_valid else ""
+        if url:
+            return f"[{text}]({url})"
+        return audio_declaration + base64_data if base64_data else base64_data
+    if (CONFIG.IFLYTEK_APP_ID and CONFIG.IFLYTEK_API_KEY and CONFIG.IFLYTEK_API_SECRET) or (
+        iflytek_app_id and iflytek_api_key and iflytek_api_secret
+    ):
+        audio_declaration = "data:audio/mp3;base64,"
+        base64_data = await oas3_iflytek_tts(
+            text=text, app_id=iflytek_app_id, api_key=iflytek_api_key, api_secret=iflytek_api_secret
+        )
+        s3 = S3()
+        url = await s3.cache(data=base64_data, file_ext=".mp3", format=BASE64_FORMAT) if s3.is_valid else ""
+        if url:
+            return f"[{text}]({url})"
+        return audio_declaration + base64_data if base64_data else base64_data
+
+    raise ValueError(
+        "AZURE_TTS_SUBSCRIPTION_KEY, AZURE_TTS_REGION, IFLYTEK_APP_ID, IFLYTEK_API_KEY, IFLYTEK_API_SECRET error"
+    )

metagpt/llm.py

@@ -6,27 +6,19 @@
 @File    : llm.py
 """
 
-from metagpt.logs import logger
-from metagpt.config import CONFIG
-from metagpt.provider.anthropic_api import Claude2 as Claude
-from metagpt.provider.openai_api import OpenAIGPTAPI
-from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI
-from metagpt.provider.spark_api import SparkAPI
+from typing import Optional
+
+from metagpt.config import CONFIG, LLMProviderEnum
+from metagpt.provider.base_llm import BaseLLM
 from metagpt.provider.human_provider import HumanProvider
+from metagpt.provider.llm_provider_registry import LLM_REGISTRY
+
+_ = HumanProvider()  # Avoid pre-commit error
 
 
-def LLM() -> "BaseGPTAPI":
-    """ initialize different LLM instance according to the key field existence"""
-    # TODO a little trick, can use registry to initialize LLM instance further
-    if CONFIG.openai_api_key:
-        llm = OpenAIGPTAPI()
-    elif CONFIG.claude_api_key:
-        llm = Claude()
-    elif CONFIG.spark_api_key:
-        llm = SparkAPI()
-    elif CONFIG.zhipuai_api_key:
-        llm = ZhiPuAIGPTAPI()
-    else:
-        raise RuntimeError("You should config a LLM configuration first")
+def LLM(provider: Optional[LLMProviderEnum] = None) -> BaseLLM:
+    """get the default llm provider"""
+    if provider is None:
+        provider = CONFIG.get_default_llm_provider_enum()
 
-    return llm
+    return LLM_REGISTRY.get_provider(provider)

metagpt/logs.py

@@ -7,18 +7,35 @@
 """
 
 import sys
+from datetime import datetime
+from functools import partial
 
 from loguru import logger as _logger
 
-from metagpt.const import PROJECT_ROOT
+from metagpt.const import METAGPT_ROOT
+
 
 def define_log_level(print_level="INFO", logfile_level="DEBUG"):
-    """调整日志级别到level之上
-       Adjust the log level to above level
-    """
+    """Adjust the log level to above level"""
+    current_date = datetime.now()
+    formatted_date = current_date.strftime("%Y%m%d")
+
     _logger.remove()
     _logger.add(sys.stderr, level=print_level)
-    _logger.add(PROJECT_ROOT / 'logs/log.txt', level=logfile_level)
+    _logger.add(METAGPT_ROOT / f"logs/{formatted_date}.txt", level=logfile_level)
     return _logger
 
+
 logger = define_log_level()
+
+
+def log_llm_stream(msg):
+    _llm_stream_log(msg)
+
+
+def set_llm_stream_logfunc(func):
+    global _llm_stream_log
+    _llm_stream_log = func
+
+
+_llm_stream_log = partial(print, end="")

metagpt/management/skill_manager.py

@@ -4,11 +4,11 @@
 @Time    : 2023/6/5 01:44
 @Author  : alexanderwu
 @File    : skill_manager.py
+@Modified By: mashenquan, 2023/8/20. Remove useless `llm`
 """
 from metagpt.actions import Action
 from metagpt.const import PROMPT_PATH
 from metagpt.document_store.chromadb_store import ChromaStore
-from metagpt.llm import LLM
 from metagpt.logs import logger
 
 Skill = Action
@@ -18,9 +18,8 @@ class SkillManager:
     """Used to manage all skills"""
 
     def __init__(self):
-        self._llm = LLM()
-        self._store = ChromaStore('skill_manager')
-        self._skills: dict[str: Skill] = {}
+        self._store = ChromaStore("skill_manager")
+        self._skills: dict[str:Skill] = {}
 
     def add_skill(self, skill: Skill):
         """
@@ -29,7 +28,7 @@ class SkillManager:
         :return:
         """
         self._skills[skill.name] = skill
-        self._store.add(skill.desc, {}, skill.name)
+        self._store.add(skill.desc, {"name": skill.name, "desc": skill.desc}, skill.name)
 
     def del_skill(self, skill_name: str):
         """
@@ -54,7 +53,7 @@ class SkillManager:
         :param desc: Skill description
         :return: Multiple skills
         """
-        return self._store.search(desc, n_results=n_results)['ids'][0]
+        return self._store.search(desc, n_results=n_results)["ids"][0]
 
     def retrieve_skill_scored(self, desc: str, n_results: int = 2) -> dict:
         """
@@ -75,6 +74,6 @@ class SkillManager:
         logger.info(text)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     manager = SkillManager()
     manager.generate_skill_desc(Action())

metagpt/manager.py

@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/11 14:42
-@Author  : alexanderwu
-@File    : manager.py
-"""
-from metagpt.llm import LLM
-from metagpt.logs import logger
-from metagpt.schema import Message
-
-
-class Manager:
-    def __init__(self, llm: LLM = LLM()):
-        self.llm = llm  # Large Language Model
-        self.role_directions = {
-            "BOSS": "Product Manager",
-            "Product Manager": "Architect",
-            "Architect": "Engineer",
-            "Engineer": "QA Engineer",
-            "QA Engineer": "Product Manager"
-        }
-        self.prompt_template = """
-        Given the following message:
-        {message}
-
-        And the current status of roles:
-        {roles}
-
-        Which role should handle this message?
-        """
-
-    async def handle(self, message: Message, environment):
-        """
-        管理员处理信息，现在简单的将信息递交给下一个人
-        The administrator processes the information, now simply passes the information on to the next person
-        :param message:
-        :param environment:
-        :return:
-        """
-        # Get all roles from the environment
-        roles = environment.get_roles()
-        # logger.debug(f"{roles=}, {message=}")
-
-        # Build a context for the LLM to understand the situation
-        # context = {
-        #     "message": str(message),
-        #     "roles": {role.name: role.get_info() for role in roles},
-        # }
-        # Ask the LLM to decide which role should handle the message
-        # chosen_role_name = self.llm.ask(self.prompt_template.format(context))
-
-        # FIXME: 现在通过简单的字典决定流向，但之后还是应该有思考过程
-        #The direction of flow is now determined by a simple dictionary, but there should still be a thought process afterwards
-        next_role_profile = self.role_directions[message.role]
-        # logger.debug(f"{next_role_profile}")
-        for _, role in roles.items():
-            if next_role_profile == role.profile:
-                next_role = role
-                break
-        else:
-            logger.error(f"No available role can handle message: {message}.")
-            return
-
-        # Find the chosen role and handle the message
-        return await next_role.handle(message)

metagpt/memory/__init__.py

@@ -7,10 +7,11 @@
 """
 
 from metagpt.memory.memory import Memory
-from metagpt.memory.longterm_memory import LongTermMemory
+
+# from metagpt.memory.longterm_memory import LongTermMemory
 
 
 __all__ = [
     "Memory",
-    "LongTermMemory",
+    # "LongTermMemory",
 ]

metagpt/memory/brain_memory.py

@@ -0,0 +1,331 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/8/18
+@Author  : mashenquan
+@File    : brain_memory.py
+@Desc    : Used by AgentStore. Used for long-term storage and automatic compression.
+@Modified By: mashenquan, 2023/9/4. + redis memory cache.
+@Modified By: mashenquan, 2023/12/25. Simplify Functionality.
+"""
+import json
+import re
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+from metagpt.config import CONFIG
+from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS, DEFAULT_TOKEN_SIZE
+from metagpt.logs import logger
+from metagpt.provider import MetaGPTLLM
+from metagpt.provider.base_llm import BaseLLM
+from metagpt.schema import Message, SimpleMessage
+from metagpt.utils.redis import Redis
+
+
+class BrainMemory(BaseModel):
+    history: List[Message] = Field(default_factory=list)
+    knowledge: List[Message] = Field(default_factory=list)
+    historical_summary: str = ""
+    last_history_id: str = ""
+    is_dirty: bool = False
+    last_talk: str = None
+    cacheable: bool = True
+    llm: Optional[BaseLLM] = None
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def add_talk(self, msg: Message):
+        """
+        Add message from user.
+        """
+        msg.role = "user"
+        self.add_history(msg)
+        self.is_dirty = True
+
+    def add_answer(self, msg: Message):
+        """Add message from LLM"""
+        msg.role = "assistant"
+        self.add_history(msg)
+        self.is_dirty = True
+
+    def get_knowledge(self) -> str:
+        texts = [m.content for m in self.knowledge]
+        return "\n".join(texts)
+
+    @staticmethod
+    async def loads(redis_key: str) -> "BrainMemory":
+        redis = Redis()
+        if not redis.is_valid or not redis_key:
+            return BrainMemory()
+        v = await redis.get(key=redis_key)
+        logger.debug(f"REDIS GET {redis_key} {v}")
+        if v:
+            bm = BrainMemory.parse_raw(v)
+            bm.is_dirty = False
+            return bm
+        return BrainMemory()
+
+    async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60):
+        if not self.is_dirty:
+            return
+        redis = Redis()
+        if not redis.is_valid or not redis_key:
+            return False
+        v = self.model_dump_json()
+        if self.cacheable:
+            await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec)
+            logger.debug(f"REDIS SET {redis_key} {v}")
+        self.is_dirty = False
+
+    @staticmethod
+    def to_redis_key(prefix: str, user_id: str, chat_id: str):
+        return f"{prefix}:{user_id}:{chat_id}"
+
+    async def set_history_summary(self, history_summary, redis_key, redis_conf):
+        if self.historical_summary == history_summary:
+            if self.is_dirty:
+                await self.dumps(redis_key=redis_key)
+                self.is_dirty = False
+            return
+
+        self.historical_summary = history_summary
+        self.history = []
+        await self.dumps(redis_key=redis_key)
+        self.is_dirty = False
+
+    def add_history(self, msg: Message):
+        if msg.id:
+            if self.to_int(msg.id, 0) <= self.to_int(self.last_history_id, -1):
+                return
+
+        self.history.append(msg)
+        self.last_history_id = str(msg.id)
+        self.is_dirty = True
+
+    def exists(self, text) -> bool:
+        for m in reversed(self.history):
+            if m.content == text:
+                return True
+        return False
+
+    @staticmethod
+    def to_int(v, default_value):
+        try:
+            return int(v)
+        except:
+            return default_value
+
+    def pop_last_talk(self):
+        v = self.last_talk
+        self.last_talk = None
+        return v
+
+    async def summarize(self, llm, max_words=200, keep_language: bool = False, limit: int = -1, **kwargs):
+        if isinstance(llm, MetaGPTLLM):
+            return await self._metagpt_summarize(max_words=max_words)
+
+        self.llm = llm
+        return await self._openai_summarize(llm=llm, max_words=max_words, keep_language=keep_language, limit=limit)
+
+    async def _openai_summarize(self, llm, max_words=200, keep_language: bool = False, limit: int = -1):
+        texts = [self.historical_summary]
+        for m in self.history:
+            texts.append(m.content)
+        text = "\n".join(texts)
+
+        text_length = len(text)
+        if limit > 0 and text_length < limit:
+            return text
+        summary = await self._summarize(text=text, max_words=max_words, keep_language=keep_language, limit=limit)
+        if summary:
+            await self.set_history_summary(history_summary=summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS)
+            return summary
+        raise ValueError(f"text too long:{text_length}")
+
+    async def _metagpt_summarize(self, max_words=200):
+        if not self.history:
+            return ""
+
+        total_length = 0
+        msgs = []
+        for m in reversed(self.history):
+            delta = len(m.content)
+            if total_length + delta > max_words:
+                left = max_words - total_length
+                if left == 0:
+                    break
+                m.content = m.content[0:left]
+                msgs.append(m)
+                break
+            msgs.append(m)
+            total_length += delta
+        msgs.reverse()
+        self.history = msgs
+        self.is_dirty = True
+        await self.dumps(redis_key=CONFIG.REDIS_KEY)
+        self.is_dirty = False
+
+        return BrainMemory.to_metagpt_history_format(self.history)
+
+    @staticmethod
+    def to_metagpt_history_format(history) -> str:
+        mmsg = [SimpleMessage(role=m.role, content=m.content).model_dump() for m in history]
+        return json.dumps(mmsg, ensure_ascii=False)
+
+    async def get_title(self, llm, max_words=5, **kwargs) -> str:
+        """Generate text title"""
+        if isinstance(llm, MetaGPTLLM):
+            return self.history[0].content if self.history else "New"
+
+        summary = await self.summarize(llm=llm, max_words=500)
+
+        language = CONFIG.language or DEFAULT_LANGUAGE
+        command = f"Translate the above summary into a {language} title of less than {max_words} words."
+        summaries = [summary, command]
+        msg = "\n".join(summaries)
+        logger.debug(f"title ask:{msg}")
+        response = await llm.aask(msg=msg, system_msgs=[])
+        logger.debug(f"title rsp: {response}")
+        return response
+
+    async def is_related(self, text1, text2, llm):
+        if isinstance(llm, MetaGPTLLM):
+            return await self._metagpt_is_related(text1=text1, text2=text2, llm=llm)
+        return await self._openai_is_related(text1=text1, text2=text2, llm=llm)
+
+    @staticmethod
+    async def _metagpt_is_related(**kwargs):
+        return False
+
+    @staticmethod
+    async def _openai_is_related(text1, text2, llm, **kwargs):
+        command = (
+            f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there "
+            "any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear."
+        )
+        rsp = await llm.aask(msg=command, system_msgs=[])
+        result = True if "TRUE" in rsp else False
+        p2 = text2.replace("\n", "")
+        p1 = text1.replace("\n", "")
+        logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}\n")
+        return result
+
+    async def rewrite(self, sentence: str, context: str, llm):
+        if isinstance(llm, MetaGPTLLM):
+            return await self._metagpt_rewrite(sentence=sentence, context=context, llm=llm)
+        return await self._openai_rewrite(sentence=sentence, context=context, llm=llm)
+
+    @staticmethod
+    async def _metagpt_rewrite(sentence: str, **kwargs):
+        return sentence
+
+    @staticmethod
+    async def _openai_rewrite(sentence: str, context: str, llm):
+        command = (
+            f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly "
+            f"supplement or rewrite the following text in brief and clear:\n{sentence}"
+        )
+        rsp = await llm.aask(msg=command, system_msgs=[])
+        logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}\n")
+        return rsp
+
+    @staticmethod
+    def extract_info(input_string, pattern=r"\[([A-Z]+)\]:\s*(.+)"):
+        match = re.match(pattern, input_string)
+        if match:
+            return match.group(1), match.group(2)
+        else:
+            return None, input_string
+
+    @property
+    def is_history_available(self):
+        return bool(self.history or self.historical_summary)
+
+    @property
+    def history_text(self):
+        if len(self.history) == 0 and not self.historical_summary:
+            return ""
+        texts = [self.historical_summary] if self.historical_summary else []
+        for m in self.history[:-1]:
+            if isinstance(m, Dict):
+                t = Message(**m).content
+            elif isinstance(m, Message):
+                t = m.content
+            else:
+                continue
+            texts.append(t)
+
+        return "\n".join(texts)
+
+    async def _summarize(self, text: str, max_words=200, keep_language: bool = False, limit: int = -1) -> str:
+        max_token_count = DEFAULT_MAX_TOKENS
+        max_count = 100
+        text_length = len(text)
+        if limit > 0 and text_length < limit:
+            return text
+        summary = ""
+        while max_count > 0:
+            if text_length < max_token_count:
+                summary = await self._get_summary(text=text, max_words=max_words, keep_language=keep_language)
+                break
+
+            padding_size = 20 if max_token_count > 20 else 0
+            text_windows = self.split_texts(text, window_size=max_token_count - padding_size)
+            part_max_words = min(int(max_words / len(text_windows)) + 1, 100)
+            summaries = []
+            for ws in text_windows:
+                response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language)
+                summaries.append(response)
+            if len(summaries) == 1:
+                summary = summaries[0]
+                break
+
+            # Merged and retry
+            text = "\n".join(summaries)
+            text_length = len(text)
+
+            max_count -= 1  # safeguard
+        return summary
+
+    async def _get_summary(self, text: str, max_words=20, keep_language: bool = False):
+        """Generate text summary"""
+        if len(text) < max_words:
+            return text
+        if keep_language:
+            command = f".Translate the above content into a summary of less than {max_words} words in language of the content strictly."
+        else:
+            command = f"Translate the above content into a summary of less than {max_words} words."
+        msg = text + "\n\n" + command
+        logger.debug(f"summary ask:{msg}")
+        response = await self.llm.aask(msg=msg, system_msgs=[])
+        logger.debug(f"summary rsp: {response}")
+        return response
+
+    @staticmethod
+    def split_texts(text: str, window_size) -> List[str]:
+        """Splitting long text into sliding windows text"""
+        if window_size <= 0:
+            window_size = DEFAULT_TOKEN_SIZE
+        total_len = len(text)
+        if total_len <= window_size:
+            return [text]
+
+        padding_size = 20 if window_size > 20 else 0
+        windows = []
+        idx = 0
+        data_len = window_size - padding_size
+        while idx < total_len:
+            if window_size + idx > total_len:  # 不足一个滑窗
+                windows.append(text[idx:])
+                break
+            # 每个窗口少算padding_size自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....]
+            # window_size=3, padding_size=1：
+            # [1, 2, 3], [3, 4, 5], [5, 6, 7], ....
+            #   idx=2,  |  idx=5   |  idx=8  | ...
+            w = text[idx : idx + window_size]
+            windows.append(w)
+            idx += data_len
+
+        return windows

metagpt/memory/longterm_memory.py

@@ -1,10 +1,18 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# @Desc   : the implement of Long-term memory
+"""
+@Desc   : the implement of Long-term memory
+@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation.
+"""
+
+from typing import Optional
+
+from pydantic import ConfigDict, Field
 
 from metagpt.logs import logger
 from metagpt.memory import Memory
 from metagpt.memory.memory_storage import MemoryStorage
+from metagpt.roles.role import RoleContext
 from metagpt.schema import Message
 
 
@@ -15,27 +23,27 @@ class LongTermMemory(Memory):
     - update memory when it changed
     """
 
-    def __init__(self):
-        self.memory_storage: MemoryStorage = MemoryStorage()
-        super(LongTermMemory, self).__init__()
-        self.rc = None  # RoleContext
-        self.msg_from_recover = False
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
-    def recover_memory(self, role_id: str, rc: "RoleContext"):
+    memory_storage: MemoryStorage = Field(default_factory=MemoryStorage)
+    rc: Optional[RoleContext] = None
+    msg_from_recover: bool = False
+
+    def recover_memory(self, role_id: str, rc: RoleContext):
         messages = self.memory_storage.recover_memory(role_id)
         self.rc = rc
         if not self.memory_storage.is_initialized:
             logger.warning(f"It may the first time to run Agent {role_id}, the long-term memory is empty")
         else:
             logger.warning(
-                f"Agent {role_id} has existed memory storage with {len(messages)} messages " f"and has recovered them."
+                f"Agent {role_id} has existing memory storage with {len(messages)} messages " f"and has recovered them."
             )
         self.msg_from_recover = True
         self.add_batch(messages)
         self.msg_from_recover = False
 
     def add(self, message: Message):
-        super(LongTermMemory, self).add(message)
+        super().add(message)
         for action in self.rc.watch:
             if message.cause_by == action and not self.msg_from_recover:
                 # currently, only add role's watching messages to its memory_storage
@@ -48,7 +56,7 @@ class LongTermMemory(Memory):
             1. find the short-term memory(stm) news
             2. furthermore, filter out similar messages based on ltm(long-term memory), get the final news
         """
-        stm_news = super(LongTermMemory, self).find_news(observed, k=k)  # shot-term memory news
+        stm_news = super().find_news(observed, k=k)  # shot-term memory news
         if not self.memory_storage.is_initialized:
             # memory_storage hasn't initialized, use default `find_news` to get stm_news
             return stm_news
@@ -62,10 +70,9 @@ class LongTermMemory(Memory):
         return ltm_news[-k:]
 
     def delete(self, message: Message):
-        super(LongTermMemory, self).delete(message)
+        super().delete(message)
         # TODO delete message in memory_storage
 
     def clear(self):
-        super(LongTermMemory, self).clear()
+        super().clear()
         self.memory_storage.clean()
-        

metagpt/memory/memory.py

@@ -4,24 +4,51 @@
 @Time    : 2023/5/20 12:15
 @Author  : alexanderwu
 @File    : memory.py
+@Modified By: mashenquan, 2023-11-1. According to RFC 116: Updated the type of index key.
 """
 from collections import defaultdict
-from typing import Iterable, Type
+from pathlib import Path
+from typing import DefaultDict, Iterable, Set
 
-from metagpt.actions import Action
+from pydantic import BaseModel, Field, SerializeAsAny
+
+from metagpt.const import IGNORED_MESSAGE_ID
 from metagpt.schema import Message
+from metagpt.utils.common import (
+    any_to_str,
+    any_to_str_set,
+    read_json_file,
+    write_json_file,
+)
 
 
-class Memory:
+class Memory(BaseModel):
     """The most basic memory: super-memory"""
 
-    def __init__(self):
-        """Initialize an empty storage list and an empty index dictionary"""
-        self.storage: list[Message] = []
-        self.index: dict[Type[Action], list[Message]] = defaultdict(list)
+    storage: list[SerializeAsAny[Message]] = []
+    index: DefaultDict[str, list[SerializeAsAny[Message]]] = Field(default_factory=lambda: defaultdict(list))
+    ignore_id: bool = False
+
+    def serialize(self, stg_path: Path):
+        """stg_path = ./storage/team/environment/ or ./storage/team/environment/roles/{role_class}_{role_name}/"""
+        memory_path = stg_path.joinpath("memory.json")
+        storage = self.model_dump()
+        write_json_file(memory_path, storage)
+
+    @classmethod
+    def deserialize(cls, stg_path: Path) -> "Memory":
+        """stg_path = ./storage/team/environment/ or ./storage/team/environment/roles/{role_class}_{role_name}/"""
+        memory_path = stg_path.joinpath("memory.json")
+
+        memory_dict = read_json_file(memory_path)
+        memory = Memory(**memory_dict)
+
+        return memory
 
     def add(self, message: Message):
         """Add a new message to storage, while updating the index"""
+        if self.ignore_id:
+            message.id = IGNORED_MESSAGE_ID
         if message in self.storage:
             return
         self.storage.append(message)
@@ -40,8 +67,20 @@ class Memory:
         """Return all messages containing a specified content"""
         return [message for message in self.storage if content in message.content]
 
+    def delete_newest(self) -> "Message":
+        """delete the newest message from the storage"""
+        if len(self.storage) > 0:
+            newest_msg = self.storage.pop()
+            if newest_msg.cause_by and newest_msg in self.index[newest_msg.cause_by]:
+                self.index[newest_msg.cause_by].remove(newest_msg)
+        else:
+            newest_msg = None
+        return newest_msg
+
     def delete(self, message: Message):
         """Delete the specified message from storage, while updating the index"""
+        if self.ignore_id:
+            message.id = IGNORED_MESSAGE_ID
         self.storage.remove(message)
         if message.cause_by and message in self.index[message.cause_by]:
             self.index[message.cause_by].remove(message)
@@ -73,16 +112,17 @@ class Memory:
             news.append(i)
         return news
 
-    def get_by_action(self, action: Type[Action]) -> list[Message]:
+    def get_by_action(self, action) -> list[Message]:
         """Return all messages triggered by a specified Action"""
-        return self.index[action]
+        index = any_to_str(action)
+        return self.index[index]
 
-    def get_by_actions(self, actions: Iterable[Type[Action]]) -> list[Message]:
+    def get_by_actions(self, actions: Set) -> list[Message]:
         """Return all messages triggered by specified Actions"""
         rsp = []
-        for action in actions:
+        indices = any_to_str_set(actions)
+        for action in indices:
             if action not in self.index:
                 continue
             rsp += self.index[action]
         return rsp
-    

metagpt/memory/memory_storage.py

@@ -1,17 +1,22 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# @Desc   : the implement of memory storage
+"""
+@Desc   : the implement of memory storage
+@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation.
+"""
 
-from typing import List
 from pathlib import Path
+from typing import Optional
 
+from langchain.embeddings import OpenAIEmbeddings
 from langchain.vectorstores.faiss import FAISS
+from langchain_core.embeddings import Embeddings
 
 from metagpt.const import DATA_PATH, MEM_TTL
+from metagpt.document_store.faiss_store import FaissStore
 from metagpt.logs import logger
 from metagpt.schema import Message
-from metagpt.utils.serialize import serialize_message, deserialize_message
-from metagpt.document_store.faiss_store import FaissStore
+from metagpt.utils.serialize import deserialize_message, serialize_message
 
 
 class MemoryStorage(FaissStore):
@@ -19,22 +24,32 @@ class MemoryStorage(FaissStore):
     The memory storage with Faiss as ANN search engine
     """
 
-    def __init__(self, mem_ttl: int = MEM_TTL):
+    def __init__(self, mem_ttl: int = MEM_TTL, embedding: Embeddings = None):
         self.role_id: str = None
         self.role_mem_path: str = None
         self.mem_ttl: int = mem_ttl  # later use
         self.threshold: float = 0.1  # experience value. TODO The threshold to filter similar memories
         self._initialized: bool = False
 
+        self.embedding = embedding or OpenAIEmbeddings()
         self.store: FAISS = None  # Faiss engine
 
     @property
     def is_initialized(self) -> bool:
         return self._initialized
 
-    def recover_memory(self, role_id: str) -> List[Message]:
+    def _load(self) -> Optional["FaissStore"]:
+        index_file, store_file = self._get_index_and_store_fname(index_ext=".faiss")  # langchain FAISS using .faiss
+
+        if not (index_file.exists() and store_file.exists()):
+            logger.info("Missing at least one of index_file/store_file, load failed and return None")
+            return None
+
+        return FAISS.load_local(self.role_mem_path, self.embedding, self.role_id)
+
+    def recover_memory(self, role_id: str) -> list[Message]:
         self.role_id = role_id
-        self.role_mem_path = Path(DATA_PATH / f'role_mem/{self.role_id}/')
+        self.role_mem_path = Path(DATA_PATH / f"role_mem/{self.role_id}/")
         self.role_mem_path.mkdir(parents=True, exist_ok=True)
 
         self.store = self._load()
@@ -49,20 +64,20 @@ class MemoryStorage(FaissStore):
 
         return messages
 
-    def _get_index_and_store_fname(self):
+    def _get_index_and_store_fname(self, index_ext=".index", pkl_ext=".pkl"):
         if not self.role_mem_path:
-            logger.error(f'You should call {self.__class__.__name__}.recover_memory fist when using LongTermMemory')
+            logger.error(f"You should call {self.__class__.__name__}.recover_memory fist when using LongTermMemory")
             return None, None
-        index_fpath = Path(self.role_mem_path / f'{self.role_id}.index')
-        storage_fpath = Path(self.role_mem_path / f'{self.role_id}.pkl')
+        index_fpath = Path(self.role_mem_path / f"{self.role_id}{index_ext}")
+        storage_fpath = Path(self.role_mem_path / f"{self.role_id}{pkl_ext}")
         return index_fpath, storage_fpath
 
     def persist(self):
-        super(MemoryStorage, self).persist()
-        logger.debug(f'Agent {self.role_id} persist memory into local')
+        self.store.save_local(self.role_mem_path, self.role_id)
+        logger.debug(f"Agent {self.role_id} persist memory into local")
 
     def add(self, message: Message) -> bool:
-        """ add message into memory storage"""
+        """add message into memory storage"""
         docs = [message.content]
         metadatas = [{"message_ser": serialize_message(message)}]
         if not self.store:
@@ -74,15 +89,12 @@ class MemoryStorage(FaissStore):
         self.persist()
         logger.info(f"Agent {self.role_id}'s memory_storage add a message")
 
-    def search_dissimilar(self, message: Message, k=4) -> List[Message]:
+    def search_dissimilar(self, message: Message, k=4) -> list[Message]:
         """search for dissimilar messages"""
         if not self.store:
             return []
 
-        resp = self.store.similarity_search_with_score(
-            query=message.content,
-            k=k
-        )
+        resp = self.store.similarity_search_with_score(query=message.content, k=k)
         # filter the result which score is smaller than the threshold
         filtered_resp = []
         for item, score in resp:
@@ -104,4 +116,3 @@ class MemoryStorage(FaissStore):
 
         self.store = None
         self._initialized = False
-        

metagpt/prompts/decompose.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/30 10:09
-@Author  : alexanderwu
-@File    : decompose.py
-"""
-
-DECOMPOSE_SYSTEM = """SYSTEM:
-You serve as an assistant that helps me play Minecraft.
-I will give you my goal in the game, please break it down as a tree-structure plan to achieve this goal.
-The requirements of the tree-structure plan are:
-1. The plan tree should be exactly of depth 2.
-2. Describe each step in one line.
-3. You should index the two levels like ’1.’, ’1.1.’, ’1.2.’, ’2.’, ’2.1.’, etc.
-4. The sub-goals at the bottom level should be basic actions so that I can easily execute them in the game.
-"""
-
-
-DECOMPOSE_USER = """USER:
-The goal is to {goal description}. Generate the plan according to the requirements.
-"""

metagpt/prompts/generate_skill.md

@@ -10,7 +10,7 @@
 from typing import Optional
 from abc import ABC
 from metagpt.llm import LLM # Large language model, similar to GPT
-n
+
 class Action(ABC):
     def __init__(self, name='', context=None, llm: LLM = LLM()):
         self.name = name

metagpt/prompts/invoice_ocr.py

@@ -10,7 +10,9 @@
 
 COMMON_PROMPT = "Now I will provide you with the OCR text recognition results for the invoice."
 
-EXTRACT_OCR_MAIN_INFO_PROMPT = COMMON_PROMPT + """
+EXTRACT_OCR_MAIN_INFO_PROMPT = (
+    COMMON_PROMPT
+    + """
 Please extract the payee, city, total cost, and invoicing date of the invoice.
 
 The OCR data of the invoice are as follows:
@@ -22,8 +24,11 @@ Mandatory restrictions are returned according to the following requirements:
 2. The returned JSON dictionary must be returned in {language}
 3. Mandatory requirement to output in JSON format: {{"收款人":"x","城市":"x","总费用/元":"","开票日期":""}}.
 """
+)
 
-REPLY_OCR_QUESTION_PROMPT = COMMON_PROMPT + """
+REPLY_OCR_QUESTION_PROMPT = (
+    COMMON_PROMPT
+    + """
 Please answer the question: {query}
 
 The OCR data of the invoice are as follows:
@@ -34,6 +39,6 @@ Mandatory restrictions are returned according to the following requirements:
 2. Enforce restrictions on not returning OCR data sent to you.
 3. Return with markdown syntax layout.
 """
+)
 
 INVOICE_OCR_SUCCESS = "Successfully completed OCR text recognition invoice."
-

metagpt/prompts/sales.py

@@ -54,10 +54,12 @@ Conversation history:
 {salesperson_name}: 
 """
 
-conversation_stages = {'1' : "Introduction: Start the conversation by introducing yourself and your company. Be polite and respectful while keeping the tone of the conversation professional. Your greeting should be welcoming. Always clarify in your greeting the reason why you are contacting the prospect.",
-'2': "Qualification: Qualify the prospect by confirming if they are the right person to talk to regarding your product/service. Ensure that they have the authority to make purchasing decisions.",
-'3': "Value proposition: Briefly explain how your product/service can benefit the prospect. Focus on the unique selling points and value proposition of your product/service that sets it apart from competitors.",
-'4': "Needs analysis: Ask open-ended questions to uncover the prospect's needs and pain points. Listen carefully to their responses and take notes.",
-'5': "Solution presentation: Based on the prospect's needs, present your product/service as the solution that can address their pain points.",
-'6': "Objection handling: Address any objections that the prospect may have regarding your product/service. Be prepared to provide evidence or testimonials to support your claims.",
-'7': "Close: Ask for the sale by proposing a next step. This could be a demo, a trial or a meeting with decision-makers. Ensure to summarize what has been discussed and reiterate the benefits."}
+conversation_stages = {
+    "1": "Introduction: Start the conversation by introducing yourself and your company. Be polite and respectful while keeping the tone of the conversation professional. Your greeting should be welcoming. Always clarify in your greeting the reason why you are contacting the prospect.",
+    "2": "Qualification: Qualify the prospect by confirming if they are the right person to talk to regarding your product/service. Ensure that they have the authority to make purchasing decisions.",
+    "3": "Value proposition: Briefly explain how your product/service can benefit the prospect. Focus on the unique selling points and value proposition of your product/service that sets it apart from competitors.",
+    "4": "Needs analysis: Ask open-ended questions to uncover the prospect's needs and pain points. Listen carefully to their responses and take notes.",
+    "5": "Solution presentation: Based on the prospect's needs, present your product/service as the solution that can address their pain points.",
+    "6": "Objection handling: Address any objections that the prospect may have regarding your product/service. Be prepared to provide evidence or testimonials to support your claims.",
+    "7": "Close: Ask for the sale by proposing a next step. This could be a demo, a trial or a meeting with decision-makers. Ensure to summarize what has been discussed and reiterate the benefits.",
+}

metagpt/prompts/structure_action.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/30 10:12
-@Author  : alexanderwu
-@File    : structure_action.py
-"""
-
-ACTION_SYSTEM = """SYSTEM:
-You serve as an assistant that helps me play Minecraft.
-I will give you a sentence. Please convert this sentence into one or several actions according to the following instructions.
-Each action should be a tuple of four items, written in the form (’verb’, ’object’, ’tools’, ’materials’)
-’verb’ is the verb of this action.
-’object’ refers to the target object of the action.
-’tools’ specifies the tools required for the action.
-’material’ specifies the materials required for the action.
-If some of the items are not required, set them to be ’None’.
-"""
-
-ACTION_USER = """USER:
-The sentence is {sentence}. Generate the action tuple according to the requirements.
-"""

metagpt/prompts/structure_goal.py

@@ -1,46 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/30 09:51
-@Author  : alexanderwu
-@File    : structure_goal.py
-"""
-
-GOAL_SYSTEM = """SYSTEM:
-You are an assistant for the game Minecraft.
-I will give you some target object and some knowledge related to the object. Please write the obtaining of the object as a goal in the standard form.
-The standard form of the goal is as follows:
-{
-"object": "the name of the target object",
-"count": "the target quantity",
-"material": "the materials required for this goal, a dictionary in the form {material_name: material_quantity}. If no material is required, set it to None",
-"tool": "the tool used for this goal. If multiple tools can be used for this goal, only write the most basic one. If no tool is required, set it to None",
-"info": "the knowledge related to this goal"
-}
-The information I will give you:
-Target object: the name and the quantity of the target object
-Knowledge: some knowledge related to the object.
-Requirements:
-1. You must generate the goal based on the provided knowledge instead of purely depending on your own knowledge.
-2. The "info" should be as compact as possible, at most 3 sentences. The knowledge I give you may be raw texts from Wiki documents. Please extract and summarize important information instead of directly copying all the texts.
-Goal Example:
-{
-"object": "iron_ore",
-"count": 1,
-"material": None,
-"tool": "stone_pickaxe",
-"info": "iron ore is obtained by mining iron ore. iron ore is most found in level 53. iron ore can only be mined with a stone pickaxe or better; using a wooden or gold pickaxe will yield nothing."
-}
-{
-"object": "wooden_pickaxe",
-"count": 1,
-"material": {"planks": 3, "stick": 2},
-"tool": "crafting_table",
-"info": "wooden pickaxe can be crafted with 3 planks and 2 stick as the material and crafting table as the tool."
-}
-"""
-
-GOAL_USER = """USER:
-Target object: {object quantity} {object name}
-Knowledge: {related knowledge}
-"""

metagpt/prompts/tutorial_assistant.py

@@ -12,7 +12,9 @@ You are now a seasoned technical professional in the field of the internet.
 We need you to write a technical tutorial with the topic "{topic}".
 """
 
-DIRECTORY_PROMPT = COMMON_PROMPT + """
+DIRECTORY_PROMPT = (
+    COMMON_PROMPT
+    + """
 Please provide the specific table of contents for this tutorial, strictly following the following requirements:
 1. The output must be strictly in the specified language, {language}.
 2. Answer strictly in the dictionary format like {{"title": "xxx", "directory": [{{"dir 1": ["sub dir 1", "sub dir 2"]}}, {{"dir 2": ["sub dir 3", "sub dir 4"]}}]}}.
@@ -20,8 +22,11 @@ Please provide the specific table of contents for this tutorial, strictly follow
 4. Do not have extra spaces or line breaks.
 5. Each directory title has practical significance.
 """
+)
 
-CONTENT_PROMPT = COMMON_PROMPT + """
+CONTENT_PROMPT = (
+    COMMON_PROMPT
+    + """
 Now I will give you the module directory titles for the topic. 
 Please output the detailed principle content of this title in detail. 
 If there are code examples, please provide them according to standard code specifications. 
@@ -36,4 +41,5 @@ Strictly limit output according to the following requirements:
 3. The output must be strictly in the specified language, {language}.
 4. Do not have redundant output, including concluding remarks.
 5. Strict requirement not to output the topic "{topic}".
-"""
+"""
+)

metagpt/prompts/use_lib_sop.py

@@ -1,88 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/30 10:45
-@Author  : alexanderwu
-@File    : use_lib_sop.py
-"""
-
-SOP_SYSTEM = """SYSTEM:
-You serve as an assistant that helps me play the game Minecraft.
-I will give you a goal in the game. Please think of a plan to achieve the goal, and then write a sequence of actions to realize the plan. The requirements and instructions are as follows:
-1. You can only use the following functions. Don’t make plans purely based on your experience, think about how to use these functions.
-explore(object, strategy)
-Move around to find the object with the strategy: used to find objects including block items and entities. This action is finished once the object is visible (maybe at the distance).
-Augments:
-- object: a string, the object to explore.
-- strategy: a string, the strategy for exploration.
-approach(object)
-Move close to a visible object: used to approach the object you want to attack or mine. It may fail if the target object is not accessible.
-Augments:
-- object: a string, the object to approach.
-craft(object, materials, tool)
-Craft the object with the materials and tool: used for crafting new object that is not in the inventory or is not enough. The required materials must be in the inventory and will be consumed, and the newly crafted objects will be added to the inventory. The tools like the crafting table and furnace should be in the inventory and this action will directly use them. Don’t try to place or approach the crafting table or furnace, you will get failed since this action does not support using tools placed on the ground. You don’t need to collect the items after crafting. If the quantity you require is more than a unit, this action will craft the objects one unit by one unit. If the materials run out halfway through, this action will stop, and you will only get part of the objects you want that have been crafted.
-Augments:
-- object: a dict, whose key is the name of the object and value is the object quantity.
-- materials: a dict, whose keys are the names of the materials and values are the quantities.
-- tool: a string, the tool used for crafting. Set to null if no tool is required.
-mine(object, tool)
-Mine the object with the tool: can only mine the object within reach, cannot mine object from a distance. If there are enough objects within reach, this action will mine as many as you specify. The obtained objects will be added to the inventory.
-Augments:
-- object: a string, the object to mine.
-- tool: a string, the tool used for mining. Set to null if no tool is required.
-attack(object, tool)
-Attack the object with the tool: used to attack the object within reach. This action will keep track of and attack the object until it is killed.
-Augments:
-- object: a string, the object to attack.
-- tool: a string, the tool used for mining. Set to null if no tool is required.
-equip(object)
-Equip the object from the inventory: used to equip equipment, including tools, weapons, and armor. The object must be in the inventory and belong to the items for equipping.
-Augments:
-- object: a string, the object to equip.
-digdown(object, tool)
-Dig down to the y-level with the tool: the only action you can take if you want to go underground for mining some ore.
-Augments:
-- object: an int, the y-level (absolute y coordinate) to dig to.
-- tool: a string, the tool used for digging. Set to null if no tool is required.
-go_back_to_ground(tool)
-Go back to the ground from underground: the only action you can take for going back to the ground if you are underground.
-Augments:
-- tool: a string, the tool used for digging. Set to null if no tool is required.
-apply(object, tool)
-Apply the tool on the object: used for fetching water, milk, lava with the tool bucket, pooling water or lava to the object with the tool water bucket or lava bucket, shearing sheep with the tool shears, blocking attacks with the tool shield.
-Augments:
-- object: a string, the object to apply to.
-- tool: a string, the tool used to apply.
-2. You cannot define any new function. Note that the "Generated structures" world creation option is turned off.
-3. There is an inventory that stores all the objects I have. It is not an entity, but objects can be added to it or retrieved from it anytime at anywhere without specific actions. The mined or crafted objects will be added to this inventory, and the materials and tools to use are also from this inventory. Objects in the inventory can be directly used. Don’t write the code to obtain them. If you plan to use some object not in the inventory, you should first plan to obtain it. You can view the inventory as one of my states, and it is written in form of a dictionary whose keys are the name of the objects I have and the values are their quantities.
-4. You will get the following information about my current state:
-- inventory: a dict representing the inventory mentioned above, whose keys are the name of the objects and the values are their quantities
-- environment: a string including my surrounding biome, the y-level of my current location, and whether I am on the ground or underground
-Pay attention to this information. Choose the easiest way to achieve the goal conditioned on my current state. Do not provide options, always make the final decision.
-5. You must describe your thoughts on the plan in natural language at the beginning. After that, you should write all the actions together. The response should follow the format:
-{
-"explanation": "explain why the last action failed, set to null for the first planning",
-"thoughts": "Your thoughts on the plan in natural languag",
-"action_list": [
-{"name": "action name", "args": {"arg name": value}, "expectation": "describe the expected results of this action"},
-{"name": "action name", "args": {"arg name": value}, "expectation": "describe the expected results of this action"},
-{"name": "action name", "args": {"arg name": value}, "expectation": "describe the expected results of this action"}
-]
-}
-The action_list can contain arbitrary number of actions. The args of each action should correspond to the type mentioned in the Arguments part. Remember to add “‘dict“‘ at the beginning and the end of the dict. Ensure that you response can be parsed by Python json.loads
-6. I will execute your code step by step and give you feedback. If some action fails, I will stop at that action and will not execute its following actions. The feedback will include error messages about the failed action. At that time, you should replan and write the new code just starting from that failed action.
-"""
-
-
-SOP_USER = """USER:
-My current state:
-- inventory: {inventory}
-- environment: {environment}
-The goal is to {goal}.
-Here is one plan to achieve similar goal for reference: {reference plan}.
-Begin your plan. Remember to follow the response format.
-or Action {successful action} succeeded, and {feedback message}. Continue your
-plan. Do not repeat successful action. Remember to follow the response format.
-or Action {failed action} failed, because {feedback message}. Revise your plan from
-the failed action. Remember to follow the response format.
-"""

metagpt/provider/__init__.py

@@ -6,7 +6,22 @@
 @File    : __init__.py
 """
 
-from metagpt.provider.openai_api import OpenAIGPTAPI
+from metagpt.provider.fireworks_api import FireworksLLM
+from metagpt.provider.google_gemini_api import GeminiLLM
+from metagpt.provider.ollama_api import OllamaLLM
+from metagpt.provider.open_llm_api import OpenLLM
+from metagpt.provider.openai_api import OpenAILLM
+from metagpt.provider.zhipuai_api import ZhiPuAILLM
+from metagpt.provider.azure_openai_api import AzureOpenAILLM
+from metagpt.provider.metagpt_api import MetaGPTLLM
 
-
-__all__ = ["OpenAIGPTAPI"]
+__all__ = [
+    "FireworksLLM",
+    "GeminiLLM",
+    "OpenLLM",
+    "OpenAILLM",
+    "ZhiPuAILLM",
+    "AzureOpenAILLM",
+    "MetaGPTLLM",
+    "OllamaLLM",
+]

metagpt/provider/anthropic_api.py

@@ -7,14 +7,14 @@
 """
 
 import anthropic
-from anthropic import Anthropic
+from anthropic import Anthropic, AsyncAnthropic
 
 from metagpt.config import CONFIG
 
 
 class Claude2:
-    def ask(self, prompt):
-        client = Anthropic(api_key=CONFIG.claude_api_key)
+    def ask(self, prompt: str) -> str:
+        client = Anthropic(api_key=CONFIG.anthropic_api_key)
 
         res = client.completions.create(
             model="claude-2",
@@ -23,13 +23,12 @@ class Claude2:
         )
         return res.completion
 
-    async def aask(self, prompt):
-        client = Anthropic(api_key=CONFIG.claude_api_key)
+    async def aask(self, prompt: str) -> str:
+        aclient = AsyncAnthropic(api_key=CONFIG.anthropic_api_key)
 
-        res = client.completions.create(
+        res = await aclient.completions.create(
             model="claude-2",
             prompt=f"{anthropic.HUMAN_PROMPT} {prompt} {anthropic.AI_PROMPT}",
             max_tokens_to_sample=1000,
         )
         return res.completion
-    

metagpt/provider/azure_openai_api.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/5/5 23:08
+@Author  : alexanderwu
+@File    : openai.py
+@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation;
+            Change cost control from global to company level.
+@Modified By: mashenquan, 2023/11/21. Fix bug: ReadTimeout.
+@Modified By: mashenquan, 2023/12/1. Fix bug: Unclosed connection caused by openai 0.x.
+"""
+
+
+from openai import AsyncAzureOpenAI
+from openai._base_client import AsyncHttpxClientWrapper
+
+from metagpt.config import LLMProviderEnum
+from metagpt.provider.llm_provider_registry import register_provider
+from metagpt.provider.openai_api import OpenAILLM
+
+
+@register_provider(LLMProviderEnum.AZURE_OPENAI)
+class AzureOpenAILLM(OpenAILLM):
+    """
+    Check https://platform.openai.com/examples for examples
+    """
+
+    def _init_client(self):
+        kwargs = self._make_client_kwargs()
+        # https://learn.microsoft.com/zh-cn/azure/ai-services/openai/how-to/migration?tabs=python-new%2Cdalle-fix
+        self.aclient = AsyncAzureOpenAI(**kwargs)
+        self.model = self.config.DEPLOYMENT_NAME  # Used in _calc_usage & _cons_kwargs
+
+    def _make_client_kwargs(self) -> dict:
+        kwargs = dict(
+            api_key=self.config.OPENAI_API_KEY,
+            api_version=self.config.OPENAI_API_VERSION,
+            azure_endpoint=self.config.OPENAI_BASE_URL,
+        )
+
+        # to use proxy, openai v1 needs http_client
+        proxy_params = self._get_proxy_params()
+        if proxy_params:
+            kwargs["http_client"] = AsyncHttpxClientWrapper(**proxy_params)
+
+        return kwargs

metagpt/provider/base_chatbot.py

@@ -1,29 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/5 23:00
-@Author  : alexanderwu
-@File    : base_chatbot.py
-"""
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
-
-
-@dataclass
-class BaseChatbot(ABC):
-    """Abstract GPT class"""
-    mode: str = "API"
-    use_system_prompt: bool = True
-
-    @abstractmethod
-    def ask(self, msg: str) -> str:
-        """Ask GPT a question and get an answer"""
-
-    @abstractmethod
-    def ask_batch(self, msgs: list) -> str:
-        """Ask GPT multiple questions and get a series of answers"""
-
-    @abstractmethod
-    def ask_code(self, msgs: list) -> str:
-        """Ask GPT multiple questions and get a piece of code"""
-        

metagpt/provider/base_llm.py

@@ -3,19 +3,18 @@
 """
 @Time    : 2023/5/5 23:04
 @Author  : alexanderwu
-@File    : base_gpt_api.py
+@File    : base_llm.py
+@Desc    : mashenquan, 2023/8/22. + try catch
 """
 import json
-from abc import abstractmethod
+from abc import ABC, abstractmethod
 from typing import Optional
 
-from metagpt.logs import logger
-from metagpt.provider.base_chatbot import BaseChatbot
 
+class BaseLLM(ABC):
+    """LLM API abstract class, requiring all inheritors to provide a series of standard capabilities"""
 
-class BaseGPTAPI(BaseChatbot):
-    """GPT API abstract class, requiring all inheritors to provide a series of standard capabilities"""
-
+    use_system_prompt: bool = True
     system_prompt = "You are a helpful assistant."
 
     def _user_msg(self, msg: str) -> dict[str, str]:
@@ -33,68 +32,44 @@ class BaseGPTAPI(BaseChatbot):
     def _default_system_msg(self):
         return self._system_msg(self.system_prompt)
 
-    def ask(self, msg: str) -> str:
-        message = [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)]
-        rsp = self.completion(message)
-        return self.get_choice_text(rsp)
-
-    async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str:
+    async def aask(
+        self,
+        msg: str,
+        system_msgs: Optional[list[str]] = None,
+        format_msgs: Optional[list[dict[str, str]]] = None,
+        timeout=3,
+        stream=True,
+    ) -> str:
         if system_msgs:
-            message = self._system_msgs(system_msgs) + [self._user_msg(msg)] if self.use_system_prompt \
-                else [self._user_msg(msg)]
+            message = self._system_msgs(system_msgs)
         else:
-            message = [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt \
-                else [self._user_msg(msg)]
-        rsp = await self.acompletion_text(message, stream=True)
-        logger.debug(message)
-        # logger.debug(rsp)
+            message = [self._default_system_msg()] if self.use_system_prompt else []
+        if format_msgs:
+            message.extend(format_msgs)
+        message.append(self._user_msg(msg))
+        rsp = await self.acompletion_text(message, stream=stream, timeout=timeout)
         return rsp
 
     def _extract_assistant_rsp(self, context):
         return "\n".join([i["content"] for i in context if i["role"] == "assistant"])
 
-    def ask_batch(self, msgs: list) -> str:
-        context = []
-        for msg in msgs:
-            umsg = self._user_msg(msg)
-            context.append(umsg)
-            rsp = self.completion(context)
-            rsp_text = self.get_choice_text(rsp)
-            context.append(self._assistant_msg(rsp_text))
-        return self._extract_assistant_rsp(context)
-
-    async def aask_batch(self, msgs: list) -> str:
+    async def aask_batch(self, msgs: list, timeout=3) -> str:
         """Sequential questioning"""
         context = []
         for msg in msgs:
             umsg = self._user_msg(msg)
             context.append(umsg)
-            rsp_text = await self.acompletion_text(context)
+            rsp_text = await self.acompletion_text(context, timeout=timeout)
             context.append(self._assistant_msg(rsp_text))
         return self._extract_assistant_rsp(context)
 
-    def ask_code(self, msgs: list[str]) -> str:
+    async def aask_code(self, msgs: list[str], timeout=3) -> str:
         """FIXME: No code segment filtering has been done here, and all results are actually displayed"""
-        rsp_text = self.ask_batch(msgs)
-        return rsp_text
-
-    async def aask_code(self, msgs: list[str]) -> str:
-        """FIXME: No code segment filtering has been done here, and all results are actually displayed"""
-        rsp_text = await self.aask_batch(msgs)
+        rsp_text = await self.aask_batch(msgs, timeout=timeout)
         return rsp_text
 
     @abstractmethod
-    def completion(self, messages: list[dict]):
-        """All GPTAPIs are required to provide the standard OpenAI completion interface
-        [
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": "hello, show me python hello world code"},
-            # {"role": "assistant", "content": ...}, # If there is an answer in the history, also include it
-        ]
-        """
-
-    @abstractmethod
-    async def acompletion(self, messages: list[dict]):
+    async def acompletion(self, messages: list[dict], timeout=3):
         """Asynchronous version of completion
         All GPTAPIs are required to provide the standard OpenAI completion interface
         [
@@ -105,7 +80,7 @@ class BaseGPTAPI(BaseChatbot):
         """
 
     @abstractmethod
-    async def acompletion_text(self, messages: list[dict], stream=False) -> str:
+    async def acompletion_text(self, messages: list[dict], stream=False, timeout=3) -> str:
         """Asynchronous version of completion. Return str. Support stream-print"""
 
     def get_choice_text(self, rsp: dict) -> str:
@@ -141,7 +116,7 @@ class BaseGPTAPI(BaseChatbot):
         :return dict: return first function of choice, for exmaple,
             {'name': 'execute', 'arguments': '{\n  "language": "python",\n  "code": "print(\'Hello, World!\')"\n}'}
         """
-        return rsp.get("choices")[0]["message"]["tool_calls"][0]["function"].to_dict()
+        return rsp.get("choices")[0]["message"]["tool_calls"][0]["function"]
 
     def get_choice_function_arguments(self, rsp: dict) -> dict:
         """Required to provide the first function arguments of choice.

metagpt/provider/fireworks_api.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : fireworks.ai's api
+
+import re
+
+from openai import APIConnectionError, AsyncStream
+from openai.types import CompletionUsage
+from openai.types.chat import ChatCompletionChunk
+from tenacity import (
+    after_log,
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+
+from metagpt.config import CONFIG, Config, LLMProviderEnum
+from metagpt.logs import logger
+from metagpt.provider.llm_provider_registry import register_provider
+from metagpt.provider.openai_api import OpenAILLM, log_and_reraise
+from metagpt.utils.cost_manager import CostManager, Costs
+
+MODEL_GRADE_TOKEN_COSTS = {
+    "-1": {"prompt": 0.0, "completion": 0.0},  # abnormal condition
+    "16": {"prompt": 0.2, "completion": 0.8},  # 16 means model size <= 16B; 0.2 means $0.2/1M tokens
+    "80": {"prompt": 0.7, "completion": 2.8},  # 80 means 16B < model size <= 80B
+    "mixtral-8x7b": {"prompt": 0.4, "completion": 1.6},
+}
+
+
+class FireworksCostManager(CostManager):
+    def model_grade_token_costs(self, model: str) -> dict[str, float]:
+        def _get_model_size(model: str) -> float:
+            size = re.findall(".*-([0-9.]+)b", model)
+            size = float(size[0]) if len(size) > 0 else -1
+            return size
+
+        if "mixtral-8x7b" in model:
+            token_costs = MODEL_GRADE_TOKEN_COSTS["mixtral-8x7b"]
+        else:
+            model_size = _get_model_size(model)
+            if 0 < model_size <= 16:
+                token_costs = MODEL_GRADE_TOKEN_COSTS["16"]
+            elif 16 < model_size <= 80:
+                token_costs = MODEL_GRADE_TOKEN_COSTS["80"]
+            else:
+                token_costs = MODEL_GRADE_TOKEN_COSTS["-1"]
+        return token_costs
+
+    def update_cost(self, prompt_tokens: int, completion_tokens: int, model: str):
+        """
+        Refs to `https://app.fireworks.ai/pricing` **Developer pricing**
+        Update the total cost, prompt tokens, and completion tokens.
+
+        Args:
+        prompt_tokens (int): The number of tokens used in the prompt.
+        completion_tokens (int): The number of tokens used in the completion.
+        model (str): The model used for the API call.
+        """
+        self.total_prompt_tokens += prompt_tokens
+        self.total_completion_tokens += completion_tokens
+
+        token_costs = self.model_grade_token_costs(model)
+        cost = (prompt_tokens * token_costs["prompt"] + completion_tokens * token_costs["completion"]) / 1000000
+        self.total_cost += cost
+        max_budget = CONFIG.max_budget if CONFIG.max_budget else CONFIG.cost_manager.max_budget
+        logger.info(
+            f"Total running cost: ${self.total_cost:.4f} | Max budget: ${max_budget:.3f} | "
+            f"Current cost: ${cost:.4f}, prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}"
+        )
+        CONFIG.total_cost = self.total_cost
+
+
+@register_provider(LLMProviderEnum.FIREWORKS)
+class FireworksLLM(OpenAILLM):
+    def __init__(self):
+        self.config: Config = CONFIG
+        self.__init_fireworks()
+        self.auto_max_tokens = False
+        self._cost_manager = FireworksCostManager()
+
+    def __init_fireworks(self):
+        self.is_azure = False
+        self.rpm = int(self.config.get("RPM", 10))
+        self._init_client()
+        self.model = self.config.fireworks_api_model  # `self.model` should after `_make_client` to rewrite it
+
+    def _make_client_kwargs(self) -> dict:
+        kwargs = dict(api_key=self.config.fireworks_api_key, base_url=self.config.fireworks_api_base)
+        return kwargs
+
+    def _update_costs(self, usage: CompletionUsage):
+        if self.config.calc_usage and usage:
+            try:
+                # use FireworksCostManager not CONFIG.cost_manager
+                self._cost_manager.update_cost(usage.prompt_tokens, usage.completion_tokens, self.model)
+            except Exception as e:
+                logger.error(f"updating costs failed!, exp: {e}")
+
+    def get_costs(self) -> Costs:
+        return self._cost_manager.get_costs()
+
+    async def _achat_completion_stream(self, messages: list[dict]) -> str:
+        response: AsyncStream[ChatCompletionChunk] = await self.aclient.chat.completions.create(
+            **self._cons_kwargs(messages), stream=True
+        )
+
+        collected_content = []
+        usage = CompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
+        # iterate through the stream of events
+        async for chunk in response:
+            if chunk.choices:
+                choice = chunk.choices[0]
+                choice_delta = choice.delta
+                finish_reason = choice.finish_reason if hasattr(choice, "finish_reason") else None
+                if choice_delta.content:
+                    collected_content.append(choice_delta.content)
+                    print(choice_delta.content, end="")
+                if finish_reason:
+                    # fireworks api return usage when finish_reason is not None
+                    usage = CompletionUsage(**chunk.usage)
+
+        full_content = "".join(collected_content)
+        self._update_costs(usage)
+        return full_content
+
+    @retry(
+        wait=wait_random_exponential(min=1, max=60),
+        stop=stop_after_attempt(6),
+        after=after_log(logger, logger.level("WARNING").name),
+        retry=retry_if_exception_type(APIConnectionError),
+        retry_error_callback=log_and_reraise,
+    )
+    async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = 3) -> str:
+        """when streaming, print each token in place."""
+        if stream:
+            return await self._achat_completion_stream(messages)
+        rsp = await self._achat_completion(messages)
+        return self.get_choice_text(rsp)

metagpt/provider/general_api_base.py

@@ -0,0 +1,622 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : refs to openai 0.x sdk
+
+import asyncio
+import json
+import os
+import platform
+import re
+import sys
+import threading
+import time
+from contextlib import asynccontextmanager
+from enum import Enum
+from typing import (
+    AsyncGenerator,
+    AsyncIterator,
+    Dict,
+    Iterator,
+    Optional,
+    Tuple,
+    Union,
+    overload,
+)
+from urllib.parse import urlencode, urlsplit, urlunsplit
+
+import aiohttp
+import requests
+
+if sys.version_info >= (3, 8):
+    from typing import Literal
+else:
+    from typing_extensions import Literal
+
+import logging
+
+import openai
+from openai import version
+
+logger = logging.getLogger("openai")
+
+TIMEOUT_SECS = 600
+MAX_SESSION_LIFETIME_SECS = 180
+MAX_CONNECTION_RETRIES = 2
+
+# Has one attribute per thread, 'session'.
+_thread_context = threading.local()
+
+LLM_LOG = os.environ.get("LLM_LOG", "debug")
+
+
+class ApiType(Enum):
+    AZURE = 1
+    OPEN_AI = 2
+    AZURE_AD = 3
+
+    @staticmethod
+    def from_str(label):
+        if label.lower() == "azure":
+            return ApiType.AZURE
+        elif label.lower() in ("azure_ad", "azuread"):
+            return ApiType.AZURE_AD
+        elif label.lower() in ("open_ai", "openai"):
+            return ApiType.OPEN_AI
+        else:
+            raise openai.OpenAIError(
+                "The API type provided in invalid. Please select one of the supported API types: 'azure', 'azure_ad', 'open_ai'"
+            )
+
+
+api_key_to_header = (
+    lambda api, key: {"Authorization": f"Bearer {key}"}
+    if api in (ApiType.OPEN_AI, ApiType.AZURE_AD)
+    else {"api-key": f"{key}"}
+)
+
+
+def _console_log_level():
+    if LLM_LOG in ["debug", "info"]:
+        return LLM_LOG
+    else:
+        return None
+
+
+def log_debug(message, **params):
+    msg = logfmt(dict(message=message, **params))
+    if _console_log_level() == "debug":
+        print(msg, file=sys.stderr)
+    logger.debug(msg)
+
+
+def log_info(message, **params):
+    msg = logfmt(dict(message=message, **params))
+    if _console_log_level() in ["debug", "info"]:
+        print(msg, file=sys.stderr)
+    logger.info(msg)
+
+
+def log_warn(message, **params):
+    msg = logfmt(dict(message=message, **params))
+    print(msg, file=sys.stderr)
+    logger.warning(msg)
+
+
+def logfmt(props):
+    def fmt(key, val):
+        # Handle case where val is a bytes or bytesarray
+        if hasattr(val, "decode"):
+            val = val.decode("utf-8")
+        # Check if val is already a string to avoid re-encoding into ascii.
+        if not isinstance(val, str):
+            val = str(val)
+        if re.search(r"\s", val):
+            val = repr(val)
+        # key should already be a string
+        if re.search(r"\s", key):
+            key = repr(key)
+        return "{key}={val}".format(key=key, val=val)
+
+    return " ".join([fmt(key, val) for key, val in sorted(props.items())])
+
+
+class OpenAIResponse:
+    def __init__(self, data, headers):
+        self._headers = headers
+        self.data = data
+
+    @property
+    def request_id(self) -> Optional[str]:
+        return self._headers.get("request-id")
+
+    @property
+    def retry_after(self) -> Optional[int]:
+        try:
+            return int(self._headers.get("retry-after"))
+        except TypeError:
+            return None
+
+    @property
+    def operation_location(self) -> Optional[str]:
+        return self._headers.get("operation-location")
+
+    @property
+    def organization(self) -> Optional[str]:
+        return self._headers.get("LLM-Organization")
+
+    @property
+    def response_ms(self) -> Optional[int]:
+        h = self._headers.get("Openai-Processing-Ms")
+        return None if h is None else round(float(h))
+
+
+def _build_api_url(url, query):
+    scheme, netloc, path, base_query, fragment = urlsplit(url)
+
+    if base_query:
+        query = "%s&%s" % (base_query, query)
+
+    return urlunsplit((scheme, netloc, path, query, fragment))
+
+
+def _requests_proxies_arg(proxy) -> Optional[Dict[str, str]]:
+    """Returns a value suitable for the 'proxies' argument to 'requests.request."""
+    if proxy is None:
+        return None
+    elif isinstance(proxy, str):
+        return {"http": proxy, "https": proxy}
+    elif isinstance(proxy, dict):
+        return proxy.copy()
+    else:
+        raise ValueError(
+            "'openai.proxy' must be specified as either a string URL or a dict with string URL under the https and/or http keys."
+        )
+
+
+def _aiohttp_proxies_arg(proxy) -> Optional[str]:
+    """Returns a value suitable for the 'proxies' argument to 'aiohttp.ClientSession.request."""
+    if proxy is None:
+        return None
+    elif isinstance(proxy, str):
+        return proxy
+    elif isinstance(proxy, dict):
+        return proxy["https"] if "https" in proxy else proxy["http"]
+    else:
+        raise ValueError(
+            "'openai.proxy' must be specified as either a string URL or a dict with string URL under the https and/or http keys."
+        )
+
+
+def _make_session() -> requests.Session:
+    s = requests.Session()
+    s.mount(
+        "https://",
+        requests.adapters.HTTPAdapter(max_retries=MAX_CONNECTION_RETRIES),
+    )
+    return s
+
+
+def parse_stream_helper(line: bytes) -> Optional[str]:
+    if line:
+        if line.strip() == b"data: [DONE]":
+            # return here will cause GeneratorExit exception in urllib3
+            # and it will close http connection with TCP Reset
+            return None
+        if line.startswith(b"data: "):
+            line = line[len(b"data: ") :]
+            return line.decode("utf-8")
+        else:
+            return None
+    return None
+
+
+def parse_stream(rbody: Iterator[bytes]) -> Iterator[str]:
+    for line in rbody:
+        _line = parse_stream_helper(line)
+        if _line is not None:
+            yield _line
+
+
+async def parse_stream_async(rbody: aiohttp.StreamReader):
+    async for line in rbody:
+        _line = parse_stream_helper(line)
+        if _line is not None:
+            yield _line
+
+
+class APIRequestor:
+    def __init__(
+        self,
+        key=None,
+        base_url=None,
+        api_type=None,
+        api_version=None,
+        organization=None,
+    ):
+        self.base_url = base_url or openai.base_url
+        self.api_key = key or openai.api_key
+        self.api_type = ApiType.from_str(api_type) if api_type else ApiType.from_str("openai")
+        self.api_version = api_version or openai.api_version
+        self.organization = organization or openai.organization
+
+    @overload
+    def request(
+        self,
+        method,
+        url,
+        params,
+        headers,
+        files,
+        stream: Literal[True],
+        request_id: Optional[str] = ...,
+        request_timeout: Optional[Union[float, Tuple[float, float]]] = ...,
+    ) -> Tuple[Iterator[OpenAIResponse], bool, str]:
+        pass
+
+    @overload
+    def request(
+        self,
+        method,
+        url,
+        params=...,
+        headers=...,
+        files=...,
+        *,
+        stream: Literal[True],
+        request_id: Optional[str] = ...,
+        request_timeout: Optional[Union[float, Tuple[float, float]]] = ...,
+    ) -> Tuple[Iterator[OpenAIResponse], bool, str]:
+        pass
+
+    @overload
+    def request(
+        self,
+        method,
+        url,
+        params=...,
+        headers=...,
+        files=...,
+        stream: Literal[False] = ...,
+        request_id: Optional[str] = ...,
+        request_timeout: Optional[Union[float, Tuple[float, float]]] = ...,
+    ) -> Tuple[OpenAIResponse, bool, str]:
+        pass
+
+    @overload
+    def request(
+        self,
+        method,
+        url,
+        params=...,
+        headers=...,
+        files=...,
+        stream: bool = ...,
+        request_id: Optional[str] = ...,
+        request_timeout: Optional[Union[float, Tuple[float, float]]] = ...,
+    ) -> Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], bool, str]:
+        pass
+
+    def request(
+        self,
+        method,
+        url,
+        params=None,
+        headers=None,
+        files=None,
+        stream: bool = False,
+        request_id: Optional[str] = None,
+        request_timeout: Optional[Union[float, Tuple[float, float]]] = None,
+    ) -> Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], bool, str]:
+        result = self.request_raw(
+            method.lower(),
+            url,
+            params=params,
+            supplied_headers=headers,
+            files=files,
+            stream=stream,
+            request_id=request_id,
+            request_timeout=request_timeout,
+        )
+        resp, got_stream = self._interpret_response(result, stream)
+        return resp, got_stream, self.api_key
+
+    @overload
+    async def arequest(
+        self,
+        method,
+        url,
+        params,
+        headers,
+        files,
+        stream: Literal[True],
+        request_id: Optional[str] = ...,
+        request_timeout: Optional[Union[float, Tuple[float, float]]] = ...,
+    ) -> Tuple[AsyncGenerator[OpenAIResponse, None], bool, str]:
+        pass
+
+    @overload
+    async def arequest(
+        self,
+        method,
+        url,
+        params=...,
+        headers=...,
+        files=...,
+        *,
+        stream: Literal[True],
+        request_id: Optional[str] = ...,
+        request_timeout: Optional[Union[float, Tuple[float, float]]] = ...,
+    ) -> Tuple[AsyncGenerator[OpenAIResponse, None], bool, str]:
+        pass
+
+    @overload
+    async def arequest(
+        self,
+        method,
+        url,
+        params=...,
+        headers=...,
+        files=...,
+        stream: Literal[False] = ...,
+        request_id: Optional[str] = ...,
+        request_timeout: Optional[Union[float, Tuple[float, float]]] = ...,
+    ) -> Tuple[OpenAIResponse, bool, str]:
+        pass
+
+    @overload
+    async def arequest(
+        self,
+        method,
+        url,
+        params=...,
+        headers=...,
+        files=...,
+        stream: bool = ...,
+        request_id: Optional[str] = ...,
+        request_timeout: Optional[Union[float, Tuple[float, float]]] = ...,
+    ) -> Tuple[Union[OpenAIResponse, AsyncGenerator[OpenAIResponse, None]], bool, str]:
+        pass
+
+    async def arequest(
+        self,
+        method,
+        url,
+        params=None,
+        headers=None,
+        files=None,
+        stream: bool = False,
+        request_id: Optional[str] = None,
+        request_timeout: Optional[Union[float, Tuple[float, float]]] = None,
+    ) -> Tuple[Union[OpenAIResponse, AsyncGenerator[OpenAIResponse, None]], bool, str]:
+        ctx = aiohttp_session()
+        session = await ctx.__aenter__()
+        try:
+            result = await self.arequest_raw(
+                method.lower(),
+                url,
+                session,
+                params=params,
+                supplied_headers=headers,
+                files=files,
+                request_id=request_id,
+                request_timeout=request_timeout,
+            )
+            resp, got_stream = await self._interpret_async_response(result, stream)
+        except Exception:
+            await ctx.__aexit__(None, None, None)
+            raise
+        if got_stream:
+
+            async def wrap_resp():
+                assert isinstance(resp, AsyncGenerator)
+                try:
+                    async for r in resp:
+                        yield r
+                finally:
+                    await ctx.__aexit__(None, None, None)
+
+            return wrap_resp(), got_stream, self.api_key
+        else:
+            await ctx.__aexit__(None, None, None)
+            return resp, got_stream, self.api_key
+
+    def request_headers(self, method: str, extra, request_id: Optional[str]) -> Dict[str, str]:
+        user_agent = "LLM/v1 PythonBindings/%s" % (version.VERSION,)
+
+        uname_without_node = " ".join(v for k, v in platform.uname()._asdict().items() if k != "node")
+        ua = {
+            "bindings_version": version.VERSION,
+            "httplib": "requests",
+            "lang": "python",
+            "lang_version": platform.python_version(),
+            "platform": platform.platform(),
+            "publisher": "openai",
+            "uname": uname_without_node,
+        }
+
+        headers = {
+            "X-LLM-Client-User-Agent": json.dumps(ua),
+            "User-Agent": user_agent,
+        }
+
+        headers.update(api_key_to_header(self.api_type, self.api_key))
+
+        if self.organization:
+            headers["LLM-Organization"] = self.organization
+
+        if self.api_version is not None and self.api_type == ApiType.OPEN_AI:
+            headers["LLM-Version"] = self.api_version
+        if request_id is not None:
+            headers["X-Request-Id"] = request_id
+        headers.update(extra)
+
+        return headers
+
+    def _validate_headers(self, supplied_headers: Optional[Dict[str, str]]) -> Dict[str, str]:
+        headers: Dict[str, str] = {}
+        if supplied_headers is None:
+            return headers
+
+        if not isinstance(supplied_headers, dict):
+            raise TypeError("Headers must be a dictionary")
+
+        for k, v in supplied_headers.items():
+            if not isinstance(k, str):
+                raise TypeError("Header keys must be strings")
+            if not isinstance(v, str):
+                raise TypeError("Header values must be strings")
+            headers[k] = v
+
+        # NOTE: It is possible to do more validation of the headers, but a request could always
+        # be made to the API manually with invalid headers, so we need to handle them server side.
+
+        return headers
+
+    def _prepare_request_raw(
+        self,
+        url,
+        supplied_headers,
+        method,
+        params,
+        files,
+        request_id: Optional[str],
+    ) -> Tuple[str, Dict[str, str], Optional[bytes]]:
+        abs_url = "%s%s" % (self.base_url, url)
+        headers = self._validate_headers(supplied_headers)
+
+        data = None
+        if method == "get" or method == "delete":
+            if params:
+                encoded_params = urlencode([(k, v) for k, v in params.items() if v is not None])
+                abs_url = _build_api_url(abs_url, encoded_params)
+        elif method in {"post", "put"}:
+            if params and files:
+                data = params
+            if params and not files:
+                data = json.dumps(params).encode()
+                headers["Content-Type"] = "application/json"
+        else:
+            raise openai.APIConnectionError(
+                message=f"Unrecognized HTTP method {method}. This may indicate a bug in the LLM bindings.",
+                request=None,
+            )
+
+        headers = self.request_headers(method, headers, request_id)
+
+        # log_debug("Request to LLM API", method=method, path=abs_url)
+        # log_debug("Post details", data=data, api_version=self.api_version)
+
+        return abs_url, headers, data
+
+    def request_raw(
+        self,
+        method,
+        url,
+        *,
+        params=None,
+        supplied_headers: Optional[Dict[str, str]] = None,
+        files=None,
+        stream: bool = False,
+        request_id: Optional[str] = None,
+        request_timeout: Optional[Union[float, Tuple[float, float]]] = None,
+    ) -> requests.Response:
+        abs_url, headers, data = self._prepare_request_raw(url, supplied_headers, method, params, files, request_id)
+
+        if not hasattr(_thread_context, "session"):
+            _thread_context.session = _make_session()
+            _thread_context.session_create_time = time.time()
+        elif time.time() - getattr(_thread_context, "session_create_time", 0) >= MAX_SESSION_LIFETIME_SECS:
+            _thread_context.session.close()
+            _thread_context.session = _make_session()
+            _thread_context.session_create_time = time.time()
+        try:
+            result = _thread_context.session.request(
+                method,
+                abs_url,
+                headers=headers,
+                data=data,
+                files=files,
+                stream=stream,
+                timeout=request_timeout if request_timeout else TIMEOUT_SECS,
+                proxies=_thread_context.session.proxies,
+            )
+        except requests.exceptions.Timeout as e:
+            raise openai.APITimeoutError("Request timed out: {}".format(e)) from e
+        except requests.exceptions.RequestException as e:
+            raise openai.APIConnectionError(message="Error communicating with LLM: {}".format(e), request=None) from e
+        # log_debug(
+        #     "LLM API response",
+        #     path=abs_url,
+        #     response_code=result.status_code,
+        #     processing_ms=result.headers.get("LLM-Processing-Ms"),
+        #     request_id=result.headers.get("X-Request-Id"),
+        # )
+        return result
+
+    async def arequest_raw(
+        self,
+        method,
+        url,
+        session,
+        *,
+        params=None,
+        supplied_headers: Optional[Dict[str, str]] = None,
+        files=None,
+        request_id: Optional[str] = None,
+        request_timeout: Optional[Union[float, Tuple[float, float]]] = None,
+    ) -> aiohttp.ClientResponse:
+        abs_url, headers, data = self._prepare_request_raw(url, supplied_headers, method, params, files, request_id)
+
+        if isinstance(request_timeout, tuple):
+            timeout = aiohttp.ClientTimeout(
+                connect=request_timeout[0],
+                total=request_timeout[1],
+            )
+        else:
+            timeout = aiohttp.ClientTimeout(total=request_timeout if request_timeout else TIMEOUT_SECS)
+
+        if files:
+            # TODO: Use `aiohttp.MultipartWriter` to create the multipart form data here.
+            # For now we use the private `requests` method that is known to have worked so far.
+            data, content_type = requests.models.RequestEncodingMixin._encode_files(files, data)  # type: ignore
+            headers["Content-Type"] = content_type
+        request_kwargs = {
+            "method": method,
+            "url": abs_url,
+            "headers": headers,
+            "data": data,
+            "timeout": timeout,
+        }
+        try:
+            result = await session.request(**request_kwargs)
+            # log_info(
+            #     "LLM API response",
+            #     path=abs_url,
+            #     response_code=result.status,
+            #     processing_ms=result.headers.get("LLM-Processing-Ms"),
+            #     request_id=result.headers.get("X-Request-Id"),
+            # )
+            return result
+        except (aiohttp.ServerTimeoutError, asyncio.TimeoutError) as e:
+            raise openai.APITimeoutError("Request timed out") from e
+        except aiohttp.ClientError as e:
+            raise openai.APIConnectionError(message="Error communicating with LLM", request=None) from e
+
+    def _interpret_response(
+        self, result: requests.Response, stream: bool
+    ) -> Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], bool]:
+        """Returns the response(s) and a bool indicating whether it is a stream."""
+
+    async def _interpret_async_response(
+        self, result: aiohttp.ClientResponse, stream: bool
+    ) -> Tuple[Union[OpenAIResponse, AsyncGenerator[OpenAIResponse, None]], bool]:
+        """Returns the response(s) and a bool indicating whether it is a stream."""
+
+    def _interpret_response_line(self, rbody: str, rcode: int, rheaders, stream: bool) -> OpenAIResponse:
+        ...
+
+
+@asynccontextmanager
+async def aiohttp_session() -> AsyncIterator[aiohttp.ClientSession]:
+    async with aiohttp.ClientSession() as session:
+        yield session

metagpt/provider/general_api_requestor.py

@@ -2,20 +2,44 @@
 # -*- coding: utf-8 -*-
 # @Desc   : General Async API for http-based LLM model
 
-from typing import AsyncGenerator, Tuple, Union, Optional, Literal
-import aiohttp
 import asyncio
+from typing import AsyncGenerator, Generator, Iterator, Tuple, Union
 
-from openai.api_requestor import APIRequestor
+import aiohttp
+import requests
 
 from metagpt.logs import logger
+from metagpt.provider.general_api_base import APIRequestor
+
+
+def parse_stream_helper(line: bytes) -> Union[bytes, None]:
+    if line and line.startswith(b"data:"):
+        if line.startswith(b"data: "):
+            # SSE event may be valid when it contain whitespace
+            line = line[len(b"data: ") :]
+        else:
+            line = line[len(b"data:") :]
+        if line.strip() == b"[DONE]":
+            # return here will cause GeneratorExit exception in urllib3
+            # and it will close http connection with TCP Reset
+            return None
+        else:
+            return line
+    return None
+
+
+def parse_stream(rbody: Iterator[bytes]) -> Iterator[bytes]:
+    for line in rbody:
+        _line = parse_stream_helper(line)
+        if _line is not None:
+            yield _line
 
 
 class GeneralAPIRequestor(APIRequestor):
     """
     usage
-        # full_url = "{api_base}{url}"
-        requester = GeneralAPIRequestor(api_base=api_base)
+        # full_url = "{base_url}{url}"
+        requester = GeneralAPIRequestor(base_url=base_url)
         result, _, api_key = await requester.arequest(
             method=method,
             url=url,
@@ -26,24 +50,44 @@ class GeneralAPIRequestor(APIRequestor):
         )
     """
 
-    def _interpret_response_line(
-        self, rbody: str, rcode: int, rheaders, stream: bool
-    ) -> str:
+    def _interpret_response_line(self, rbody: bytes, rcode: int, rheaders, stream: bool) -> bytes:
         # just do nothing to meet the APIRequestor process and return the raw data
         # due to the openai sdk will convert the data into OpenAIResponse which we don't need in general cases.
 
         return rbody
 
-    async def _interpret_async_response(
-        self, result: aiohttp.ClientResponse, stream: bool
-    ) -> Tuple[Union[str, AsyncGenerator[str, None]], bool]:
+    def _interpret_response(
+        self, result: requests.Response, stream: bool
+    ) -> Tuple[Union[bytes, Iterator[Generator]], bytes]:
+        """Returns the response(s) and a bool indicating whether it is a stream."""
         if stream and "text/event-stream" in result.headers.get("Content-Type", ""):
             return (
-                   self._interpret_response_line(
-                       line, result.status, result.headers, stream=True
-                   )
-                   async for line in result.content
-               ), True
+                self._interpret_response_line(line, result.status_code, result.headers, stream=True)
+                for line in parse_stream(result.iter_lines())
+            ), True
+        else:
+            return (
+                self._interpret_response_line(
+                    result.content,  # let the caller to decode the msg
+                    result.status_code,
+                    result.headers,
+                    stream=False,
+                ),
+                False,
+            )
+
+    async def _interpret_async_response(
+        self, result: aiohttp.ClientResponse, stream: bool
+    ) -> Tuple[Union[bytes, AsyncGenerator[bytes, None]], bool]:
+        if stream and (
+            "text/event-stream" in result.headers.get("Content-Type", "")
+            or "application/x-ndjson" in result.headers.get("Content-Type", "")
+        ):
+            # the `Content-Type` of ollama stream resp is "application/x-ndjson"
+            return (
+                self._interpret_response_line(line, result.status, result.headers, stream=True)
+                async for line in result.content
+            ), True
         else:
             try:
                 await result.read()

metagpt/provider/google_gemini_api.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : Google Gemini LLM from https://ai.google.dev/tutorials/python_quickstart
+
+import google.generativeai as genai
+from google.ai import generativelanguage as glm
+from google.generativeai.generative_models import GenerativeModel
+from google.generativeai.types import content_types
+from google.generativeai.types.generation_types import (
+    AsyncGenerateContentResponse,
+    GenerateContentResponse,
+    GenerationConfig,
+)
+from tenacity import (
+    after_log,
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+
+from metagpt.config import CONFIG, LLMProviderEnum
+from metagpt.logs import log_llm_stream, logger
+from metagpt.provider.base_llm import BaseLLM
+from metagpt.provider.llm_provider_registry import register_provider
+from metagpt.provider.openai_api import log_and_reraise
+
+
+class GeminiGenerativeModel(GenerativeModel):
+    """
+    Due to `https://github.com/google/generative-ai-python/pull/123`, inherit a new class.
+    Will use default GenerativeModel if it fixed.
+    """
+
+    def count_tokens(self, contents: content_types.ContentsType) -> glm.CountTokensResponse:
+        contents = content_types.to_contents(contents)
+        return self._client.count_tokens(model=self.model_name, contents=contents)
+
+    async def count_tokens_async(self, contents: content_types.ContentsType) -> glm.CountTokensResponse:
+        contents = content_types.to_contents(contents)
+        return await self._async_client.count_tokens(model=self.model_name, contents=contents)
+
+
+@register_provider(LLMProviderEnum.GEMINI)
+class GeminiLLM(BaseLLM):
+    """
+    Refs to `https://ai.google.dev/tutorials/python_quickstart`
+    """
+
+    def __init__(self):
+        self.use_system_prompt = False  # google gemini has no system prompt when use api
+
+        self.__init_gemini(CONFIG)
+        self.model = "gemini-pro"  # so far only one model
+        self.llm = GeminiGenerativeModel(model_name=self.model)
+
+    def __init_gemini(self, config: CONFIG):
+        genai.configure(api_key=config.gemini_api_key)
+
+    def _user_msg(self, msg: str) -> dict[str, str]:
+        # Not to change BaseLLM default functions but update with Gemini's conversation format.
+        # You should follow the format.
+        return {"role": "user", "parts": [msg]}
+
+    def _assistant_msg(self, msg: str) -> dict[str, str]:
+        return {"role": "model", "parts": [msg]}
+
+    def _const_kwargs(self, messages: list[dict], stream: bool = False) -> dict:
+        kwargs = {"contents": messages, "generation_config": GenerationConfig(temperature=0.3), "stream": stream}
+        return kwargs
+
+    def _update_costs(self, usage: dict):
+        """update each request's token cost"""
+        if CONFIG.calc_usage:
+            try:
+                prompt_tokens = int(usage.get("prompt_tokens", 0))
+                completion_tokens = int(usage.get("completion_tokens", 0))
+                CONFIG.cost_manager.update_cost(prompt_tokens, completion_tokens, self.model)
+            except Exception as e:
+                logger.error(f"google gemini updats costs failed! exp: {e}")
+
+    def get_choice_text(self, resp: GenerateContentResponse) -> str:
+        return resp.text
+
+    def get_usage(self, messages: list[dict], resp_text: str) -> dict:
+        req_text = messages[-1]["parts"][0] if messages else ""
+        prompt_resp = self.llm.count_tokens(contents={"role": "user", "parts": [{"text": req_text}]})
+        completion_resp = self.llm.count_tokens(contents={"role": "model", "parts": [{"text": resp_text}]})
+        usage = {"prompt_tokens": prompt_resp.total_tokens, "completion_tokens": completion_resp.total_tokens}
+        return usage
+
+    async def aget_usage(self, messages: list[dict], resp_text: str) -> dict:
+        req_text = messages[-1]["parts"][0] if messages else ""
+        prompt_resp = await self.llm.count_tokens_async(contents={"role": "user", "parts": [{"text": req_text}]})
+        completion_resp = await self.llm.count_tokens_async(contents={"role": "model", "parts": [{"text": resp_text}]})
+        usage = {"prompt_tokens": prompt_resp.total_tokens, "completion_tokens": completion_resp.total_tokens}
+        return usage
+
+    def completion(self, messages: list[dict]) -> "GenerateContentResponse":
+        resp: GenerateContentResponse = self.llm.generate_content(**self._const_kwargs(messages))
+        usage = self.get_usage(messages, resp.text)
+        self._update_costs(usage)
+        return resp
+
+    async def _achat_completion(self, messages: list[dict]) -> "AsyncGenerateContentResponse":
+        resp: AsyncGenerateContentResponse = await self.llm.generate_content_async(**self._const_kwargs(messages))
+        usage = await self.aget_usage(messages, resp.text)
+        self._update_costs(usage)
+        return resp
+
+    async def acompletion(self, messages: list[dict]) -> dict:
+        return await self._achat_completion(messages)
+
+    async def _achat_completion_stream(self, messages: list[dict]) -> str:
+        resp: AsyncGenerateContentResponse = await self.llm.generate_content_async(
+            **self._const_kwargs(messages, stream=True)
+        )
+        collected_content = []
+        async for chunk in resp:
+            content = chunk.text
+            log_llm_stream(content)
+            collected_content.append(content)
+        log_llm_stream("\n")
+
+        full_content = "".join(collected_content)
+        usage = await self.aget_usage(messages, full_content)
+        self._update_costs(usage)
+        return full_content
+
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_random_exponential(min=1, max=60),
+        after=after_log(logger, logger.level("WARNING").name),
+        retry=retry_if_exception_type(ConnectionError),
+        retry_error_callback=log_and_reraise,
+    )
+    async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = 3) -> str:
+        """response in async with stream or non-stream mode"""
+        if stream:
+            return await self._achat_completion_stream(messages)
+        resp = await self._achat_completion(messages)
+        return self.get_choice_text(resp)

metagpt/provider/human_provider.py

@@ -1,35 +1,40 @@
-'''
+"""
 Filename: MetaGPT/metagpt/provider/human_provider.py
 Created Date: Wednesday, November 8th 2023, 11:55:46 pm
 Author: garylin2099
-'''
+"""
 from typing import Optional
-from metagpt.provider.base_gpt_api import BaseGPTAPI
-from metagpt.logs import logger
 
-class HumanProvider(BaseGPTAPI):
+from metagpt.logs import logger
+from metagpt.provider.base_llm import BaseLLM
+
+
+class HumanProvider(BaseLLM):
     """Humans provide themselves as a 'model', which actually takes in human input as its response.
     This enables replacing LLM anywhere in the framework with a human, thus introducing human interaction
     """
 
-    def ask(self, msg: str) -> str:
+    def ask(self, msg: str, timeout=3) -> str:
         logger.info("It's your turn, please type in your response. You may also refer to the context below")
         rsp = input(msg)
         if rsp in ["exit", "quit"]:
             exit()
         return rsp
 
-    async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str:
-        return self.ask(msg)
+    async def aask(
+        self,
+        msg: str,
+        system_msgs: Optional[list[str]] = None,
+        format_msgs: Optional[list[dict[str, str]]] = None,
+        generator: bool = False,
+        timeout=3,
+    ) -> str:
+        return self.ask(msg, timeout=timeout)
 
-    def completion(self, messages: list[dict]):
+    async def acompletion(self, messages: list[dict], timeout=3):
         """dummy implementation of abstract method in base"""
         return []
 
-    async def acompletion(self, messages: list[dict]):
+    async def acompletion_text(self, messages: list[dict], stream=False, timeout=3) -> str:
         """dummy implementation of abstract method in base"""
-        return []
-
-    async def acompletion_text(self, messages: list[dict], stream=False) -> str:
-        """dummy implementation of abstract method in base"""
-        return []
+        return ""

metagpt/provider/llm_provider_registry.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/12/19 17:26
+@Author  : alexanderwu
+@File    : llm_provider_registry.py
+"""
+from metagpt.config import LLMProviderEnum
+
+
+class LLMProviderRegistry:
+    def __init__(self):
+        self.providers = {}
+
+    def register(self, key, provider_cls):
+        self.providers[key] = provider_cls
+
+    def get_provider(self, enum: LLMProviderEnum):
+        """get provider instance according to the enum"""
+        return self.providers[enum]()
+
+
+# Registry instance
+LLM_REGISTRY = LLMProviderRegistry()
+
+
+def register_provider(key):
+    """register provider to registry"""
+
+    def decorator(cls):
+        LLM_REGISTRY.register(key, cls)
+        return cls
+
+    return decorator

metagpt/provider/metagpt_api.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/5/5 23:08
+@Author  : alexanderwu
+@File    : metagpt_api.py
+@Desc    : MetaGPT LLM provider.
+"""
+from metagpt.config import LLMProviderEnum
+from metagpt.provider import OpenAILLM
+from metagpt.provider.llm_provider_registry import register_provider
+
+
+@register_provider(LLMProviderEnum.METAGPT)
+class MetaGPTLLM(OpenAILLM):
+    def __init__(self):
+        super().__init__()

metagpt/provider/ollama_api.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : self-host open llm model with ollama which isn't openai-api-compatible
+
+import json
+
+from requests import ConnectionError
+from tenacity import (
+    after_log,
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+
+from metagpt.config import CONFIG, LLMProviderEnum
+from metagpt.const import LLM_API_TIMEOUT
+from metagpt.logs import log_llm_stream, logger
+from metagpt.provider.base_llm import BaseLLM
+from metagpt.provider.general_api_requestor import GeneralAPIRequestor
+from metagpt.provider.llm_provider_registry import register_provider
+from metagpt.provider.openai_api import log_and_reraise
+from metagpt.utils.cost_manager import CostManager
+
+
+class OllamaCostManager(CostManager):
+    def update_cost(self, prompt_tokens, completion_tokens, model):
+        """
+        Update the total cost, prompt tokens, and completion tokens.
+        """
+        self.total_prompt_tokens += prompt_tokens
+        self.total_completion_tokens += completion_tokens
+        max_budget = CONFIG.max_budget if CONFIG.max_budget else CONFIG.cost_manager.max_budget
+        logger.info(
+            f"Max budget: ${max_budget:.3f} | "
+            f"prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}"
+        )
+        CONFIG.total_cost = self.total_cost
+
+
+@register_provider(LLMProviderEnum.OLLAMA)
+class OllamaLLM(BaseLLM):
+    """
+    Refs to `https://github.com/jmorganca/ollama/blob/main/docs/api.md#generate-a-chat-completion`
+    """
+
+    def __init__(self):
+        self.__init_ollama(CONFIG)
+        self.client = GeneralAPIRequestor(base_url=CONFIG.ollama_api_base)
+        self.suffix_url = "/chat"
+        self.http_method = "post"
+        self.use_system_prompt = False
+        self._cost_manager = OllamaCostManager()
+
+    def __init_ollama(self, config: CONFIG):
+        assert config.ollama_api_base
+        self.model = config.ollama_api_model
+
+    def _const_kwargs(self, messages: list[dict], stream: bool = False) -> dict:
+        kwargs = {"model": self.model, "messages": messages, "options": {"temperature": 0.3}, "stream": stream}
+        return kwargs
+
+    def _update_costs(self, usage: dict):
+        """update each request's token cost"""
+        if CONFIG.calc_usage:
+            try:
+                prompt_tokens = int(usage.get("prompt_tokens", 0))
+                completion_tokens = int(usage.get("completion_tokens", 0))
+                self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model)
+            except Exception as e:
+                logger.error(f"ollama updats costs failed! exp: {e}")
+
+    def get_choice_text(self, resp: dict) -> str:
+        """get the resp content from llm response"""
+        assist_msg = resp.get("message", {})
+        assert assist_msg.get("role", None) == "assistant"
+        return assist_msg.get("content")
+
+    def get_usage(self, resp: dict) -> dict:
+        return {"prompt_tokens": resp.get("prompt_eval_count", 0), "completion_tokens": resp.get("eval_count", 0)}
+
+    def _decode_and_load(self, chunk: bytes, encoding: str = "utf-8") -> dict:
+        chunk = chunk.decode(encoding)
+        return json.loads(chunk)
+
+    async def _achat_completion(self, messages: list[dict]) -> dict:
+        resp, _, _ = await self.client.arequest(
+            method=self.http_method,
+            url=self.suffix_url,
+            params=self._const_kwargs(messages),
+            request_timeout=LLM_API_TIMEOUT,
+        )
+        resp = self._decode_and_load(resp)
+        usage = self.get_usage(resp)
+        self._update_costs(usage)
+        return resp
+
+    async def acompletion(self, messages: list[dict], timeout=3) -> dict:
+        return await self._achat_completion(messages)
+
+    async def _achat_completion_stream(self, messages: list[dict]) -> str:
+        stream_resp, _, _ = await self.client.arequest(
+            method=self.http_method,
+            url=self.suffix_url,
+            stream=True,
+            params=self._const_kwargs(messages, stream=True),
+            request_timeout=LLM_API_TIMEOUT,
+        )
+
+        collected_content = []
+        usage = {}
+        async for raw_chunk in stream_resp:
+            chunk = self._decode_and_load(raw_chunk)
+
+            if not chunk.get("done", False):
+                content = self.get_choice_text(chunk)
+                collected_content.append(content)
+                log_llm_stream(content)
+            else:
+                # stream finished
+                usage = self.get_usage(chunk)
+        log_llm_stream("\n")
+
+        self._update_costs(usage)
+        full_content = "".join(collected_content)
+        return full_content
+
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_random_exponential(min=1, max=60),
+        after=after_log(logger, logger.level("WARNING").name),
+        retry=retry_if_exception_type(ConnectionError),
+        retry_error_callback=log_and_reraise,
+    )
+    async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = 3) -> str:
+        """response in async with stream or non-stream mode"""
+        if stream:
+            return await self._achat_completion_stream(messages)
+        resp = await self._achat_completion(messages)
+        return self.get_choice_text(resp)

metagpt/provider/open_llm_api.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : self-host open llm model with openai-compatible interface
+
+from openai.types import CompletionUsage
+
+from metagpt.config import CONFIG, Config, LLMProviderEnum
+from metagpt.logs import logger
+from metagpt.provider.llm_provider_registry import register_provider
+from metagpt.provider.openai_api import OpenAILLM
+from metagpt.utils.cost_manager import CostManager, Costs
+from metagpt.utils.token_counter import count_message_tokens, count_string_tokens
+
+
+class OpenLLMCostManager(CostManager):
+    """open llm model is self-host, it's free and without cost"""
+
+    def update_cost(self, prompt_tokens, completion_tokens, model):
+        """
+        Update the total cost, prompt tokens, and completion tokens.
+
+        Args:
+        prompt_tokens (int): The number of tokens used in the prompt.
+        completion_tokens (int): The number of tokens used in the completion.
+        model (str): The model used for the API call.
+        """
+        self.total_prompt_tokens += prompt_tokens
+        self.total_completion_tokens += completion_tokens
+        max_budget = CONFIG.max_budget if CONFIG.max_budget else CONFIG.cost_manager.max_budget
+        logger.info(
+            f"Max budget: ${max_budget:.3f} | reference "
+            f"prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}"
+        )
+
+
+@register_provider(LLMProviderEnum.OPEN_LLM)
+class OpenLLM(OpenAILLM):
+    def __init__(self):
+        self.config: Config = CONFIG
+        self.__init_openllm()
+        self.auto_max_tokens = False
+        self._cost_manager = OpenLLMCostManager()
+
+    def __init_openllm(self):
+        self.is_azure = False
+        self.rpm = int(self.config.get("RPM", 10))
+        self._init_client()
+        self.model = self.config.open_llm_api_model  # `self.model` should after `_make_client` to rewrite it
+
+    def _make_client_kwargs(self) -> dict:
+        kwargs = dict(api_key="sk-xxx", base_url=self.config.open_llm_api_base)
+        return kwargs
+
+    def _calc_usage(self, messages: list[dict], rsp: str) -> CompletionUsage:
+        usage = CompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
+        if not CONFIG.calc_usage:
+            return usage
+
+        try:
+            usage.prompt_tokens = count_message_tokens(messages, "open-llm-model")
+            usage.completion_tokens = count_string_tokens(rsp, "open-llm-model")
+        except Exception as e:
+            logger.error(f"usage calculation failed!: {e}")
+
+        return usage
+
+    def _update_costs(self, usage: CompletionUsage):
+        if self.config.calc_usage and usage:
+            try:
+                # use OpenLLMCostManager not CONFIG.cost_manager
+                self._cost_manager.update_cost(usage.prompt_tokens, usage.completion_tokens, self.model)
+            except Exception as e:
+                logger.error(f"updating costs failed!, exp: {e}")
+
+    def get_costs(self) -> Costs:
+        return self._cost_manager.get_costs()

metagpt/provider/openai_api.py

@@ -3,13 +3,19 @@
 @Time    : 2023/5/5 23:08
 @Author  : alexanderwu
 @File    : openai.py
+@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for isolation;
+            Change cost control from global to company level.
+@Modified By: mashenquan, 2023/11/21. Fix bug: ReadTimeout.
+@Modified By: mashenquan, 2023/12/1. Fix bug: Unclosed connection caused by openai 0.x.
 """
-import asyncio
-import time
-from typing import NamedTuple, Union
 
-import openai
-from openai.error import APIConnectionError
+import json
+from typing import AsyncIterator, Union
+
+from openai import APIConnectionError, AsyncOpenAI, AsyncStream
+from openai._base_client import AsyncHttpxClientWrapper
+from openai.types import CompletionUsage
+from openai.types.chat import ChatCompletion, ChatCompletionChunk
 from tenacity import (
     after_log,
     retry,
@@ -19,114 +25,21 @@ from tenacity import (
     wait_fixed,
 )
 
-from metagpt.config import CONFIG
-from metagpt.logs import logger
-from metagpt.provider.base_gpt_api import BaseGPTAPI
+from metagpt.config import CONFIG, Config, LLMProviderEnum
+from metagpt.logs import log_llm_stream, logger
+from metagpt.provider.base_llm import BaseLLM
 from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA, GENERAL_TOOL_CHOICE
+from metagpt.provider.llm_provider_registry import register_provider
 from metagpt.schema import Message
-from metagpt.utils.singleton import Singleton
+from metagpt.utils.cost_manager import Costs
+from metagpt.utils.exceptions import handle_exception
 from metagpt.utils.token_counter import (
-    TOKEN_COSTS,
     count_message_tokens,
     count_string_tokens,
     get_max_completion_tokens,
 )
 
 
-class RateLimiter:
-    """Rate control class, each call goes through wait_if_needed, sleep if rate control is needed"""
-
-    def __init__(self, rpm):
-        self.last_call_time = 0
-        # Here 1.1 is used because even if the calls are made strictly according to time,
-        # they will still be QOS'd; consider switching to simple error retry later
-        self.interval = 1.1 * 60 / rpm
-        self.rpm = rpm
-
-    def split_batches(self, batch):
-        return [batch[i : i + self.rpm] for i in range(0, len(batch), self.rpm)]
-
-    async def wait_if_needed(self, num_requests):
-        current_time = time.time()
-        elapsed_time = current_time - self.last_call_time
-
-        if elapsed_time < self.interval * num_requests:
-            remaining_time = self.interval * num_requests - elapsed_time
-            logger.info(f"sleep {remaining_time}")
-            await asyncio.sleep(remaining_time)
-
-        self.last_call_time = time.time()
-
-
-class Costs(NamedTuple):
-    total_prompt_tokens: int
-    total_completion_tokens: int
-    total_cost: float
-    total_budget: float
-
-
-class CostManager(metaclass=Singleton):
-    """计算使用接口的开销"""
-
-    def __init__(self):
-        self.total_prompt_tokens = 0
-        self.total_completion_tokens = 0
-        self.total_cost = 0
-        self.total_budget = 0
-
-    def update_cost(self, prompt_tokens, completion_tokens, model):
-        """
-        Update the total cost, prompt tokens, and completion tokens.
-
-        Args:
-        prompt_tokens (int): The number of tokens used in the prompt.
-        completion_tokens (int): The number of tokens used in the completion.
-        model (str): The model used for the API call.
-        """
-        self.total_prompt_tokens += prompt_tokens
-        self.total_completion_tokens += completion_tokens
-        cost = (
-            prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model]["completion"]
-        ) / 1000
-        self.total_cost += cost
-        logger.info(
-            f"Total running cost: ${self.total_cost:.3f} | Max budget: ${CONFIG.max_budget:.3f} | "
-            f"Current cost: ${cost:.3f}, prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}"
-        )
-        CONFIG.total_cost = self.total_cost
-
-    def get_total_prompt_tokens(self):
-        """
-        Get the total number of prompt tokens.
-
-        Returns:
-        int: The total number of prompt tokens.
-        """
-        return self.total_prompt_tokens
-
-    def get_total_completion_tokens(self):
-        """
-        Get the total number of completion tokens.
-
-        Returns:
-        int: The total number of completion tokens.
-        """
-        return self.total_completion_tokens
-
-    def get_total_cost(self):
-        """
-        Get the total cost of API calls.
-
-        Returns:
-        float: The total cost of API calls.
-        """
-        return self.total_cost
-
-    def get_costs(self) -> Costs:
-        """Get all costs"""
-        return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget)
-
-
 def log_and_reraise(retry_state):
     logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}")
     logger.warning(
@@ -138,115 +51,102 @@ See FAQ 5.8
     raise retry_state.outcome.exception()
 
 
-class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
-    """
-    Check https://platform.openai.com/examples for examples
-    """
+@register_provider(LLMProviderEnum.OPENAI)
+class OpenAILLM(BaseLLM):
+    """Check https://platform.openai.com/examples for examples"""
 
     def __init__(self):
-        self.__init_openai(CONFIG)
-        self.llm = openai
-        self.model = CONFIG.openai_api_model
+        self.config: Config = CONFIG
+        self._init_openai()
+        self._init_client()
         self.auto_max_tokens = False
-        self._cost_manager = CostManager()
-        RateLimiter.__init__(self, rpm=self.rpm)
 
-    def __init_openai(self, config):
-        openai.api_key = config.openai_api_key
-        if config.openai_api_base:
-            openai.api_base = config.openai_api_base
-        if config.openai_api_type:
-            openai.api_type = config.openai_api_type
-            openai.api_version = config.openai_api_version
-        self.rpm = int(config.get("RPM", 10))
+    def _init_openai(self):
+        self.model = self.config.OPENAI_API_MODEL  # Used in _calc_usage & _cons_kwargs
 
-    async def _achat_completion_stream(self, messages: list[dict]) -> str:
-        response = await openai.ChatCompletion.acreate(**self._cons_kwargs(messages), stream=True)
+    def _init_client(self):
+        """https://github.com/openai/openai-python#async-usage"""
+        kwargs = self._make_client_kwargs()
+        self.aclient = AsyncOpenAI(**kwargs)
+
+    def _make_client_kwargs(self) -> dict:
+        kwargs = {"api_key": self.config.openai_api_key, "base_url": self.config.openai_base_url}
+
+        # to use proxy, openai v1 needs http_client
+        if proxy_params := self._get_proxy_params():
+            kwargs["http_client"] = AsyncHttpxClientWrapper(**proxy_params)
+
+        return kwargs
+
+    def _get_proxy_params(self) -> dict:
+        params = {}
+        if self.config.openai_proxy:
+            params = {"proxies": self.config.openai_proxy}
+            if self.config.openai_base_url:
+                params["base_url"] = self.config.openai_base_url
+
+        return params
+
+    async def _achat_completion_stream(self, messages: list[dict], timeout=3) -> AsyncIterator[str]:
+        response: AsyncStream[ChatCompletionChunk] = await self.aclient.chat.completions.create(
+            **self._cons_kwargs(messages, timeout=timeout), stream=True
+        )
 
-        # create variables to collect the stream of chunks
-        collected_chunks = []
-        collected_messages = []
-        # iterate through the stream of events
         async for chunk in response:
-            collected_chunks.append(chunk)  # save the event response
-            choices = chunk["choices"]
-            if len(choices) > 0:
-                chunk_message = chunk["choices"][0].get("delta", {})  # extract the message
-                collected_messages.append(chunk_message)  # save the message
-                if "content" in chunk_message:
-                    print(chunk_message["content"], end="")
-        print()
+            chunk_message = chunk.choices[0].delta.content or "" if chunk.choices else ""  # extract the message
+            yield chunk_message
 
-        full_reply_content = "".join([m.get("content", "") for m in collected_messages])
-        usage = self._calc_usage(messages, full_reply_content)
-        self._update_costs(usage)
-        return full_reply_content
-
-    def _cons_kwargs(self, messages: list[dict], **configs) -> dict:
+    def _cons_kwargs(self, messages: list[dict], timeout=3, **extra_kwargs) -> dict:
         kwargs = {
             "messages": messages,
-            "max_tokens": self.get_max_tokens(messages),
+            "max_tokens": self._get_max_tokens(messages),
             "n": 1,
             "stop": None,
             "temperature": 0.3,
-            "timeout": 3,
+            "model": self.model,
+            "timeout": max(CONFIG.timeout, timeout),
         }
-        if configs:
-            kwargs.update(configs)
-
-        if CONFIG.openai_api_type == "azure":
-            if CONFIG.deployment_name and CONFIG.deployment_id:
-                raise ValueError("You can only use one of the `deployment_id` or `deployment_name` model")
-            elif not CONFIG.deployment_name and not CONFIG.deployment_id:
-                raise ValueError("You must specify `DEPLOYMENT_NAME` or `DEPLOYMENT_ID` parameter")
-            kwargs_mode = (
-                {"engine": CONFIG.deployment_name}
-                if CONFIG.deployment_name
-                else {"deployment_id": CONFIG.deployment_id}
-            )
-        else:
-            kwargs_mode = {"model": self.model}
-        kwargs.update(kwargs_mode)
+        if extra_kwargs:
+            kwargs.update(extra_kwargs)
         return kwargs
 
-    async def _achat_completion(self, messages: list[dict]) -> dict:
-        rsp = await self.llm.ChatCompletion.acreate(**self._cons_kwargs(messages))
-        self._update_costs(rsp.get("usage"))
+    async def _achat_completion(self, messages: list[dict], timeout=3) -> ChatCompletion:
+        kwargs = self._cons_kwargs(messages, timeout=timeout)
+        rsp: ChatCompletion = await self.aclient.chat.completions.create(**kwargs)
+        self._update_costs(rsp.usage)
         return rsp
 
-    def _chat_completion(self, messages: list[dict]) -> dict:
-        rsp = self.llm.ChatCompletion.create(**self._cons_kwargs(messages))
-        self._update_costs(rsp)
-        return rsp
-
-    def completion(self, messages: list[dict]) -> dict:
-        # if isinstance(messages[0], Message):
-        #     messages = self.messages_to_dict(messages)
-        return self._chat_completion(messages)
-
-    async def acompletion(self, messages: list[dict]) -> dict:
-        # if isinstance(messages[0], Message):
-        #     messages = self.messages_to_dict(messages)
-        return await self._achat_completion(messages)
+    async def acompletion(self, messages: list[dict], timeout=3) -> ChatCompletion:
+        return await self._achat_completion(messages, timeout=timeout)
 
     @retry(
-        stop=stop_after_attempt(3),
-        wait=wait_fixed(1),
+        wait=wait_random_exponential(min=1, max=60),
+        stop=stop_after_attempt(6),
         after=after_log(logger, logger.level("WARNING").name),
         retry=retry_if_exception_type(APIConnectionError),
         retry_error_callback=log_and_reraise,
     )
-    async def acompletion_text(self, messages: list[dict], stream=False) -> str:
+    async def acompletion_text(self, messages: list[dict], stream=False, timeout=3) -> str:
         """when streaming, print each token in place."""
         if stream:
-            return await self._achat_completion_stream(messages)
-        rsp = await self._achat_completion(messages)
+            resp = self._achat_completion_stream(messages, timeout=timeout)
+
+            collected_messages = []
+            async for i in resp:
+                log_llm_stream(i)
+                collected_messages.append(i)
+            log_llm_stream("\n")
+
+            full_reply_content = "".join(collected_messages)
+            usage = self._calc_usage(messages, full_reply_content)
+            self._update_costs(usage)
+            return full_reply_content
+
+        rsp = await self._achat_completion(messages, timeout=timeout)
         return self.get_choice_text(rsp)
 
-    def _func_configs(self, messages: list[dict], **kwargs) -> dict:
-        """
-        Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create
-        """
+    def _func_configs(self, messages: list[dict], timeout=3, **kwargs) -> dict:
+        """Note: Keep kwargs consistent with https://platform.openai.com/docs/api-reference/chat/create"""
         if "tools" not in kwargs:
             configs = {
                 "tools": [{"type": "function", "function": GENERAL_FUNCTION_SCHEMA}],
@@ -254,23 +154,18 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
             }
             kwargs.update(configs)
 
-        return self._cons_kwargs(messages, **kwargs)
+        return self._cons_kwargs(messages=messages, timeout=timeout, **kwargs)
 
-    def _chat_completion_function(self, messages: list[dict], **kwargs) -> dict:
-        rsp = self.llm.ChatCompletion.create(**self._func_configs(messages, **kwargs))
-        self._update_costs(rsp.get("usage"))
-        return rsp
-    
-    @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
-    async def _achat_completion_function(self, messages: list[dict], **chat_configs) -> dict:
-        rsp = await self.llm.ChatCompletion.acreate(**self._func_configs(messages, **chat_configs))
-        self._update_costs(rsp.get("usage"))
+    async def _achat_completion_function(self, messages: list[dict], timeout=3, **chat_configs) -> ChatCompletion:
+        kwargs = self._func_configs(messages=messages, timeout=timeout, **chat_configs)
+        rsp: ChatCompletion = await self.aclient.chat.completions.create(**kwargs)
+        self._update_costs(rsp.usage)
         return rsp
 
     def _process_message(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]:
         """convert messages to list[dict]."""
         if isinstance(messages, list):
-            messages = [Message(msg) if isinstance(msg, str) else msg for msg in messages]
+            messages = [Message(content=msg) if isinstance(msg, str) else msg for msg in messages]
             return [msg if isinstance(msg, dict) else msg.to_dict() for msg in messages]
 
         if isinstance(messages, Message):
@@ -283,123 +178,60 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
             )
         return messages
 
-    def ask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict:
-        """Use function of tools to ask a code.
-
-        Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create
-
-        Examples:
-
-        >>> llm = OpenAIGPTAPI()
-        >>> llm.ask_code("Write a python hello world code.")
-        {'language': 'python', 'code': "print('Hello, World!')"}
-        >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}]
-        >>> llm.ask_code(msg)
-        {'language': 'python', 'code': "print('Hello, World!')"}
-        """
-        messages = self._process_message(messages)
-        rsp = self._chat_completion_function(messages, **kwargs)
-        return self.get_choice_function_arguments(rsp)
-
     async def aask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict:
         """Use function of tools to ask a code.
-
-        Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create
+        Note: Keep kwargs consistent with https://platform.openai.com/docs/api-reference/chat/create
 
         Examples:
-
-        >>> llm = OpenAIGPTAPI()
-        >>> rsp = await llm.ask_code("Write a python hello world code.")
-        >>> rsp
-        {'language': 'python', 'code': "print('Hello, World!')"}
+        >>> llm = OpenAILLM()
         >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}]
-        >>> rsp = await llm.aask_code(msg)   # -> {'language': 'python', 'code': "print('Hello, World!')"}
+        >>> rsp = await llm.aask_code(msg)
+        # -> {'language': 'python', 'code': "print('Hello, World!')"}
         """
         messages = self._process_message(messages)
         rsp = await self._achat_completion_function(messages, **kwargs)
         return self.get_choice_function_arguments(rsp)
 
-    def _calc_usage(self, messages: list[dict], rsp: str) -> dict:
-        usage = {}
-        if CONFIG.calc_usage:
-            try:
-                prompt_tokens = count_message_tokens(messages, self.model)
-                completion_tokens = count_string_tokens(rsp, self.model)
-                usage["prompt_tokens"] = prompt_tokens
-                usage["completion_tokens"] = completion_tokens
-                return usage
-            except Exception as e:
-                logger.error("usage calculation failed!", e)
-        else:
+    @handle_exception
+    def get_choice_function_arguments(self, rsp: ChatCompletion) -> dict:
+        """Required to provide the first function arguments of choice.
+
+        :return dict: return the first function arguments of choice, for example,
+            {'language': 'python', 'code': "print('Hello, World!')"}
+        """
+        return json.loads(rsp.choices[0].message.tool_calls[0].function.arguments)
+
+    def get_choice_text(self, rsp: ChatCompletion) -> str:
+        """Required to provide the first text of choice"""
+        return rsp.choices[0].message.content if rsp.choices else ""
+
+    def _calc_usage(self, messages: list[dict], rsp: str) -> CompletionUsage:
+        usage = CompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
+        if not CONFIG.calc_usage:
             return usage
 
-    async def acompletion_batch(self, batch: list[list[dict]]) -> list[dict]:
-        """Return full JSON"""
-        split_batches = self.split_batches(batch)
-        all_results = []
+        try:
+            usage.prompt_tokens = count_message_tokens(messages, self.model)
+            usage.completion_tokens = count_string_tokens(rsp, self.model)
+        except Exception as e:
+            logger.error(f"usage calculation failed: {e}")
 
-        for small_batch in split_batches:
-            logger.info(small_batch)
-            await self.wait_if_needed(len(small_batch))
+        return usage
 
-            future = [self.acompletion(prompt) for prompt in small_batch]
-            results = await asyncio.gather(*future)
-            logger.info(results)
-            all_results.extend(results)
-
-        return all_results
-
-    async def acompletion_batch_text(self, batch: list[list[dict]]) -> list[str]:
-        """Only return plain text"""
-        raw_results = await self.acompletion_batch(batch)
-        results = []
-        for idx, raw_result in enumerate(raw_results, start=1):
-            result = self.get_choice_text(raw_result)
-            results.append(result)
-            logger.info(f"Result of task {idx}: {result}")
-        return results
-
-    def _update_costs(self, usage: dict):
-        if CONFIG.calc_usage:
-            try:
-                prompt_tokens = int(usage["prompt_tokens"])
-                completion_tokens = int(usage["completion_tokens"])
-                self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model)
-            except Exception as e:
-                logger.error("updating costs failed!", e)
+    @handle_exception
+    def _update_costs(self, usage: CompletionUsage):
+        if CONFIG.calc_usage and usage:
+            CONFIG.cost_manager.update_cost(usage.prompt_tokens, usage.completion_tokens, self.model)
 
     def get_costs(self) -> Costs:
-        return self._cost_manager.get_costs()
+        return CONFIG.cost_manager.get_costs()
 
-    def get_max_tokens(self, messages: list[dict]):
+    def _get_max_tokens(self, messages: list[dict]):
         if not self.auto_max_tokens:
             return CONFIG.max_tokens_rsp
         return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp)
 
-    def moderation(self, content: Union[str, list[str]]):
-        try:
-            if not content:
-                logger.error("content cannot be empty!")
-            else:
-                rsp = self._moderation(content=content)
-                return rsp
-        except Exception as e:
-            logger.error(f"moderating failed:{e}")
-
-    def _moderation(self, content: Union[str, list[str]]):
-        rsp = self.llm.Moderation.create(input=content)
-        return rsp
-
+    @handle_exception
     async def amoderation(self, content: Union[str, list[str]]):
-        try:
-            if not content:
-                logger.error("content cannot be empty!")
-            else:
-                rsp = await self._amoderation(content=content)
-                return rsp
-        except Exception as e:
-            logger.error(f"moderating failed:{e}")
-
-    async def _amoderation(self, content: Union[str, list[str]]):
-        rsp = await self.llm.Moderation.acreate(input=content)
-        return rsp
+        """Moderate content."""
+        return await self.aclient.moderations.create(input=content)

metagpt/provider/postprocess/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

metagpt/provider/postprocess/base_postprocess_plugin.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : base llm postprocess plugin to do the operations like repair the raw llm output
+
+from typing import Union
+
+from metagpt.utils.repair_llm_raw_output import (
+    RepairType,
+    extract_content_from_output,
+    repair_llm_raw_output,
+    retry_parse_json_text,
+)
+
+
+class BasePostProcessPlugin(object):
+    model = None  # the plugin of the `model`, use to judge in `llm_postprocess`
+
+    def run_repair_llm_output(self, output: str, schema: dict, req_key: str = "[/CONTENT]") -> Union[dict, list]:
+        """
+        repair steps
+            1. repair the case sensitive problem using the schema's fields
+            2. extract the content from the req_key pair( xx[REQ_KEY]xxx[/REQ_KEY]xx )
+            3. repair the invalid json text in the content
+            4. parse the json text and repair it according to the exception with retry loop
+        """
+        output_class_fields = list(schema["properties"].keys())  # Custom ActionOutput's fields
+
+        content = self.run_repair_llm_raw_output(output, req_keys=output_class_fields + [req_key])
+        content = self.run_extract_content_from_output(content, right_key=req_key)
+        # # req_keys mocked
+        content = self.run_repair_llm_raw_output(content, req_keys=[None], repair_type=RepairType.JSON)
+        parsed_data = self.run_retry_parse_json_text(content)
+
+        return parsed_data
+
+    def run_repair_llm_raw_output(self, content: str, req_keys: list[str], repair_type: str = None) -> str:
+        """inherited class can re-implement the function"""
+        return repair_llm_raw_output(content, req_keys=req_keys, repair_type=repair_type)
+
+    def run_extract_content_from_output(self, content: str, right_key: str) -> str:
+        """inherited class can re-implement the function"""
+        return extract_content_from_output(content, right_key=right_key)
+
+    def run_retry_parse_json_text(self, content: str) -> Union[dict, list]:
+        """inherited class can re-implement the function"""
+        # logger.info(f"extracted json CONTENT from output:\n{content}")
+        parsed_data = retry_parse_json_text(output=content)  # should use output=content
+        return parsed_data
+
+    def run(self, output: str, schema: dict, req_key: str = "[/CONTENT]") -> Union[dict, list]:
+        """
+        this is used for prompt with a json-format output requirement and outer pair key, like
+            [REQ_KEY]
+                {
+                    "Key": "value"
+                }
+            [/REQ_KEY]
+
+        Args
+            outer (str): llm raw output
+            schema: output json schema
+            req_key: outer pair right key, usually in `[/REQ_KEY]` format
+        """
+        assert len(schema.get("properties")) > 0
+        assert "/" in req_key
+
+        # current, postprocess only deal the repair_llm_raw_output
+        new_output = self.run_repair_llm_output(output=output, schema=schema, req_key=req_key)
+        return new_output

metagpt/provider/postprocess/llm_output_postprocess.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : the entry of choosing which PostProcessPlugin to deal particular LLM model's output
+
+from typing import Union
+
+from metagpt.provider.postprocess.base_postprocess_plugin import BasePostProcessPlugin
+
+
+def llm_output_postprocess(
+    output: str, schema: dict, req_key: str = "[/CONTENT]", model_name: str = None
+) -> Union[dict, str]:
+    """
+    default use BasePostProcessPlugin if there is not matched plugin.
+    """
+    # TODO choose different model's plugin according to the model_name
+    postprocess_plugin = BasePostProcessPlugin()
+
+    result = postprocess_plugin.run(output=output, schema=schema, req_key=req_key)
+    return result

metagpt/provider/spark_api.py

@@ -1,9 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """
-@Time    : 2023/7/21 11:15
-@Author  : Leo Xiao
-@File    : anthropic_api.py
+@File    : spark_api.py
 """
 import _thread as thread
 import base64
@@ -13,55 +11,36 @@ import hmac
 import json
 import ssl
 from time import mktime
-from typing import Optional
-from urllib.parse import urlencode
-from urllib.parse import urlparse
+from urllib.parse import urlencode, urlparse
 from wsgiref.handlers import format_date_time
 
 import websocket  # 使用websocket_client
 
-from metagpt.config import CONFIG
+from metagpt.config import CONFIG, LLMProviderEnum
 from metagpt.logs import logger
-from metagpt.provider.base_gpt_api import BaseGPTAPI
+from metagpt.provider.base_llm import BaseLLM
+from metagpt.provider.llm_provider_registry import register_provider
 
 
-class SparkAPI(BaseGPTAPI):
-
+@register_provider(LLMProviderEnum.SPARK)
+class SparkLLM(BaseLLM):
     def __init__(self):
-        logger.warning('当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。')
-
-    def ask(self, msg: str) -> str:
-        message = [self._default_system_msg(), self._user_msg(msg)]
-        rsp = self.completion(message)
-        return rsp
-
-    async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str:
-        if system_msgs:
-            message = self._system_msgs(system_msgs) + [self._user_msg(msg)]
-        else:
-            message = [self._default_system_msg(), self._user_msg(msg)]
-        rsp = await self.acompletion(message)
-        logger.debug(message)
-        return rsp
+        logger.warning("当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。")
 
     def get_choice_text(self, rsp: dict) -> str:
         return rsp["payload"]["choices"]["text"][-1]["content"]
 
-    async def acompletion_text(self, messages: list[dict], stream=False) -> str:
+    async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = 3) -> str:
         # 不支持
-        logger.error('该功能禁用。')
+        logger.error("该功能禁用。")
         w = GetMessageFromWeb(messages)
         return w.run()
 
-    async def acompletion(self, messages: list[dict]):
+    async def acompletion(self, messages: list[dict], timeout=3):
         # 不支持异步
         w = GetMessageFromWeb(messages)
         return w.run()
 
-    def completion(self, messages: list[dict]):
-        w = GetMessageFromWeb(messages)
-        return w.run()
-
 
 class GetMessageFromWeb:
     class WsParam:
@@ -93,29 +72,26 @@ class GetMessageFromWeb:
             signature_origin += "GET " + self.path + " HTTP/1.1"
 
             # 进行hmac-sha256进行加密
-            signature_sha = hmac.new(self.api_secret.encode('utf-8'), signature_origin.encode('utf-8'),
-                                     digestmod=hashlib.sha256).digest()
+            signature_sha = hmac.new(
+                self.api_secret.encode("utf-8"), signature_origin.encode("utf-8"), digestmod=hashlib.sha256
+            ).digest()
 
-            signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding='utf-8')
+            signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding="utf-8")
 
             authorization_origin = f'api_key="{self.api_key}", algorithm="hmac-sha256", headers="host date request-line", signature="{signature_sha_base64}"'
 
-            authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8')
+            authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode(encoding="utf-8")
 
             # 将请求的鉴权参数组合为字典
-            v = {
-                "authorization": authorization,
-                "date": date,
-                "host": self.host
-            }
+            v = {"authorization": authorization, "date": date, "host": self.host}
             # 拼接鉴权参数，生成url
-            url = self.spark_url + '?' + urlencode(v)
+            url = self.spark_url + "?" + urlencode(v)
             # 此处打印出建立连接时候的url,参考本demo的时候可取消上方打印的注释，比对相同参数时生成的url与自己代码生成的url是否一致
             return url
 
     def __init__(self, text):
         self.text = text
-        self.ret = ''
+        self.ret = ""
         self.spark_appid = CONFIG.spark_appid
         self.spark_api_secret = CONFIG.spark_api_secret
         self.spark_api_key = CONFIG.spark_api_key
@@ -124,15 +100,15 @@ class GetMessageFromWeb:
 
     def on_message(self, ws, message):
         data = json.loads(message)
-        code = data['header']['code']
+        code = data["header"]["code"]
 
         if code != 0:
             ws.close()  # 请求错误，则关闭socket
-            logger.critical(f'回答获取失败，响应信息反序列化之后为： {data}')
+            logger.critical(f"回答获取失败，响应信息反序列化之后为： {data}")
             return
         else:
             choices = data["payload"]["choices"]
-            seq = choices["seq"]  # 服务端是流式返回，seq为返回的数据序号
+            # seq = choices["seq"]  # 服务端是流式返回，seq为返回的数据序号
             status = choices["status"]  # 服务端是流式返回，status用于判断信息是否传送完毕
             content = choices["text"][0]["content"]  # 本次接收到的回答文本
             self.ret += content
@@ -142,7 +118,7 @@ class GetMessageFromWeb:
     # 收到websocket错误的处理
     def on_error(self, ws, error):
         # on_message方法处理接收到的信息，出现任何错误，都会调用这个方法
-        logger.critical(f'通讯连接出错，【错误提示: {error}】')
+        logger.critical(f"通讯连接出错，【错误提示: {error}】")
 
     # 收到websocket关闭的处理
     def on_close(self, ws, one, two):
@@ -150,17 +126,12 @@ class GetMessageFromWeb:
 
     # 处理请求数据
     def gen_params(self):
-
         data = {
-            "header": {
-                "app_id": self.spark_appid,
-                "uid": "1234"
-            },
+            "header": {"app_id": self.spark_appid, "uid": "1234"},
             "parameter": {
                 "chat": {
                     # domain为必传参数
                     "domain": self.domain,
-
                     # 以下为可微调，非必传参数
                     # 注意：官方建议，temperature和top_k修改一个即可
                     "max_tokens": 2048,  # 默认2048，模型回答的tokens的最大长度，即允许它输出文本的最长字数
@@ -168,11 +139,7 @@ class GetMessageFromWeb:
                     "top_k": 4,  # 取值为[1，6],默认为4。从k个候选中随机选择一个（非等概率）
                 }
             },
-            "payload": {
-                "message": {
-                    "text": self.text
-                }
-            }
+            "payload": {"message": {"text": self.text}},
         }
         return data
 
@@ -189,17 +156,12 @@ class GetMessageFromWeb:
         return self._run(self.text)
 
     def _run(self, text_list):
-
-        ws_param = self.WsParam(
-            self.spark_appid,
-            self.spark_api_key,
-            self.spark_api_secret,
-            self.spark_url,
-            text_list)
+        ws_param = self.WsParam(self.spark_appid, self.spark_api_key, self.spark_api_secret, self.spark_url, text_list)
         ws_url = ws_param.create_url()
 
         websocket.enableTrace(False)  # 默认禁用 WebSocket 的跟踪功能
-        ws = websocket.WebSocketApp(ws_url, on_message=self.on_message, on_error=self.on_error, on_close=self.on_close,
-                                    on_open=self.on_open)
+        ws = websocket.WebSocketApp(
+            ws_url, on_message=self.on_message, on_error=self.on_error, on_close=self.on_close, on_open=self.on_open
+        )
         ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE})
         return self.ret

metagpt/provider/zhipuai/async_sse_client.py

@@ -3,11 +3,10 @@
 # @Desc   : async_sse_client to make keep the use of Event to access response
 #           refs to `https://github.com/zhipuai/zhipuai-sdk-python/blob/main/zhipuai/utils/sse_client.py`
 
-from zhipuai.utils.sse_client import SSEClient, Event, _FIELD_SEPARATOR
+from zhipuai.utils.sse_client import _FIELD_SEPARATOR, Event, SSEClient
 
 
 class AsyncSSEClient(SSEClient):
-
     async def _aread(self):
         data = b""
         async for chunk in self._event_source:
@@ -37,9 +36,7 @@ class AsyncSSEClient(SSEClient):
 
                 # Ignore unknown fields.
                 if field not in event.__dict__:
-                    self._logger.debug(
-                        "Saw invalid field %s while parsing " "Server Side Event", field
-                    )
+                    self._logger.debug("Saw invalid field %s while parsing " "Server Side Event", field)
                     continue
 
                 if len(data) > 1:

metagpt/provider/zhipuai/zhipu_model_api.py

@@ -2,16 +2,17 @@
 # -*- coding: utf-8 -*-
 # @Desc   : zhipu model api to support sync & async for invoke & sse_invoke
 
+import json
+
 import zhipuai
-from zhipuai.model_api.api import ModelAPI, InvokeType
+from zhipuai.model_api.api import InvokeType, ModelAPI
 from zhipuai.utils.http_client import headers as zhipuai_default_headers
 
-from metagpt.provider.zhipuai.async_sse_client import AsyncSSEClient
 from metagpt.provider.general_api_requestor import GeneralAPIRequestor
+from metagpt.provider.zhipuai.async_sse_client import AsyncSSEClient
 
 
 class ZhiPuModelAPI(ModelAPI):
-
     @classmethod
     def get_header(cls) -> dict:
         token = cls._generate_token()
@@ -21,9 +22,7 @@ class ZhiPuModelAPI(ModelAPI):
     @classmethod
     def get_sse_header(cls) -> dict:
         token = cls._generate_token()
-        headers = {
-            "Authorization": token
-        }
+        headers = {"Authorization": token}
         return headers
 
     @classmethod
@@ -36,7 +35,7 @@ class ZhiPuModelAPI(ModelAPI):
                 zhipu_api_url: https://open.bigmodel.cn/api/paas/v3/model-api/{model}/{invoke_method}
         """
         arr = zhipu_api_url.split("/api/")
-        # ("https://open.bigmodel.cn/api/" , "/paas/v3/model-api/chatglm_turbo/invoke")
+        # ("https://open.bigmodel.cn/api" , "/paas/v3/model-api/chatglm_turbo/invoke")
         return f"{arr[0]}/api", f"/{arr[1]}"
 
     @classmethod
@@ -44,36 +43,33 @@ class ZhiPuModelAPI(ModelAPI):
         # TODO to make the async request to be more generic for models in http mode.
         assert method in ["post", "get"]
 
-        api_base, url = cls.split_zhipu_api_url(invoke_type, kwargs)
-        requester = GeneralAPIRequestor(api_base=api_base)
+        base_url, url = cls.split_zhipu_api_url(invoke_type, kwargs)
+        requester = GeneralAPIRequestor(base_url=base_url)
         result, _, api_key = await requester.arequest(
             method=method,
             url=url,
             headers=headers,
             stream=stream,
             params=kwargs,
-            request_timeout=zhipuai.api_timeout_seconds
+            request_timeout=zhipuai.api_timeout_seconds,
         )
-
         return result
 
     @classmethod
     async def ainvoke(cls, **kwargs) -> dict:
-        """ async invoke different from raw method `async_invoke` which get the final result by task_id"""
+        """async invoke different from raw method `async_invoke` which get the final result by task_id"""
         headers = cls.get_header()
-        resp = await cls.arequest(invoke_type=InvokeType.SYNC,
-                                  stream=False,
-                                  method="post",
-                                  headers=headers,
-                                  kwargs=kwargs)
+        resp = await cls.arequest(
+            invoke_type=InvokeType.SYNC, stream=False, method="post", headers=headers, kwargs=kwargs
+        )
+        resp = resp.decode("utf-8")
+        resp = json.loads(resp)
         return resp
 
     @classmethod
     async def asse_invoke(cls, **kwargs) -> AsyncSSEClient:
-        """ async sse_invoke """
+        """async sse_invoke"""
         headers = cls.get_sse_header()
-        return AsyncSSEClient(await cls.arequest(invoke_type=InvokeType.SSE,
-                                                 stream=True,
-                                                 method="post",
-                                                 headers=headers,
-                                                 kwargs=kwargs))
+        return AsyncSSEClient(
+            await cls.arequest(invoke_type=InvokeType.SSE, stream=True, method="post", headers=headers, kwargs=kwargs)
+        )

metagpt/provider/zhipuai_api.py

@@ -2,24 +2,25 @@
 # -*- coding: utf-8 -*-
 # @Desc   : zhipuai LLM from https://open.bigmodel.cn/dev/api#sdk
 
-from enum import Enum
 import json
+from enum import Enum
+
+import openai
+import zhipuai
+from requests import ConnectionError
 from tenacity import (
     after_log,
     retry,
     retry_if_exception_type,
     stop_after_attempt,
-    wait_fixed,
+    wait_random_exponential,
 )
-from requests import ConnectionError
 
-import openai
-import zhipuai
-
-from metagpt.config import CONFIG
-from metagpt.logs import logger
-from metagpt.provider.base_gpt_api import BaseGPTAPI
-from metagpt.provider.openai_api import CostManager, log_and_reraise
+from metagpt.config import CONFIG, LLMProviderEnum
+from metagpt.logs import log_llm_stream, logger
+from metagpt.provider.base_llm import BaseLLM
+from metagpt.provider.llm_provider_registry import register_provider
+from metagpt.provider.openai_api import log_and_reraise
 from metagpt.provider.zhipuai.zhipu_model_api import ZhiPuModelAPI
 
 
@@ -30,65 +31,64 @@ class ZhiPuEvent(Enum):
     FINISH = "finish"
 
 
-class ZhiPuAIGPTAPI(BaseGPTAPI):
+@register_provider(LLMProviderEnum.ZHIPUAI)
+class ZhiPuAILLM(BaseLLM):
     """
     Refs to `https://open.bigmodel.cn/dev/api#chatglm_turbo`
     From now, there is only one model named `chatglm_turbo`
     """
 
-    use_system_prompt: bool = False  # zhipuai has no system prompt when use api
-
     def __init__(self):
         self.__init_zhipuai(CONFIG)
         self.llm = ZhiPuModelAPI
         self.model = "chatglm_turbo"  # so far only one model, just use it
-        self._cost_manager = CostManager()
+        self.use_system_prompt: bool = False  # zhipuai has no system prompt when use api
 
     def __init_zhipuai(self, config: CONFIG):
         assert config.zhipuai_api_key
         zhipuai.api_key = config.zhipuai_api_key
-        openai.api_key = zhipuai.api_key  # due to use openai sdk, set the api_key but it will't be used.
+        # due to use openai sdk, set the api_key but it will't be used.
+        # openai.api_key = zhipuai.api_key  # due to use openai sdk, set the api_key but it will't be used.
+        if config.openai_proxy:
+            # FIXME: openai v1.x sdk has no proxy support
+            openai.proxy = config.openai_proxy
 
     def _const_kwargs(self, messages: list[dict]) -> dict:
-        kwargs = {
-            "model": self.model,
-            "prompt": messages,
-            "temperature": 0.3
-        }
+        kwargs = {"model": self.model, "prompt": messages, "temperature": 0.3}
         return kwargs
 
     def _update_costs(self, usage: dict):
-        """ update each request's token cost """
+        """update each request's token cost"""
         if CONFIG.calc_usage:
             try:
                 prompt_tokens = int(usage.get("prompt_tokens", 0))
                 completion_tokens = int(usage.get("completion_tokens", 0))
-                self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model)
+                CONFIG.cost_manager.update_cost(prompt_tokens, completion_tokens, self.model)
             except Exception as e:
-                logger.error("zhipuai updats costs failed!", e)
+                logger.error(f"zhipuai updats costs failed! exp: {e}")
 
     def get_choice_text(self, resp: dict) -> str:
-        """ get the first text of choice from llm response """
+        """get the first text of choice from llm response"""
         assist_msg = resp.get("data", {}).get("choices", [{"role": "error"}])[-1]
         assert assist_msg["role"] == "assistant"
         return assist_msg.get("content")
 
-    def completion(self, messages: list[dict]) -> dict:
+    def completion(self, messages: list[dict], timeout=3) -> dict:
         resp = self.llm.invoke(**self._const_kwargs(messages))
         usage = resp.get("data").get("usage")
         self._update_costs(usage)
         return resp
 
-    async def _achat_completion(self, messages: list[dict]) -> dict:
+    async def _achat_completion(self, messages: list[dict], timeout=3) -> dict:
         resp = await self.llm.ainvoke(**self._const_kwargs(messages))
         usage = resp.get("data").get("usage")
         self._update_costs(usage)
         return resp
 
-    async def acompletion(self, messages: list[dict]) -> dict:
-        return await self._achat_completion(messages)
+    async def acompletion(self, messages: list[dict], timeout=3) -> dict:
+        return await self._achat_completion(messages, timeout=timeout)
 
-    async def _achat_completion_stream(self, messages: list[dict]) -> str:
+    async def _achat_completion_stream(self, messages: list[dict], timeout=3) -> str:
         response = await self.llm.asse_invoke(**self._const_kwargs(messages))
         collected_content = []
         usage = {}
@@ -96,11 +96,10 @@ class ZhiPuAIGPTAPI(BaseGPTAPI):
             if event.event == ZhiPuEvent.ADD.value:
                 content = event.data
                 collected_content.append(content)
-                print(content, end="")
+                log_llm_stream(content)
             elif event.event == ZhiPuEvent.ERROR.value or event.event == ZhiPuEvent.INTERRUPTED.value:
                 content = event.data
                 logger.error(f"event error: {content}", end="")
-                collected_content.append([content])
             elif event.event == ZhiPuEvent.FINISH.value:
                 """
                 event.meta
@@ -119,6 +118,7 @@ class ZhiPuAIGPTAPI(BaseGPTAPI):
                 usage = meta.get("usage")
             else:
                 print(f"zhipuapi else event: {event.data}", end="")
+        log_llm_stream("\n")
 
         self._update_costs(usage)
         full_content = "".join(collected_content)
@@ -126,13 +126,13 @@ class ZhiPuAIGPTAPI(BaseGPTAPI):
 
     @retry(
         stop=stop_after_attempt(3),
-        wait=wait_fixed(1),
+        wait=wait_random_exponential(min=1, max=60),
         after=after_log(logger, logger.level("WARNING").name),
         retry=retry_if_exception_type(ConnectionError),
-        retry_error_callback=log_and_reraise
+        retry_error_callback=log_and_reraise,
     )
-    async def acompletion_text(self, messages: list[dict], stream=False) -> str:
-        """ response in async with stream or non-stream mode """
+    async def acompletion_text(self, messages: list[dict], stream=False, timeout=3) -> str:
+        """response in async with stream or non-stream mode"""
         if stream:
             return await self._achat_completion_stream(messages)
         resp = await self._achat_completion(messages)

metagpt/repo_parser.py

@@ -0,0 +1,421 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/11/17 17:58
+@Author  : alexanderwu
+@File    : repo_parser.py
+"""
+from __future__ import annotations
+
+import ast
+import json
+import re
+import subprocess
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import pandas as pd
+from pydantic import BaseModel, Field
+
+from metagpt.const import AGGREGATION, COMPOSITION, GENERALIZATION
+from metagpt.logs import logger
+from metagpt.utils.common import any_to_str, aread
+from metagpt.utils.exceptions import handle_exception
+
+
+class RepoFileInfo(BaseModel):
+    file: str
+    classes: List = Field(default_factory=list)
+    functions: List = Field(default_factory=list)
+    globals: List = Field(default_factory=list)
+    page_info: List = Field(default_factory=list)
+
+
+class CodeBlockInfo(BaseModel):
+    lineno: int
+    end_lineno: int
+    type_name: str
+    tokens: List = Field(default_factory=list)
+    properties: Dict = Field(default_factory=dict)
+
+
+class ClassInfo(BaseModel):
+    name: str
+    package: Optional[str] = None
+    attributes: Dict[str, str] = Field(default_factory=dict)
+    methods: Dict[str, str] = Field(default_factory=dict)
+
+
+class ClassRelationship(BaseModel):
+    src: str = ""
+    dest: str = ""
+    relationship: str = ""
+    label: Optional[str] = None
+
+
+class RepoParser(BaseModel):
+    base_directory: Path = Field(default=None)
+
+    @classmethod
+    @handle_exception(exception_type=Exception, default_return=[])
+    def _parse_file(cls, file_path: Path) -> list:
+        """Parse a Python file in the repository."""
+        return ast.parse(file_path.read_text()).body
+
+    def extract_class_and_function_info(self, tree, file_path) -> RepoFileInfo:
+        """Extract class, function, and global variable information from the AST."""
+        file_info = RepoFileInfo(file=str(file_path.relative_to(self.base_directory)))
+        for node in tree:
+            info = RepoParser.node_to_str(node)
+            if info:
+                file_info.page_info.append(info)
+            if isinstance(node, ast.ClassDef):
+                class_methods = [m.name for m in node.body if is_func(m)]
+                file_info.classes.append({"name": node.name, "methods": class_methods})
+            elif is_func(node):
+                file_info.functions.append(node.name)
+            elif isinstance(node, (ast.Assign, ast.AnnAssign)):
+                for target in node.targets if isinstance(node, ast.Assign) else [node.target]:
+                    if isinstance(target, ast.Name):
+                        file_info.globals.append(target.id)
+        return file_info
+
+    def generate_symbols(self) -> List[RepoFileInfo]:
+        files_classes = []
+        directory = self.base_directory
+
+        matching_files = []
+        extensions = ["*.py", "*.js"]
+        for ext in extensions:
+            matching_files += directory.rglob(ext)
+        for path in matching_files:
+            tree = self._parse_file(path)
+            file_info = self.extract_class_and_function_info(tree, path)
+            files_classes.append(file_info)
+
+        return files_classes
+
+    def generate_json_structure(self, output_path):
+        """Generate a JSON file documenting the repository structure."""
+        files_classes = [i.model_dump() for i in self.generate_symbols()]
+        output_path.write_text(json.dumps(files_classes, indent=4))
+
+    def generate_dataframe_structure(self, output_path):
+        """Generate a DataFrame documenting the repository structure and save as CSV."""
+        files_classes = [i.model_dump() for i in self.generate_symbols()]
+        df = pd.DataFrame(files_classes)
+        df.to_csv(output_path, index=False)
+
+    def generate_structure(self, output_path=None, mode="json") -> Path:
+        """Generate the structure of the repository as a specified format."""
+        output_file = self.base_directory / f"{self.base_directory.name}-structure.{mode}"
+        output_path = Path(output_path) if output_path else output_file
+
+        if mode == "json":
+            self.generate_json_structure(output_path)
+        elif mode == "csv":
+            self.generate_dataframe_structure(output_path)
+        return output_path
+
+    @staticmethod
+    def node_to_str(node) -> CodeBlockInfo | None:
+        if isinstance(node, ast.Try):
+            return None
+        if any_to_str(node) == any_to_str(ast.Expr):
+            return CodeBlockInfo(
+                lineno=node.lineno,
+                end_lineno=node.end_lineno,
+                type_name=any_to_str(node),
+                tokens=RepoParser._parse_expr(node),
+            )
+        mappings = {
+            any_to_str(ast.Import): lambda x: [RepoParser._parse_name(n) for n in x.names],
+            any_to_str(ast.Assign): RepoParser._parse_assign,
+            any_to_str(ast.ClassDef): lambda x: x.name,
+            any_to_str(ast.FunctionDef): lambda x: x.name,
+            any_to_str(ast.ImportFrom): lambda x: {
+                "module": x.module,
+                "names": [RepoParser._parse_name(n) for n in x.names],
+            },
+            any_to_str(ast.If): RepoParser._parse_if,
+            any_to_str(ast.AsyncFunctionDef): lambda x: x.name,
+            any_to_str(ast.AnnAssign): lambda x: RepoParser._parse_variable(x.target),
+        }
+        func = mappings.get(any_to_str(node))
+        if func:
+            code_block = CodeBlockInfo(lineno=node.lineno, end_lineno=node.end_lineno, type_name=any_to_str(node))
+            val = func(node)
+            if isinstance(val, dict):
+                code_block.properties = val
+            elif isinstance(val, list):
+                code_block.tokens = val
+            elif isinstance(val, str):
+                code_block.tokens = [val]
+            else:
+                raise NotImplementedError(f"Not implement:{val}")
+            return code_block
+        logger.warning(f"Unsupported code block:{node.lineno}, {node.end_lineno}, {any_to_str(node)}")
+        return None
+
+    @staticmethod
+    def _parse_expr(node) -> List:
+        funcs = {
+            any_to_str(ast.Constant): lambda x: [any_to_str(x.value), RepoParser._parse_variable(x.value)],
+            any_to_str(ast.Call): lambda x: [any_to_str(x.value), RepoParser._parse_variable(x.value.func)],
+        }
+        func = funcs.get(any_to_str(node.value))
+        if func:
+            return func(node)
+        raise NotImplementedError(f"Not implement: {node.value}")
+
+    @staticmethod
+    def _parse_name(n):
+        if n.asname:
+            return f"{n.name} as {n.asname}"
+        return n.name
+
+    @staticmethod
+    def _parse_if(n):
+        tokens = []
+        try:
+            if isinstance(n.test, ast.BoolOp):
+                tokens = []
+                for v in n.test.values:
+                    tokens.extend(RepoParser._parse_if_compare(v))
+                return tokens
+            if isinstance(n.test, ast.Compare):
+                v = RepoParser._parse_variable(n.test.left)
+                if v:
+                    tokens.append(v)
+            for item in n.test.comparators:
+                v = RepoParser._parse_variable(item)
+                if v:
+                    tokens.append(v)
+            return tokens
+        except Exception as e:
+            logger.warning(f"Unsupported if: {n}, err:{e}")
+        return tokens
+
+    @staticmethod
+    def _parse_if_compare(n):
+        if hasattr(n, "left"):
+            return RepoParser._parse_variable(n.left)
+        else:
+            return []
+
+    @staticmethod
+    def _parse_variable(node):
+        try:
+            funcs = {
+                any_to_str(ast.Constant): lambda x: x.value,
+                any_to_str(ast.Name): lambda x: x.id,
+                any_to_str(ast.Attribute): lambda x: f"{x.value.id}.{x.attr}"
+                if hasattr(x.value, "id")
+                else f"{x.attr}",
+                any_to_str(ast.Call): lambda x: RepoParser._parse_variable(x.func),
+                any_to_str(ast.Tuple): lambda x: "",
+            }
+            func = funcs.get(any_to_str(node))
+            if not func:
+                raise NotImplementedError(f"Not implement:{node}")
+            return func(node)
+        except Exception as e:
+            logger.warning(f"Unsupported variable:{node}, err:{e}")
+
+    @staticmethod
+    def _parse_assign(node):
+        return [RepoParser._parse_variable(t) for t in node.targets]
+
+    async def rebuild_class_views(self, path: str | Path = None):
+        if not path:
+            path = self.base_directory
+        path = Path(path)
+        if not path.exists():
+            return
+        command = f"pyreverse {str(path)} -o dot"
+        result = subprocess.run(command, shell=True, check=True, cwd=str(path))
+        if result.returncode != 0:
+            raise ValueError(f"{result}")
+        class_view_pathname = path / "classes.dot"
+        class_views = await self._parse_classes(class_view_pathname)
+        relationship_views = await self._parse_class_relationships(class_view_pathname)
+        packages_pathname = path / "packages.dot"
+        class_views, relationship_views, package_root = RepoParser._repair_namespaces(
+            class_views=class_views, relationship_views=relationship_views, path=path
+        )
+        class_view_pathname.unlink(missing_ok=True)
+        packages_pathname.unlink(missing_ok=True)
+        return class_views, relationship_views, package_root
+
+    async def _parse_classes(self, class_view_pathname):
+        class_views = []
+        if not class_view_pathname.exists():
+            return class_views
+        data = await aread(filename=class_view_pathname, encoding="utf-8")
+        lines = data.split("\n")
+        for line in lines:
+            package_name, info = RepoParser._split_class_line(line)
+            if not package_name:
+                continue
+            class_name, members, functions = re.split(r"(?<!\\)\|", info)
+            class_info = ClassInfo(name=class_name)
+            class_info.package = package_name
+            for m in members.split("\n"):
+                if not m:
+                    continue
+                member_name = m.split(":", 1)[0].strip() if ":" in m else m.strip()
+                class_info.attributes[member_name] = m
+            for f in functions.split("\n"):
+                if not f:
+                    continue
+                function_name, _ = f.split("(", 1)
+                class_info.methods[function_name] = f
+            class_views.append(class_info)
+        return class_views
+
+    async def _parse_class_relationships(self, class_view_pathname) -> List[ClassRelationship]:
+        relationship_views = []
+        if not class_view_pathname.exists():
+            return relationship_views
+        data = await aread(filename=class_view_pathname, encoding="utf-8")
+        lines = data.split("\n")
+        for line in lines:
+            relationship = RepoParser._split_relationship_line(line)
+            if not relationship:
+                continue
+            relationship_views.append(relationship)
+        return relationship_views
+
+    @staticmethod
+    def _split_class_line(line):
+        part_splitor = '" ['
+        if part_splitor not in line:
+            return None, None
+        ix = line.find(part_splitor)
+        class_name = line[0:ix].replace('"', "")
+        left = line[ix:]
+        begin_flag = "label=<{"
+        end_flag = "}>"
+        if begin_flag not in left or end_flag not in left:
+            return None, None
+        bix = left.find(begin_flag)
+        eix = left.rfind(end_flag)
+        info = left[bix + len(begin_flag) : eix]
+        info = re.sub(r"<br[^>]*>", "\n", info)
+        return class_name, info
+
+    @staticmethod
+    def _split_relationship_line(line):
+        splitters = [" -> ", " [", "];"]
+        idxs = []
+        for tag in splitters:
+            if tag not in line:
+                return None
+            idxs.append(line.find(tag))
+        ret = ClassRelationship()
+        ret.src = line[0 : idxs[0]].strip('"')
+        ret.dest = line[idxs[0] + len(splitters[0]) : idxs[1]].strip('"')
+        properties = line[idxs[1] + len(splitters[1]) : idxs[2]].strip(" ")
+        mappings = {
+            'arrowhead="empty"': GENERALIZATION,
+            'arrowhead="diamond"': COMPOSITION,
+            'arrowhead="odiamond"': AGGREGATION,
+        }
+        for k, v in mappings.items():
+            if k in properties:
+                ret.relationship = v
+                if v != GENERALIZATION:
+                    ret.label = RepoParser._get_label(properties)
+                break
+        return ret
+
+    @staticmethod
+    def _get_label(line):
+        tag = 'label="'
+        if tag not in line:
+            return ""
+        ix = line.find(tag)
+        eix = line.find('"', ix + len(tag))
+        return line[ix + len(tag) : eix]
+
+    @staticmethod
+    def _create_path_mapping(path: str | Path) -> Dict[str, str]:
+        mappings = {
+            str(path).replace("/", "."): str(path),
+        }
+        files = []
+        try:
+            directory_path = Path(path)
+            if not directory_path.exists():
+                return mappings
+            for file_path in directory_path.iterdir():
+                if file_path.is_file():
+                    files.append(str(file_path))
+                else:
+                    subfolder_files = RepoParser._create_path_mapping(path=file_path)
+                    mappings.update(subfolder_files)
+        except Exception as e:
+            logger.error(f"Error: {e}")
+        for f in files:
+            mappings[str(Path(f).with_suffix("")).replace("/", ".")] = str(f)
+
+        return mappings
+
+    @staticmethod
+    def _repair_namespaces(
+        class_views: List[ClassInfo], relationship_views: List[ClassRelationship], path: str | Path
+    ) -> (List[ClassInfo], List[ClassRelationship], str):
+        if not class_views:
+            return [], [], ""
+        c = class_views[0]
+        full_key = str(path).lstrip("/").replace("/", ".")
+        root_namespace = RepoParser._find_root(full_key, c.package)
+        root_path = root_namespace.replace(".", "/")
+
+        mappings = RepoParser._create_path_mapping(path=path)
+        new_mappings = {}
+        ix_root_namespace = len(root_namespace)
+        ix_root_path = len(root_path)
+        for k, v in mappings.items():
+            nk = k[ix_root_namespace:]
+            nv = v[ix_root_path:]
+            new_mappings[nk] = nv
+
+        for c in class_views:
+            c.package = RepoParser._repair_ns(c.package, new_mappings)
+        for i in range(len(relationship_views)):
+            v = relationship_views[i]
+            v.src = RepoParser._repair_ns(v.src, new_mappings)
+            v.dest = RepoParser._repair_ns(v.dest, new_mappings)
+            relationship_views[i] = v
+        return class_views, relationship_views, root_path
+
+    @staticmethod
+    def _repair_ns(package, mappings):
+        file_ns = package
+        while file_ns != "":
+            if file_ns not in mappings:
+                ix = file_ns.rfind(".")
+                file_ns = file_ns[0:ix]
+                continue
+            break
+        internal_ns = package[ix + 1 :]
+        ns = mappings[file_ns] + ":" + internal_ns.replace(".", ":")
+        return ns
+
+    @staticmethod
+    def _find_root(full_key, package) -> str:
+        left = full_key
+        while left != "":
+            if left in package:
+                break
+            if "." not in left:
+                break
+            ix = left.find(".")
+            left = left[ix + 1 :]
+        ix = full_key.rfind(left)
+        return "." + full_key[0:ix]
+
+
+def is_func(node):
+    return isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))

metagpt/roles/__init__.py

@@ -12,7 +12,7 @@ from metagpt.roles.project_manager import ProjectManager
 from metagpt.roles.product_manager import ProductManager
 from metagpt.roles.engineer import Engineer
 from metagpt.roles.qa_engineer import QaEngineer
-from metagpt.roles.seacher import Searcher
+from metagpt.roles.searcher import Searcher
 from metagpt.roles.sales import Sales
 from metagpt.roles.customer_service import CustomerService