Merge remote-tracking branch 'upstream/main'

Dockerfile

@@ -8,7 +8,8 @@ RUN apt update &&\
 
 # Install Mermaid CLI globally
 ENV CHROME_BIN="/usr/bin/chromium" \
-    AM_I_IN_A_DOCKER_CONTAINER="true"
+    PUPPETEER_CONFIG="/app/metagpt/config/puppeteer-config.json"\
+    PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true"
 RUN npm install -g @mermaid-js/mermaid-cli &&\
     npm cache clean --force
 

README.md

@@ -75,25 +75,25 @@ ### Installation by Docker
 
 ```bash
 # Step 1: Download metagpt official image and prepare config.yaml
-docker pull metagpt/metagpt:v0.3
+docker pull metagpt/metagpt:v0.3.1
 mkdir -p /opt/metagpt/{config,workspace}
-docker run --rm metagpt/metagpt:v0.3 cat /app/metagpt/config/config.yaml > /opt/metagpt/config/config.yaml
-vim /opt/metagpt/config/config.yaml # Change the config
+docker run --rm metagpt/metagpt:v0.3.1 cat /app/metagpt/config/config.yaml > /opt/metagpt/config/key.yaml
+vim /opt/metagpt/config/key.yaml # Change the config
 
 # Step 2: Run metagpt demo with container
 docker run --rm \
     --privileged \
-    -v /opt/metagpt/config:/app/metagpt/config \
+    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
-    metagpt/metagpt:v0.3 \
+    metagpt/metagpt:v0.3.1 \
     python startup.py "Write a cli snake game"
 
 # You can also start a container and execute commands in it
 docker run --name metagpt -d \
     --privileged \
-    -v /opt/metagpt/config:/app/metagpt/config \
+    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
-    metagpt/metagpt:v0.3
+    metagpt/metagpt:v0.3.1
 
 docker exec -it metagpt /bin/bash
 $ python startup.py "Write a cli snake game"
@@ -111,7 +111,7 @@ ### Build image by yourself
 ```bash
 # You can also build metagpt image by yourself.
 git clone https://github.com/geekan/MetaGPT.git
-cd MetaGPT && docker build -t metagpt:v0.3 .
+cd MetaGPT && docker build -t metagpt:custom .
 ```
 
 ## Configuration
@@ -191,6 +191,25 @@ ### Code walkthrough
 
 You can check `examples` for more details on single role (with knowledge base) and LLM only examples.
 
+## QuickStart
+It is difficult to install and configure the local environment for some users. The following tutorials will allow you to quickly experience the charm of MetaGPT.  
+
+- [MetaGPT quickstart](https://deepwisdom.feishu.cn/wiki/CyY9wdJc4iNqArku3Lncl4v8n2b)
+
+## Citation
+
+For now, cite the [Arxiv paper](https://arxiv.org/abs/2308.00352):
+```bibtex
+@misc{hong2023metagpt,
+      title={MetaGPT: Meta Programming for Multi-Agent Collaborative Framework}, 
+      author={Sirui Hong and Xiawu Zheng and Jonathan Chen and Yuheng Cheng and Jinlin Wang and Ceyao Zhang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu},
+      year={2023},
+      eprint={2308.00352},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI}
+}
+```
+
 ## Contact Information
 
 If you have any questions or feedback about this project, please feel free to contact us. We highly appreciate your suggestions!

docs/README_CN.md

@@ -175,6 +175,11 @@ ### 代码实现
 
 你可以查看`examples`，其中有单角色（带知识库）的使用例子与仅LLM的使用例子。
 
+## 快速体验
+对一些用户来说，安装配置本地环境是有困难的，下面这些教程能够让你快速体验到MetaGPT的魅力。  
+
+- [MetaGPT快速体验](https://deepwisdom.feishu.cn/wiki/Q8ycw6J9tiNXdHk66MRcIN8Pnlg)
+
 ## 联系信息
 
 如果您对这个项目有任何问题或反馈，欢迎联系我们。我们非常欢迎您的建议！
@@ -190,8 +195,6 @@ ## 演示
 
 ## 加入微信讨论群
 
-<img src="resources/MetaGPT-WeChat-Group4.jpeg" width = "30%" height = "30%" alt="MetaGPT WeChat Discuss Group" align=center />
+添加运营小姐姐，拉你入群
 
-如果群已满，请添加负责人微信，会邀请进群
-
-<img src="resources/MetaGPT-WeChat-Personal.jpeg" width = "30%" height = "30%" alt="MetaGPT WeChat Discuss Group" align=center />
+<img src="resources/20230808-220924.jpg" width = "30%" height = "30%" alt="MetaGPT WeChat Discuss Group" align=center />

docs/README_JA.md

@@ -75,25 +75,25 @@ ### Docker によるインストール
 
 ```bash
 # ステップ 1: metagpt 公式イメージをダウンロードし、config.yaml を準備する
-docker pull metagpt/metagpt:v0.3
+docker pull metagpt/metagpt:v0.3.1
 mkdir -p /opt/metagpt/{config,workspace}
-docker run --rm metagpt/metagpt:v0.3 cat /app/metagpt/config/config.yaml > /opt/metagpt/config/config.yaml
-vim /opt/metagpt/config/config.yaml # 設定を変更する
+docker run --rm metagpt/metagpt:v0.3.1 cat /app/metagpt/config/config.yaml > /opt/metagpt/config/key.yaml
+vim /opt/metagpt/config/key.yaml # 設定を変更する
 
 # ステップ 2: コンテナで metagpt デモを実行する
 docker run --rm \
     --privileged \
-    -v /opt/metagpt/config:/app/metagpt/config \
+    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
-    metagpt/metagpt:v0.3 \
+    metagpt/metagpt:v0.3.1 \
     python startup.py "Write a cli snake game"
 
 # コンテナを起動し、その中でコマンドを実行することもできます
 docker run --name metagpt -d \
     --privileged \
-    -v /opt/metagpt/config:/app/metagpt/config \
+    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
-    metagpt/metagpt:v0.3
+    metagpt/metagpt:v0.3.1
 
 docker exec -it metagpt /bin/bash
 $ python startup.py "Write a cli snake game"
@@ -111,7 +111,7 @@ ### 自分でイメージをビルドする
 ```bash
 # また、自分で metagpt イメージを構築することもできます。
 git clone https://github.com/geekan/MetaGPT.git
-cd MetaGPT && docker build -t metagpt:v0.3 .
+cd MetaGPT && docker build -t metagpt:custom .
 ```
 
 ## 設定
@@ -142,37 +142,36 @@ ### プラットフォームまたはツールの設定
 
 要件を述べるときに、どのプラットフォームまたはツールを使用するかを指定できます。
 ```shell
-python startup.py "Write a cli snake game based on pygame"
+python startup.py "pygame をベースとした cli ヘビゲームを書く"
 ```
-
 ### 使用方法
 
 ```
-NAME
-    startup.py - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities.
+会社名
+    startup.py - 私たちは AI で構成されたソフトウェア・スタートアップです。私たちに投資することは、無限の可能性に満ちた未来に力を与えることです。
 
-SYNOPSIS
+シノプシス
     startup.py IDEA <flags>
 
-DESCRIPTION
-    We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities.
+説明
+    私たちは AI で構成されたソフトウェア・スタートアップです。私たちに投資することは、無限の可能性に満ちた未来に力を与えることです。
 
-POSITIONAL ARGUMENTS
+位置引数
     IDEA
-        Type: str
-        Your innovative idea, such as "Creating a snake game."
+        型: str
+        あなたの革新的なアイデア、例えば"スネークゲームを作る。"など
 
-FLAGS
+フラグ
     --investment=INVESTMENT
-        Type: float
-        Default: 3.0
-        As an investor, you have the opportunity to contribute a certain dollar amount to this AI company.
+        型: float
+        デフォルト: 3.0
+        投資家として、あなたはこの AI 企業に一定の金額を拠出する機会がある。
     --n_round=N_ROUND
-        Type: int
-        Default: 5
+        型: int
+        デフォルト: 5
 
-NOTES
-    You can also use flags syntax for POSITIONAL ARGUMENTS
+注意事項
+    位置引数にフラグ構文を使うこともできます
 ```
 
 ### コードウォークスルー
@@ -192,6 +191,11 @@ ### コードウォークスルー
 
 `examples` でシングル・ロール（ナレッジ・ベース付き）と LLM のみの例を詳しく見ることができます。
 
+## クイックスタート
+ローカル環境のインストールや設定は、ユーザーによっては難しいものです。以下のチュートリアルで MetaGPT の魅力をすぐに体験できます。
+
+- [MetaGPT クイックスタート](https://deepwisdom.feishu.cn/wiki/Q8ycw6J9tiNXdHk66MRcIN8Pnlg)
+
 ## お問い合わせ先
 
 このプロジェクトに関するご質問やご意見がございましたら、お気軽にお問い合わせください。皆様のご意見をお待ちしております！

metagpt/actions/write_docstring.py

@@ -0,0 +1,214 @@
+"""Code Docstring Generator.
+
+This script provides a tool to automatically generate docstrings for Python code. It uses the specified style to create
+docstrings for the given code and system text.
+
+Usage:
+    python3 -m metagpt.actions.write_docstring <filename> [--overwrite] [--style=<docstring_style>]
+
+Arguments:
+    filename           The path to the Python file for which you want to generate docstrings.
+
+Options:
+    --overwrite        If specified, overwrite the original file with the code containing docstrings.
+    --style=<docstring_style>   Specify the style of the generated docstrings.
+                                Valid values: 'google', 'numpy', or 'sphinx'.
+                                Default: 'google'
+
+Example:
+    python3 -m metagpt.actions.write_docstring startup.py --overwrite False --style=numpy
+
+This script uses the 'fire' library to create a command-line interface. It generates docstrings for the given Python code using
+the specified docstring style and adds them to the code.
+"""
+import ast
+from typing import Literal
+
+from metagpt.actions.action import Action
+from metagpt.utils.common import OutputParser
+from metagpt.utils.pycst import merge_docstring
+
+PYTHON_DOCSTRING_SYSTEM = '''### Requirements
+1. Add docstrings to the given code following the {style} style.
+2. Replace the function body with an Ellipsis object(...) to reduce output.
+3. If the types are already annotated, there is no need to include them in the docstring.
+4. Extract only class, function or the docstrings for the module parts from the given Python code, avoiding any other text.
+
+### Input Example
+```python
+def function_with_pep484_type_annotations(param1: int) -> bool:
+    return isinstance(param1, int)
+
+class ExampleError(Exception):
+    def __init__(self, msg: str):
+        self.msg = msg
+```
+
+### Output Example
+```python
+{example}
+```
+'''
+
+# https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html
+
+PYTHON_DOCSTRING_EXAMPLE_GOOGLE = '''
+def function_with_pep484_type_annotations(param1: int) -> bool:
+    """Example function with PEP 484 type annotations.
+
+    Extended description of function.
+
+    Args:
+        param1: The first parameter.
+
+    Returns:
+        The return value. True for success, False otherwise.
+    """
+    ...
+
+class ExampleError(Exception):
+    """Exceptions are documented in the same way as classes.
+
+    The __init__ method was documented in the class level docstring.
+
+    Args:
+        msg: Human readable string describing the exception.
+
+    Attributes:
+        msg: Human readable string describing the exception.
+    """
+    ...
+'''
+
+PYTHON_DOCSTRING_EXAMPLE_NUMPY = '''
+def function_with_pep484_type_annotations(param1: int) -> bool:
+    """
+    Example function with PEP 484 type annotations.
+
+    Extended description of function.
+
+    Parameters
+    ----------
+    param1
+        The first parameter.
+
+    Returns
+    -------
+    bool
+        The return value. True for success, False otherwise.
+    """
+    ...
+
+class ExampleError(Exception):
+    """
+    Exceptions are documented in the same way as classes.
+
+    The __init__ method was documented in the class level docstring.
+
+    Parameters
+    ----------
+    msg
+        Human readable string describing the exception.
+
+    Attributes
+    ----------
+    msg
+        Human readable string describing the exception.
+    """
+    ...
+'''
+
+PYTHON_DOCSTRING_EXAMPLE_SPHINX = '''
+def function_with_pep484_type_annotations(param1: int) -> bool:
+    """Example function with PEP 484 type annotations.
+
+    Extended description of function.
+
+    :param param1: The first parameter.
+    :type param1: int
+
+    :return: The return value. True for success, False otherwise.
+    :rtype: bool
+    """
+    ...
+
+class ExampleError(Exception):
+    """Exceptions are documented in the same way as classes.
+
+    The __init__ method was documented in the class level docstring.
+
+    :param msg: Human-readable string describing the exception.
+    :type msg: str
+    """
+    ...
+'''
+
+_python_docstring_style = {
+    "google": PYTHON_DOCSTRING_EXAMPLE_GOOGLE.strip(),
+    "numpy": PYTHON_DOCSTRING_EXAMPLE_NUMPY.strip(),
+    "sphinx": PYTHON_DOCSTRING_EXAMPLE_SPHINX.strip(),
+}
+
+
+class WriteDocstring(Action):
+    """This class is used to write docstrings for code.
+
+    Attributes:
+        desc: A string describing the action.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.desc = "Write docstring for code."
+
+    async def run(
+        self, code: str,
+        system_text: str = PYTHON_DOCSTRING_SYSTEM,
+        style: Literal["google", "numpy", "sphinx"] = "google",
+    ) -> str:
+        """Writes docstrings for the given code and system text in the specified style.
+
+        Args:
+            code: A string of Python code.
+            system_text: A string of system text.
+            style: A string specifying the style of the docstring. Can be 'google', 'numpy', or 'sphinx'.
+
+        Returns:
+            The Python code with docstrings added.
+        """
+        system_text = system_text.format(style=style, example=_python_docstring_style[style])
+        simplified_code = _simplify_python_code(code)
+        documented_code = await self._aask(f"```python\n{simplified_code}\n```", [system_text])
+        documented_code = OutputParser.parse_python_code(documented_code)
+        return merge_docstring(code, documented_code)
+
+
+def _simplify_python_code(code: str) -> None:
+    """Simplifies the given Python code by removing expressions and the last if statement.
+
+    Args:
+        code: A string of Python code.
+
+    Returns:
+        The simplified Python code.
+    """
+    code_tree = ast.parse(code)
+    code_tree.body = [i for i in code_tree.body if not isinstance(i, ast.Expr)]
+    if isinstance(code_tree.body[-1], ast.If):
+        code_tree.body.pop()
+    return ast.unparse(code_tree)
+
+
+if __name__ == "__main__":
+    import fire
+
+    async def run(filename: str, overwrite: bool = False, style: Literal["google", "numpy", "sphinx"] = "google"):
+        with open(filename) as f:
+            code = f.read()
+        code = await WriteDocstring().run(code, style=style)
+        if overwrite:
+            with open(filename, "w") as f:
+                f.write(code)
+        return code
+
+    fire.Fire(run)

metagpt/memory/longterm_memory.py

@@ -43,13 +43,13 @@ class LongTermMemory(Memory):
                 # and ignore adding messages from recover repeatedly
                 self.memory_storage.add(message)
 
-    def remember(self, observed: list[Message], k=10) -> list[Message]:
+    def remember(self, observed: list[Message], k=0) -> list[Message]:
         """
         remember the most similar k memories from observed Messages, return all when k=0
             1. remember the short-term memory(stm) news
             2. integrate the stm news with ltm(long-term memory) news
         """
-        stm_news = super(LongTermMemory, self).remember(observed)  # shot-term memory news
+        stm_news = super(LongTermMemory, self).remember(observed, k=k)  # shot-term memory news
         if not self.memory_storage.is_initialized:
             # memory_storage hasn't initialized, use default `remember` to get stm_news
             return stm_news

metagpt/memory/memory.py

@@ -63,7 +63,7 @@ class Memory:
         """Return the most recent k memories, return all when k=0"""
         return self.storage[-k:]
 
-    def remember(self, observed: list[Message], k=10) -> list[Message]:
+    def remember(self, observed: list[Message], k=0) -> list[Message]:
         """remember the most recent k memories from observed Messages, return all when k=0"""
         already_observed = self.get(k)
         news: list[Message] = []

metagpt/roles/engineer.py

@@ -16,6 +16,7 @@ from metagpt.roles import Role
 from metagpt.actions import WriteCode, WriteCodeReview, WriteTasks, WriteDesign
 from metagpt.schema import Message
 from metagpt.utils.common import CodeParser
+from metagpt.utils.special_tokens import MSG_SEP, FILENAME_CODE_SEP
 
 
 async def gather_ordered_k(coros, k) -> list:
@@ -78,7 +79,7 @@ class Engineer(Role):
 
     @classmethod
     def parse_tasks(self, task_msg: Message) -> list[str]:
-        if not task_msg.instruct_content:
+        if task_msg.instruct_content:
             return task_msg.instruct_content.dict().get("Task list")
         return CodeParser.parse_file_list(block="Task list", text=task_msg.content)
 
@@ -88,8 +89,8 @@ class Engineer(Role):
 
     @classmethod
     def parse_workspace(cls, system_design_msg: Message) -> str:
-        if not system_design_msg.instruct_content:
-            return system_design_msg.instruct_content.dict().get("Python package name")
+        if system_design_msg.instruct_content:
+            return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip("\"")
         return CodeParser.parse_str(block="Python package name", text=system_design_msg.content)
 
     def get_workspace(self) -> Path:
@@ -110,9 +111,11 @@ class Engineer(Role):
 
     def write_file(self, filename: str, code: str):
         workspace = self.get_workspace()
+        filename = filename.replace('"', '').replace('\n', '')
         file = workspace / filename
         file.parent.mkdir(parents=True, exist_ok=True)
         file.write_text(code)
+        return file
 
     def recv(self, message: Message) -> None:
         self._rc.memory.add(message)
@@ -144,23 +147,33 @@ class Engineer(Role):
         return msg
 
     async def _act_sp(self) -> Message:
+        code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
         for todo in self.todos:
-            code_rsp = await WriteCode().run(
+            code = await WriteCode().run(
                 context=self._rc.history,
                 filename=todo
             )
             # logger.info(todo)
             # logger.info(code_rsp)
             # code = self.parse_code(code_rsp)
-            self.write_file(todo, code_rsp)
-            msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo))
+            file_path = self.write_file(todo, code)
+            msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo))
             self._rc.memory.add(msg)
 
+            code_msg = todo + FILENAME_CODE_SEP + str(file_path)
+            code_msg_all.append(code_msg)
+
         logger.info(f'Done {self.get_workspace()} generating.')
-        msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo))
+        msg = Message(
+            content=MSG_SEP.join(code_msg_all),
+            role=self.profile,
+            cause_by=type(self._rc.todo),
+            send_to="QaEngineer"
+        )
         return msg
 
     async def _act_sp_precision(self) -> Message:
+        code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
         for todo in self.todos:
             """
             # Select essential information from the historical data to reduce the length of the prompt (summarized from human experience):
@@ -191,12 +204,20 @@ class Engineer(Role):
                 except Exception as e:
                     logger.error("code review failed!", e)
                     pass
-            self.write_file(todo, code)
+            file_path = self.write_file(todo, code)
             msg = Message(content=code, role=self.profile, cause_by=WriteCode)
             self._rc.memory.add(msg)
 
+            code_msg = todo + FILENAME_CODE_SEP + str(file_path)
+            code_msg_all.append(code_msg)
+
         logger.info(f'Done {self.get_workspace()} generating.')
-        msg = Message(content="all done.", role=self.profile, cause_by=WriteCode)
+        msg = Message(
+            content=MSG_SEP.join(code_msg_all),
+            role=self.profile,
+            cause_by=type(self._rc.todo),
+            send_to="QaEngineer"
+        )
         return msg
 
     async def _act(self) -> Message:

metagpt/utils/common.py

@@ -253,4 +253,4 @@ def print_members(module, indent=0):
 def parse_recipient(text):
     pattern = r"## Send To:\s*([A-Za-z]+)\s*?"  # hard code for now
     recipient = re.search(pattern, text)
-    return recipient.group(1) if recipient else ""
+    return recipient.group(1) if recipient else ""

metagpt/utils/mermaid.py

@@ -13,8 +13,6 @@ from metagpt.const import PROJECT_ROOT
 from metagpt.logs import logger
 from metagpt.utils.common import check_cmd_exists
 
-IS_DOCKER = os.environ.get('AM_I_IN_A_DOCKER_CONTAINER', 'false').lower()
-
 
 def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
     """suffix: png/svg/pdf
@@ -38,16 +36,13 @@ def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height
         output_file = f'{output_file_without_suffix}.{suffix}'
         # Call the `mmdc` command to convert the Mermaid code to a PNG
         logger.info(f"Generating {output_file}..")
-        if IS_DOCKER == 'true':
-            subprocess.run(['mmdc', '-p', '/app/metagpt/config/puppeteer-config.json', '-i',
-                           str(tmp), '-o', output_file, '-w', str(width), '-H', str(height)])
+
+        if CONFIG.puppeteer_config:
+            subprocess.run([CONFIG.mmdc, '-p', CONFIG.puppeteer_config, '-i', str(tmp), '-o',
+                            output_file, '-w', str(width), '-H', str(height)])
         else:
-            if CONFIG.puppeteer_config:
-                subprocess.run([CONFIG.mmdc, '-p', CONFIG.puppeteer_config, '-i', str(tmp), '-o',
-                                output_file, '-w', str(width), '-H', str(height)])
-            else:
-                subprocess.run([CONFIG.mmdc, '-i', str(tmp), '-o',
-                                output_file, '-w', str(width), '-H', str(height)])
+            subprocess.run([CONFIG.mmdc, '-i', str(tmp), '-o',
+                            output_file, '-w', str(width), '-H', str(height)])
     return 0
 
 

metagpt/utils/pycst.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+from typing import Union
+
+import libcst as cst
+from libcst._nodes.module import Module
+
+DocstringNode = Union[cst.Module, cst.ClassDef, cst.FunctionDef]
+
+
+def get_docstring_statement(body: DocstringNode) -> cst.SimpleStatementLine:
+    """Extracts the docstring from the body of a node.
+
+    Args:
+        body: The body of a node.
+
+    Returns:
+        The docstring statement if it exists, None otherwise.
+    """
+    if isinstance(body, cst.Module):
+        body = body.body
+    else:
+        body = body.body.body
+
+    if not body:
+        return
+
+    statement = body[0]
+    if not isinstance(statement, cst.SimpleStatementLine):
+        return
+
+    expr = statement
+    while isinstance(expr, (cst.BaseSuite, cst.SimpleStatementLine)):
+        if len(expr.body) == 0:
+            return None
+        expr = expr.body[0]
+
+    if not isinstance(expr, cst.Expr):
+        return None
+    
+    val = expr.value
+    if not isinstance(val, (cst.SimpleString, cst.ConcatenatedString)):
+        return None
+    
+    evaluated_value = val.evaluated_value    
+    if isinstance(evaluated_value, bytes):
+        return None
+
+    return statement
+
+
+class DocstringCollector(cst.CSTVisitor):
+    """A visitor class for collecting docstrings from a CST.
+
+    Attributes:
+        stack: A list to keep track of the current path in the CST.
+        docstrings: A dictionary mapping paths in the CST to their corresponding docstrings.
+    """
+    def __init__(self):
+        self.stack: list[str] = []
+        self.docstrings: dict[tuple[str, ...], cst.SimpleStatementLine] = {}
+
+    def visit_Module(self, node: cst.Module) -> bool | None:
+        self.stack.append("")
+
+    def leave_Module(self, node: cst.Module) -> None:
+        return self._leave(node)
+
+    def visit_ClassDef(self, node: cst.ClassDef) -> bool | None:
+        self.stack.append(node.name.value)
+
+    def leave_ClassDef(self, node: cst.ClassDef) -> None:
+        return self._leave(node)
+
+    def visit_FunctionDef(self, node: cst.FunctionDef) -> bool | None:
+        self.stack.append(node.name.value)
+
+    def leave_FunctionDef(self, node: cst.FunctionDef) -> None:
+        return self._leave(node)
+
+    def _leave(self, node: DocstringNode) -> None:
+        key = tuple(self.stack)
+        self.stack.pop()
+        if hasattr(node, "decorators") and any(i.decorator.value == "overload" for i in node.decorators):
+            return
+
+        statement = get_docstring_statement(node)
+        if statement:
+            self.docstrings[key] = statement
+
+
+class DocstringTransformer(cst.CSTTransformer):
+    """A transformer class for replacing docstrings in a CST.
+
+    Attributes:
+        stack: A list to keep track of the current path in the CST.
+        docstrings: A dictionary mapping paths in the CST to their corresponding docstrings.
+    """
+    def __init__(
+        self,
+        docstrings: dict[tuple[str, ...], cst.SimpleStatementLine],
+    ):
+        self.stack: list[str] = []
+        self.docstrings = docstrings
+
+    def visit_Module(self, node: cst.Module) -> bool | None:
+        self.stack.append("")
+
+    def leave_Module(self, original_node: Module, updated_node: Module) -> Module:
+        return self._leave(original_node, updated_node)
+
+    def visit_ClassDef(self, node: cst.ClassDef) -> bool | None:
+        self.stack.append(node.name.value)
+
+    def leave_ClassDef(self, original_node: cst.ClassDef, updated_node: cst.ClassDef) -> cst.CSTNode:
+        return self._leave(original_node, updated_node)
+
+    def visit_FunctionDef(self, node: cst.FunctionDef) -> bool | None:
+        self.stack.append(node.name.value)
+
+    def leave_FunctionDef(self, original_node: cst.FunctionDef, updated_node: cst.FunctionDef) -> cst.CSTNode:
+        return self._leave(original_node, updated_node)
+
+    def _leave(self, original_node: DocstringNode, updated_node: DocstringNode) -> DocstringNode:
+        key = tuple(self.stack)
+        self.stack.pop()
+
+        if hasattr(updated_node, "decorators") and any((i.decorator.value == "overload") for i in updated_node.decorators):
+            return updated_node
+
+        statement = self.docstrings.get(key)
+        if not statement:
+            return updated_node
+
+        original_statement = get_docstring_statement(original_node)
+
+        if isinstance(updated_node, cst.Module):
+            body = updated_node.body
+            if original_statement:
+                return updated_node.with_changes(body=(statement, *body[1:]))
+            else:
+                updated_node = updated_node.with_changes(body=(statement, cst.EmptyLine(), *body))
+                return updated_node
+
+        body = updated_node.body.body[1:] if original_statement else updated_node.body.body
+        return updated_node.with_changes(body=updated_node.body.with_changes(body=(statement, *body)))
+
+
+def merge_docstring(code: str, documented_code: str) -> str:
+    """Merges the docstrings from the documented code into the original code.
+
+    Args:
+        code: The original code.
+        documented_code: The documented code.
+
+    Returns:
+        The original code with the docstrings from the documented code.
+    """
+    code_tree = cst.parse_module(code)
+    documented_code_tree = cst.parse_module(documented_code)
+
+    visitor = DocstringCollector()
+    documented_code_tree.visit(visitor)
+    transformer = DocstringTransformer(visitor.docstrings)
+    modified_tree = code_tree.visit(transformer)
+    return modified_tree.code

metagpt/utils/special_tokens.py

@@ -0,0 +1,4 @@
+# token to separate different code messages in a WriteCode Message content
+MSG_SEP = "#*000*#" 
+# token to seperate file name and the actual code text in a code message
+FILENAME_CODE_SEP = "#*001*#"

requirements.txt

@@ -35,3 +35,4 @@ tqdm==4.64.0
 anthropic==0.3.6
 typing-inspect==0.8.0
 typing_extensions==4.5.0
+libcst==1.0.1

startup.py

@@ -4,23 +4,27 @@ import asyncio
 
 import fire
 
-from metagpt.roles import Architect, Engineer, ProductManager, ProjectManager
+from metagpt.roles import Architect, Engineer, ProductManager, ProjectManager, QaEngineer
 from metagpt.software_company import SoftwareCompany
 
 
-async def startup(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False):
+async def startup(idea: str, investment: float = 3.0, n_round: int = 5,
+                  code_review: bool = False, run_tests: bool = False):
     """Run a startup. Be a boss."""
     company = SoftwareCompany()
     company.hire([ProductManager(),
                   Architect(),
                   ProjectManager(),
                   Engineer(n_borg=5, use_code_review=code_review)])
+    if run_tests:
+        # developing features: run tests on the spot and identify bugs (bug fixing capability comes soon!)
+        company.hire([QaEngineer()])
     company.invest(investment)
     company.start_project(idea)
     await company.run(n_round=n_round)
 
 
-def main(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False):
+def main(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False, run_tests: bool = False):
     """
     We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities.
     :param idea: Your innovative idea, such as "Creating a snake game."
@@ -29,7 +33,7 @@ def main(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool
     :param code_review: Whether to use code review.
     :return:
     """
-    asyncio.run(startup(idea, investment, n_round, code_review))
+    asyncio.run(startup(idea, investment, n_round, code_review, run_tests))
 
 
 if __name__ == '__main__':

tests/metagpt/actions/test_debug_error.py

@@ -9,15 +9,147 @@ import pytest
 
 from metagpt.actions.debug_error import DebugError
 
+EXAMPLE_MSG_CONTENT = '''
+---
+## Development Code File Name
+player.py
+## Development Code
+```python
+from typing import List
+from deck import Deck
+from card import Card
+
+class Player:
+    """
+    A class representing a player in the Black Jack game.
+    """
+
+    def __init__(self, name: str):
+        """
+        Initialize a Player object.
+        
+        Args:
+            name (str): The name of the player.
+        """
+        self.name = name
+        self.hand: List[Card] = []
+        self.score = 0
+
+    def draw(self, deck: Deck):
+        """
+        Draw a card from the deck and add it to the player's hand.
+        
+        Args:
+            deck (Deck): The deck of cards.
+        """
+        card = deck.draw_card()
+        self.hand.append(card)
+        self.calculate_score()
+
+    def calculate_score(self) -> int:
+        """
+        Calculate the score of the player's hand.
+        
+        Returns:
+            int: The score of the player's hand.
+        """
+        self.score = sum(card.value for card in self.hand)
+        # Handle the case where Ace is counted as 11 and causes the score to exceed 21
+        if self.score > 21 and any(card.rank == 'A' for card in self.hand):
+            self.score -= 10
+        return self.score
+
+```
+## Test File Name
+test_player.py
+## Test Code
+```python
+import unittest
+from blackjack_game.player import Player
+from blackjack_game.deck import Deck
+from blackjack_game.card import Card
+
+class TestPlayer(unittest.TestCase):
+    ## Test the Player's initialization
+    def test_player_initialization(self):
+        player = Player("Test Player")
+        self.assertEqual(player.name, "Test Player")
+        self.assertEqual(player.hand, [])
+        self.assertEqual(player.score, 0)
+
+    ## Test the Player's draw method
+    def test_player_draw(self):
+        deck = Deck()
+        player = Player("Test Player")
+        player.draw(deck)
+        self.assertEqual(len(player.hand), 1)
+        self.assertEqual(player.score, player.hand[0].value)
+
+    ## Test the Player's calculate_score method
+    def test_player_calculate_score(self):
+        deck = Deck()
+        player = Player("Test Player")
+        player.draw(deck)
+        player.draw(deck)
+        self.assertEqual(player.score, sum(card.value for card in player.hand))
+
+    ## Test the Player's calculate_score method with Ace card
+    def test_player_calculate_score_with_ace(self):
+        deck = Deck()
+        player = Player("Test Player")
+        player.hand.append(Card('A', 'Hearts', 11))
+        player.hand.append(Card('K', 'Hearts', 10))
+        player.calculate_score()
+        self.assertEqual(player.score, 21)
+
+    ## Test the Player's calculate_score method with multiple Aces
+    def test_player_calculate_score_with_multiple_aces(self):
+        deck = Deck()
+        player = Player("Test Player")
+        player.hand.append(Card('A', 'Hearts', 11))
+        player.hand.append(Card('A', 'Diamonds', 11))
+        player.calculate_score()
+        self.assertEqual(player.score, 12)
+
+if __name__ == '__main__':
+    unittest.main()
+
+```
+## Running Command
+python tests/test_player.py
+## Running Output
+standard output: ;
+standard errors: ..F..
+======================================================================
+FAIL: test_player_calculate_score_with_multiple_aces (__main__.TestPlayer)
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "tests/test_player.py", line 46, in test_player_calculate_score_with_multiple_aces
+    self.assertEqual(player.score, 12)
+AssertionError: 22 != 12
+
+----------------------------------------------------------------------
+Ran 5 tests in 0.007s
+
+FAILED (failures=1)
+;
+## instruction:
+The error is in the development code, specifically in the calculate_score method of the Player class. The method is not correctly handling the case where there are multiple Aces in the player's hand. The current implementation only subtracts 10 from the score once if the score is over 21 and there's an Ace in the hand. However, in the case of multiple Aces, it should subtract 10 for each Ace until the score is 21 or less.
+## File To Rewrite:
+player.py
+## Status:
+FAIL
+## Send To:
+Engineer
+---
+'''
 
 @pytest.mark.asyncio
 async def test_debug_error():
-    code = "def add(a, b):\n    return a - b"
-    error = "AssertionError: Expected add(1, 1) to equal 2 but got 0"
 
     debug_error = DebugError("debug_error")
 
-    result = await debug_error.run(code, error)
+    file_name, rewritten_code = await debug_error.run(context=EXAMPLE_MSG_CONTENT)
 
-    # mock_llm.ask.assert_called_once_with(prompt)
-    assert len(result) > 0
+    assert "class Player" in rewritten_code # rewrite the same class
+    assert "while self.score > 21" in rewritten_code # a key logic to rewrite to (original one is "if self.score > 12")

tests/metagpt/actions/test_run_code.py

@@ -6,33 +6,65 @@
 @File    : test_run_code.py
 """
 import pytest
-
+import asyncio
 from metagpt.actions.run_code import RunCode
 
+@pytest.mark.asyncio
+async def test_run_text():
+    action = RunCode()
+    result, errs = await RunCode.run_text('result = 1 + 1')
+    assert result == 2
+    assert errs == ""
+
+    result, errs = await RunCode.run_text('result = 1 / 0')
+    assert result == ""
+    assert "ZeroDivisionError" in errs
 
 @pytest.mark.asyncio
-async def test_run_code():
-    code = """
-def add(a, b):
-    return a + b
-result = add(1, 2)
-"""
-    run_code = RunCode("run_code")
-
-    result = await run_code.run(code)
-
-    assert result == 3
+async def test_run_script():
+    action = RunCode()
+    
+    # Successful command
+    out, err = await RunCode.run_script(".", command=["echo", "Hello World"])
+    assert out.strip() == "Hello World"
+    assert err == ""
 
+    # Unsuccessful command
+    out, err = await RunCode.run_script(".", command=["python", "-c", "print(1/0)"])
+    assert "ZeroDivisionError" in err
 
 @pytest.mark.asyncio
-async def test_run_code_with_error():
-    code = """
-def add(a, b):
-    return a + b
-result = add(1, '2')
-"""
-    run_code = RunCode("run_code")
+async def test_run():
+    action = RunCode()
+    result = await action.run(mode="text", code="print('Hello, World')")
+    assert "PASS" in result
 
-    result = await run_code.run(code)
+    result = await action.run(
+        mode="script",
+        code="echo 'Hello World'",
+        code_file_name="",
+        test_code="",
+        test_file_name="",
+        command=["echo", "Hello World"],
+        working_directory=".",
+        additional_python_paths=[]
+    )
+    assert "PASS" in result
 
-    assert "TypeError: unsupported operand type(s) for +" in result
+@pytest.mark.asyncio
+async def test_run_failure():
+    action = RunCode()
+    result = await action.run(mode="text", code="result = 1 / 0")
+    assert "FAIL" in result
+
+    result = await action.run(
+        mode="script",
+        code='python -c "print(1/0)"',
+        code_file_name="",
+        test_code="",
+        test_file_name="",
+        command=["python", "-c", "print(1/0)"],
+        working_directory=".",
+        additional_python_paths=[]
+    )
+    assert "FAIL" in result

tests/metagpt/actions/test_write_docstring.py

@@ -0,0 +1,32 @@
+import pytest
+
+from metagpt.actions.write_docstring import WriteDocstring
+
+code = '''
+def add_numbers(a: int, b: int):
+    return a + b
+
+
+class Person:
+    def __init__(self, name: str, age: int):
+        self.name = name
+        self.age = age
+
+    def greet(self):
+        return f"Hello, my name is {self.name} and I am {self.age} years old."
+'''
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("style", "part"),
+    [
+        ("google", "Args:"),
+        ("numpy", "Parameters"),
+        ("sphinx", ":param name:"),
+    ],
+    ids=["google", "numpy", "sphinx"]
+)
+async def test_write_docstring(style: str, part: str):
+    ret = await WriteDocstring().run(code, style=style)
+    assert part in ret

tests/metagpt/actions/test_write_test.py

@@ -8,19 +8,35 @@
 import pytest
 
 from metagpt.actions.write_test import WriteTest
+from metagpt.logs import logger
 
 
 @pytest.mark.asyncio
 async def test_write_test():
     code = """
-    def add(a, b):
-        return a + b
+    import random
+    from typing import Tuple
+
+    class Food:
+        def __init__(self, position: Tuple[int, int]):
+            self.position = position
+
+        def generate(self, max_y: int, max_x: int):
+            self.position = (random.randint(1, max_y - 1), random.randint(1, max_x - 1))
     """
 
-    write_test = WriteTest("write_test")
+    write_test = WriteTest()
 
-    test_cases = await write_test.run(code)
+    test_code = await write_test.run(
+        code_to_test=code,
+        test_file_name="test_food.py",
+        source_file_path="/some/dummy/path/cli_snake_game/cli_snake_game/food.py",
+        workspace="/some/dummy/path/cli_snake_game"
+    )
+    logger.info(test_code)
 
     # We cannot exactly predict the generated test cases, but we can check if it is a string and if it is not empty
-    assert isinstance(test_cases, str)
-    assert len(test_cases) > 0
+    assert isinstance(test_code, str)
+    assert "from cli_snake_game.food import Food" in test_code
+    assert "class TestFood(unittest.TestCase)" in test_code
+    assert "def test_generate" in test_code

tests/metagpt/utils/test_output_parser.py

@@ -19,7 +19,7 @@ def test_parse_blocks():
 
 
 def test_parse_code():
-    test_text = "```python\nprint('Hello, world!')\n```"
+    test_text = "```python\nprint('Hello, world!')```"
     expected_result = "print('Hello, world!')"
     assert OutputParser.parse_code(test_text, 'python') == expected_result
 
@@ -27,6 +27,22 @@ def test_parse_code():
         OutputParser.parse_code(test_text, 'java')
 
 
+def test_parse_python_code():
+    expected_result = "print('Hello, world!')"
+    assert OutputParser.parse_python_code("```python\nprint('Hello, world!')```") == expected_result
+    assert OutputParser.parse_python_code("```python\nprint('Hello, world!')") == expected_result
+    assert OutputParser.parse_python_code("print('Hello, world!')") == expected_result
+    assert OutputParser.parse_python_code("print('Hello, world!')```") == expected_result
+    assert OutputParser.parse_python_code("print('Hello, world!')```") == expected_result
+    expected_result = "print('```Hello, world!```')"
+    assert OutputParser.parse_python_code("```python\nprint('```Hello, world!```')```") == expected_result
+    assert OutputParser.parse_python_code("The code is: ```python\nprint('```Hello, world!```')```") == expected_result
+    assert OutputParser.parse_python_code("xxx.\n```python\nprint('```Hello, world!```')```\nxxx") == expected_result
+
+    with pytest.raises(ValueError):
+        OutputParser.parse_python_code("xxx =")
+
+
 def test_parse_str():
     test_text = "name = 'Alice'"
     expected_result = 'Alice'

tests/metagpt/utils/test_pycst.py

@@ -0,0 +1,136 @@
+from metagpt.utils import pycst
+
+code = '''
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from typing import overload
+
+@overload
+def add_numbers(a: int, b: int):
+    ...
+
+@overload
+def add_numbers(a: float, b: float):
+    ...
+
+def add_numbers(a: int, b: int):
+    return a + b
+
+
+class Person:
+    def __init__(self, name: str, age: int):
+        self.name = name
+        self.age = age
+
+    def greet(self):
+        return f"Hello, my name is {self.name} and I am {self.age} years old."
+'''
+
+documented_code = '''
+"""
+This is an example module containing a function and a class definition.
+"""
+
+
+def add_numbers(a: int, b: int):
+    """This function is used to add two numbers and return the result.
+
+    Parameters:
+        a: The first integer.
+        b: The second integer.
+
+    Returns:
+        int: The sum of the two numbers.
+    """
+    return a + b
+
+class Person:
+    """This class represents a person's information, including name and age.
+
+    Attributes:
+        name: The person's name.
+        age: The person's age.
+    """
+
+    def __init__(self, name: str, age: int):
+        """Creates a new instance of the Person class.
+
+        Parameters:
+            name: The person's name.
+            age: The person's age.
+        """
+        ...
+
+    def greet(self):
+        """
+        Returns a greeting message including the name and age.
+
+        Returns:
+            str: The greeting message.
+        """
+        ...
+'''
+
+
+merged_code = '''
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+This is an example module containing a function and a class definition.
+"""
+
+from typing import overload
+
+@overload
+def add_numbers(a: int, b: int):
+    ...
+
+@overload
+def add_numbers(a: float, b: float):
+    ...
+
+def add_numbers(a: int, b: int):
+    """This function is used to add two numbers and return the result.
+
+    Parameters:
+        a: The first integer.
+        b: The second integer.
+
+    Returns:
+        int: The sum of the two numbers.
+    """
+    return a + b
+
+
+class Person:
+    """This class represents a person's information, including name and age.
+
+    Attributes:
+        name: The person's name.
+        age: The person's age.
+    """
+    def __init__(self, name: str, age: int):
+        """Creates a new instance of the Person class.
+
+        Parameters:
+            name: The person's name.
+            age: The person's age.
+        """
+        self.name = name
+        self.age = age
+
+    def greet(self):
+        """
+        Returns a greeting message including the name and age.
+
+        Returns:
+            str: The greeting message.
+        """
+        return f"Hello, my name is {self.name} and I am {self.age} years old."
+'''
+
+
+def test_merge_docstring():
+    data = pycst.merge_docstring(code, documented_code)
+    print(data)
+    assert data == merged_code