diff --git a/.gitignore b/.gitignore index e03eab3d3..0ac318ff5 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ cover/ # Django stuff: *.log +logs local_settings.py db.sqlite3 db.sqlite3-journal diff --git a/README.md b/README.md index e80082a3a..b3473a12c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + # MetaGPT: The Multi-Agent Framework

@@ -50,9 +51,9 @@ # Step 2: Clone the repository to your local machine for latest version, and ins cd MetaGPT pip3 install -e. # or pip3 install metagpt # for stable version -# Step 3: run the startup.py +# Step 3: run metagpt cli # setup your OPENAI_API_KEY in key.yaml copy from config.yaml -python3 startup.py "Write a cli snake game" +metagpt "Write a cli snake game" # Step 4 [Optional]: If you want to save the artifacts like diagrams such as quadrant chart, system designs, sequence flow in the workspace, you can execute the step before Step 3. By default, the framework is compatible, and the entire process can be run completely without executing this step. # If executing, ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) @@ -78,7 +79,7 @@ # Step 2: Run metagpt demo with container -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ - python startup.py "Write a cli snake game" + metagpt "Write a cli snake game" ``` detail installation please refer to [docker_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-with-docker) @@ -117,7 +118,7 @@ ### Contact Information If you have any questions or feedback about this project, please feel free to contact us. We highly appreciate your suggestions! -- **Email:** alexanderwu@fuzhi.ai +- **Email:** alexanderwu@deepwisdom.ai - **GitHub Issues:** For more technical inquiries, you can also create a new issue in our [GitHub repository](https://github.com/geekan/metagpt/issues). We will respond to all questions within 2-3 business days. diff --git a/config/config.yaml b/config/config.yaml index 080de4000..8fd208c59 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,9 +7,9 @@ ## Or, you can configure OPENAI_PROXY to access official OPENAI_API_BASE. OPENAI_API_BASE: "https://api.openai.com/v1" #OPENAI_PROXY: "http://127.0.0.1:8118" -#OPENAI_API_KEY: "YOUR_API_KEY" # set the value to sk-xxx if you host the openai interface for open llm model -OPENAI_API_MODEL: "gpt-4" -MAX_TOKENS: 1500 +#OPENAI_API_KEY: "YOUR_API_KEY" # set the value to sk-xxx if you host the openai interface for open llm model +OPENAI_API_MODEL: "gpt-4-1106-preview" +MAX_TOKENS: 4096 RPM: 10 #### if Spark @@ -77,8 +77,8 @@ RPM: 10 #### for Stable Diffusion ## Use SD service, based on https://github.com/AUTOMATIC1111/stable-diffusion-webui -SD_URL: "YOUR_SD_URL" -SD_T2I_API: "/sdapi/v1/txt2img" +#SD_URL: "YOUR_SD_URL" +#SD_T2I_API: "/sdapi/v1/txt2img" #### for Execution #LONG_TERM_MEMORY: false @@ -93,8 +93,8 @@ SD_T2I_API: "/sdapi/v1/txt2img" # CALC_USAGE: false ### for Research -MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo -MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k +# MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo +# MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k ### choose the engine for mermaid conversion, # default is nodejs, you can change it to playwright,pyppeteer or ink @@ -108,4 +108,4 @@ MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k ### repair operation on the content extracted from LLM's raw output. Warning, it improves the result but not fix all cases. # REPAIR_LLM_OUTPUT: false -PROMPT_FORMAT: json #json or markdown \ No newline at end of file +# PROMPT_FORMAT: json #json or markdown \ No newline at end of file diff --git a/docs/FAQ-EN.md b/docs/FAQ-EN.md index f9df50caf..af6868509 100644 --- a/docs/FAQ-EN.md +++ b/docs/FAQ-EN.md @@ -98,7 +98,7 @@ 1. How to change the investment amount? - 1. You can view all commands by typing `python startup.py --help` + 1. You can view all commands by typing `metagpt --help` 1. Which version of Python is more stable? @@ -134,7 +134,7 @@ 1. Configuration instructions for SD Skills: The SD interface is currently deployed based on *https://github.com/AUTOMATIC1111/stable-diffusion-webui* **For environmental configurations and model downloads, please refer to the aforementioned GitHub repository. To initiate the SD service that supports API calls, run the command specified in cmd with the parameter nowebui, i.e., - 1. > python webui.py --enable-insecure-extension-access --port xxx --no-gradio-queue --nowebui + 1. > python3 webui.py --enable-insecure-extension-access --port xxx --no-gradio-queue --nowebui 1.     Once it runs without errors, the interface will be accessible after approximately 1 minute when the model finishes loading. 1. Configure SD_URL and SD_T2I_API in the config.yaml/key.yaml files. 1. ![](https://p3-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/065295a67b0b4feea665d1372722d49d~tplv-k3u1fbpfcp-zoom-1.image) diff --git a/docs/README_CN.md b/docs/README_CN.md index 038925184..dd65c2a25 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -47,9 +47,9 @@ # 第 2 步：克隆最新仓库到您的本地机器，并进行安装。 cd MetaGPT pip3 install -e. # 或者 pip3 install metagpt # 安装稳定版本 -# 第 3 步：执行startup.py +# 第 3 步：执行metagpt # 拷贝config.yaml为key.yaml，并设置你自己的OPENAI_API_KEY -python3 startup.py "Write a cli snake game" +metagpt "Write a cli snake game" # 第 4 步【可选的】：如果你想在执行过程中保存像象限图、系统设计、序列流程等图表这些产物，可以在第3步前执行该步骤。默认的，框架做了兼容，在不执行该步的情况下，也可以完整跑完整个流程。 # 如果执行，确保您的系统上安装了 NPM。并使用npm安装mermaid-js @@ -75,7 +75,7 @@ # 步骤2: 使用容器运行metagpt演示 -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ - python startup.py "Write a cli snake game" + metagpt "Write a cli snake game" ``` 详细的安装请安装 [docker_install](https://docs.deepwisdom.ai/zhcn/guide/get_started/installation.html#%E4%BD%BF%E7%94%A8docker%E5%AE%89%E8%A3%85) @@ -114,7 +114,7 @@ ### 联系信息 如果您对这个项目有任何问题或反馈，欢迎联系我们。我们非常欢迎您的建议！ -- **邮箱：** alexanderwu@fuzhi.ai +- **邮箱：** alexanderwu@deepwisdom.ai - **GitHub 问题：** 对于更技术性的问题，您也可以在我们的 [GitHub 仓库](https://github.com/geekan/metagpt/issues) 中创建一个新的问题。 我们会在2-3个工作日内回复所有问题。 diff --git a/docs/README_JA.md b/docs/README_JA.md index 411d190b4..05f718635 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -41,7 +41,7 @@ ## MetaGPT の能力 ## 例（GPT-4 で完全生成） -例えば、`python startup.py "Toutiao のような RecSys をデザインする"`と入力すると、多くの出力が得られます +例えば、`metagpt "Toutiao のような RecSys をデザインする"`と入力すると、多くの出力が得られます ![Jinri Toutiao Recsys データと API デザイン](resources/workspace/content_rec_sys/resources/data_api_design.png) @@ -60,16 +60,16 @@ ### 伝統的なインストール ```bash # ステップ 1: Python 3.9+ がシステムにインストールされていることを確認してください。これを確認するには: -python --version +python3 --version # ステップ 2: リポジトリをローカルマシンにクローンし、インストールする。 git clone https://github.com/geekan/MetaGPT.git cd MetaGPT pip install -e. -# ステップ 3: startup.py を実行する +# ステップ 3: metagpt を実行する # config.yaml を key.yaml にコピーし、独自の OPENAI_API_KEY を設定します -python3 startup.py "Write a cli snake game" +metagpt "Write a cli snake game" # ステップ 4 [オプション]: 実行中に PRD ファイルなどのアーティファクトを保存する場合は、ステップ 3 の前にこのステップを実行できます。デフォルトでは、フレームワークには互換性があり、この手順を実行しなくてもプロセス全体を完了できます。 # NPM がシステムにインストールされていることを確認してください。次に mermaid-js をインストールします。(お使いのコンピューターに npm がない場合は、Node.js 公式サイトで Node.js https://nodejs.org/ をインストールしてください。） @@ -178,7 +178,7 @@ # ステップ 2: コンテナで metagpt デモを実行する -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ - python startup.py "Write a cli snake game" + metagpt "Write a cli snake game" # コンテナを起動し、その中でコマンドを実行することもできます docker run --name metagpt -d \ @@ -188,7 +188,7 @@ # コンテナを起動し、その中でコマンドを実行することもで metagpt/metagpt:latest docker exec -it metagpt /bin/bash -$ python startup.py "Write a cli snake game" +$ metagpt "Write a cli snake game" ``` コマンド `docker run ...` は以下のことを行います: @@ -196,7 +196,7 @@ # コンテナを起動し、その中でコマンドを実行することもで - 特権モードで実行し、ブラウザの実行権限を得る - ホスト設定ファイル `/opt/metagpt/config/key.yaml` をコンテナ `/app/metagpt/config/key.yaml` にマップします - ホストディレクトリ `/opt/metagpt/workspace` をコンテナディレクトリ `/app/metagpt/workspace` にマップするs -- デモコマンド `python startup.py "Write a cli snake game"` を実行する +- デモコマンド `metagpt "Write a cli snake game"` を実行する ### 自分でイメージをビルドする @@ -225,11 +225,11 @@ ## チュートリアル: スタートアップの開始 ```shell # スクリプトの実行 -python startup.py "Write a cli snake game" +metagpt "Write a cli snake game" # プロジェクトの実施にエンジニアを雇わないこと -python startup.py "Write a cli snake game" --implement False +metagpt "Write a cli snake game" --no-implement # エンジニアを雇い、コードレビューを行う -python startup.py "Write a cli snake game" --code_review True +metagpt "Write a cli snake game" --code_review ``` スクリプトを実行すると、`workspace/` ディレクトリに新しいプロジェクトが見つかります。 @@ -239,17 +239,17 @@ ### プラットフォームまたはツールの設定 要件を述べるときに、どのプラットフォームまたはツールを使用するかを指定できます。 ```shell -python startup.py "pygame をベースとした cli ヘビゲームを書く" +metagpt "pygame をベースとした cli ヘビゲームを書く" ``` ### 使用方法 ``` 会社名 - startup.py - 私たちは AI で構成されたソフトウェア・スタートアップです。私たちに投資することは、無限の可能性に満ちた未来に力を与えることです。 + metagpt - 私たちは AI で構成されたソフトウェア・スタートアップです。私たちに投資することは、無限の可能性に満ちた未来に力を与えることです。 シノプシス - startup.py IDEA + metagpt IDEA 説明 私たちは AI で構成されたソフトウェア・スタートアップです。私たちに投資することは、無限の可能性に満ちた未来に力を与えることです。 @@ -317,7 +317,7 @@ ## お問い合わせ先 このプロジェクトに関するご質問やご意見がございましたら、お気軽にお問い合わせください。皆様のご意見をお待ちしております！ -- **Email:** alexanderwu@fuzhi.ai +- **Email:** alexanderwu@deepwisdom.ai - **GitHub Issues:** 技術的なお問い合わせについては、[GitHub リポジトリ](https://github.com/geekan/metagpt/issues) に新しい issue を作成することもできます。 ご質問には 2-3 営業日以内に回答いたします。 diff --git a/docs/install/cli_install_cn.md b/docs/install/cli_install_cn.md index f351090ed..b1da1b813 100644 --- a/docs/install/cli_install_cn.md +++ b/docs/install/cli_install_cn.md @@ -15,7 +15,7 @@ # 第 1 步：确保您的系统上安装了 NPM。并使用npm安装mermaid-js sudo npm install -g @mermaid-js/mermaid-cli # 第 2 步：确保您的系统上安装了 Python 3.9+。您可以使用以下命令进行检查： -python --version +python3 --version # 第 3 步：克隆仓库到您的本地机器，并进行安装。 git clone https://github.com/geekan/MetaGPT.git diff --git a/docs/install/docker_install.md b/docs/install/docker_install.md index b803a5dae..37125bdbe 100644 --- a/docs/install/docker_install.md +++ b/docs/install/docker_install.md @@ -15,7 +15,7 @@ # Step 2: Run metagpt demo with container -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ - python3 startup.py "Write a cli snake game" + metagpt "Write a cli snake game" # You can also start a container and execute commands in it docker run --name metagpt -d \ @@ -25,7 +25,7 @@ # You can also start a container and execute commands in it metagpt/metagpt:latest docker exec -it metagpt /bin/bash -$ python3 startup.py "Write a cli snake game" +$ metagpt "Write a cli snake game" ``` The command `docker run ...` do the following things: @@ -33,7 +33,7 @@ # You can also start a container and execute commands in it - Run in privileged mode to have permission to run the browser - Map host configure file `/opt/metagpt/config/key.yaml` to container `/app/metagpt/config/key.yaml` - Map host directory `/opt/metagpt/workspace` to container `/app/metagpt/workspace` -- Execute the demo command `python3 startup.py "Write a cli snake game"` +- Execute the demo command `metagpt "Write a cli snake game"` ### Build image by yourself diff --git a/docs/install/docker_install_cn.md b/docs/install/docker_install_cn.md index 347fae10c..f360b49ed 100644 --- a/docs/install/docker_install_cn.md +++ b/docs/install/docker_install_cn.md @@ -15,7 +15,7 @@ # 步骤2: 使用容器运行metagpt演示 -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ - python startup.py "Write a cli snake game" + metagpt "Write a cli snake game" # 您也可以启动一个容器并在其中执行命令 docker run --name metagpt -d \ @@ -25,7 +25,7 @@ # 您也可以启动一个容器并在其中执行命令 metagpt/metagpt:latest docker exec -it metagpt /bin/bash -$ python startup.py "Write a cli snake game" +$ metagpt "Write a cli snake game" ``` `docker run ...`做了以下事情: @@ -33,7 +33,7 @@ # 您也可以启动一个容器并在其中执行命令 - 以特权模式运行，有权限运行浏览器 - 将主机文件 `/opt/metagpt/config/key.yaml` 映射到容器文件 `/app/metagpt/config/key.yaml` - 将主机目录 `/opt/metagpt/workspace` 映射到容器目录 `/app/metagpt/workspace` -- 执行示例命令 `python startup.py "Write a cli snake game"` +- 执行示例命令 `metagpt "Write a cli snake game"` ### 自己构建镜像 diff --git a/docs/tutorial/usage.md b/docs/tutorial/usage.md index ee87b65c9..fbe4a8311 100644 --- a/docs/tutorial/usage.md +++ b/docs/tutorial/usage.md @@ -19,11 +19,11 @@ ### Initiating a startup ```shell # Run the script -python startup.py "Write a cli snake game" +metagpt "Write a cli snake game" # Do not hire an engineer to implement the project -python startup.py "Write a cli snake game" --implement False +metagpt "Write a cli snake game" --no-implement # Hire an engineer and perform code reviews -python startup.py "Write a cli snake game" --code_review True +metagpt "Write a cli snake game" --code_review ``` After running the script, you can find your new project in the `workspace/` directory. @@ -33,17 +33,17 @@ ### Preference of Platform or Tool You can tell which platform or tool you want to use when stating your requirements. ```shell -python startup.py "Write a cli snake game based on pygame" +metagpt "Write a cli snake game based on pygame" ``` ### Usage ``` NAME - startup.py - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. + metagpt - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. SYNOPSIS - startup.py IDEA + metagpt IDEA DESCRIPTION We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. diff --git a/docs/tutorial/usage_cn.md b/docs/tutorial/usage_cn.md index 4b3bdd2c3..1ef50d633 100644 --- a/docs/tutorial/usage_cn.md +++ b/docs/tutorial/usage_cn.md @@ -18,9 +18,9 @@ # 复制配置文件并进行必要的修改 ### 示例：启动一个创业公司 ```shell -python startup.py "写一个命令行贪吃蛇" +metagpt "写一个命令行贪吃蛇" # 开启code review模式会花费更多的金钱, 但是会提升代码质量和成功率 -python startup.py "写一个命令行贪吃蛇" --code_review True +metagpt "写一个命令行贪吃蛇" --code_review ``` 运行脚本后，您可以在 `workspace/` 目录中找到您的新项目。 @@ -29,17 +29,17 @@ ### 平台或工具的倾向性 可以在阐述需求时说明想要使用的平台或工具。 例如： ```shell -python startup.py "写一个基于pygame的命令行贪吃蛇" +metagpt "写一个基于pygame的命令行贪吃蛇" ``` ### 使用 ``` 名称 - startup.py - 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 + metagpt - 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 概要 - startup.py IDEA + metagpt IDEA 描述 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 diff --git a/examples/agent_creator.py b/examples/agent_creator.py index 6269dac10..05417d24a 100644 --- a/examples/agent_creator.py +++ b/examples/agent_creator.py @@ -1,22 +1,23 @@ -''' +""" Filename: MetaGPT/examples/agent_creator.py Created Date: Tuesday, September 12th 2023, 3:28:37 pm Author: garylin2099 -''' +""" import re -from metagpt.const import PROJECT_ROOT, WORKSPACE_ROOT from metagpt.actions import Action +from metagpt.config import CONFIG +from metagpt.const import METAGPT_ROOT +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.logs import logger -with open(PROJECT_ROOT / "examples/build_customized_agent.py", "r") as f: +with open(METAGPT_ROOT / "examples/build_customized_agent.py", "r") as f: # use official example script to guide AgentCreator MULTI_ACTION_AGENT_CODE_EXAMPLE = f.read() -class CreateAgent(Action): +class CreateAgent(Action): PROMPT_TEMPLATE = """ ### BACKGROUND You are using an agent framework called metagpt to write agents capable of different actions, @@ -34,7 +35,6 @@ class CreateAgent(Action): """ async def run(self, example: str, instruction: str): - prompt = self.PROMPT_TEMPLATE.format(example=example, instruction=instruction) # logger.info(prompt) @@ -46,15 +46,15 @@ class CreateAgent(Action): @staticmethod def parse_code(rsp): - pattern = r'```python(.*)```' + pattern = r"```python(.*)```" match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else "" - if not WORKSPACE_ROOT.exists(): - WORKSPACE_ROOT.mkdir(parents=True) - with open(WORKSPACE_ROOT / "agent_created_agent.py", "w") as f: + CONFIG.workspace_path.mkdir(parents=True, exist_ok=True) + with open(CONFIG.workspace_path / "agent_created_agent.py", "w") as f: f.write(code_text) return code_text + class AgentCreator(Role): def __init__( self, @@ -78,11 +78,11 @@ class AgentCreator(Role): return msg + if __name__ == "__main__": import asyncio async def main(): - agent_template = MULTI_ACTION_AGENT_CODE_EXAMPLE creator = AgentCreator(agent_template=agent_template) diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py index be34e5e5e..6805fd460 100644 --- a/examples/build_customized_agent.py +++ b/examples/build_customized_agent.py @@ -1,22 +1,22 @@ -''' +""" Filename: MetaGPT/examples/build_customized_agent.py Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 -''' +""" +import asyncio import re import subprocess -import asyncio import fire -from metagpt.llm import LLM from metagpt.actions import Action +from metagpt.llm import LLM +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.logs import logger + class SimpleWriteCode(Action): - PROMPT_TEMPLATE = """ Write a python function that can {instruction} and provide two runnnable test cases. Return ```python your_code_here ``` with NO other texts, @@ -27,7 +27,6 @@ class SimpleWriteCode(Action): super().__init__(name, context, llm) async def run(self, instruction: str): - prompt = self.PROMPT_TEMPLATE.format(instruction=instruction) rsp = await self._aask(prompt) @@ -38,7 +37,7 @@ class SimpleWriteCode(Action): @staticmethod def parse_code(rsp): - pattern = r'```python(.*)```' + pattern = r"```python(.*)```" match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else rsp return code_text @@ -67,10 +66,9 @@ class SimpleCoder(Role): async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - todo = self._rc.todo # todo will be SimpleWriteCode() - - msg = self.get_memories(k=1)[0] # find the most recent messages + todo = self._rc.todo # todo will be SimpleWriteCode() + msg = self.get_memories(k=1)[0] # find the most recent messages code_text = await todo.run(msg.content) msg = Message(content=code_text, role=self.profile, cause_by=type(todo)) @@ -94,7 +92,7 @@ class RunnableCoder(Role): # todo will be first SimpleWriteCode() then SimpleRunCode() todo = self._rc.todo - msg = self.get_memories(k=1)[0] # find the most k recent messages + msg = self.get_memories(k=1)[0] # find the most k recent messages result = await todo.run(msg.content) msg = Message(content=result, role=self.profile, cause_by=type(todo)) @@ -109,5 +107,6 @@ def main(msg="write a function that calculates the product of a list and run it" result = asyncio.run(role.run(msg)) logger.info(result) -if __name__ == '__main__': + +if __name__ == "__main__": fire.Fire(main) diff --git a/examples/build_customized_multi_agents.py b/examples/build_customized_multi_agents.py index 0df927e32..030a4b339 100644 --- a/examples/build_customized_multi_agents.py +++ b/examples/build_customized_multi_agents.py @@ -1,27 +1,28 @@ -''' +""" Filename: MetaGPT/examples/build_customized_multi_agents.py Created Date: Wednesday, November 15th 2023, 7:12:39 pm Author: garylin2099 -''' +""" import re -import asyncio + import fire +from metagpt.actions import Action, UserRequirement from metagpt.llm import LLM -from metagpt.actions import Action, BossRequirement -from metagpt.roles import Role -from metagpt.team import Team -from metagpt.schema import Message from metagpt.logs import logger +from metagpt.roles import Role +from metagpt.schema import Message +from metagpt.team import Team + def parse_code(rsp): - pattern = r'```python(.*)```' + pattern = r"```python(.*)```" match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else rsp return code_text -class SimpleWriteCode(Action): +class SimpleWriteCode(Action): PROMPT_TEMPLATE = """ Write a python function that can {instruction}. Return ```python your_code_here ``` with NO other texts, @@ -32,7 +33,6 @@ class SimpleWriteCode(Action): super().__init__(name, context, llm) async def run(self, instruction: str): - prompt = self.PROMPT_TEMPLATE.format(instruction=instruction) rsp = await self._aask(prompt) @@ -50,12 +50,11 @@ class SimpleCoder(Role): **kwargs, ): super().__init__(name, profile, **kwargs) - self._watch([BossRequirement]) + self._watch([UserRequirement]) self._init_actions([SimpleWriteCode]) class SimpleWriteTest(Action): - PROMPT_TEMPLATE = """ Context: {context} Write {k} unit tests using pytest for the given function, assuming you have imported it. @@ -67,7 +66,6 @@ class SimpleWriteTest(Action): super().__init__(name, context, llm) async def run(self, context: str, k: int = 3): - prompt = self.PROMPT_TEMPLATE.format(context=context, k=k) rsp = await self._aask(prompt) @@ -87,23 +85,22 @@ class SimpleTester(Role): super().__init__(name, profile, **kwargs) self._init_actions([SimpleWriteTest]) # self._watch([SimpleWriteCode]) - self._watch([SimpleWriteCode, SimpleWriteReview]) # feel free to try this too + self._watch([SimpleWriteCode, SimpleWriteReview]) # feel free to try this too async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") todo = self._rc.todo # context = self.get_memories(k=1)[0].content # use the most recent memory as context - context = self.get_memories() # use all memories as context + context = self.get_memories() # use all memories as context - code_text = await todo.run(context, k=5) # specify arguments + code_text = await todo.run(context, k=5) # specify arguments msg = Message(content=code_text, role=self.profile, cause_by=type(todo)) return msg class SimpleWriteReview(Action): - PROMPT_TEMPLATE = """ Context: {context} Review the test cases and provide one critical comments: @@ -113,7 +110,6 @@ class SimpleWriteReview(Action): super().__init__(name, context, llm) async def run(self, context: str): - prompt = self.PROMPT_TEMPLATE.format(context=context) rsp = await self._aask(prompt) @@ -154,5 +150,6 @@ async def main( team.start_project(idea) await team.run(n_round=n_round) -if __name__ == '__main__': + +if __name__ == "__main__": fire.Fire(main) diff --git a/examples/debate.py b/examples/debate.py index a37e60848..52f49e00e 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -1,17 +1,21 @@ -''' +""" Filename: MetaGPT/examples/debate.py Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 -''' +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `send_to` + value of the `Message` object; modify the argument type of `get_by_actions`. +""" import asyncio import platform + import fire -from metagpt.team import Team -from metagpt.actions import Action, BossRequirement +from metagpt.actions import Action, UserRequirement +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.logs import logger +from metagpt.team import Team + class SpeakAloud(Action): """Action: Speak out aloud in a debate (quarrel)""" @@ -31,7 +35,6 @@ class SpeakAloud(Action): super().__init__(name, context, llm) async def run(self, context: str, name: str, opponent_name: str): - prompt = self.PROMPT_TEMPLATE.format(context=context, name=name, opponent_name=opponent_name) # logger.info(prompt) @@ -39,6 +42,7 @@ class SpeakAloud(Action): return rsp + class Debator(Role): def __init__( self, @@ -49,19 +53,18 @@ class Debator(Role): ): super().__init__(name, profile, **kwargs) self._init_actions([SpeakAloud]) - self._watch([BossRequirement, SpeakAloud]) - self.name = name + self._watch([UserRequirement, SpeakAloud]) self.opponent_name = opponent_name async def _observe(self) -> int: await super()._observe() # accept messages sent (from opponent) to self, disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.send_to == self.name] + self._rc.news = [msg for msg in self._rc.news if msg.send_to == {self.name}] return len(self._rc.news) async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - todo = self._rc.todo # An instance of SpeakAloud + todo = self._rc.todo # An instance of SpeakAloud memories = self.get_memories() context = "\n".join(f"{msg.sent_from}: {msg.content}" for msg in memories) @@ -76,25 +79,25 @@ class Debator(Role): sent_from=self.name, send_to=self.opponent_name, ) - self._rc.memory.add(msg) return msg + async def debate(idea: str, investment: float = 3.0, n_round: int = 5): - """Run a team of presidents and watch they quarrel. :) """ + """Run a team of presidents and watch they quarrel. :)""" Biden = Debator(name="Biden", profile="Democrat", opponent_name="Trump") Trump = Debator(name="Trump", profile="Republican", opponent_name="Biden") team = Team() team.hire([Biden, Trump]) team.invest(investment) - team.start_project(idea, send_to="Biden") # send debate topic to Biden and let him speak first + team.run_project(idea, send_to="Biden") # send debate topic to Biden and let him speak first await team.run(n_round=n_round) def main(idea: str, investment: float = 3.0, n_round: int = 10): """ - :param idea: Debate topic, such as "Topic: The U.S. should commit more in climate change fighting" + :param idea: Debate topic, such as "Topic: The U.S. should commit more in climate change fighting" or "Trump: Climate change is a hoax" :param investment: contribute a certain dollar amount to watch the debate :param n_round: maximum rounds of the debate @@ -105,5 +108,5 @@ def main(idea: str, investment: float = 3.0, n_round: int = 10): asyncio.run(debate(idea, investment, n_round)) -if __name__ == '__main__': +if __name__ == "__main__": fire.Fire(main) diff --git a/examples/invoice_ocr.py b/examples/invoice_ocr.py index 11656ed52..a6e565772 100644 --- a/examples/invoice_ocr.py +++ b/examples/invoice_ocr.py @@ -19,19 +19,15 @@ async def main(): Path("../tests/data/invoices/invoice-1.pdf"), Path("../tests/data/invoices/invoice-2.png"), Path("../tests/data/invoices/invoice-3.jpg"), - Path("../tests/data/invoices/invoice-4.zip") + Path("../tests/data/invoices/invoice-4.zip"), ] # The absolute path of the file absolute_file_paths = [Path.cwd() / path for path in relative_paths] for path in absolute_file_paths: role = InvoiceOCRAssistant() - await role.run(Message( - content="Invoicing date", - instruct_content={"file_path": path} - )) + await role.run(Message(content="Invoicing date", instruct_content={"file_path": path})) -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) - diff --git a/examples/llm_hello_world.py b/examples/llm_hello_world.py index 3ba03eea0..677098399 100644 --- a/examples/llm_hello_world.py +++ b/examples/llm_hello_world.py @@ -14,11 +14,11 @@ from metagpt.logs import logger async def main(): llm = LLM() claude = Claude() - logger.info(await claude.aask('你好，请进行自我介绍')) - logger.info(await llm.aask('hello world')) - logger.info(await llm.aask_batch(['hi', 'write python hello world.'])) + logger.info(await claude.aask("你好，请进行自我介绍")) + logger.info(await llm.aask("hello world")) + logger.info(await llm.aask_batch(["hi", "write python hello world."])) - hello_msg = [{'role': 'user', 'content': 'count from 1 to 10. split by newline.'}] + hello_msg = [{"role": "user", "content": "count from 1 to 10. split by newline."}] logger.info(await llm.acompletion(hello_msg)) logger.info(await llm.acompletion_batch([hello_msg])) logger.info(await llm.acompletion_batch_text([hello_msg])) @@ -27,5 +27,5 @@ async def main(): await llm.acompletion_text(hello_msg, stream=True) -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/examples/research.py b/examples/research.py index 344f8d0e9..5c371cdd2 100644 --- a/examples/research.py +++ b/examples/research.py @@ -12,5 +12,5 @@ async def main(): print(f"save report to {RESEARCH_PATH / f'{topic}.md'}.") -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/examples/search_google.py b/examples/search_google.py index 9e9521b9c..73d04bf87 100644 --- a/examples/search_google.py +++ b/examples/search_google.py @@ -15,5 +15,5 @@ async def main(): await Searcher().run("What are some good sun protection products?") -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/examples/search_with_specific_engine.py b/examples/search_with_specific_engine.py index 7cc431cd4..334a7821f 100644 --- a/examples/search_with_specific_engine.py +++ b/examples/search_with_specific_engine.py @@ -6,11 +6,12 @@ from metagpt.tools import SearchEngineType async def main(): # Serper API - #await Searcher(engine = SearchEngineType.SERPER_GOOGLE).run(["What are some good sun protection products?","What are some of the best beaches?"]) + # await Searcher(engine = SearchEngineType.SERPER_GOOGLE).run(["What are some good sun protection products?","What are some of the best beaches?"]) # SerpAPI - #await Searcher(engine=SearchEngineType.SERPAPI_GOOGLE).run("What are the best ski brands for skiers?") + # await Searcher(engine=SearchEngineType.SERPAPI_GOOGLE).run("What are the best ski brands for skiers?") # Google API await Searcher(engine=SearchEngineType.DIRECT_GOOGLE).run("What are the most interesting human facts?") -if __name__ == '__main__': + +if __name__ == "__main__": asyncio.run(main()) diff --git a/examples/sk_agent.py b/examples/sk_agent.py index a7513e838..647ea4380 100644 --- a/examples/sk_agent.py +++ b/examples/sk_agent.py @@ -13,7 +13,7 @@ from semantic_kernel.planning import SequentialPlanner # from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.const import SKILL_DIRECTORY from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -39,7 +39,7 @@ async def basic_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def sequential_planner_example(): @@ -53,7 +53,7 @@ async def sequential_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def basic_planner_web_search_example(): @@ -64,7 +64,7 @@ async def basic_planner_web_search_example(): role.import_skill(SkSearchEngine(), "WebSearchSkill") # role.import_semantic_skill_from_directory(skills_directory, "QASkill") - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def action_planner_example(): @@ -75,7 +75,7 @@ async def action_planner_example(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - await role.run(Message(content=task, cause_by=BossRequirement)) # it will choose mathskill.Add + await role.run(Message(content=task, cause_by=UserRequirement)) # it will choose mathskill.Add if __name__ == "__main__": diff --git a/examples/use_off_the_shelf_agent.py b/examples/use_off_the_shelf_agent.py index 2e10068bd..4445a6c62 100644 --- a/examples/use_off_the_shelf_agent.py +++ b/examples/use_off_the_shelf_agent.py @@ -1,12 +1,13 @@ -''' +""" Filename: MetaGPT/examples/use_off_the_shelf_agent.py Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 -''' +""" import asyncio -from metagpt.roles.product_manager import ProductManager from metagpt.logs import logger +from metagpt.roles.product_manager import ProductManager + async def main(): msg = "Write a PRD for a snake game" @@ -14,5 +15,6 @@ async def main(): result = await role.run(msg) logger.info(result.content[:100]) -if __name__ == '__main__': + +if __name__ == "__main__": asyncio.run(main()) diff --git a/examples/write_tutorial.py b/examples/write_tutorial.py index 71ece5527..0dba3cdb7 100644 --- a/examples/write_tutorial.py +++ b/examples/write_tutorial.py @@ -16,6 +16,5 @@ async def main(): await role.run(topic) -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) - diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index b004bd58e..79ff94b3e 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -9,7 +9,7 @@ from enum import Enum from metagpt.actions.action import Action from metagpt.actions.action_output import ActionOutput -from metagpt.actions.add_requirement import BossRequirement +from metagpt.actions.add_requirement import UserRequirement from metagpt.actions.debug_error import DebugError from metagpt.actions.design_api import WriteDesign from metagpt.actions.design_api_review import DesignReview @@ -28,7 +28,7 @@ from metagpt.actions.write_test import WriteTest class ActionType(Enum): """All types of Actions, used for indexing.""" - ADD_REQUIREMENT = BossRequirement + ADD_REQUIREMENT = UserRequirement WRITE_PRD = WritePRD WRITE_PRD_REVIEW = WritePRDReview WRITE_DESIGN = WriteDesign diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 27d8db649..1534b1f4d 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -14,9 +14,9 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action_output import ActionOutput from metagpt.llm import LLM from metagpt.logs import logger +from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess from metagpt.utils.common import OutputParser from metagpt.utils.utils import general_after_log -from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess class Action(ABC): @@ -26,16 +26,24 @@ class Action(ABC): llm = LLM() self.llm = llm self.context = context - self.prefix = "" - self.profile = "" - self.desc = "" - self.content = "" - self.instruct_content = None + self.prefix = "" # aask*时会加上prefix，作为system_message + self.profile = "" # FIXME: USELESS + self.desc = "" # for skill manager + self.nodes = ... + + # Output, useless + # self.content = "" + # self.instruct_content = None + # self.env = None + + # def set_env(self, env): + # self.env = env def set_prefix(self, prefix, profile): """Set prefix for later usage""" self.prefix = prefix self.profile = profile + return self def __str__(self): return self.__class__.__name__ @@ -63,10 +71,6 @@ class Action(ABC): system_msgs: Optional[list[str]] = None, format="markdown", # compatible to original format ) -> ActionOutput: - """Append default prefix""" - if not system_msgs: - system_msgs = [] - system_msgs.append(self.prefix) content = await self.llm.aask(prompt, system_msgs) logger.debug(f"llm raw output:\n{content}") output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py new file mode 100644 index 000000000..fb7d621d8 --- /dev/null +++ b/metagpt/actions/action_node.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/11 18:45 +@Author : alexanderwu +@File : action_node.py +""" +import json +import re +from typing import Any, Dict, List, Optional, Type + +from pydantic import BaseModel, create_model, root_validator, validator +from tenacity import retry, stop_after_attempt, wait_random_exponential + +from metagpt.actions import ActionOutput +from metagpt.llm import BaseGPTAPI +from metagpt.logs import logger +from metagpt.utils.common import OutputParser +from metagpt.utils.custom_decoder import CustomDecoder + +CONSTRAINT = """ +- Language: Please use the same language as the user input. +- Format: output wrapped inside [CONTENT][/CONTENT] as format example, nothing else. +""" + +SIMPLE_TEMPLATE = """ +## context +{context} + +----- + +## format example +{example} + +## nodes: ": # " +{instruction} + +## constraint +{constraint} + +## action +Fill in the above nodes based on the format example. +""" + + +def dict_to_markdown(d, prefix="-", postfix="\n"): + markdown_str = "" + for key, value in d.items(): + markdown_str += f"{prefix} {key}: {value}{postfix}" + return markdown_str + + +class ActionNode: + """ActionNode is a tree of nodes.""" + mode: str + + # Action Context + context: str # all the context, including all necessary info + llm: BaseGPTAPI # LLM with aask interface + children: dict[str, "ActionNode"] + + # Action Input + key: str # Product Requirement / File list / Code + expected_type: Type # such as str / int / float etc. + # context: str # everything in the history. + instruction: str # the instructions should be followed. + example: Any # example for In Context-Learning. + + # Action Output + content: str + instruct_content: BaseModel + + def __init__(self, key: str, expected_type: Type, instruction: str, example: str, content: str = "", + children: dict[str, "ActionNode"] = None): + self.key = key + self.expected_type = expected_type + self.instruction = instruction + self.example = example + self.content = content + self.children = children if children is not None else {} + + def __str__(self): + return ( + f"{self.key}, {self.expected_type}, {self.instruction}, {self.example}" f", {self.content}, {self.children}" + ) + + def __repr__(self): + return self.__str__() + + def add_child(self, node: "ActionNode"): + """增加子ActionNode""" + self.children[node.key] = node + + def add_children(self, nodes: List["ActionNode"]): + """批量增加子ActionNode""" + for node in nodes: + self.add_child(node) + + @classmethod + def from_children(cls, key, nodes: List["ActionNode"]): + """直接从一系列的子nodes初始化""" + obj = cls(key, str, "", "") + obj.add_children(nodes) + return obj + + def get_children_mapping(self) -> Dict[str, Type]: + """获得子ActionNode的字典，以key索引""" + return {k: (v.expected_type, ...) for k, v in self.children.items()} + + def get_self_mapping(self) -> Dict[str, Type]: + """get self key: type mapping""" + return {self.key: (self.expected_type, ...)} + + def get_mapping(self, mode="children") -> Dict[str, Type]: + """get key: type mapping under mode""" + if mode == "children" or (mode == "auto" and self.children): + return self.get_children_mapping() + return self.get_self_mapping() + + @classmethod + def create_model_class(cls, class_name: str, mapping: Dict[str, Type]): + """基于pydantic v1的模型动态生成，用来检验结果类型正确性""" + new_class = create_model(class_name, **mapping) + + @validator("*", allow_reuse=True) + def check_name(v, field): + if field.name not in mapping.keys(): + raise ValueError(f"Unrecognized block: {field.name}") + return v + + @root_validator(pre=True, allow_reuse=True) + def check_missing_fields(values): + required_fields = set(mapping.keys()) + missing_fields = required_fields - set(values.keys()) + if missing_fields: + raise ValueError(f"Missing fields: {missing_fields}") + return values + + new_class.__validator_check_name = classmethod(check_name) + new_class.__root_validator_check_missing_fields = classmethod(check_missing_fields) + return new_class + + @classmethod + def create_model_class_v2(cls, class_name: str, mapping: Dict[str, Type]): + """基于pydantic v2的模型动态生成，用来检验结果类型正确性，待验证""" + new_class = create_model(class_name, **mapping) + + @model_validator(mode="before") + def check_missing_fields(data): + required_fields = set(mapping.keys()) + missing_fields = required_fields - set(data.keys()) + if missing_fields: + raise ValueError(f"Missing fields: {missing_fields}") + return data + + @field_validator("*") + def check_name(v: Any, field: str) -> Any: + if field not in mapping.keys(): + raise ValueError(f"Unrecognized block: {field}") + return v + + new_class.__model_validator_check_missing_fields = classmethod(check_missing_fields) + new_class.__field_validator_check_name = classmethod(check_name) + return new_class + + def create_children_class(self): + """使用object内有的字段直接生成model_class""" + class_name = f"{self.key}_AN" + mapping = self.get_children_mapping() + return self.create_model_class(class_name, mapping) + + def to_dict(self, format_func=None, mode="auto") -> Dict: + """将当前节点与子节点都按照node: format的格式组织成字典""" + + # 如果没有提供格式化函数，使用默认的格式化方式 + if format_func is None: + format_func = lambda node: f"{node.instruction}" + + # 使用提供的格式化函数来格式化当前节点的值 + formatted_value = format_func(self) + + # 创建当前节点的键值对 + if mode == "children" or (mode == "auto" and self.children): + node_dict = {} + else: + node_dict = {self.key: formatted_value} + + if mode == "root": + return node_dict + + # 遍历子节点并递归调用 to_dict 方法 + for child_key, child_node in self.children.items(): + node_dict.update(child_node.to_dict(format_func)) + + return node_dict + + def compile_to(self, i: Dict, to) -> str: + if to == "json": + return json.dumps(i, indent=4) + elif to == "markdown": + return dict_to_markdown(i) + else: + return str(i) + + def tagging(self, text, to, tag="") -> str: + if not tag: + return text + if to == "json": + return f"[{tag}]\n" + text + f"\n[/{tag}]" + else: + return f"[{tag}]\n" + text + f"\n[/{tag}]" + + def _compile_f(self, to, mode, tag, format_func) -> str: + nodes = self.to_dict(format_func=format_func, mode=mode) + text = self.compile_to(nodes, to) + return self.tagging(text, to, tag) + + def compile_instruction(self, to="raw", mode="children", tag="") -> str: + """compile to raw/json/markdown template with all/root/children nodes""" + format_func = lambda i: f"{i.expected_type} # {i.instruction}" + return self._compile_f(to, mode, tag, format_func) + + def compile_example(self, to="raw", mode="children", tag="") -> str: + """compile to raw/json/markdown examples with all/root/children nodes""" + + # 这里不能使用f-string，因为转译为str后再json.dumps会额外加上引号，无法作为有效的example + # 错误示例："File list": "['main.py', 'const.py', 'game.py']", 注意这里值不是list，而是str + format_func = lambda i: i.example + return self._compile_f(to, mode, tag, format_func) + + def compile(self, context, to="json", mode="children", template=SIMPLE_TEMPLATE) -> str: + """ + mode: all/root/children + mode="children": 编译所有子节点为一个统一模板，包括instruction与example + mode="all": NotImplemented + mode="root": NotImplemented + """ + + # FIXME: json instruction会带来格式问题，如："Project name": "web_2048 # 项目名称使用下划线", + self.instruction = self.compile_instruction(to="markdown", mode=mode) + self.example = self.compile_example(to=to, tag="CONTENT", mode=mode) + prompt = template.format( + context=context, example=self.example, instruction=self.instruction, constraint=CONSTRAINT + ) + return prompt + + @retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(6)) + async def _aask_v1( + self, + prompt: str, + output_class_name: str, + output_data_mapping: dict, + system_msgs: Optional[list[str]] = None, + format="markdown", # compatible to original format + ) -> ActionOutput: + content = await self.llm.aask(prompt, system_msgs) + logger.debug(content) + output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) + + if format == "json": + pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]" + matches = re.findall(pattern, content, re.DOTALL) + + for match in matches: + if match: + content = match + break + + parsed_data = CustomDecoder(strict=False).decode(content) + + else: # using markdown parser + parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping) + + logger.debug(parsed_data) + instruct_content = output_class(**parsed_data) + return ActionOutput(content, instruct_content) + + def get(self, key): + return self.instruct_content.dict()[key] + + def set_recursive(self, name, value): + setattr(self, name, value) + for _, i in self.children.items(): + i.set_recursive(name, value) + + def set_llm(self, llm): + self.set_recursive("llm", llm) + + def set_context(self, context): + self.set_recursive("context", context) + + async def simple_fill(self, to, mode): + prompt = self.compile(context=self.context, to=to, mode=mode) + mapping = self.get_mapping(mode) + + class_name = f"{self.key}_AN" + output = await self._aask_v1(prompt, class_name, mapping, format=to) + self.content = output.content + self.instruct_content = output.instruct_content + return self + + async def fill(self, context, llm, to="json", mode="auto", strgy="simple"): + """Fill the node(s) with mode. + + :param context: Everything we should know when filling node. + :param llm: Large Language Model with pre-defined system message. + :param to: json/markdown, determine example and output format. + - json: it's easy to open source LLM with json format + - markdown: when generating code, markdown is always better + :param mode: auto/children/root + - auto: automated fill children's nodes and gather outputs, if no children, fill itself + - children: fill children's nodes and gather outputs + - root: fill root's node and gather output + :param strgy: simple/complex + - simple: run only once + - complex: run each node + :return: self + """ + self.set_llm(llm) + self.set_context(context) + + if strgy == "simple": + return await self.simple_fill(to, mode) + elif strgy == "complex": + # 这里隐式假设了拥有children + tmp = {} + for _, i in self.children.items(): + child = await i.simple_fill(to, mode) + tmp.update(child.instruct_content.dict()) + cls = self.create_children_class() + self.instruct_content = cls(**tmp) + return self + + +def action_node_from_tuple_example(): + # 示例：列表中包含元组 + list_of_tuples = [("key1", str, "Instruction 1", "Example 1")] + + # 从列表中创建 ActionNode 实例 + nodes = [ActionNode(*data) for data in list_of_tuples] + for i in nodes: + logger.info(i) + + +if __name__ == "__main__": + action_node_from_tuple_example() diff --git a/metagpt/actions/action_output.py b/metagpt/actions/action_output.py index ea7f4fb80..25326d43b 100644 --- a/metagpt/actions/action_output.py +++ b/metagpt/actions/action_output.py @@ -23,10 +23,10 @@ class ActionOutput: def create_model_class(cls, class_name: str, mapping: Dict[str, Type]): new_class = create_model(class_name, **mapping) - @validator('*', allow_reuse=True) + @validator("*", allow_reuse=True) def check_name(v, field): if field.name not in mapping.keys(): - raise ValueError(f'Unrecognized block: {field.name}') + raise ValueError(f"Unrecognized block: {field.name}") return v @root_validator(pre=True, allow_reuse=True) @@ -34,10 +34,9 @@ class ActionOutput: required_fields = set(mapping.keys()) missing_fields = required_fields - set(values.keys()) if missing_fields: - raise ValueError(f'Missing fields: {missing_fields}') + raise ValueError(f"Missing fields: {missing_fields}") return values new_class.__validator_check_name = classmethod(check_name) new_class.__root_validator_check_missing_fields = classmethod(check_missing_fields) return new_class - \ No newline at end of file diff --git a/metagpt/actions/add_requirement.py b/metagpt/actions/add_requirement.py index 7dc09d062..d77d423ba 100644 --- a/metagpt/actions/add_requirement.py +++ b/metagpt/actions/add_requirement.py @@ -8,7 +8,8 @@ from metagpt.actions import Action -class BossRequirement(Action): - """Boss Requirement without any implementation details""" +class UserRequirement(Action): + """User Requirement without any implementation details""" + async def run(self, *args, **kwargs): raise NotImplementedError diff --git a/metagpt/actions/azure_tts.py b/metagpt/actions/azure_tts.py index c13a4750d..daa3f6892 100644 --- a/metagpt/actions/azure_tts.py +++ b/metagpt/actions/azure_tts.py @@ -18,16 +18,13 @@ class AzureTTS(Action): # Parameters reference: https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles def synthesize_speech(self, lang, voice, role, text, output_file): - subscription_key = self.config.get('AZURE_TTS_SUBSCRIPTION_KEY') - region = self.config.get('AZURE_TTS_REGION') - speech_config = SpeechConfig( - subscription=subscription_key, region=region) + subscription_key = self.config.get("AZURE_TTS_SUBSCRIPTION_KEY") + region = self.config.get("AZURE_TTS_REGION") + speech_config = SpeechConfig(subscription=subscription_key, region=region) speech_config.speech_synthesis_voice_name = voice audio_config = AudioConfig(filename=output_file) - synthesizer = SpeechSynthesizer( - speech_config=speech_config, - audio_config=audio_config) + synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=audio_config) # if voice=="zh-CN-YunxiNeural": ssml_string = f""" @@ -45,9 +42,4 @@ class AzureTTS(Action): if __name__ == "__main__": azure_tts = AzureTTS("azure_tts") - azure_tts.synthesize_speech( - "zh-CN", - "zh-CN-YunxiNeural", - "Boy", - "Hello, I am Kaka", - "output.wav") + azure_tts.synthesize_speech("zh-CN", "zh-CN-YunxiNeural", "Boy", "Hello, I am Kaka", "output.wav") diff --git a/metagpt/actions/clone_function.py b/metagpt/actions/clone_function.py index cf7d22f04..1447e8dbf 100644 --- a/metagpt/actions/clone_function.py +++ b/metagpt/actions/clone_function.py @@ -1,5 +1,5 @@ -from pathlib import Path import traceback +from pathlib import Path from metagpt.actions.write_code import WriteCode from metagpt.logs import logger @@ -42,7 +42,7 @@ class CloneFunction(WriteCode): prompt = CLONE_PROMPT.format(source_code=source_code, template_func=template_func) logger.info(f"query for CloneFunction: \n {prompt}") code = await self.write_code(prompt) - logger.info(f'CloneFunction code is \n {highlight(code)}') + logger.info(f"CloneFunction code is \n {highlight(code)}") return code @@ -61,5 +61,5 @@ def run_function_script(code_script_path: str, func_name: str, *args, **kwargs): """Run function code from script.""" if isinstance(code_script_path, str): code_path = Path(code_script_path) - code = code_path.read_text(encoding='utf-8') + code = code_path.read_text(encoding="utf-8") return run_function_code(code, func_name, *args, **kwargs) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 101cc2025..39f3bc1bc 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -4,12 +4,19 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : debug_error.py +@Modified By: mashenquan, 2023/11/27. + 1. Divide the context into three components: legacy code, unit test code, and console log. + 2. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. """ import re -from metagpt.logs import logger from metagpt.actions.action import Action +from metagpt.config import CONFIG +from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO +from metagpt.logs import logger +from metagpt.schema import RunCodeResult from metagpt.utils.common import CodeParser +from metagpt.utils.file_repository import FileRepository PROMPT_TEMPLATE = """ NOTICE @@ -19,33 +26,56 @@ Based on the message, first, figure out your own role, i.e. Engineer or QaEngine then rewrite the development code or the test code based on your role, the error, and the summary, such that all bugs are fixed and the code performs well. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. The message is as follows: -{context} +# Legacy Code +```python +{code} +``` +--- +# Unit Test Code +```python +{test_code} +``` +--- +# Console logs +```text +{logs} +``` --- Now you should start rewriting the code: ## file name of the code to rewrite: Write code with triple quote. Do your best to implement THIS IN ONLY ONE FILE. """ + + class DebugError(Action): def __init__(self, name="DebugError", context=None, llm=None): super().__init__(name, context, llm) - # async def run(self, code, error): - # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ - # f"\n\n{error}\n\nPlease try to fix the error in this code." - # fixed_code = await self._aask(prompt) - # return fixed_code - - async def run(self, context): - if "PASS" in context: - return "", "the original code works fine, no need to debug" - - file_name = re.search("## File To Rewrite:\s*(.+\\.py)", context).group(1) + async def run(self, *args, **kwargs) -> str: + output_doc = await FileRepository.get_file( + filename=self.context.output_filename, relative_path=TEST_OUTPUTS_FILE_REPO + ) + if not output_doc: + return "" + output_detail = RunCodeResult.loads(output_doc.content) + pattern = r"Ran (\d+) tests in ([\d.]+)s\n\nOK" + matches = re.search(pattern, output_detail.stderr) + if matches: + return "" - logger.info(f"Debug and rewrite {file_name}") + logger.info(f"Debug and rewrite {self.context.test_filename}") + code_doc = await FileRepository.get_file( + filename=self.context.code_filename, relative_path=CONFIG.src_workspace + ) + if not code_doc: + return "" + test_doc = await FileRepository.get_file( + filename=self.context.test_filename, relative_path=TEST_CODES_FILE_REPO + ) + if not test_doc: + return "" + prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr) - prompt = PROMPT_TEMPLATE.format(context=context) - rsp = await self._aask(prompt) - code = CodeParser.parse_code(block="", text=rsp) - return file_name, code + return code diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index fc3be602b..5a5f52de7 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -4,149 +4,41 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : design_api.py +@Modified By: mashenquan, 2023/11/27. + 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. + 2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ -import shutil +import json from pathlib import Path -from typing import List from metagpt.actions import Action, ActionOutput +from metagpt.actions.design_api_an import DESIGN_API_NODE from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT +from metagpt.const import ( + DATA_API_DESIGN_FILE_REPO, + PRDS_FILE_REPO, + SEQ_FLOW_FILE_REPO, + SYSTEM_DESIGN_FILE_REPO, + SYSTEM_DESIGN_PDF_FILE_REPO, +) from metagpt.logs import logger -from metagpt.utils.common import CodeParser -from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown +from metagpt.schema import Document, Documents +from metagpt.utils.file_repository import FileRepository + +# from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file -templates = { - "json": { - "PROMPT_TEMPLATE": """ -# Context +# from typing import List + + +NEW_REQ_TEMPLATE = """ +### Legacy Content +{old_design} + +### New Requirements {context} - -## Format example -{format_example} ------ -Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools -Requirement: Fill in the following missing information based on the context, each section name is a key in json -Max Output: 8192 chars or 2048 tokens. Try to use them up. - -## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. - -## Python package name: Provide as Python str with python triple quote, concise and clear, characters only use a combination of all lowercase and underscores - -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here - -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. - -## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. - -## Anything UNCLEAR: Provide as Plain text. Make clear here. - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, -and only output the json inside this tag, nothing else -""", - "FORMAT_EXAMPLE": """ -[CONTENT] -{ - "Implementation approach": "We will ...", - "Python package name": "snake_game", - "File list": ["main.py"], - "Data structures and interface definitions": ' - classDiagram - class Game{ - +int score - } - ... - Game "1" -- "1" Food: has - ', - "Program call flow": ' - sequenceDiagram - participant M as Main - ... - G->>M: end game - ', - "Anything UNCLEAR": "The requirement is clear to me." -} -[/CONTENT] -""", - }, - "markdown": { - "PROMPT_TEMPLATE": """ -# Context -{context} - -## Format example -{format_example} ------ -Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools -Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately -Max Output: 8192 chars or 2048 tokens. Try to use them up. -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. - -## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. - -## Python package name: Provide as Python str with python triple quote, concise and clear, characters only use a combination of all lowercase and underscores - -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here - -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. - -## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. - -## Anything UNCLEAR: Provide as Plain text. Make clear here. - -""", - "FORMAT_EXAMPLE": """ ---- -## Implementation approach -We will ... - -## Python package name -```python -"snake_game" -``` - -## File list -```python -[ - "main.py", -] -``` - -## Data structures and interface definitions -```mermaid -classDiagram - class Game{ - +int score - } - ... - Game "1" -- "1" Food: has -``` - -## Program call flow -```mermaid -sequenceDiagram - participant M as Main - ... - G->>M: end game -``` - -## Anything UNCLEAR -The requirement is clear to me. ---- -""", - }, -} - -OUTPUT_MAPPING = { - "Implementation approach": (str, ...), - "Python package name": (str, ...), - "File list": (List[str], ...), - "Data structures and interface definitions": (str, ...), - "Program call flow": (str, ...), - "Anything UNCLEAR": (str, ...), -} +""" class WriteDesign(Action): @@ -158,60 +50,91 @@ class WriteDesign(Action): "clearly and in detail." ) - def recreate_workspace(self, workspace: Path): - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # Folder does not exist, but we don't care - workspace.mkdir(parents=True, exist_ok=True) + async def run(self, with_messages, format=CONFIG.prompt_format): + # Use `git diff` to identify which PRD documents have been modified in the `docs/prds` directory. + prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) + changed_prds = prds_file_repo.changed_files + # Use `git diff` to identify which design documents in the `docs/system_designs` directory have undergone + # changes. + system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) + changed_system_designs = system_design_file_repo.changed_files - async def _save_prd(self, docs_path, resources_path, context): - prd_file = docs_path / "prd.md" - if context[-1].instruct_content and context[-1].instruct_content.dict()["Competitive Quadrant Chart"]: - quadrant_chart = context[-1].instruct_content.dict()["Competitive Quadrant Chart"] - await mermaid_to_file(quadrant_chart, resources_path / "competitive_analysis") + # For those PRDs and design documents that have undergone changes, regenerate the design content. + changed_files = Documents() + for filename in changed_prds.keys(): + doc = await self._update_system_design( + filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo + ) + changed_files.docs[filename] = doc - if context[-1].instruct_content: - logger.info(f"Saving PRD to {prd_file}") - prd_file.write_text(json_to_markdown(context[-1].instruct_content.dict())) + for filename in changed_system_designs.keys(): + if filename in changed_files.docs: + continue + doc = await self._update_system_design( + filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo + ) + changed_files.docs[filename] = doc + if not changed_files.docs: + logger.info("Nothing has changed.") + # Wait until all files under `docs/system_designs/` are processed before sending the publish message, + # leaving room for global optimization in subsequent steps. + return ActionOutput(content=changed_files.json(), instruct_content=changed_files) - async def _save_system_design(self, docs_path, resources_path, system_design): - data_api_design = system_design.instruct_content.dict()[ - "Data structures and interface definitions" - ] # CodeParser.parse_code(block="Data structures and interface definitions", text=content) - seq_flow = system_design.instruct_content.dict()[ - "Program call flow" - ] # CodeParser.parse_code(block="Program call flow", text=content) - await mermaid_to_file(data_api_design, resources_path / "data_api_design") - await mermaid_to_file(seq_flow, resources_path / "seq_flow") - system_design_file = docs_path / "system_design.md" - logger.info(f"Saving System Designs to {system_design_file}") - system_design_file.write_text((json_to_markdown(system_design.instruct_content.dict()))) + async def _new_system_design(self, context, format=CONFIG.prompt_format): + node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=format) + return node - async def _save(self, context, system_design): - if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["Python package name"] + async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): + context = NEW_REQ_TEMPLATE.format(old_design=system_design_doc.content, context=prd_doc.content) + node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=format) + system_design_doc.content = node.instruct_content.json(ensure_ascii=False) + return system_design_doc + + async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document: + prd = await prds_file_repo.get(filename) + old_system_design_doc = await system_design_file_repo.get(filename) + if not old_system_design_doc: + system_design = await self._new_system_design(context=prd.content) + doc = Document( + root_path=SYSTEM_DESIGN_FILE_REPO, + filename=filename, + content=system_design.instruct_content.json(ensure_ascii=False), + ) else: - ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = WORKSPACE_ROOT / ws_name - self.recreate_workspace(workspace) - docs_path = workspace / "docs" - resources_path = workspace / "resources" - docs_path.mkdir(parents=True, exist_ok=True) - resources_path.mkdir(parents=True, exist_ok=True) - await self._save_prd(docs_path, resources_path, context) - await self._save_system_design(docs_path, resources_path, system_design) - - async def run(self, context, format=CONFIG.prompt_format): - prompt_template, format_example = get_template(templates, format) - prompt = prompt_template.format(context=context, format_example=format_example) - # system_design = await self._aask(prompt) - system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python package name" contain space, have to use setattr - setattr( - system_design.instruct_content, - "Python package name", - system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), + doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) + await system_design_file_repo.save( + filename=filename, content=doc.content, dependencies={prd.root_relative_path} ) - await self._save(context, system_design) - return system_design + await self._save_data_api_design(doc) + await self._save_seq_flow(doc) + await self._save_pdf(doc) + return doc + + @staticmethod + async def _save_data_api_design(design_doc): + m = json.loads(design_doc.content) + data_api_design = m.get("Data structures and interfaces") + if not data_api_design: + return + pathname = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("") + await WriteDesign._save_mermaid_file(data_api_design, pathname) + logger.info(f"Save class view to {str(pathname)}") + + @staticmethod + async def _save_seq_flow(design_doc): + m = json.loads(design_doc.content) + seq_flow = m.get("Program call flow") + if not seq_flow: + return + pathname = CONFIG.git_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc.filename).with_suffix("") + await WriteDesign._save_mermaid_file(seq_flow, pathname) + logger.info(f"Saving sequence flow to {str(pathname)}") + + @staticmethod + async def _save_pdf(design_doc): + await FileRepository.save_as(doc=design_doc, with_suffix=".md", relative_path=SYSTEM_DESIGN_PDF_FILE_REPO) + + @staticmethod + async def _save_mermaid_file(data: str, pathname: Path): + pathname.parent.mkdir(parents=True, exist_ok=True) + await mermaid_to_file(data, pathname) diff --git a/metagpt/actions/design_api_an.py b/metagpt/actions/design_api_an.py new file mode 100644 index 000000000..0a303cdd5 --- /dev/null +++ b/metagpt/actions/design_api_an.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/12 22:24 +@Author : alexanderwu +@File : design_api_an.py +""" +from metagpt.actions.action_node import ActionNode +from metagpt.logs import logger +from metagpt.utils.mermaid import MMC1, MMC2 + +IMPLEMENTATION_APPROACH = ActionNode( + key="Implementation approach", + expected_type=str, + instruction="Analyze the difficult points of the requirements, select the appropriate open-source framework", + example="We will ...", +) + +PROJECT_NAME = ActionNode( + key="Project name", expected_type=str, instruction="The project name with underline", example="game_2048" +) + +FILE_LIST = ActionNode( + key="File list", + expected_type=list[str], + instruction="Only need relative paths. ALWAYS write a main.py or app.py here", + example=["main.py", "game.py"], +) + +DATA_STRUCTURES_AND_INTERFACES = ActionNode( + key="Data structures and interfaces", + expected_type=str, + instruction="Use mermaid classDiagram code syntax, including classes, method(__init__ etc.) and functions with type" + " annotations, CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. " + "The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.", + example=MMC1, +) + +PROGRAM_CALL_FLOW = ActionNode( + key="Program call flow", + expected_type=str, + instruction="Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE " + "accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.", + example=MMC2, +) + +ANYTHING_UNCLEAR = ActionNode( + key="Anything UNCLEAR", + expected_type=str, + instruction="Mention unclear project aspects, then try to clarify it.", + example="Clarification needed on third-party API integration, ...", +) + +NODES = [ + IMPLEMENTATION_APPROACH, + # PROJECT_NAME, + FILE_LIST, + DATA_STRUCTURES_AND_INTERFACES, + PROGRAM_CALL_FLOW, + ANYTHING_UNCLEAR, +] + +DESIGN_API_NODE = ActionNode.from_children("DesignAPI", NODES) + + +def main(): + prompt = DESIGN_API_NODE.compile(context="") + logger.info(prompt) + + +if __name__ == "__main__": + main() diff --git a/metagpt/actions/design_api_review.py b/metagpt/actions/design_api_review.py index 9bb822a62..7f25bb9a3 100644 --- a/metagpt/actions/design_api_review.py +++ b/metagpt/actions/design_api_review.py @@ -13,10 +13,11 @@ class DesignReview(Action): super().__init__(name, context, llm) async def run(self, prd, api_design): - prompt = f"Here is the Product Requirement Document (PRD):\n\n{prd}\n\nHere is the list of APIs designed " \ - f"based on this PRD:\n\n{api_design}\n\nPlease review whether this API design meets the requirements" \ - f" of the PRD, and whether it complies with good design practices." + prompt = ( + f"Here is the Product Requirement Document (PRD):\n\n{prd}\n\nHere is the list of APIs designed " + f"based on this PRD:\n\n{api_design}\n\nPlease review whether this API design meets the requirements" + f" of the PRD, and whether it complies with good design practices." + ) api_review = await self._aask(prompt) return api_review - \ No newline at end of file diff --git a/metagpt/actions/design_filenames.py b/metagpt/actions/design_filenames.py index 29400e950..ffa171d7b 100644 --- a/metagpt/actions/design_filenames.py +++ b/metagpt/actions/design_filenames.py @@ -17,8 +17,10 @@ Do not add any other explanations, just return a Python string list.""" class DesignFilenames(Action): def __init__(self, name, context=None, llm=None): super().__init__(name, context, llm) - self.desc = "Based on the PRD, consider system design, and carry out the basic design of the corresponding " \ - "APIs, data structures, and database tables. Please give your design, feedback clearly and in detail." + self.desc = ( + "Based on the PRD, consider system design, and carry out the basic design of the corresponding " + "APIs, data structures, and database tables. Please give your design, feedback clearly and in detail." + ) async def run(self, prd): prompt = f"The following is the Product Requirement Document (PRD):\n\n{prd}\n\n{PROMPT}" @@ -26,4 +28,3 @@ class DesignFilenames(Action): logger.debug(prompt) logger.debug(design_filenames) return design_filenames - \ No newline at end of file diff --git a/metagpt/actions/detail_mining.py b/metagpt/actions/detail_mining.py index e29d6911b..5afcf52c6 100644 --- a/metagpt/actions/detail_mining.py +++ b/metagpt/actions/detail_mining.py @@ -6,7 +6,6 @@ @File : detail_mining.py """ from metagpt.actions import Action, ActionOutput -from metagpt.logs import logger PROMPT_TEMPLATE = """ ##TOPIC @@ -41,8 +40,8 @@ OUTPUT_MAPPING = { class DetailMining(Action): - """This class allows LLM to further mine noteworthy details based on specific "##TOPIC"(discussion topic) and "##RECORD" (discussion records), thereby deepening the discussion. - """ + """This class allows LLM to further mine noteworthy details based on specific "##TOPIC"(discussion topic) and "##RECORD" (discussion records), thereby deepening the discussion.""" + def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) diff --git a/metagpt/actions/fix_bug.py b/metagpt/actions/fix_bug.py new file mode 100644 index 000000000..6bd550d3d --- /dev/null +++ b/metagpt/actions/fix_bug.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +""" +@Time : 2023-12-12 +@Author : mashenquan +@File : fix_bug.py +""" +from metagpt.actions import Action + + +class FixBug(Action): + """Fix bug action without any implementation details""" + + async def run(self, *args, **kwargs): + raise NotImplementedError diff --git a/metagpt/actions/invoice_ocr.py b/metagpt/actions/invoice_ocr.py index b37aa6885..dcf537a58 100644 --- a/metagpt/actions/invoice_ocr.py +++ b/metagpt/actions/invoice_ocr.py @@ -10,8 +10,8 @@ import os import zipfile -from pathlib import Path from datetime import datetime +from pathlib import Path import pandas as pd from paddleocr import PaddleOCR @@ -19,7 +19,10 @@ from paddleocr import PaddleOCR from metagpt.actions import Action from metagpt.const import INVOICE_OCR_TABLE_PATH from metagpt.logs import logger -from metagpt.prompts.invoice_ocr import EXTRACT_OCR_MAIN_INFO_PROMPT, REPLY_OCR_QUESTION_PROMPT +from metagpt.prompts.invoice_ocr import ( + EXTRACT_OCR_MAIN_INFO_PROMPT, + REPLY_OCR_QUESTION_PROMPT, +) from metagpt.utils.common import OutputParser from metagpt.utils.file import File @@ -183,4 +186,3 @@ class ReplyQuestion(Action): prompt = REPLY_OCR_QUESTION_PROMPT.format(query=query, ocr_result=ocr_result, language=self.language) resp = await self._aask(prompt=prompt) return resp - diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py new file mode 100644 index 000000000..8d3445ae4 --- /dev/null +++ b/metagpt/actions/prepare_documents.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : prepare_documents.py +@Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt. + RFC 135 2.2.3.5.1. +""" +import shutil +from pathlib import Path + +from metagpt.actions import Action, ActionOutput +from metagpt.config import CONFIG +from metagpt.const import DEFAULT_WORKSPACE_ROOT, DOCS_FILE_REPO, REQUIREMENT_FILENAME +from metagpt.schema import Document +from metagpt.utils.file_repository import FileRepository +from metagpt.utils.git_repository import GitRepository + + +class PrepareDocuments(Action): + def __init__(self, name="", context=None, llm=None): + super().__init__(name, context, llm) + + async def run(self, with_messages, **kwargs): + if not CONFIG.git_repo: + # Create and initialize the workspace folder, initialize the Git environment. + project_name = CONFIG.project_name or FileRepository.new_filename() + workdir = CONFIG.project_path + if not workdir and CONFIG.workspace_path: + workdir = Path(CONFIG.workspace_path) / project_name + workdir = Path(workdir or DEFAULT_WORKSPACE_ROOT / project_name) + if not CONFIG.inc and workdir.exists(): + shutil.rmtree(workdir) + CONFIG.git_repo = GitRepository() + CONFIG.git_repo.open(local_path=workdir, auto_init=True) + + # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. + doc = Document(root_path=DOCS_FILE_REPO, filename=REQUIREMENT_FILENAME, content=with_messages[0].content) + await FileRepository.save_file(filename=REQUIREMENT_FILENAME, content=doc.content, relative_path=DOCS_FILE_REPO) + + # Send a Message notification to the WritePRD action, instructing it to process requirements using + # `docs/requirement.txt` and `docs/prds/`. + return ActionOutput(content=doc.content, instruct_content=doc) diff --git a/metagpt/actions/prepare_interview.py b/metagpt/actions/prepare_interview.py index 5db3a9f37..b2704616e 100644 --- a/metagpt/actions/prepare_interview.py +++ b/metagpt/actions/prepare_interview.py @@ -38,4 +38,3 @@ class PrepareInterview(Action): prompt = PROMPT_TEMPLATE.format(context=context) question_list = await self._aask_v1(prompt) return question_list - diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index b395fa64e..1f14e7944 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -4,186 +4,122 @@ @Time : 2023/5/11 19:12 @Author : alexanderwu @File : project_management.py +@Modified By: mashenquan, 2023/11/27. + 1. Divide the context into three components: legacy code, unit test code, and console log. + 2. Move the document storage operations related to WritePRD from the save operation of WriteDesign. + 3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality. """ -from typing import List +import json +from metagpt.actions import ActionOutput from metagpt.actions.action import Action +from metagpt.actions.project_management_an import PM_NODE from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT -from metagpt.utils.common import CodeParser -from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown +from metagpt.const import ( + PACKAGE_REQUIREMENTS_FILENAME, + SYSTEM_DESIGN_FILE_REPO, + TASK_FILE_REPO, + TASK_PDF_FILE_REPO, +) +from metagpt.logs import logger +from metagpt.schema import Document, Documents +from metagpt.utils.file_repository import FileRepository -templates = { - "json": { - "PROMPT_TEMPLATE": """ -# Context +# from typing import List + +# from metagpt.utils.get_template import get_template + +NEW_REQ_TEMPLATE = """ +### Legacy Content +{old_tasks} + +### New Requirements {context} - -## Format example -{format_example} ------ -Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules -Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. - -## Required Python third-party packages: Provided in requirements.txt format - -## Required Other language third-party packages: Provided in requirements.txt format - -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. - -## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first - -## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first - -## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. - -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, -and only output the json inside this tag, nothing else -""", - "FORMAT_EXAMPLE": ''' -{ - "Required Python third-party packages": [ - "flask==1.1.2", - "bcrypt==3.2.0" - ], - "Required Other language third-party packages": [ - "No third-party ..." - ], - "Full API spec": """ - openapi: 3.0.0 - ... - description: A JSON object ... - """, - "Logic Analysis": [ - ["game.py","Contains..."] - ], - "Task list": [ - "game.py" - ], - "Shared Knowledge": """ - 'game.py' contains ... - """, - "Anything UNCLEAR": "We need ... how to start." -} -''', - }, - "markdown": { - "PROMPT_TEMPLATE": """ -# Context -{context} - -## Format example -{format_example} ------ -Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules -Requirements: Based on the context, fill in the following missing information, note that all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file, if there are any missing files, you can supplement them -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. - -## Required Python third-party packages: Provided in requirements.txt format - -## Required Other language third-party packages: Provided in requirements.txt format - -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. - -## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first - -## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first - -## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. - -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. - -""", - "FORMAT_EXAMPLE": ''' ---- -## Required Python third-party packages -```python """ -flask==1.1.2 -bcrypt==3.2.0 -""" -``` - -## Required Other language third-party packages -```python -""" -No third-party ... -""" -``` - -## Full API spec -```python -""" -openapi: 3.0.0 -... -description: A JSON object ... -""" -``` - -## Logic Analysis -```python -[ - ["game.py", "Contains ..."], -] -``` - -## Task list -```python -[ - "game.py", -] -``` - -## Shared Knowledge -```python -""" -'game.py' contains ... -""" -``` - -## Anything UNCLEAR -We need ... how to start. ---- -''', - }, -} -OUTPUT_MAPPING = { - "Required Python third-party packages": (List[str], ...), - "Required Other language third-party packages": (List[str], ...), - "Full API spec": (str, ...), - "Logic Analysis": (List[List[str]], ...), - "Task list": (List[str], ...), - "Shared Knowledge": (str, ...), - "Anything UNCLEAR": (str, ...), -} class WriteTasks(Action): def __init__(self, name="CreateTasks", context=None, llm=None): super().__init__(name, context, llm) - def _save(self, context, rsp): - if context[-1].instruct_content: - ws_name = context[-1].instruct_content.dict()["Python package name"] + async def run(self, with_messages, format=CONFIG.prompt_format): + system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) + changed_system_designs = system_design_file_repo.changed_files + + tasks_file_repo = CONFIG.git_repo.new_file_repository(TASK_FILE_REPO) + changed_tasks = tasks_file_repo.changed_files + change_files = Documents() + # Rewrite the system designs that have undergone changes based on the git head diff under + # `docs/system_designs/`. + for filename in changed_system_designs: + task_doc = await self._update_tasks( + filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo + ) + change_files.docs[filename] = task_doc + + # Rewrite the task files that have undergone changes based on the git head diff under `docs/tasks/`. + for filename in changed_tasks: + if filename in change_files.docs: + continue + task_doc = await self._update_tasks( + filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo + ) + change_files.docs[filename] = task_doc + + if not change_files.docs: + logger.info("Nothing has changed.") + # Wait until all files under `docs/tasks/` are processed before sending the publish_message, leaving room for + # global optimization in subsequent steps. + return ActionOutput(content=change_files.json(), instruct_content=change_files) + + async def _update_tasks(self, filename, system_design_file_repo, tasks_file_repo): + system_design_doc = await system_design_file_repo.get(filename) + task_doc = await tasks_file_repo.get(filename) + if task_doc: + task_doc = await self._merge(system_design_doc=system_design_doc, task_doc=task_doc) else: - ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) - file_path = WORKSPACE_ROOT / ws_name / "docs/api_spec_and_tasks.md" - file_path.write_text(json_to_markdown(rsp.instruct_content.dict())) + rsp = await self._run_new_tasks(context=system_design_doc.content) + task_doc = Document( + root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json(ensure_ascii=False) + ) + await tasks_file_repo.save( + filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + ) + await self._update_requirements(task_doc) + await self._save_pdf(task_doc=task_doc) + return task_doc - # Write requirements.txt - requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt" - requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) + async def _run_new_tasks(self, context, format=CONFIG.prompt_format): + node = await PM_NODE.fill(context, self.llm, format) + # prompt_template, format_example = get_template(templates, format) + # prompt = prompt_template.format(context=context, format_example=format_example) + # rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) + return node - async def run(self, context, format=CONFIG.prompt_format): - prompt_template, format_example = get_template(templates, format) - prompt = prompt_template.format(context=context, format_example=format_example) - rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) - self._save(context, rsp) - return rsp + async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: + context = NEW_REQ_TEMPLATE.format(context=system_design_doc.content, old_tasks=task_doc.content) + node = await PM_NODE.fill(context, self.llm, format) + task_doc.content = node.instruct_content.json(ensure_ascii=False) + return task_doc + + @staticmethod + async def _update_requirements(doc): + m = json.loads(doc.content) + packages = set(m.get("Required Python third-party packages", set())) + file_repo = CONFIG.git_repo.new_file_repository() + requirement_doc = await file_repo.get(filename=PACKAGE_REQUIREMENTS_FILENAME) + if not requirement_doc: + requirement_doc = Document(filename=PACKAGE_REQUIREMENTS_FILENAME, root_path=".", content="") + lines = requirement_doc.content.splitlines() + for pkg in lines: + if pkg == "": + continue + packages.add(pkg) + await file_repo.save(PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages)) + + @staticmethod + async def _save_pdf(task_doc): + await FileRepository.save_as(doc=task_doc, with_suffix=".md", relative_path=TASK_PDF_FILE_REPO) class AssignTasks(Action): diff --git a/metagpt/actions/project_management_an.py b/metagpt/actions/project_management_an.py new file mode 100644 index 000000000..970cb0594 --- /dev/null +++ b/metagpt/actions/project_management_an.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/14 15:28 +@Author : alexanderwu +@File : project_management_an.py +""" +from metagpt.actions.action_node import ActionNode +from metagpt.logs import logger + +REQUIRED_PYTHON_PACKAGES = ActionNode( + key="Required Python packages", + expected_type=list[str], + instruction="Provide required Python packages in requirements.txt format.", + example=["flask==1.1.2", "bcrypt==3.2.0"], +) + +REQUIRED_OTHER_LANGUAGE_PACKAGES = ActionNode( + key="Required Other language third-party packages", + expected_type=list[str], + instruction="List down the required packages for languages other than Python.", + example=["No third-party dependencies required"], +) + +LOGIC_ANALYSIS = ActionNode( + key="Logic Analysis", + expected_type=list[list[str]], + instruction="Provide a list of files with the classes/methods/functions to be implemented, " + "including dependency analysis and imports.", + example=[ + ["game.py", "Contains Game class and ... functions"], + ["main.py", "Contains main function, from game import Game"], + ], +) + +TASK_LIST = ActionNode( + key="Task list", + expected_type=list[str], + instruction="Break down the tasks into a list of filenames, prioritized by dependency order.", + example=["game.py", "main.py"], +) + +FULL_API_SPEC = ActionNode( + key="Full API spec", + expected_type=str, + instruction="Describe all APIs using OpenAPI 3.0 spec that may be used by both frontend and backend. If front-end " + "and back-end communication is not required, leave it blank.", + example="openapi: 3.0.0 ...", +) + +SHARED_KNOWLEDGE = ActionNode( + key="Shared Knowledge", + expected_type=str, + instruction="Detail any shared knowledge, like common utility functions or configuration variables.", + example="'game.py' contains functions shared across the project.", +) + +ANYTHING_UNCLEAR_PM = ActionNode( + key="Anything UNCLEAR", + expected_type=str, + instruction="Mention any unclear aspects in the project management context and try to clarify them.", + example="Clarification needed on how to start and initialize third-party libraries.", +) + +NODES = [ + REQUIRED_PYTHON_PACKAGES, + REQUIRED_OTHER_LANGUAGE_PACKAGES, + LOGIC_ANALYSIS, + TASK_LIST, + FULL_API_SPEC, + SHARED_KNOWLEDGE, + ANYTHING_UNCLEAR_PM, +] + + +PM_NODE = ActionNode.from_children("PM_NODE", NODES) + + +def main(): + prompt = PM_NODE.compile(context="") + logger.info(prompt) + + +if __name__ == "__main__": + main() diff --git a/metagpt/actions/research.py b/metagpt/actions/research.py index 49a981e86..a70038c51 100644 --- a/metagpt/actions/research.py +++ b/metagpt/actions/research.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -import json from typing import Callable from pydantic import parse_obj_as @@ -49,7 +48,7 @@ based on the link credibility. If two results have equal credibility, prioritize ranked results' indices in JSON format, like [0, 1, 3, 4, ...], without including other words. """ -WEB_BROWSE_AND_SUMMARIZE_PROMPT = '''### Requirements +WEB_BROWSE_AND_SUMMARIZE_PROMPT = """### Requirements 1. Utilize the text in the "Reference Information" section to respond to the question "{query}". 2. If the question cannot be directly answered using the text, but the text is related to the research topic, please provide \ a comprehensive summary of the text. @@ -58,10 +57,10 @@ a comprehensive summary of the text. ### Reference Information {content} -''' +""" -CONDUCT_RESEARCH_PROMPT = '''### Reference Information +CONDUCT_RESEARCH_PROMPT = """### Reference Information {content} ### Requirements @@ -73,11 +72,12 @@ above. The report must meet the following requirements: - Present data and findings in an intuitive manner, utilizing feature comparative tables, if applicable. - The report should have a minimum word count of 2,000 and be formatted with Markdown syntax following APA style guidelines. - Include all source URLs in APA format at the end of the report. -''' +""" class CollectLinks(Action): """Action class to collect links from a search engine.""" + def __init__( self, name: str = "", @@ -114,19 +114,24 @@ class CollectLinks(Action): keywords = OutputParser.extract_struct(keywords, list) keywords = parse_obj_as(list[str], keywords) except Exception as e: - logger.exception(f"fail to get keywords related to the research topic \"{topic}\" for {e}") + logger.exception(f"fail to get keywords related to the research topic '{topic}' for {e}") keywords = [topic] results = await asyncio.gather(*(self.search_engine.run(i, as_string=False) for i in keywords)) def gen_msg(): while True: - search_results = "\n".join(f"#### Keyword: {i}\n Search Result: {j}\n" for (i, j) in zip(keywords, results)) - prompt = SUMMARIZE_SEARCH_PROMPT.format(decomposition_nums=decomposition_nums, search_results=search_results) + search_results = "\n".join( + f"#### Keyword: {i}\n Search Result: {j}\n" for (i, j) in zip(keywords, results) + ) + prompt = SUMMARIZE_SEARCH_PROMPT.format( + decomposition_nums=decomposition_nums, search_results=search_results + ) yield prompt remove = max(results, key=len) remove.pop() if len(remove) == 0: break + prompt = reduce_message_length(gen_msg(), self.llm.model, system_text, CONFIG.max_tokens_rsp) logger.debug(prompt) queries = await self._aask(prompt, [system_text]) @@ -172,6 +177,7 @@ class CollectLinks(Action): class WebBrowseAndSummarize(Action): """Action class to explore the web and provide summaries of articles and webpages.""" + def __init__( self, *args, @@ -214,7 +220,9 @@ class WebBrowseAndSummarize(Action): for u, content in zip([url, *urls], contents): content = content.inner_text chunk_summaries = [] - for prompt in generate_prompt_chunk(content, prompt_template, self.llm.model, system_text, CONFIG.max_tokens_rsp): + for prompt in generate_prompt_chunk( + content, prompt_template, self.llm.model, system_text, CONFIG.max_tokens_rsp + ): logger.debug(prompt) summary = await self._aask(prompt, [system_text]) if summary == "Not relevant.": @@ -238,6 +246,7 @@ class WebBrowseAndSummarize(Action): class ConductResearch(Action): """Action class to conduct research and generate a research report.""" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if CONFIG.model_for_researcher_report: diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index f69d2cd1a..fa13a0980 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -4,14 +4,25 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : run_code.py +@Modified By: mashenquan, 2023/11/27. + 1. Mark the location of Console logs in the PROMPT_TEMPLATE with markdown code-block formatting to enhance + the understanding for the LLM. + 2. Fix bug: Add the "install dependency" operation. + 3. Encapsulate the input of RunCode into RunCodeContext and encapsulate the output of RunCode into + RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. + 4. According to section 2.2.3.5.7 of RFC 135, change the method of transferring file content + (code files, unit test files, log files) from using the message to using the file name. + 5. Merged the `Config` class of send18:dev branch to take over the set/get operations of the Environment + class. """ -import os import subprocess import traceback from typing import Tuple from metagpt.actions.action import Action +from metagpt.config import CONFIG from metagpt.logs import logger +from metagpt.schema import RunCodeResult PROMPT_TEMPLATE = """ Role: You are a senior development and qa engineer, your role is summarize the code running result. @@ -51,8 +62,14 @@ CONTEXT = """ ## Running Command {command} ## Running Output -standard output: {outs}; -standard errors: {errs}; +standard output: +```text +{outs} +``` +standard errors: +```text +{errs} +``` """ @@ -77,17 +94,19 @@ class RunCode(Action): additional_python_paths = [str(path) for path in additional_python_paths] # Copy the current environment variables - env = os.environ.copy() + env = CONFIG.new_environ() # Modify the PYTHONPATH environment variable additional_python_paths = [working_directory] + additional_python_paths additional_python_paths = ":".join(additional_python_paths) env["PYTHONPATH"] = additional_python_paths + ":" + env.get("PYTHONPATH", "") + RunCode._install_dependencies(working_directory=working_directory, env=env) # Start the subprocess process = subprocess.Popen( command, cwd=working_directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env ) + logger.info(" ".join(command)) try: # Wait for the process to complete, with a timeout @@ -98,31 +117,46 @@ class RunCode(Action): stdout, stderr = process.communicate() return stdout.decode("utf-8"), stderr.decode("utf-8") - async def run( - self, code, mode="script", code_file_name="", test_code="", test_file_name="", command=[], **kwargs - ) -> str: - logger.info(f"Running {' '.join(command)}") - if mode == "script": - outs, errs = await self.run_script(command=command, **kwargs) - elif mode == "text": - outs, errs = await self.run_text(code=code) + async def run(self, *args, **kwargs) -> RunCodeResult: + logger.info(f"Running {' '.join(self.context.command)}") + if self.context.mode == "script": + outs, errs = await self.run_script( + command=self.context.command, + working_directory=self.context.working_directory, + additional_python_paths=self.context.additional_python_paths, + ) + elif self.context.mode == "text": + outs, errs = await self.run_text(code=self.context.code) logger.info(f"{outs=}") logger.info(f"{errs=}") context = CONTEXT.format( - code=code, - code_file_name=code_file_name, - test_code=test_code, - test_file_name=test_file_name, - command=" ".join(command), + code=self.context.code, + code_file_name=self.context.code_filename, + test_code=self.context.test_code, + test_file_name=self.context.test_filename, + command=" ".join(self.context.command), outs=outs[:500], # outs might be long but they are not important, truncate them to avoid token overflow errs=errs[:10000], # truncate errors to avoid token overflow ) prompt = PROMPT_TEMPLATE.format(context=context) rsp = await self._aask(prompt) + return RunCodeResult(summary=rsp, stdout=outs, stderr=errs) - result = context + rsp + @staticmethod + def _install_dependencies(working_directory, env): + install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"] + logger.info(" ".join(install_command)) + try: + subprocess.run(install_command, check=True, cwd=working_directory, env=env) + except subprocess.CalledProcessError as e: + logger.warning(f"{e}") - return result + install_pytest_command = ["python", "-m", "pip", "install", "pytest"] + logger.info(" ".join(install_pytest_command)) + try: + subprocess.run(install_pytest_command, check=True, cwd=working_directory, env=env) + except subprocess.CalledProcessError as e: + logger.warning(f"{e}") diff --git a/metagpt/actions/summarize_code.py b/metagpt/actions/summarize_code.py new file mode 100644 index 000000000..f8d8d2b47 --- /dev/null +++ b/metagpt/actions/summarize_code.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Author : alexanderwu +@File : summarize_code.py +@Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode. +""" +from pathlib import Path + +from tenacity import retry, stop_after_attempt, wait_random_exponential + +from metagpt.actions.action import Action +from metagpt.config import CONFIG +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO +from metagpt.logs import logger +from metagpt.utils.file_repository import FileRepository + +PROMPT_TEMPLATE = """ +NOTICE +Role: You are a professional software engineer, and your main task is to review the code. +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. +ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". + +----- +# System Design +```text +{system_design} +``` +----- +# Tasks +```text +{tasks} +``` +----- +{code_blocks} + +## Code Review All: Please read all historical files and find possible bugs in the files, such as unimplemented functions, calling errors, unreferences, etc. + +## Call flow: mermaid code, based on the implemented function, use mermaid to draw a complete call chain + +## Summary: Summary based on the implementation of historical files + +## TODOs: Python dict[str, str], write down the list of files that need to be modified and the reasons. We will modify them later. + +""" + +FORMAT_EXAMPLE = """ + +## Code Review All + +### a.py +- It fulfills less of xxx requirements... +- Field yyy is not given... +-... + +### b.py +... + +### c.py +... + +## Call flow +```mermaid +flowchart TB + c1-->a2 + subgraph one + a1-->a2 + end + subgraph two + b1-->b2 + end + subgraph three + c1-->c2 + end +``` + +## Summary +- a.py:... +- b.py:... +- c.py:... +- ... + +## TODOs +{ + "a.py": "implement requirement xxx...", +} + +""" + + +class SummarizeCode(Action): + def __init__(self, name="SummarizeCode", context=None, llm=None): + super().__init__(name, context, llm) + + @retry(stop=stop_after_attempt(2), wait=wait_random_exponential(min=1, max=60)) + async def summarize_code(self, prompt): + code_rsp = await self._aask(prompt) + return code_rsp + + async def run(self): + design_pathname = Path(self.context.design_filename) + design_doc = await FileRepository.get_file(filename=design_pathname.name, relative_path=SYSTEM_DESIGN_FILE_REPO) + task_pathname = Path(self.context.task_filename) + task_doc = await FileRepository.get_file(filename=task_pathname.name, relative_path=TASK_FILE_REPO) + src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) + code_blocks = [] + for filename in self.context.codes_filenames: + code_doc = await src_file_repo.get(filename) + code_block = f"```python\n{code_doc.content}\n```\n-----" + code_blocks.append(code_block) + format_example = FORMAT_EXAMPLE + prompt = PROMPT_TEMPLATE.format( + system_design=design_doc.content, + tasks=task_doc.content, + code_blocks="\n".join(code_blocks), + format_example=format_example, + ) + logger.info("Summarize code..") + rsp = await self.summarize_code(prompt) + return rsp diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index a89bce60f..5960e2621 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -4,79 +4,147 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code.py +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `cause_by` + value of the `Message` object. +@Modified By: mashenquan, 2023-11-27. + 1. Mark the location of Design, Tasks, Legacy Code and Debug logs in the PROMPT_TEMPLATE with markdown + code-block formatting to enhance the understanding for the LLM. + 2. Following the think-act principle, solidify the task parameters when creating the WriteCode object, rather + than passing them in when calling the run function. + 3. Encapsulate the input of RunCode into RunCodeContext and encapsulate the output of RunCode into + RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. """ -from metagpt.actions import WriteDesign -from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT -from metagpt.logs import logger -from metagpt.schema import Message -from metagpt.utils.common import CodeParser +import json + from tenacity import retry, stop_after_attempt, wait_random_exponential +from metagpt.actions.action import Action +from metagpt.config import CONFIG +from metagpt.const import ( + BUGFIX_FILENAME, + CODE_SUMMARIES_FILE_REPO, + DOCS_FILE_REPO, + TASK_FILE_REPO, + TEST_OUTPUTS_FILE_REPO, +) +from metagpt.logs import logger +from metagpt.schema import CodingContext, Document, RunCodeResult +from metagpt.utils.common import CodeParser +from metagpt.utils.file_repository import FileRepository + PROMPT_TEMPLATE = """ NOTICE -Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language) +Role: You are a professional engineer; the main goal is to write google-style, elegant, modular, easy to read and maintain code +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". -## Code: {filename} Write code with triple quote, based on the following list and context. -1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. -2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets -3. Attention1: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -4. Attention2: YOU MUST FOLLOW "Data structures and interface definitions". DONT CHANGE ANY DESIGN. -5. Think before writing: What should be implemented and provided in this document? -6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. -7. Do not use public member functions that do not exist in your design. - ------ # Context -{context} ------ -## Format example ------ +## Design +{design} + +## Tasks +{tasks} + +## Legacy Code +```Code +{code} +``` + +## Debug logs +```text +{logs} + +{summary_log} +``` + +## Bug Feedback logs +```text +{feedback} +``` + +# Format example ## Code: {filename} ```python ## {filename} ... ``` ------ + +# Instruction: Based on the context, follow "Format example", write code. + +## Code: {filename}. Write code with triple quoto, based on the following attentions and context. +1. Only One file: do your best to implement THIS ONLY ONE FILE. +2. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets. +3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. AVOID circular import. +4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. +5. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. +6. Before using a external variable/module, make sure you import it first. +7. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. + """ class WriteCode(Action): - def __init__(self, name="WriteCode", context: list[Message] = None, llm=None): + def __init__(self, name="WriteCode", context=None, llm=None): super().__init__(name, context, llm) - def _is_invalid(self, filename): - return any(i in filename for i in ["mp3", "wav"]) - - def _save(self, context, filename, code): - # logger.info(filename) - # logger.info(code_rsp) - if self._is_invalid(filename): - return - - design = [i for i in context if i.cause_by == WriteDesign][0] - - ws_name = CodeParser.parse_str(block="Python package name", text=design.content) - ws_path = WORKSPACE_ROOT / ws_name - if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): - ws_path = ws_path / ws_name - code_path = ws_path / filename - code_path.parent.mkdir(parents=True, exist_ok=True) - code_path.write_text(code) - logger.info(f"Saving Code to {code_path}") - @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) - async def write_code(self, prompt): + async def write_code(self, prompt) -> str: code_rsp = await self._aask(prompt) code = CodeParser.parse_code(block="", text=code_rsp) return code - async def run(self, context, filename): - prompt = PROMPT_TEMPLATE.format(context=context, filename=filename) - logger.info(f'Writing {filename}..') + async def run(self, *args, **kwargs) -> CodingContext: + bug_feedback = await FileRepository.get_file(filename=BUGFIX_FILENAME, relative_path=DOCS_FILE_REPO) + coding_context = CodingContext.loads(self.context.content) + test_doc = await FileRepository.get_file( + filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO + ) + summary_doc = None + if coding_context.design_doc and coding_context.design_doc.filename: + summary_doc = await FileRepository.get_file( + filename=coding_context.design_doc.filename, relative_path=CODE_SUMMARIES_FILE_REPO + ) + logs = "" + if test_doc: + test_detail = RunCodeResult.loads(test_doc.content) + logs = test_detail.stderr + + if bug_feedback: + code_context = coding_context.code_doc.content + else: + code_context = await self.get_codes(coding_context.task_doc, exclude=self.context.filename) + + prompt = PROMPT_TEMPLATE.format( + design=coding_context.design_doc.content if coding_context.design_doc else "", + tasks=coding_context.task_doc.content if coding_context.task_doc else "", + code=code_context, + logs=logs, + feedback=bug_feedback.content if bug_feedback else "", + filename=self.context.filename, + summary_log=summary_doc.content if summary_doc else "", + ) + logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) - # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) - # self._save(context, filename, code) - return code - \ No newline at end of file + if not coding_context.code_doc: + coding_context.code_doc = Document(filename=coding_context.filename, root_path=CONFIG.src_workspace) + coding_context.code_doc.content = code + return coding_context + + @staticmethod + async def get_codes(task_doc, exclude) -> str: + if not task_doc: + return "" + if not task_doc.content: + task_doc.content = FileRepository.get_file(filename=task_doc.filename, relative_path=TASK_FILE_REPO) + m = json.loads(task_doc.content) + code_filenames = m.get("Task list", []) + codes = [] + src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) + for filename in code_filenames: + if filename == exclude: + continue + doc = await src_file_repo.get(filename=filename) + if not doc: + continue + codes.append(f"----- {filename}\n" + doc.content) + return "\n".join(codes) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 06282411a..4b3e9aece 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -4,57 +4,114 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code_review.py +@Modified By: mashenquan, 2023/11/27. Following the think-act principle, solidify the task parameters when creating the + WriteCode object, rather than passing them in when calling the run function. """ -from metagpt.actions.action import Action -from metagpt.logs import logger -from metagpt.schema import Message -from metagpt.utils.common import CodeParser from tenacity import retry, stop_after_attempt, wait_random_exponential +from metagpt.actions import WriteCode +from metagpt.actions.action import Action +from metagpt.config import CONFIG +from metagpt.logs import logger +from metagpt.schema import CodingContext +from metagpt.utils.common import CodeParser + PROMPT_TEMPLATE = """ -NOTICE -Role: You are a professional software engineer, and your main task is to review the code. You need to ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). +# System +Role: You are a professional software engineer, and your main task is to review and revise the code. You need to ensure that the code conforms to the google-style standards, is elegantly designed and modularized, easy to read and maintain. +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". -## Code Review: Based on the following context and code, and following the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5. -``` -1. Check 0: Is the code implemented as per the requirements? -2. Check 1: Are there any issues with the code logic? -3. Check 2: Does the existing code follow the "Data structures and interface definitions"? -4. Check 3: Is there a function in the code that is omitted or not fully implemented that needs to be implemented? -5. Check 4: Does the code have unnecessary or lack dependencies? -``` - -## Rewrite Code: {filename} Base on "Code Review" and the source code, rewrite code with triple quotes. Do your utmost to optimize THIS SINGLE FILE. ------ # Context {context} -## Code: {filename} -``` +## Code to be Reviewed: {filename} +```Code {code} ``` ------ +""" + + +EXAMPLE_AND_INSTRUCTION = """ -## Format example ------ {format_example} ------ + + +# Instruction: Based on the actual code situation, follow one of the "Format example". + +## Code Review: Ordered List. Based on the "Code to be Reviewed", provide key, clear, concise, and specific answer. If any answer is no, explain how to fix it step by step. +1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step. +2. Is the code logic completely correct? If there are errors, please indicate how to correct them. +3. Does the existing code follow the "Data structures and interfaces"? +4. Are all functions implemented? If there is no implementation, please indicate how to achieve it step by step. +5. Have all necessary pre-dependencies been imported? If not, indicate which ones need to be imported +6. Are methods from other files being reused correctly? + +## Actions: Ordered List. Things that should be done after CR, such as implementing class A and function B + +## Code Review Result: str. If the code doesn't have bugs, we don't need to rewrite it, so answer LGTM and stop. ONLY ANSWER LGTM/LBTM. +LGTM/LBTM """ FORMAT_EXAMPLE = """ - -## Code Review -1. The code ... +# Format example 1 +## Code Review: {filename} +1. No, we should fix the logic of class A due to ... 2. ... 3. ... -4. ... +4. No, function B is not implemented, ... 5. ... +6. ... -## Rewrite Code: {filename} -```python +## Actions +1. Fix the `handle_events` method to update the game state only if a move is successful. + ```python + def handle_events(self): + for event in pygame.event.get(): + if event.type == pygame.QUIT: + return False + if event.type == pygame.KEYDOWN: + moved = False + if event.key == pygame.K_UP: + moved = self.game.move('UP') + elif event.key == pygame.K_DOWN: + moved = self.game.move('DOWN') + elif event.key == pygame.K_LEFT: + moved = self.game.move('LEFT') + elif event.key == pygame.K_RIGHT: + moved = self.game.move('RIGHT') + if moved: + # Update the game state only if a move was successful + self.render() + return True + ``` +2. Implement function B + +## Code Review Result +LBTM + +# Format example 2 +## Code Review: {filename} +1. Yes. +2. Yes. +3. Yes. +4. Yes. +5. Yes. +6. Yes. + +## Actions +pass + +## Code Review Result +LGTM +""" + +REWRITE_CODE_TEMPLATE = """ +# Instruction: rewrite code based on the Code Review and Actions +## Rewrite Code: CodeBlock. If it still has some bugs, rewrite {filename} with triple quotes. Do your utmost to optimize THIS SINGLE FILE. Return all completed codes and prohibit the return of unfinished codes. +```Code ## {filename} ... ``` @@ -62,21 +119,53 @@ FORMAT_EXAMPLE = """ class WriteCodeReview(Action): - def __init__(self, name="WriteCodeReview", context: list[Message] = None, llm=None): + def __init__(self, name="WriteCodeReview", context=None, llm=None): super().__init__(name, context, llm) @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) - async def write_code(self, prompt): - code_rsp = await self._aask(prompt) - code = CodeParser.parse_code(block="", text=code_rsp) - return code + async def write_code_review_and_rewrite(self, context_prompt, cr_prompt, filename): + cr_rsp = await self._aask(context_prompt + cr_prompt) + result = CodeParser.parse_block("Code Review Result", cr_rsp) + if "LGTM" in result: + return result, None - async def run(self, context, code, filename): - format_example = FORMAT_EXAMPLE.format(filename=filename) - prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=filename, format_example=format_example) - logger.info(f'Code review {filename}..') - code = await self.write_code(prompt) + # if LBTM, rewrite code + rewrite_prompt = f"{context_prompt}\n{cr_rsp}\n{REWRITE_CODE_TEMPLATE.format(filename=filename)}" + code_rsp = await self._aask(rewrite_prompt) + code = CodeParser.parse_code(block="", text=code_rsp) + return result, code + + async def run(self, *args, **kwargs) -> CodingContext: + iterative_code = self.context.code_doc.content + k = CONFIG.code_review_k_times or 1 + for i in range(k): + format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) + task_content = self.context.task_doc.content if self.context.task_doc else "" + code_context = await WriteCode.get_codes(self.context.task_doc, exclude=self.context.filename) + context = "\n".join( + [ + "## System Design\n" + str(self.context.design_doc) + "\n", + "## Tasks\n" + task_content + "\n", + "## Code Files\n" + code_context + "\n", + ] + ) + context_prompt = PROMPT_TEMPLATE.format( + context=context, + code=iterative_code, + filename=self.context.code_doc.filename, + ) + cr_prompt = EXAMPLE_AND_INSTRUCTION.format(format_example=format_example, ) + logger.info( + f"Code review and rewrite {self.context.code_doc.filename}: {i+1}/{k} | {len(iterative_code)=}, {len(self.context.code_doc.content)=}" + ) + result, rewrited_code = await self.write_code_review_and_rewrite(context_prompt, cr_prompt, self.context.code_doc.filename) + if "LBTM" in result: + iterative_code = rewrited_code + elif "LGTM" in result: + self.context.code_doc.content = iterative_code + return self.context # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) # self._save(context, filename, code) - return code - \ No newline at end of file + # 如果rewrited_code是None（原code perfect），那么直接返回code + self.context.code_doc.content = iterative_code + return self.context diff --git a/metagpt/actions/write_docstring.py b/metagpt/actions/write_docstring.py index 5c7815793..0ad134157 100644 --- a/metagpt/actions/write_docstring.py +++ b/metagpt/actions/write_docstring.py @@ -16,7 +16,7 @@ Options: Default: 'google' Example: - python3 -m metagpt.actions.write_docstring startup.py --overwrite False --style=numpy + python3 -m metagpt.actions.write_docstring ./metagpt/startup.py --overwrite False --style=numpy This script uses the 'fire' library to create a command-line interface. It generates docstrings for the given Python code using the specified docstring style and adds them to the code. @@ -28,7 +28,7 @@ from metagpt.actions.action import Action from metagpt.utils.common import OutputParser from metagpt.utils.pycst import merge_docstring -PYTHON_DOCSTRING_SYSTEM = '''### Requirements +PYTHON_DOCSTRING_SYSTEM = """### Requirements 1. Add docstrings to the given code following the {style} style. 2. Replace the function body with an Ellipsis object(...) to reduce output. 3. If the types are already annotated, there is no need to include them in the docstring. @@ -48,7 +48,7 @@ class ExampleError(Exception): ```python {example} ``` -''' +""" # https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html @@ -162,7 +162,8 @@ class WriteDocstring(Action): self.desc = "Write docstring for code." async def run( - self, code: str, + self, + code: str, system_text: str = PYTHON_DOCSTRING_SYSTEM, style: Literal["google", "numpy", "sphinx"] = "google", ) -> str: diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 52a99dafc..bb0cf8fb9 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -4,238 +4,195 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_prd.py +@Modified By: mashenquan, 2023/11/27. + 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. + 2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality. + 3. Move the document storage operations related to WritePRD from the save operation of WriteDesign. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ -from typing import List +from __future__ import annotations + +import json +from pathlib import Path from metagpt.actions import Action, ActionOutput -from metagpt.actions.search_and_summarize import SearchAndSummarize +from metagpt.actions.action_node import ActionNode +from metagpt.actions.fix_bug import FixBug +from metagpt.actions.write_prd_an import ( + WP_IS_RELATIVE_NODE, + WP_ISSUE_TYPE_NODE, + WRITE_PRD_NODE, +) from metagpt.config import CONFIG +from metagpt.const import ( + BUGFIX_FILENAME, + COMPETITIVE_ANALYSIS_FILE_REPO, + DOCS_FILE_REPO, + PRD_PDF_FILE_REPO, + PRDS_FILE_REPO, + REQUIREMENT_FILENAME, +) from metagpt.logs import logger -from metagpt.utils.get_template import get_template +from metagpt.schema import BugFixContext, Document, Documents, Message +from metagpt.utils.common import CodeParser +from metagpt.utils.file_repository import FileRepository -templates = { - "json": { - "PROMPT_TEMPLATE": """ -# Context -## Original Requirements +# from metagpt.utils.get_template import get_template +from metagpt.utils.mermaid import mermaid_to_file + +# from typing import List + + +CONTEXT_TEMPLATE = """ +### Project Name +{project_name} + +### Original Requirements {requirements} -## Search Information -{search_information} +### Search Information +- +""" -## mermaid quadrantChart code syntax example. DONT USE QUOTE IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME -```mermaid -quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - "Campaign: A": [0.3, 0.6] - "Campaign B": [0.45, 0.23] - "Campaign C": [0.57, 0.69] - "Campaign D": [0.78, 0.34] - "Campaign E": [0.40, 0.34] - "Campaign F": [0.35, 0.78] - "Our Target Product": [0.5, 0.6] -``` +NEW_REQ_TEMPLATE = """ +### Legacy Content +{old_prd} -## Format example -{format_example} ------ -Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design - -## Original Requirements: Provide as Plain text, place the polished complete original requirements here - -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple - -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less - -## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible - -## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. - -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower - -## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, -and only output the json inside this tag, nothing else -""", - "FORMAT_EXAMPLE": """ -[CONTENT] -{ - "Original Requirements": "", - "Search Information": "", - "Requirements": "", - "Product Goals": [], - "User Stories": [], - "Competitive Analysis": [], - "Competitive Quadrant Chart": "quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - Campaign A: [0.3, 0.6] - Campaign B: [0.45, 0.23] - Campaign C: [0.57, 0.69] - Campaign D: [0.78, 0.34] - Campaign E: [0.40, 0.34] - Campaign F: [0.35, 0.78]", - "Requirement Analysis": "", - "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], - "UI Design draft": "", - "Anything UNCLEAR": "", -} -[/CONTENT] -""", - }, - "markdown": { - "PROMPT_TEMPLATE": """ -# Context -## Original Requirements +### New Requirements {requirements} - -## Search Information -{search_information} - -## mermaid quadrantChart code syntax example. DONT USE QUOTE IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME -```mermaid -quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - "Campaign: A": [0.3, 0.6] - "Campaign B": [0.45, 0.23] - "Campaign C": [0.57, 0.69] - "Campaign D": [0.78, 0.34] - "Campaign E": [0.40, 0.34] - "Campaign F": [0.35, 0.78] - "Our Target Product": [0.5, 0.6] -``` - -## Format example -{format_example} ------ -Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design -ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. - -## Original Requirements: Provide as Plain text, place the polished complete original requirements here - -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple - -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less - -## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible - -## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. - -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower - -## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. -""", - "FORMAT_EXAMPLE": """ ---- -## Original Requirements -The boss ... - -## Product Goals -```python -[ - "Create a ...", -] -``` - -## User Stories -```python -[ - "As a user, ...", -] -``` - -## Competitive Analysis -```python -[ - "Python Snake Game: ...", -] -``` - -## Competitive Quadrant Chart -```mermaid -quadrantChart - title Reach and engagement of campaigns - ... - "Our Target Product": [0.6, 0.7] -``` - -## Requirement Analysis -The product should be a ... - -## Requirement Pool -```python -[ - ["End game ...", "P0"] -] -``` - -## UI Design draft -Give a basic function description, and a draft - -## Anything UNCLEAR -There are no unclear points. ---- -""", - }, -} - -OUTPUT_MAPPING = { - "Original Requirements": (str, ...), - "Product Goals": (List[str], ...), - "User Stories": (List[str], ...), - "Competitive Analysis": (List[str], ...), - "Competitive Quadrant Chart": (str, ...), - "Requirement Analysis": (str, ...), - "Requirement Pool": (List[List[str]], ...), - "UI Design draft": (str, ...), - "Anything UNCLEAR": (str, ...), -} +""" class WritePRD(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: - sas = SearchAndSummarize() - # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) - rsp = "" - info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" - if sas.result: - logger.info(sas.result) - logger.info(rsp) + async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput | Message: + # Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are + # related to the PRD. If they are related, rewrite the PRD. + docs_file_repo = CONFIG.git_repo.new_file_repository(relative_path=DOCS_FILE_REPO) + requirement_doc = await docs_file_repo.get(filename=REQUIREMENT_FILENAME) + if await self._is_bugfix(requirement_doc.content): + await docs_file_repo.save(filename=BUGFIX_FILENAME, content=requirement_doc.content) + await docs_file_repo.save(filename=REQUIREMENT_FILENAME, content="") + bug_fix = BugFixContext(filename=BUGFIX_FILENAME) + return Message( + content=bug_fix.json(), + instruct_content=bug_fix, + role="", + cause_by=FixBug, + sent_from=self, + send_to="Alex", # the name of Engineer + ) + else: + await docs_file_repo.delete(filename=BUGFIX_FILENAME) - prompt_template, format_example = get_template(templates, format) - prompt = prompt_template.format( - requirements=requirements, search_information=info, format_example=format_example + prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) + prd_docs = await prds_file_repo.get_all() + change_files = Documents() + for prd_doc in prd_docs: + prd_doc = await self._update_prd( + requirement_doc=requirement_doc, prd_doc=prd_doc, prds_file_repo=prds_file_repo, *args, **kwargs + ) + if not prd_doc: + continue + change_files.docs[prd_doc.filename] = prd_doc + logger.info(f"rewrite prd: {prd_doc.filename}") + # If there is no existing PRD, generate one using 'docs/requirement.txt'. + if not change_files.docs: + prd_doc = await self._update_prd( + requirement_doc=requirement_doc, prd_doc=None, prds_file_repo=prds_file_repo, *args, **kwargs + ) + if prd_doc: + change_files.docs[prd_doc.filename] = prd_doc + logger.debug(f"new prd: {prd_doc.filename}") + # Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the + # 'publish' message to transition the workflow to the next stage. This design allows room for global + # optimization in subsequent steps. + return ActionOutput(content=change_files.json(), instruct_content=change_files) + + async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format) -> ActionOutput: + # sas = SearchAndSummarize() + # # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) + # rsp = "" + # info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" + # if sas.result: + # logger.info(sas.result) + # logger.info(rsp) + project_name = CONFIG.project_name if CONFIG.project_name else "" + context = CONTEXT_TEMPLATE.format(requirements=requirements, project_name=project_name) + node = await WRITE_PRD_NODE.fill(context=context, llm=self.llm, to=format) + await self._rename_workspace(node) + return node + + async def _is_relative(self, new_requirement_doc, old_prd_doc) -> bool: + context = NEW_REQ_TEMPLATE.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content) + node = await WP_IS_RELATIVE_NODE.fill(context, self.llm) + return node.get("is_relative") == "YES" + + async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document: + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + prompt = NEW_REQ_TEMPLATE.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) + node = await WRITE_PRD_NODE.fill(context=prompt, llm=self.llm, to=format) + prd_doc.content = node.instruct_content.json(ensure_ascii=False) + await self._rename_workspace(node) + return prd_doc + + async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: + if not prd_doc: + prd = await self._run_new_requirement(requirements=[requirement_doc.content], *args, **kwargs) + new_prd_doc = Document( + root_path=PRDS_FILE_REPO, + filename=FileRepository.new_filename() + ".json", + content=prd.instruct_content.json(ensure_ascii=False), + ) + elif await self._is_relative(requirement_doc, prd_doc): + new_prd_doc = await self._merge(requirement_doc, prd_doc) + else: + return None + await prds_file_repo.save(filename=new_prd_doc.filename, content=new_prd_doc.content) + await self._save_competitive_analysis(new_prd_doc) + await self._save_pdf(new_prd_doc) + return new_prd_doc + + @staticmethod + async def _save_competitive_analysis(prd_doc): + m = json.loads(prd_doc.content) + quadrant_chart = m.get("Competitive Quadrant Chart") + if not quadrant_chart: + return + pathname = ( + CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") ) - logger.debug(prompt) - # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) - prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) - return prd + if not pathname.parent.exists(): + pathname.parent.mkdir(parents=True, exist_ok=True) + await mermaid_to_file(quadrant_chart, pathname) + + @staticmethod + async def _save_pdf(prd_doc): + await FileRepository.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO) + + @staticmethod + async def _rename_workspace(prd): + if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to + # Section 2.2.3.10 of RFC 135 + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + return + + if not CONFIG.project_name: + if isinstance(prd, ActionOutput) or isinstance(prd, ActionNode): + ws_name = prd.instruct_content.dict()["Project Name"] + else: + ws_name = CodeParser.parse_str(block="Project Name", text=prd) + CONFIG.project_name = ws_name + CONFIG.git_repo.rename_root(CONFIG.project_name) + + async def _is_bugfix(self, context) -> bool: + src_workspace_path = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name + code_files = CONFIG.git_repo.get_files(relative_path=src_workspace_path) + if not code_files: + return False + node = await WP_ISSUE_TYPE_NODE.fill(context, self.llm) + return node.get("issue_type") == "BUG" diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py new file mode 100644 index 000000000..d96c0aeac --- /dev/null +++ b/metagpt/actions/write_prd_an.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/14 11:40 +@Author : alexanderwu +@File : write_prd_an.py +""" + +from metagpt.actions.action_node import ActionNode +from metagpt.logs import logger + +LANGUAGE = ActionNode( + key="Language", + expected_type=str, + instruction="Provide the language used in the project, typically matching the user's requirement language.", + example="en_us", +) + +PROGRAMMING_LANGUAGE = ActionNode( + key="Programming Language", + expected_type=str, + instruction="Python/JavaScript or other mainstream programming language.", + example="Python", +) + +ORIGINAL_REQUIREMENTS = ActionNode( + key="Original Requirements", + expected_type=str, + instruction="Place the polished, complete original requirements here.", + example="The game should have a leaderboard and multiple difficulty levels.", +) + +PROJECT_NAME = ActionNode( + key="Project Name", + expected_type=str, + instruction="Name the project using snake case style, like 'game_2048' or 'simple_crm'.", + example="game_2048", +) + +PRODUCT_GOALS = ActionNode( + key="Product Goals", + expected_type=list[str], + instruction="Provide up to three clear, orthogonal product goals.", + example=["Create an engaging user experience", "Ensure high performance", "Provide customizable features"], +) + +USER_STORIES = ActionNode( + key="User Stories", + expected_type=list[str], + instruction="Provide up to five scenario-based user stories.", + example=[ + "As a user, I want to be able to choose difficulty levels", + "As a player, I want to see my score after each game", + ], +) + +COMPETITIVE_ANALYSIS = ActionNode( + key="Competitive Analysis", + expected_type=list[str], + instruction="Provide analyses for up to seven competitive products.", + example=["Python Snake Game: Simple interface, lacks advanced features"], +) + +COMPETITIVE_QUADRANT_CHART = ActionNode( + key="Competitive Quadrant Chart", + expected_type=str, + instruction="Use mermaid quadrantChart syntax. Distribute scores evenly between 0 and 1", + example="""quadrantChart + title "Reach and engagement of campaigns" + x-axis "Low Reach" --> "High Reach" + y-axis "Low Engagement" --> "High Engagement" + quadrant-1 "We should expand" + quadrant-2 "Need to promote" + quadrant-3 "Re-evaluate" + quadrant-4 "May be improved" + "Campaign A": [0.3, 0.6] + "Campaign B": [0.45, 0.23] + "Campaign C": [0.57, 0.69] + "Campaign D": [0.78, 0.34] + "Campaign E": [0.40, 0.34] + "Campaign F": [0.35, 0.78] + "Our Target Product": [0.5, 0.6]""", +) + +REQUIREMENT_ANALYSIS = ActionNode( + key="Requirement Analysis", + expected_type=str, + instruction="Provide a detailed analysis of the requirements.", + example="The product should be user-friendly.", +) + +REQUIREMENT_POOL = ActionNode( + key="Requirement Pool", + expected_type=list[list[str]], + instruction="List down the requirements with their priority (P0, P1, P2).", + example=[["P0", "..."], ["P1", "..."]], +) + +UI_DESIGN_DRAFT = ActionNode( + key="UI Design draft", + expected_type=str, + instruction="Provide a simple description of UI elements, functions, style, and layout.", + example="Basic function description with a simple style and layout.", +) + +ANYTHING_UNCLEAR = ActionNode( + key="Anything UNCLEAR", + expected_type=str, + instruction="Mention any aspects of the project that are unclear and try to clarify them.", + example="...", +) + +ISSUE_TYPE = ActionNode( + key="issue_type", + expected_type=str, + instruction="Answer BUG/REQUIREMENT. If it is a bugfix, answer BUG, otherwise answer Requirement", + example="BUG", +) + +IS_RELATIVE = ActionNode( + key="is_relative", + expected_type=str, + instruction="Answer YES/NO. If the requirement is related to the old PRD, answer YES, otherwise NO", + example="YES", +) + +REASON = ActionNode( + key="reason", expected_type=str, instruction="Explain the reasoning process from question to answer", example="..." +) + + +NODES = [ + LANGUAGE, + PROGRAMMING_LANGUAGE, + ORIGINAL_REQUIREMENTS, + PROJECT_NAME, + PRODUCT_GOALS, + USER_STORIES, + COMPETITIVE_ANALYSIS, + COMPETITIVE_QUADRANT_CHART, + REQUIREMENT_ANALYSIS, + REQUIREMENT_POOL, + UI_DESIGN_DRAFT, + ANYTHING_UNCLEAR, +] + +WRITE_PRD_NODE = ActionNode.from_children("WritePRD", NODES) +WP_ISSUE_TYPE_NODE = ActionNode.from_children("WP_ISSUE_TYPE", [ISSUE_TYPE, REASON]) +WP_IS_RELATIVE_NODE = ActionNode.from_children("WP_IS_RELATIVE", [IS_RELATIVE, REASON]) + + +def main(): + prompt = WRITE_PRD_NODE.compile(context="") + logger.info(prompt) + + +if __name__ == "__main__": + main() diff --git a/metagpt/actions/write_prd_review.py b/metagpt/actions/write_prd_review.py index 5c922d3bc..5ff9624c5 100644 --- a/metagpt/actions/write_prd_review.py +++ b/metagpt/actions/write_prd_review.py @@ -25,4 +25,3 @@ class WritePRDReview(Action): prompt = self.prd_review_prompt_template.format(prd=self.prd) review = await self._aask(prompt) return review - \ No newline at end of file diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index e2352b641..9dd967788 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -3,10 +3,15 @@ """ @Time : 2023/5/11 22:12 @Author : alexanderwu -@File : environment.py +@File : write_test.py +@Modified By: mashenquan, 2023-11-27. Following the think-act principle, solidify the task parameters when creating the + WriteTest object, rather than passing them in when calling the run function. """ from metagpt.actions.action import Action +from metagpt.config import CONFIG +from metagpt.const import TEST_CODES_FILE_REPO from metagpt.logs import logger +from metagpt.schema import Document, TestingContext from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -15,7 +20,7 @@ NOTICE 2. Requirement: Based on the context, develop a comprehensive test suite that adequately covers all relevant aspects of the code file under review. Your test suite will be part of the overall project QA, so please develop complete, robust, and reusable test cases. 3. Attention1: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script. 4. Attention2: If there are any settings in your tests, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -5. Attention3: YOU MUST FOLLOW "Data structures and interface definitions". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. +5. Attention3: YOU MUST FOLLOW "Data structures and interfaces". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. 6. Think before writing: What should be tested and validated in this document? What edge cases could exist? What might fail? 7. CAREFULLY CHECK THAT YOU DON'T MISS ANY NECESSARY TEST CASES/SCRIPTS IN THIS FILE. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. @@ -47,12 +52,16 @@ class WriteTest(Action): code = code_rsp return code - async def run(self, code_to_test, test_file_name, source_file_path, workspace): + async def run(self, *args, **kwargs) -> TestingContext: + if not self.context.test_doc: + self.context.test_doc = Document( + filename="test_" + self.context.code_doc.filename, root_path=TEST_CODES_FILE_REPO + ) prompt = PROMPT_TEMPLATE.format( - code_to_test=code_to_test, - test_file_name=test_file_name, - source_file_path=source_file_path, - workspace=workspace, + code_to_test=self.context.code_doc.content, + test_file_name=self.context.test_doc.filename, + source_file_path=self.context.code_doc.root_relative_path, + workspace=CONFIG.git_repo.workdir, ) - code = await self.write_code(prompt) - return code + self.context.test_doc.content = await self.write_code(prompt) + return self.context diff --git a/metagpt/actions/write_tutorial.py b/metagpt/actions/write_tutorial.py index 23e3560e8..d41915de3 100644 --- a/metagpt/actions/write_tutorial.py +++ b/metagpt/actions/write_tutorial.py @@ -10,7 +10,7 @@ from typing import Dict from metagpt.actions import Action -from metagpt.prompts.tutorial_assistant import DIRECTORY_PROMPT, CONTENT_PROMPT +from metagpt.prompts.tutorial_assistant import CONTENT_PROMPT, DIRECTORY_PROMPT from metagpt.utils.common import OutputParser @@ -65,4 +65,3 @@ class WriteContent(Action): """ prompt = CONTENT_PROMPT.format(topic=topic, language=self.language, directory=self.directory) return await self._aask(prompt=prompt) - diff --git a/metagpt/config.py b/metagpt/config.py index 2ce75b013..19bd02c87 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -2,13 +2,18 @@ # -*- coding: utf-8 -*- """ Provide configuration, singleton +@Modified By: mashenquan, 2023/11/27. + 1. According to Section 2.2.3.11 of RFC 135, add git repository support. + 2. Add the parameter `src_workspace` for the old version project path. """ import os +from copy import deepcopy +from pathlib import Path +from typing import Any -import openai import yaml -from metagpt.const import PROJECT_ROOT +from metagpt.const import DEFAULT_WORKSPACE_ROOT, METAGPT_ROOT, OPTIONS from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType from metagpt.utils.singleton import Singleton @@ -35,34 +40,37 @@ class Config(metaclass=Singleton): """ _instance = None - key_yaml_file = PROJECT_ROOT / "config/key.yaml" - default_yaml_file = PROJECT_ROOT / "config/config.yaml" + home_yaml_file = Path.home() / ".metagpt/config.yaml" + key_yaml_file = METAGPT_ROOT / "config/key.yaml" + default_yaml_file = METAGPT_ROOT / "config/config.yaml" def __init__(self, yaml_file=default_yaml_file): - self._configs = {} - self._init_with_config_files_and_env(self._configs, yaml_file) - logger.info("Config loading done.") + self._init_with_config_files_and_env(yaml_file) + logger.debug("Config loading done.") + self._update() + + def _update(self): + # logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") self.zhipuai_api_key = self._get("ZHIPUAI_API_KEY") - self.open_llm_api_base = self._get("OPEN_LLM_API_BASE") self.open_llm_api_model = self._get("OPEN_LLM_API_MODEL") - self.fireworks_api_key = self._get("FIREWORKS_API_KEY") - if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and \ - (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key) and \ - (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key) and \ - (not self.open_llm_api_base) and \ - (not self.fireworks_api_key or "YOUR_API_KEY" == self.fireworks_api_key): - raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first " - "or FIREWORKS_API_KEY or OPEN_LLM_API_BASE") + if ( + (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) + and (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key) + and (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key) + and (not self.open_llm_api_base) + and (not self.fireworks_api_key or "YOUR_API_KEY" == self.fireworks_api_key) + ): + raise NotConfiguredException( + "Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first " + "or FIREWORKS_API_KEY or OPEN_LLM_API_BASE" + ) self.openai_api_base = self._get("OPENAI_API_BASE") - openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy - if openai_proxy: - openai.proxy = openai_proxy - openai.api_base = self.openai_api_base + self.openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy self.openai_api_type = self._get("OPENAI_API_TYPE") self.openai_api_version = self._get("OPENAI_API_VERSION") self.openai_api_rpm = self._get("RPM", 3) @@ -95,6 +103,7 @@ class Config(metaclass=Singleton): logger.warning("LONG_TERM_MEMORY is True") self.max_budget = self._get("MAX_BUDGET", 10.0) self.total_cost = 0.0 + self.code_review_k_times = 2 self.puppeteer_config = self._get("PUPPETEER_CONFIG", "") self.mmdc = self._get("MMDC", "mmdc") @@ -105,13 +114,19 @@ class Config(metaclass=Singleton): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.repair_llm_output = self._get("REPAIR_LLM_OUTPUT", False) - self.prompt_format = self._get("PROMPT_FORMAT", "markdown") + self.prompt_format = self._get("PROMPT_FORMAT", "json") + self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT)) + self._ensure_workspace_exists() - def _init_with_config_files_and_env(self, configs: dict, yaml_file): + def _ensure_workspace_exists(self): + self.workspace_path.mkdir(parents=True, exist_ok=True) + logger.debug(f"WORKSPACE_PATH set to {self.workspace_path}") + + def _init_with_config_files_and_env(self, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" - configs.update(os.environ) + configs = dict(os.environ) - for _yaml_file in [yaml_file, self.key_yaml_file]: + for _yaml_file in [yaml_file, self.key_yaml_file, self.home_yaml_file]: if not _yaml_file.exists(): continue @@ -120,11 +135,13 @@ class Config(metaclass=Singleton): yaml_data = yaml.safe_load(file) if not yaml_data: continue - os.environ.update({k: v for k, v in yaml_data.items() if isinstance(v, str)}) configs.update(yaml_data) + OPTIONS.set(configs) - def _get(self, *args, **kwargs): - return self._configs.get(*args, **kwargs) + @staticmethod + def _get(*args, **kwargs): + m = OPTIONS.get() + return m.get(*args, **kwargs) def get(self, key, *args, **kwargs): """Search for a value in config/key.yaml, config/config.yaml, and env; raise an error if not found""" @@ -133,5 +150,33 @@ class Config(metaclass=Singleton): raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") return value + def __setattr__(self, name: str, value: Any) -> None: + OPTIONS.get()[name] = value + + def __getattr__(self, name: str) -> Any: + m = OPTIONS.get() + return m.get(name) + + def set_context(self, options: dict): + """Update current config""" + if not options: + return + opts = deepcopy(OPTIONS.get()) + opts.update(options) + OPTIONS.set(opts) + self._update() + + @property + def options(self): + """Return all key-values""" + return OPTIONS.get() + + def new_environ(self): + """Return a new os.environ object""" + env = os.environ.copy() + m = self.options + env.update({k: v for k, v in m.items() if isinstance(v, str)}) + return env + CONFIG = Config() diff --git a/metagpt/const.py b/metagpt/const.py index 407ce803a..f6f64a27d 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -4,45 +4,92 @@ @Time : 2023/5/1 11:59 @Author : alexanderwu @File : const.py +@Modified By: mashenquan, 2023-11-1. According to Section 2.2.1 and 2.2.2 of RFC 116, added key definitions for + common properties in the Message. +@Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135. +@Modified By: mashenquan, 2023/12/5. Add directories for code summarization.. """ +import contextvars +import os from pathlib import Path + from loguru import logger -def get_project_root(): - """Search upwards to find the project root directory.""" - current_path = Path.cwd() - while True: - if ( - (current_path / ".git").exists() - or (current_path / ".project_root").exists() - or (current_path / ".gitignore").exists() - ): - # use metagpt with git clone will land here - logger.info(f"PROJECT_ROOT set to {str(current_path)}") - return current_path - parent_path = current_path.parent - if parent_path == current_path: - # use metagpt with pip install will land here - cwd = Path.cwd() - logger.info(f"PROJECT_ROOT set to current working directory: {str(cwd)}") - return cwd - current_path = parent_path +import metagpt + +OPTIONS = contextvars.ContextVar("OPTIONS") -PROJECT_ROOT = get_project_root() -DATA_PATH = PROJECT_ROOT / "data" -WORKSPACE_ROOT = PROJECT_ROOT / "workspace" -PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts" -UT_PATH = PROJECT_ROOT / "data/ut" -SWAGGER_PATH = UT_PATH / "files/api/" -UT_PY_PATH = UT_PATH / "files/ut/" -API_QUESTIONS_PATH = UT_PATH / "files/question/" -YAPI_URL = "http://yapi.deepwisdomai.com/" -TMP = PROJECT_ROOT / "tmp" +def get_metagpt_package_root(): + """Get the root directory of the installed package.""" + package_root = Path(metagpt.__file__).parent.parent + logger.info(f"Package root set to {str(package_root)}") + return package_root + + +def get_metagpt_root(): + """Get the project root directory.""" + # Check if a project root is specified in the environment variable + project_root_env = os.getenv("METAGPT_PROJECT_ROOT") + if project_root_env: + project_root = Path(project_root_env) + logger.info(f"PROJECT_ROOT set from environment variable to {str(project_root)}") + else: + # Fallback to package root if no environment variable is set + project_root = get_metagpt_package_root() + return project_root + + +# METAGPT PROJECT ROOT AND VARS + +METAGPT_ROOT = get_metagpt_root() +DEFAULT_WORKSPACE_ROOT = METAGPT_ROOT / "workspace" + +DATA_PATH = METAGPT_ROOT / "data" RESEARCH_PATH = DATA_PATH / "research" TUTORIAL_PATH = DATA_PATH / "tutorial_docx" INVOICE_OCR_TABLE_PATH = DATA_PATH / "invoice_table" +UT_PATH = DATA_PATH / "ut" +SWAGGER_PATH = UT_PATH / "files/api/" +UT_PY_PATH = UT_PATH / "files/ut/" +API_QUESTIONS_PATH = UT_PATH / "files/question/" -SKILL_DIRECTORY = PROJECT_ROOT / "metagpt/skills" +TMP = METAGPT_ROOT / "tmp" + +SOURCE_ROOT = METAGPT_ROOT / "metagpt" +PROMPT_PATH = SOURCE_ROOT / "prompts" +SKILL_DIRECTORY = SOURCE_ROOT / "skills" + + +# REAL CONSTS MEM_TTL = 24 * 30 * 3600 + + +MESSAGE_ROUTE_FROM = "sent_from" +MESSAGE_ROUTE_TO = "send_to" +MESSAGE_ROUTE_CAUSE_BY = "cause_by" +MESSAGE_META_ROLE = "role" +MESSAGE_ROUTE_TO_ALL = "" +MESSAGE_ROUTE_TO_NONE = "" + +REQUIREMENT_FILENAME = "requirement.txt" +BUGFIX_FILENAME = "bugfix.txt" +PACKAGE_REQUIREMENTS_FILENAME = "requirements.txt" + +DOCS_FILE_REPO = "docs" +PRDS_FILE_REPO = "docs/prds" +SYSTEM_DESIGN_FILE_REPO = "docs/system_design" +TASK_FILE_REPO = "docs/tasks" +COMPETITIVE_ANALYSIS_FILE_REPO = "resources/competitive_analysis" +DATA_API_DESIGN_FILE_REPO = "resources/data_api_design" +SEQ_FLOW_FILE_REPO = "resources/seq_flow" +SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design" +PRD_PDF_FILE_REPO = "resources/prd" +TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" +TEST_CODES_FILE_REPO = "tests" +TEST_OUTPUTS_FILE_REPO = "test_outputs" +CODE_SUMMARIES_FILE_REPO = "docs/code_summaries" +CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries" + +YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/document.py b/metagpt/document.py new file mode 100644 index 000000000..0af3a915c --- /dev/null +++ b/metagpt/document.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/6/8 14:03 +@Author : alexanderwu +@File : document.py +@Desc : Classes and Operations Related to Files in the File System. +""" +from enum import Enum +from pathlib import Path +from typing import Optional, Union + +import pandas as pd +from langchain.document_loaders import ( + TextLoader, + UnstructuredPDFLoader, + UnstructuredWordDocumentLoader, +) +from langchain.text_splitter import CharacterTextSplitter +from pydantic import BaseModel, Field +from tqdm import tqdm + +from metagpt.config import CONFIG +from metagpt.logs import logger +from metagpt.repo_parser import RepoParser + + +def validate_cols(content_col: str, df: pd.DataFrame): + if content_col not in df.columns: + raise ValueError("Content column not found in DataFrame.") + + +def read_data(data_path: Path): + suffix = data_path.suffix + if ".xlsx" == suffix: + data = pd.read_excel(data_path) + elif ".csv" == suffix: + data = pd.read_csv(data_path) + elif ".json" == suffix: + data = pd.read_json(data_path) + elif suffix in (".docx", ".doc"): + data = UnstructuredWordDocumentLoader(str(data_path), mode="elements").load() + elif ".txt" == suffix: + data = TextLoader(str(data_path)).load() + text_splitter = CharacterTextSplitter(separator="\n", chunk_size=256, chunk_overlap=0) + texts = text_splitter.split_documents(data) + data = texts + elif ".pdf" == suffix: + data = UnstructuredPDFLoader(str(data_path), mode="elements").load() + else: + raise NotImplementedError("File format not supported.") + return data + + +class DocumentStatus(Enum): + """Indicates document status, a mechanism similar to RFC/PEP""" + + DRAFT = "draft" + UNDERREVIEW = "underreview" + APPROVED = "approved" + DONE = "done" + + +class Document(BaseModel): + """ + Document: Handles operations related to document files. + """ + + path: Path = Field(default=None) + name: str = Field(default="") + content: str = Field(default="") + + # metadata? in content perhaps. + author: str = Field(default="") + status: DocumentStatus = Field(default=DocumentStatus.DRAFT) + reviews: list = Field(default_factory=list) + + @classmethod + def from_path(cls, path: Path): + """ + Create a Document instance from a file path. + """ + if not path.exists(): + raise FileNotFoundError(f"File {path} not found.") + content = path.read_text() + return cls(content=content, path=path) + + @classmethod + def from_text(cls, text: str, path: Optional[Path] = None): + """ + Create a Document from a text string. + """ + return cls(content=text, path=path) + + def to_path(self, path: Optional[Path] = None): + """ + Save content to the specified file path. + """ + if path is not None: + self.path = path + + if self.path is None: + raise ValueError("File path is not set.") + + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(self.content, encoding="utf-8") + + def persist(self): + """ + Persist document to disk. + """ + return self.to_path() + + +class IndexableDocument(Document): + """ + Advanced document handling: For vector databases or search engines. + """ + + data: Union[pd.DataFrame, list] + content_col: Optional[str] = Field(default="") + meta_col: Optional[str] = Field(default="") + + class Config: + arbitrary_types_allowed = True + + @classmethod + def from_path(cls, data_path: Path, content_col="content", meta_col="metadata"): + if not data_path.exists(): + raise FileNotFoundError(f"File {data_path} not found.") + data = read_data(data_path) + content = data_path.read_text() + if isinstance(data, pd.DataFrame): + validate_cols(content_col, data) + return cls(data=data, content=content, content_col=content_col, meta_col=meta_col) + + def _get_docs_and_metadatas_by_df(self) -> (list, list): + df = self.data + docs = [] + metadatas = [] + for i in tqdm(range(len(df))): + docs.append(df[self.content_col].iloc[i]) + if self.meta_col: + metadatas.append({self.meta_col: df[self.meta_col].iloc[i]}) + else: + metadatas.append({}) + return docs, metadatas + + def _get_docs_and_metadatas_by_langchain(self) -> (list, list): + data = self.data + docs = [i.page_content for i in data] + metadatas = [i.metadata for i in data] + return docs, metadatas + + def get_docs_and_metadatas(self) -> (list, list): + if isinstance(self.data, pd.DataFrame): + return self._get_docs_and_metadatas_by_df() + elif isinstance(self.data, list): + return self._get_docs_and_metadatas_by_langchain() + else: + raise NotImplementedError("Data type not supported for metadata extraction.") + + +class RepoMetadata(BaseModel): + name: str = Field(default="") + n_docs: int = Field(default=0) + n_chars: int = Field(default=0) + symbols: list = Field(default_factory=list) + + +class Repo(BaseModel): + # Name of this repo. + name: str = Field(default="") + # metadata: RepoMetadata = Field(default=RepoMetadata) + docs: dict[Path, Document] = Field(default_factory=dict) + codes: dict[Path, Document] = Field(default_factory=dict) + assets: dict[Path, Document] = Field(default_factory=dict) + path: Path = Field(default=None) + + def _path(self, filename): + return self.path / filename + + @classmethod + def from_path(cls, path: Path): + """Load documents, code, and assets from a repository path.""" + path.mkdir(parents=True, exist_ok=True) + repo = Repo(path=path, name=path.name) + for file_path in path.rglob("*"): + # FIXME: These judgments are difficult to support multiple programming languages and need to be more general + if file_path.is_file() and file_path.suffix in [".json", ".txt", ".md", ".py", ".js", ".css", ".html"]: + repo._set(file_path.read_text(), file_path) + return repo + + def to_path(self): + """Persist all documents, code, and assets to the given repository path.""" + for doc in self.docs.values(): + doc.to_path() + for code in self.codes.values(): + code.to_path() + for asset in self.assets.values(): + asset.to_path() + + def _set(self, content: str, path: Path): + """Add a document to the appropriate category based on its file extension.""" + suffix = path.suffix + doc = Document(content=content, path=path, name=str(path.relative_to(self.path))) + + # FIXME: These judgments are difficult to support multiple programming languages and need to be more general + if suffix.lower() == ".md": + self.docs[path] = doc + elif suffix.lower() in [".py", ".js", ".css", ".html"]: + self.codes[path] = doc + else: + self.assets[path] = doc + return doc + + def set(self, content: str, filename: str): + """Set a document and persist it to disk.""" + path = self._path(filename) + doc = self._set(content, path) + doc.to_path() + + def get(self, filename: str) -> Optional[Document]: + """Get a document by its filename.""" + path = self._path(filename) + return self.docs.get(path) or self.codes.get(path) or self.assets.get(path) + + def get_text_documents(self) -> list[Document]: + return list(self.docs.values()) + list(self.codes.values()) + + def eda(self) -> RepoMetadata: + n_docs = sum(len(i) for i in [self.docs, self.codes, self.assets]) + n_chars = sum(sum(len(j.content) for j in i.values()) for i in [self.docs, self.codes, self.assets]) + symbols = RepoParser(base_directory=self.path).generate_symbols() + return RepoMetadata(name=self.name, n_docs=n_docs, n_chars=n_chars, symbols=symbols) + + +def set_existing_repo(path=CONFIG.workspace_path / "t1"): + repo1 = Repo.from_path(path) + repo1.set("wtf content", "doc/wtf_file.md") + repo1.set("wtf code", "code/wtf_file.py") + logger.info(repo1) # check doc + + +def load_existing_repo(path=CONFIG.workspace_path / "web_tetris"): + repo = Repo.from_path(path) + logger.info(repo) + logger.info(repo.eda()) + + +def main(): + load_existing_repo() + + +if __name__ == "__main__": + main() diff --git a/metagpt/document_store/base_store.py b/metagpt/document_store/base_store.py index 5d7015e8b..5de377d21 100644 --- a/metagpt/document_store/base_store.py +++ b/metagpt/document_store/base_store.py @@ -28,20 +28,20 @@ class BaseStore(ABC): class LocalStore(BaseStore, ABC): - def __init__(self, raw_data: Path, cache_dir: Path = None): - if not raw_data: + def __init__(self, raw_data_path: Path, cache_dir: Path = None): + if not raw_data_path: raise FileNotFoundError self.config = Config() - self.raw_data = raw_data + self.raw_data_path = raw_data_path if not cache_dir: - cache_dir = raw_data.parent + cache_dir = raw_data_path.parent self.cache_dir = cache_dir self.store = self._load() if not self.store: self.store = self.write() def _get_index_and_store_fname(self): - fname = self.raw_data.name.split('.')[0] + fname = self.raw_data_path.name.split(".")[0] index_file = self.cache_dir / f"{fname}.index" store_file = self.cache_dir / f"{fname}.pkl" return index_file, store_file @@ -53,4 +53,3 @@ class LocalStore(BaseStore, ABC): @abstractmethod def _write(self, docs, metadatas): raise NotImplementedError - \ No newline at end of file diff --git a/metagpt/document_store/chromadb_store.py b/metagpt/document_store/chromadb_store.py index d2ecc05f6..d7344d41b 100644 --- a/metagpt/document_store/chromadb_store.py +++ b/metagpt/document_store/chromadb_store.py @@ -10,6 +10,7 @@ import chromadb class ChromaStore: """If inherited from BaseStore, or importing other modules from metagpt, a Python exception occurs, which is strange.""" + def __init__(self, name): client = chromadb.Client() collection = client.create_collection(name) @@ -22,7 +23,7 @@ class ChromaStore: query_texts=[query], n_results=n_results, where=metadata_filter, # optional filter - where_document=document_filter # optional filter + where_document=document_filter, # optional filter ) return results diff --git a/metagpt/document_store/document.py b/metagpt/document_store/document.py index e4b9473c7..90abc54de 100644 --- a/metagpt/document_store/document.py +++ b/metagpt/document_store/document.py @@ -4,6 +4,7 @@ @Time : 2023/6/8 14:03 @Author : alexanderwu @File : document.py +@Desc : Classes and Operations Related to Vector Files in the Vector Database. Still under design. """ from pathlib import Path @@ -24,20 +25,20 @@ def validate_cols(content_col: str, df: pd.DataFrame): def read_data(data_path: Path): suffix = data_path.suffix - if '.xlsx' == suffix: + if ".xlsx" == suffix: data = pd.read_excel(data_path) - elif '.csv' == suffix: + elif ".csv" == suffix: data = pd.read_csv(data_path) - elif '.json' == suffix: + elif ".json" == suffix: data = pd.read_json(data_path) - elif suffix in ('.docx', '.doc'): - data = UnstructuredWordDocumentLoader(str(data_path), mode='elements').load() - elif '.txt' == suffix: + elif suffix in (".docx", ".doc"): + data = UnstructuredWordDocumentLoader(str(data_path), mode="elements").load() + elif ".txt" == suffix: data = TextLoader(str(data_path)).load() - text_splitter = CharacterTextSplitter(separator='\n', chunk_size=256, chunk_overlap=0) + text_splitter = CharacterTextSplitter(separator="\n", chunk_size=256, chunk_overlap=0) texts = text_splitter.split_documents(data) data = texts - elif '.pdf' == suffix: + elif ".pdf" == suffix: data = UnstructuredPDFLoader(str(data_path), mode="elements").load() else: raise NotImplementedError @@ -45,8 +46,7 @@ def read_data(data_path: Path): class Document: - - def __init__(self, data_path, content_col='content', meta_col='metadata'): + def __init__(self, data_path, content_col="content", meta_col="metadata"): self.data = read_data(data_path) if isinstance(self.data, pd.DataFrame): validate_cols(content_col, self.data) @@ -79,4 +79,3 @@ class Document: return self._get_docs_and_metadatas_by_langchain() else: raise NotImplementedError - \ No newline at end of file diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 5c6459179..b1faa3538 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -15,16 +15,16 @@ from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import FAISS from metagpt.const import DATA_PATH +from metagpt.document import IndexableDocument from metagpt.document_store.base_store import LocalStore -from metagpt.document_store.document import Document from metagpt.logs import logger class FaissStore(LocalStore): - def __init__(self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output"): + def __init__(self, raw_data_path: Path, cache_dir=None, meta_col="source", content_col="output"): self.meta_col = meta_col self.content_col = content_col - super().__init__(raw_data, cache_dir) + super().__init__(raw_data_path, cache_dir) def _load(self) -> Optional["FaissStore"]: index_file, store_file = self._get_index_and_store_fname() @@ -64,9 +64,9 @@ class FaissStore(LocalStore): def write(self): """Initialize the index and library based on the Document (JSON / XLSX, etc.) file provided by the user.""" - if not self.raw_data.exists(): + if not self.raw_data_path.exists(): raise FileNotFoundError - doc = Document(self.raw_data, self.content_col, self.meta_col) + doc = IndexableDocument.from_path(self.raw_data_path, self.content_col, self.meta_col) docs, metadatas = doc.get_docs_and_metadatas() self.store = self._write(docs, metadatas) diff --git a/metagpt/document_store/milvus_store.py b/metagpt/document_store/milvus_store.py index 77a8ec141..fcfc59d79 100644 --- a/metagpt/document_store/milvus_store.py +++ b/metagpt/document_store/milvus_store.py @@ -12,12 +12,7 @@ from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connec from metagpt.document_store.base_store import BaseStore -type_mapping = { - int: DataType.INT64, - str: DataType.VARCHAR, - float: DataType.DOUBLE, - np.ndarray: DataType.FLOAT_VECTOR -} +type_mapping = {int: DataType.INT64, str: DataType.VARCHAR, float: DataType.DOUBLE, np.ndarray: DataType.FLOAT_VECTOR} def columns_to_milvus_schema(columns: dict, primary_col_name: str = "", desc: str = ""): @@ -52,17 +47,11 @@ class MilvusStore(BaseStore): self.collection = None def _create_collection(self, name, schema): - collection = Collection( - name=name, - schema=schema, - using='default', - shards_num=2, - consistency_level="Strong" - ) + collection = Collection(name=name, schema=schema, using="default", shards_num=2, consistency_level="Strong") return collection def create_collection(self, name, columns): - schema = columns_to_milvus_schema(columns, 'idx') + schema = columns_to_milvus_schema(columns, "idx") self.collection = self._create_collection(name, schema) return self.collection @@ -72,7 +61,7 @@ class MilvusStore(BaseStore): def load_collection(self): self.collection.load() - def build_index(self, field='emb'): + def build_index(self, field="emb"): self.collection.create_index(field, {"index_type": "FLAT", "metric_type": "L2", "params": {}}) def search(self, query: list[list[float]], *args, **kwargs): @@ -85,11 +74,11 @@ class MilvusStore(BaseStore): search_params = {"metric_type": "L2", "params": {"nprobe": 10}} results = self.collection.search( data=query, - anns_field=kwargs.get('field', 'emb'), + anns_field=kwargs.get("field", "emb"), param=search_params, limit=10, expr=None, - consistency_level="Strong" + consistency_level="Strong", ) # FIXME: results contain id, but to get the actual value from the id, we still need to call the query interface return results diff --git a/metagpt/document_store/qdrant_store.py b/metagpt/document_store/qdrant_store.py index 98b82cf87..4e9637aa7 100644 --- a/metagpt/document_store/qdrant_store.py +++ b/metagpt/document_store/qdrant_store.py @@ -10,13 +10,14 @@ from metagpt.document_store.base_store import BaseStore @dataclass class QdrantConnection: """ - Args: - url: qdrant url - host: qdrant host - port: qdrant port - memory: qdrant service use memory mode - api_key: qdrant cloud api_key - """ + Args: + url: qdrant url + host: qdrant host + port: qdrant port + memory: qdrant service use memory mode + api_key: qdrant cloud api_key + """ + url: str = None host: str = None port: int = None @@ -31,9 +32,7 @@ class QdrantStore(BaseStore): elif connect.url: self.client = QdrantClient(url=connect.url, api_key=connect.api_key) elif connect.host and connect.port: - self.client = QdrantClient( - host=connect.host, port=connect.port, api_key=connect.api_key - ) + self.client = QdrantClient(host=connect.host, port=connect.port, api_key=connect.api_key) else: raise Exception("please check QdrantConnection.") @@ -58,15 +57,11 @@ class QdrantStore(BaseStore): try: self.client.get_collection(collection_name) if force_recreate: - res = self.client.recreate_collection( - collection_name, vectors_config=vectors_config, **kwargs - ) + res = self.client.recreate_collection(collection_name, vectors_config=vectors_config, **kwargs) return res return True except: # noqa: E722 - return self.client.recreate_collection( - collection_name, vectors_config=vectors_config, **kwargs - ) + return self.client.recreate_collection(collection_name, vectors_config=vectors_config, **kwargs) def has_collection(self, collection_name: str): try: diff --git a/metagpt/environment.py b/metagpt/environment.py index 24e6ada2f..89b6f9d46 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -4,60 +4,77 @@ @Time : 2023/5/11 22:12 @Author : alexanderwu @File : environment.py +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.2 of RFC 116: + 1. Remove the functionality of `Environment` class as a public message buffer. + 2. Standardize the message forwarding behavior of the `Environment` class. + 3. Add the `is_idle` property. +@Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing + functionality is to be consolidated into the `Environment` class. """ import asyncio -from typing import Iterable +from typing import Iterable, Set from pydantic import BaseModel, Field -from metagpt.memory import Memory +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message +from metagpt.utils.common import is_subscribed class Environment(BaseModel): """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到 - Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles - + Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles + """ roles: dict[str, Role] = Field(default_factory=dict) - memory: Memory = Field(default_factory=Memory) - history: str = Field(default='') + members: dict[Role, Set] = Field(default_factory=dict) + history: str = Field(default="") # For debug class Config: arbitrary_types_allowed = True def add_role(self, role: Role): """增加一个在当前环境的角色 - Add a role in the current environment + Add a role in the current environment """ role.set_env(self) self.roles[role.profile] = role def add_roles(self, roles: Iterable[Role]): """增加一批在当前环境的角色 - Add a batch of characters in the current environment + Add a batch of characters in the current environment """ for role in roles: self.add_role(role) - def publish_message(self, message: Message): - """向当前环境发布信息 - Post information to the current environment + def publish_message(self, message: Message) -> bool: """ - # self.message_queue.put(message) - self.memory.add(message) - self.history += f"\n{message}" + Distribute the message to the recipients. + In accordance with the Message routing structure design in Chapter 2.2.1 of RFC 116, as already planned + in RFC 113 for the entire system, the routing information in the Message is only responsible for + specifying the message recipient, without concern for where the message recipient is located. How to + route the message to the message recipient is a problem addressed by the transport framework designed + in RFC 113. + """ + logger.debug(f"publish_message: {message.dump()}") + found = False + # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 + for role, subscription in self.members.items(): + if is_subscribed(message, subscription): + role.put_message(message) + found = True + if not found: + logger.warning(f"Message no recipients: {message.dump()}") + self.history += f"\n{message}" # For debug + + return True async def run(self, k=1): """处理一次所有信息的运行 Process all Role runs at once """ - # while not self.message_queue.empty(): - # message = self.message_queue.get() - # rsp = await self.manager.handle(message, self) - # self.message_queue.put(rsp) for _ in range(k): futures = [] for role in self.roles.values(): @@ -65,15 +82,32 @@ class Environment(BaseModel): futures.append(future) await asyncio.gather(*futures) + logger.debug(f"is idle: {self.is_idle}") def get_roles(self) -> dict[str, Role]: """获得环境内的所有角色 - Process all Role runs at once + Process all Role runs at once """ return self.roles def get_role(self, name: str) -> Role: """获得环境内的指定角色 - get all the environment roles + get all the environment roles """ return self.roles.get(name, None) + + @property + def is_idle(self): + """If true, all actions have been executed.""" + for r in self.roles.values(): + if not r.is_idle: + return False + return True + + def get_subscription(self, obj): + """Get the labels for messages to be consumed by the object.""" + return self.members.get(obj, {}) + + def set_subscription(self, obj, tags): + """Set the labels for message to be consumed by the object""" + self.members[obj] = tags diff --git a/metagpt/inspect_module.py b/metagpt/inspect_module.py index a89ac1c5e..48ceffc57 100644 --- a/metagpt/inspect_module.py +++ b/metagpt/inspect_module.py @@ -12,17 +12,17 @@ import metagpt # replace with your module def print_classes_and_functions(module): - """FIXME: NOT WORK.. """ + """FIXME: NOT WORK..""" for name, obj in inspect.getmembers(module): if inspect.isclass(obj): - print(f'Class: {name}') + print(f"Class: {name}") elif inspect.isfunction(obj): - print(f'Function: {name}') + print(f"Function: {name}") else: print(name) print(dir(module)) -if __name__ == '__main__': - print_classes_and_functions(metagpt) \ No newline at end of file +if __name__ == "__main__": + print_classes_and_functions(metagpt) diff --git a/metagpt/llm.py b/metagpt/llm.py index 7b490ec4a..7c0ad7975 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -7,17 +7,19 @@ """ from metagpt.config import CONFIG -from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.openai_api import OpenAIGPTAPI -from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI -from metagpt.provider.spark_api import SparkAPI -from metagpt.provider.open_llm_api import OpenLLMGPTAPI +from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.fireworks_api import FireWorksGPTAPI from metagpt.provider.human_provider import HumanProvider +from metagpt.provider.open_llm_api import OpenLLMGPTAPI +from metagpt.provider.openai_api import OpenAIGPTAPI +from metagpt.provider.spark_api import SparkAPI +from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI + +_ = HumanProvider() # Avoid pre-commit error -def LLM() -> "BaseGPTAPI": - """ initialize different LLM instance according to the key field existence""" +def LLM() -> BaseGPTAPI: + """initialize different LLM instance according to the key field existence""" # TODO a little trick, can use registry to initialize LLM instance further if CONFIG.openai_api_key: llm = OpenAIGPTAPI() diff --git a/metagpt/logs.py b/metagpt/logs.py index 471d57fe9..ab1bc4e94 100644 --- a/metagpt/logs.py +++ b/metagpt/logs.py @@ -11,19 +11,17 @@ from datetime import datetime from loguru import logger as _logger -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT def define_log_level(print_level="INFO", logfile_level="DEBUG"): - """调整日志级别到level之上 - Adjust the log level to above level - """ + """Adjust the log level to above level""" current_date = datetime.now() formatted_date = current_date.strftime("%Y%m%d") _logger.remove() _logger.add(sys.stderr, level=print_level) - _logger.add(PROJECT_ROOT / f"logs/{formatted_date}.log", level=logfile_level) + _logger.add(METAGPT_ROOT / f"logs/{formatted_date}.txt", level=logfile_level) return _logger diff --git a/metagpt/management/skill_manager.py b/metagpt/management/skill_manager.py index f967a0a94..b3181b64e 100644 --- a/metagpt/management/skill_manager.py +++ b/metagpt/management/skill_manager.py @@ -19,8 +19,8 @@ class SkillManager: def __init__(self): self._llm = LLM() - self._store = ChromaStore('skill_manager') - self._skills: dict[str: Skill] = {} + self._store = ChromaStore("skill_manager") + self._skills: dict[str:Skill] = {} def add_skill(self, skill: Skill): """ @@ -54,7 +54,7 @@ class SkillManager: :param desc: Skill description :return: Multiple skills """ - return self._store.search(desc, n_results=n_results)['ids'][0] + return self._store.search(desc, n_results=n_results)["ids"][0] def retrieve_skill_scored(self, desc: str, n_results: int = 2) -> dict: """ @@ -75,6 +75,6 @@ class SkillManager: logger.info(text) -if __name__ == '__main__': +if __name__ == "__main__": manager = SkillManager() manager.generate_skill_desc(Action()) diff --git a/metagpt/manager.py b/metagpt/manager.py index 9d238c621..a063608be 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -14,11 +14,11 @@ class Manager: def __init__(self, llm: LLM = LLM()): self.llm = llm # Large Language Model self.role_directions = { - "BOSS": "Product Manager", + "User": "Product Manager", "Product Manager": "Architect", "Architect": "Engineer", "Engineer": "QA Engineer", - "QA Engineer": "Product Manager" + "QA Engineer": "Product Manager", } self.prompt_template = """ Given the following message: @@ -51,7 +51,7 @@ class Manager: # chosen_role_name = self.llm.ask(self.prompt_template.format(context)) # FIXME: 现在通过简单的字典决定流向，但之后还是应该有思考过程 - #The direction of flow is now determined by a simple dictionary, but there should still be a thought process afterwards + # The direction of flow is now determined by a simple dictionary, but there should still be a thought process afterwards next_role_profile = self.role_directions[message.role] # logger.debug(f"{next_role_profile}") for _, role in roles.items(): diff --git a/metagpt/memory/__init__.py b/metagpt/memory/__init__.py index 710930626..3d8f679b2 100644 --- a/metagpt/memory/__init__.py +++ b/metagpt/memory/__init__.py @@ -7,10 +7,11 @@ """ from metagpt.memory.memory import Memory -from metagpt.memory.longterm_memory import LongTermMemory + +# from metagpt.memory.longterm_memory import LongTermMemory __all__ = [ "Memory", - "LongTermMemory", + # "LongTermMemory", ] diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index f8abea5f3..22032a86e 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : the implement of Long-term memory +""" +@Desc : the implement of Long-term memory +""" from metagpt.logs import logger from metagpt.memory import Memory @@ -28,7 +30,7 @@ class LongTermMemory(Memory): logger.warning(f"It may the first time to run Agent {role_id}, the long-term memory is empty") else: logger.warning( - f"Agent {role_id} has existed memory storage with {len(messages)} messages " f"and has recovered them." + f"Agent {role_id} has existing memory storage with {len(messages)} messages " f"and has recovered them." ) self.msg_from_recover = True self.add_batch(messages) @@ -68,4 +70,3 @@ class LongTermMemory(Memory): def clear(self): super(LongTermMemory, self).clear() self.memory_storage.clean() - \ No newline at end of file diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index c818fa707..53b65fcf7 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -4,12 +4,13 @@ @Time : 2023/5/20 12:15 @Author : alexanderwu @File : memory.py +@Modified By: mashenquan, 2023-11-1. According to RFC 116: Updated the type of index key. """ from collections import defaultdict -from typing import Iterable, Type +from typing import Iterable, Set -from metagpt.actions import Action from metagpt.schema import Message +from metagpt.utils.common import any_to_str, any_to_str_set class Memory: @@ -18,7 +19,7 @@ class Memory: def __init__(self): """Initialize an empty storage list and an empty index dictionary""" self.storage: list[Message] = [] - self.index: dict[Type[Action], list[Message]] = defaultdict(list) + self.index: dict[str, list[Message]] = defaultdict(list) def add(self, message: Message): """Add a new message to storage, while updating the index""" @@ -73,16 +74,17 @@ class Memory: news.append(i) return news - def get_by_action(self, action: Type[Action]) -> list[Message]: + def get_by_action(self, action) -> list[Message]: """Return all messages triggered by a specified Action""" - return self.index[action] + index = any_to_str(action) + return self.index[index] - def get_by_actions(self, actions: Iterable[Type[Action]]) -> list[Message]: + def get_by_actions(self, actions: Set) -> list[Message]: """Return all messages triggered by specified Actions""" rsp = [] - for action in actions: + indices = any_to_str_set(actions) + for action in indices: if action not in self.index: continue rsp += self.index[action] return rsp - \ No newline at end of file diff --git a/metagpt/memory/memory_storage.py b/metagpt/memory/memory_storage.py index 302d96aa7..a213f6d7a 100644 --- a/metagpt/memory/memory_storage.py +++ b/metagpt/memory/memory_storage.py @@ -2,16 +2,16 @@ # -*- coding: utf-8 -*- # @Desc : the implement of memory storage -from typing import List from pathlib import Path +from typing import List from langchain.vectorstores.faiss import FAISS from metagpt.const import DATA_PATH, MEM_TTL +from metagpt.document_store.faiss_store import FaissStore from metagpt.logs import logger from metagpt.schema import Message -from metagpt.utils.serialize import serialize_message, deserialize_message -from metagpt.document_store.faiss_store import FaissStore +from metagpt.utils.serialize import deserialize_message, serialize_message class MemoryStorage(FaissStore): @@ -34,7 +34,7 @@ class MemoryStorage(FaissStore): def recover_memory(self, role_id: str) -> List[Message]: self.role_id = role_id - self.role_mem_path = Path(DATA_PATH / f'role_mem/{self.role_id}/') + self.role_mem_path = Path(DATA_PATH / f"role_mem/{self.role_id}/") self.role_mem_path.mkdir(parents=True, exist_ok=True) self.store = self._load() @@ -51,18 +51,18 @@ class MemoryStorage(FaissStore): def _get_index_and_store_fname(self): if not self.role_mem_path: - logger.error(f'You should call {self.__class__.__name__}.recover_memory fist when using LongTermMemory') + logger.error(f"You should call {self.__class__.__name__}.recover_memory fist when using LongTermMemory") return None, None - index_fpath = Path(self.role_mem_path / f'{self.role_id}.index') - storage_fpath = Path(self.role_mem_path / f'{self.role_id}.pkl') + index_fpath = Path(self.role_mem_path / f"{self.role_id}.index") + storage_fpath = Path(self.role_mem_path / f"{self.role_id}.pkl") return index_fpath, storage_fpath def persist(self): super(MemoryStorage, self).persist() - logger.debug(f'Agent {self.role_id} persist memory into local') + logger.debug(f"Agent {self.role_id} persist memory into local") def add(self, message: Message) -> bool: - """ add message into memory storage""" + """add message into memory storage""" docs = [message.content] metadatas = [{"message_ser": serialize_message(message)}] if not self.store: @@ -79,10 +79,7 @@ class MemoryStorage(FaissStore): if not self.store: return [] - resp = self.store.similarity_search_with_score( - query=message.content, - k=k - ) + resp = self.store.similarity_search_with_score(query=message.content, k=k) # filter the result which score is smaller than the threshold filtered_resp = [] for item, score in resp: @@ -104,4 +101,3 @@ class MemoryStorage(FaissStore): self.store = None self._initialized = False - \ No newline at end of file diff --git a/metagpt/prompts/invoice_ocr.py b/metagpt/prompts/invoice_ocr.py index 52f628a5b..aa79651be 100644 --- a/metagpt/prompts/invoice_ocr.py +++ b/metagpt/prompts/invoice_ocr.py @@ -10,7 +10,9 @@ COMMON_PROMPT = "Now I will provide you with the OCR text recognition results for the invoice." -EXTRACT_OCR_MAIN_INFO_PROMPT = COMMON_PROMPT + """ +EXTRACT_OCR_MAIN_INFO_PROMPT = ( + COMMON_PROMPT + + """ Please extract the payee, city, total cost, and invoicing date of the invoice. The OCR data of the invoice are as follows: @@ -22,8 +24,11 @@ Mandatory restrictions are returned according to the following requirements: 2. The returned JSON dictionary must be returned in {language} 3. Mandatory requirement to output in JSON format: {{"收款人":"x","城市":"x","总费用/元":"","开票日期":""}}. """ +) -REPLY_OCR_QUESTION_PROMPT = COMMON_PROMPT + """ +REPLY_OCR_QUESTION_PROMPT = ( + COMMON_PROMPT + + """ Please answer the question: {query} The OCR data of the invoice are as follows: @@ -34,6 +39,6 @@ Mandatory restrictions are returned according to the following requirements: 2. Enforce restrictions on not returning OCR data sent to you. 3. Return with markdown syntax layout. """ +) INVOICE_OCR_SUCCESS = "Successfully completed OCR text recognition invoice." - diff --git a/metagpt/prompts/sales.py b/metagpt/prompts/sales.py index a44aacafe..30ef1ae02 100644 --- a/metagpt/prompts/sales.py +++ b/metagpt/prompts/sales.py @@ -54,10 +54,12 @@ Conversation history: {salesperson_name}: """ -conversation_stages = {'1' : "Introduction: Start the conversation by introducing yourself and your company. Be polite and respectful while keeping the tone of the conversation professional. Your greeting should be welcoming. Always clarify in your greeting the reason why you are contacting the prospect.", -'2': "Qualification: Qualify the prospect by confirming if they are the right person to talk to regarding your product/service. Ensure that they have the authority to make purchasing decisions.", -'3': "Value proposition: Briefly explain how your product/service can benefit the prospect. Focus on the unique selling points and value proposition of your product/service that sets it apart from competitors.", -'4': "Needs analysis: Ask open-ended questions to uncover the prospect's needs and pain points. Listen carefully to their responses and take notes.", -'5': "Solution presentation: Based on the prospect's needs, present your product/service as the solution that can address their pain points.", -'6': "Objection handling: Address any objections that the prospect may have regarding your product/service. Be prepared to provide evidence or testimonials to support your claims.", -'7': "Close: Ask for the sale by proposing a next step. This could be a demo, a trial or a meeting with decision-makers. Ensure to summarize what has been discussed and reiterate the benefits."} +conversation_stages = { + "1": "Introduction: Start the conversation by introducing yourself and your company. Be polite and respectful while keeping the tone of the conversation professional. Your greeting should be welcoming. Always clarify in your greeting the reason why you are contacting the prospect.", + "2": "Qualification: Qualify the prospect by confirming if they are the right person to talk to regarding your product/service. Ensure that they have the authority to make purchasing decisions.", + "3": "Value proposition: Briefly explain how your product/service can benefit the prospect. Focus on the unique selling points and value proposition of your product/service that sets it apart from competitors.", + "4": "Needs analysis: Ask open-ended questions to uncover the prospect's needs and pain points. Listen carefully to their responses and take notes.", + "5": "Solution presentation: Based on the prospect's needs, present your product/service as the solution that can address their pain points.", + "6": "Objection handling: Address any objections that the prospect may have regarding your product/service. Be prepared to provide evidence or testimonials to support your claims.", + "7": "Close: Ask for the sale by proposing a next step. This could be a demo, a trial or a meeting with decision-makers. Ensure to summarize what has been discussed and reiterate the benefits.", +} diff --git a/metagpt/prompts/tutorial_assistant.py b/metagpt/prompts/tutorial_assistant.py index d690aad83..3d4b6fa24 100644 --- a/metagpt/prompts/tutorial_assistant.py +++ b/metagpt/prompts/tutorial_assistant.py @@ -12,7 +12,9 @@ You are now a seasoned technical professional in the field of the internet. We need you to write a technical tutorial with the topic "{topic}". """ -DIRECTORY_PROMPT = COMMON_PROMPT + """ +DIRECTORY_PROMPT = ( + COMMON_PROMPT + + """ Please provide the specific table of contents for this tutorial, strictly following the following requirements: 1. The output must be strictly in the specified language, {language}. 2. Answer strictly in the dictionary format like {{"title": "xxx", "directory": [{{"dir 1": ["sub dir 1", "sub dir 2"]}}, {{"dir 2": ["sub dir 3", "sub dir 4"]}}]}}. @@ -20,8 +22,11 @@ Please provide the specific table of contents for this tutorial, strictly follow 4. Do not have extra spaces or line breaks. 5. Each directory title has practical significance. """ +) -CONTENT_PROMPT = COMMON_PROMPT + """ +CONTENT_PROMPT = ( + COMMON_PROMPT + + """ Now I will give you the module directory titles for the topic. Please output the detailed principle content of this title in detail. If there are code examples, please provide them according to standard code specifications. @@ -36,4 +41,5 @@ Strictly limit output according to the following requirements: 3. The output must be strictly in the specified language, {language}. 4. Do not have redundant output, including concluding remarks. 5. Strict requirement not to output the topic "{topic}". -""" \ No newline at end of file +""" +) diff --git a/metagpt/provider/anthropic_api.py b/metagpt/provider/anthropic_api.py index 7293e2cde..03802a716 100644 --- a/metagpt/provider/anthropic_api.py +++ b/metagpt/provider/anthropic_api.py @@ -32,4 +32,3 @@ class Claude2: max_tokens_to_sample=1000, ) return res.completion - \ No newline at end of file diff --git a/metagpt/provider/base_chatbot.py b/metagpt/provider/base_chatbot.py index 72e6c94f9..a6950f144 100644 --- a/metagpt/provider/base_chatbot.py +++ b/metagpt/provider/base_chatbot.py @@ -12,6 +12,7 @@ from dataclasses import dataclass @dataclass class BaseChatbot(ABC): """Abstract GPT class""" + mode: str = "API" use_system_prompt: bool = True @@ -26,4 +27,3 @@ class BaseChatbot(ABC): @abstractmethod def ask_code(self, msgs: list) -> str: """Ask GPT multiple questions and get a piece of code""" - \ No newline at end of file diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index b6b034329..c38576806 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -38,15 +38,19 @@ class BaseGPTAPI(BaseChatbot): rsp = self.completion(message) return self.get_choice_text(rsp) - async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: + async def aask(self, msg: str, system_msgs: Optional[list[str]] = None, stream=True) -> str: if system_msgs: - message = self._system_msgs(system_msgs) + [self._user_msg(msg)] if self.use_system_prompt \ + message = ( + self._system_msgs(system_msgs) + [self._user_msg(msg)] + if self.use_system_prompt else [self._user_msg(msg)] + ) else: - message = [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt \ - else [self._user_msg(msg)] - rsp = await self.acompletion_text(message, stream=True) + message = ( + [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)] + ) logger.debug(message) + rsp = await self.acompletion_text(message, stream=stream) # logger.debug(rsp) return rsp diff --git a/metagpt/provider/fireworks_api.py b/metagpt/provider/fireworks_api.py index 23126af2d..47ac9cf61 100644 --- a/metagpt/provider/fireworks_api.py +++ b/metagpt/provider/fireworks_api.py @@ -5,11 +5,10 @@ import openai from metagpt.config import CONFIG -from metagpt.provider.openai_api import OpenAIGPTAPI, CostManager, RateLimiter +from metagpt.provider.openai_api import CostManager, OpenAIGPTAPI, RateLimiter class FireWorksGPTAPI(OpenAIGPTAPI): - def __init__(self): self.__init_fireworks(CONFIG) self.llm = openai diff --git a/metagpt/provider/general_api_requestor.py b/metagpt/provider/general_api_requestor.py index 150f2f1e0..875122e8b 100644 --- a/metagpt/provider/general_api_requestor.py +++ b/metagpt/provider/general_api_requestor.py @@ -2,10 +2,10 @@ # -*- coding: utf-8 -*- # @Desc : General Async API for http-based LLM model -from typing import AsyncGenerator, Tuple, Union, Optional, Literal -import aiohttp import asyncio +from typing import AsyncGenerator, Tuple, Union +import aiohttp from openai.api_requestor import APIRequestor from metagpt.logs import logger @@ -26,9 +26,7 @@ class GeneralAPIRequestor(APIRequestor): ) """ - def _interpret_response_line( - self, rbody: str, rcode: int, rheaders, stream: bool - ) -> str: + def _interpret_response_line(self, rbody: str, rcode: int, rheaders, stream: bool) -> str: # just do nothing to meet the APIRequestor process and return the raw data # due to the openai sdk will convert the data into OpenAIResponse which we don't need in general cases. @@ -39,11 +37,9 @@ class GeneralAPIRequestor(APIRequestor): ) -> Tuple[Union[str, AsyncGenerator[str, None]], bool]: if stream and "text/event-stream" in result.headers.get("Content-Type", ""): return ( - self._interpret_response_line( - line, result.status, result.headers, stream=True - ) - async for line in result.content - ), True + self._interpret_response_line(line, result.status, result.headers, stream=True) + async for line in result.content + ), True else: try: await result.read() diff --git a/metagpt/provider/human_provider.py b/metagpt/provider/human_provider.py index 1d12f972f..c70a7f1a6 100644 --- a/metagpt/provider/human_provider.py +++ b/metagpt/provider/human_provider.py @@ -1,11 +1,13 @@ -''' +""" Filename: MetaGPT/metagpt/provider/human_provider.py Created Date: Wednesday, November 8th 2023, 11:55:46 pm Author: garylin2099 -''' +""" from typing import Optional -from metagpt.provider.base_gpt_api import BaseGPTAPI + from metagpt.logs import logger +from metagpt.provider.base_gpt_api import BaseGPTAPI + class HumanProvider(BaseGPTAPI): """Humans provide themselves as a 'model', which actually takes in human input as its response. diff --git a/metagpt/provider/open_llm_api.py b/metagpt/provider/open_llm_api.py index a6820b42b..f421e30c8 100644 --- a/metagpt/provider/open_llm_api.py +++ b/metagpt/provider/open_llm_api.py @@ -4,13 +4,13 @@ import openai -from metagpt.logs import logger from metagpt.config import CONFIG -from metagpt.provider.openai_api import OpenAIGPTAPI, CostManager, RateLimiter +from metagpt.logs import logger +from metagpt.provider.openai_api import CostManager, OpenAIGPTAPI, RateLimiter class OpenLLMCostManager(CostManager): - """ open llm model is self-host, it's free and without cost""" + """open llm model is self-host, it's free and without cost""" def update_cost(self, prompt_tokens, completion_tokens, model): """ @@ -32,7 +32,6 @@ class OpenLLMCostManager(CostManager): class OpenLLMGPTAPI(OpenAIGPTAPI): - def __init__(self): self.__init_openllm(CONFIG) self.llm = openai diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 2d9359b94..a73bb0aa0 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -115,7 +115,7 @@ class CostManager(metaclass=Singleton): def get_total_cost(self): """ Get the total cost of API calls. - + Returns: float: The total cost of API calls. """ @@ -157,6 +157,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if config.openai_api_type: openai.api_type = config.openai_api_type openai.api_version = config.openai_api_version + if config.openai_proxy: + openai.proxy = config.openai_proxy self.rpm = int(config.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: diff --git a/metagpt/provider/postprecess/base_postprecess_plugin.py b/metagpt/provider/postprecess/base_postprecess_plugin.py index 702a03194..0d1cfbb11 100644 --- a/metagpt/provider/postprecess/base_postprecess_plugin.py +++ b/metagpt/provider/postprecess/base_postprecess_plugin.py @@ -5,13 +5,15 @@ from typing import Union from metagpt.logs import logger -from metagpt.utils.repair_llm_raw_output import RepairType -from metagpt.utils.repair_llm_raw_output import repair_llm_raw_output, extract_content_from_output, \ - retry_parse_json_text +from metagpt.utils.repair_llm_raw_output import ( + RepairType, + extract_content_from_output, + repair_llm_raw_output, + retry_parse_json_text, +) class BasePostPrecessPlugin(object): - model = None # the plugin of the `model`, use to judge in `llm_postprecess` def run_repair_llm_output(self, output: str, schema: dict, req_key: str = "[/CONTENT]") -> Union[dict, list]: @@ -33,15 +35,15 @@ class BasePostPrecessPlugin(object): return parsed_data def run_repair_llm_raw_output(self, content: str, req_keys: list[str], repair_type: str = None) -> str: - """ inherited class can re-implement the function""" + """inherited class can re-implement the function""" return repair_llm_raw_output(content, req_keys=req_keys, repair_type=repair_type) def run_extract_content_from_output(self, content: str, right_key: str) -> str: - """ inherited class can re-implement the function""" + """inherited class can re-implement the function""" return extract_content_from_output(content, right_key=right_key) def run_retry_parse_json_text(self, content: str) -> Union[dict, list]: - """ inherited class can re-implement the function""" + """inherited class can re-implement the function""" logger.info(f"extracted json CONTENT from output:\n{content}") parsed_data = retry_parse_json_text(output=content) # should use output=content return parsed_data @@ -64,9 +66,5 @@ class BasePostPrecessPlugin(object): assert "/" in req_key # current, postprocess only deal the repair_llm_raw_output - new_output = self.run_repair_llm_output( - output=output, - schema=schema, - req_key=req_key - ) + new_output = self.run_repair_llm_output(output=output, schema=schema, req_key=req_key) return new_output diff --git a/metagpt/provider/postprecess/llm_output_postprecess.py b/metagpt/provider/postprecess/llm_output_postprecess.py index 4b5955061..85405543d 100644 --- a/metagpt/provider/postprecess/llm_output_postprecess.py +++ b/metagpt/provider/postprecess/llm_output_postprecess.py @@ -7,17 +7,14 @@ from typing import Union from metagpt.provider.postprecess.base_postprecess_plugin import BasePostPrecessPlugin -def llm_output_postprecess(output: str, schema: dict, req_key: str = "[/CONTENT]", - model_name: str = None) -> Union[dict, str]: +def llm_output_postprecess( + output: str, schema: dict, req_key: str = "[/CONTENT]", model_name: str = None +) -> Union[dict, str]: """ default use BasePostPrecessPlugin if there is not matched plugin. """ # TODO choose different model's plugin according to the model_name postprecess_plugin = BasePostPrecessPlugin() - result = postprecess_plugin.run( - output=output, - schema=schema, - req_key=req_key - ) + result = postprecess_plugin.run(output=output, schema=schema, req_key=req_key) return result diff --git a/metagpt/provider/spark_api.py b/metagpt/provider/spark_api.py index 55f7000ec..60c86f4dc 100644 --- a/metagpt/provider/spark_api.py +++ b/metagpt/provider/spark_api.py @@ -14,8 +14,7 @@ import json import ssl from time import mktime from typing import Optional -from urllib.parse import urlencode -from urllib.parse import urlparse +from urllib.parse import urlencode, urlparse from wsgiref.handlers import format_date_time import websocket # 使用websocket_client @@ -26,9 +25,8 @@ from metagpt.provider.base_gpt_api import BaseGPTAPI class SparkAPI(BaseGPTAPI): - def __init__(self): - logger.warning('当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。') + logger.warning("当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。") def ask(self, msg: str) -> str: message = [self._default_system_msg(), self._user_msg(msg)] @@ -49,7 +47,7 @@ class SparkAPI(BaseGPTAPI): async def acompletion_text(self, messages: list[dict], stream=False) -> str: # 不支持 - logger.error('该功能禁用。') + logger.error("该功能禁用。") w = GetMessageFromWeb(messages) return w.run() @@ -93,29 +91,26 @@ class GetMessageFromWeb: signature_origin += "GET " + self.path + " HTTP/1.1" # 进行hmac-sha256进行加密 - signature_sha = hmac.new(self.api_secret.encode('utf-8'), signature_origin.encode('utf-8'), - digestmod=hashlib.sha256).digest() + signature_sha = hmac.new( + self.api_secret.encode("utf-8"), signature_origin.encode("utf-8"), digestmod=hashlib.sha256 + ).digest() - signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding='utf-8') + signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding="utf-8") authorization_origin = f'api_key="{self.api_key}", algorithm="hmac-sha256", headers="host date request-line", signature="{signature_sha_base64}"' - authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8') + authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode(encoding="utf-8") # 将请求的鉴权参数组合为字典 - v = { - "authorization": authorization, - "date": date, - "host": self.host - } + v = {"authorization": authorization, "date": date, "host": self.host} # 拼接鉴权参数，生成url - url = self.spark_url + '?' + urlencode(v) + url = self.spark_url + "?" + urlencode(v) # 此处打印出建立连接时候的url,参考本demo的时候可取消上方打印的注释，比对相同参数时生成的url与自己代码生成的url是否一致 return url def __init__(self, text): self.text = text - self.ret = '' + self.ret = "" self.spark_appid = CONFIG.spark_appid self.spark_api_secret = CONFIG.spark_api_secret self.spark_api_key = CONFIG.spark_api_key @@ -124,15 +119,15 @@ class GetMessageFromWeb: def on_message(self, ws, message): data = json.loads(message) - code = data['header']['code'] + code = data["header"]["code"] if code != 0: ws.close() # 请求错误，则关闭socket - logger.critical(f'回答获取失败，响应信息反序列化之后为： {data}') + logger.critical(f"回答获取失败，响应信息反序列化之后为： {data}") return else: choices = data["payload"]["choices"] - seq = choices["seq"] # 服务端是流式返回，seq为返回的数据序号 + # seq = choices["seq"] # 服务端是流式返回，seq为返回的数据序号 status = choices["status"] # 服务端是流式返回，status用于判断信息是否传送完毕 content = choices["text"][0]["content"] # 本次接收到的回答文本 self.ret += content @@ -142,7 +137,7 @@ class GetMessageFromWeb: # 收到websocket错误的处理 def on_error(self, ws, error): # on_message方法处理接收到的信息，出现任何错误，都会调用这个方法 - logger.critical(f'通讯连接出错，【错误提示: {error}】') + logger.critical(f"通讯连接出错，【错误提示: {error}】") # 收到websocket关闭的处理 def on_close(self, ws, one, two): @@ -150,17 +145,12 @@ class GetMessageFromWeb: # 处理请求数据 def gen_params(self): - data = { - "header": { - "app_id": self.spark_appid, - "uid": "1234" - }, + "header": {"app_id": self.spark_appid, "uid": "1234"}, "parameter": { "chat": { # domain为必传参数 "domain": self.domain, - # 以下为可微调，非必传参数 # 注意：官方建议，temperature和top_k修改一个即可 "max_tokens": 2048, # 默认2048，模型回答的tokens的最大长度，即允许它输出文本的最长字数 @@ -168,11 +158,7 @@ class GetMessageFromWeb: "top_k": 4, # 取值为[1，6],默认为4。从k个候选中随机选择一个（非等概率） } }, - "payload": { - "message": { - "text": self.text - } - } + "payload": {"message": {"text": self.text}}, } return data @@ -189,17 +175,12 @@ class GetMessageFromWeb: return self._run(self.text) def _run(self, text_list): - - ws_param = self.WsParam( - self.spark_appid, - self.spark_api_key, - self.spark_api_secret, - self.spark_url, - text_list) + ws_param = self.WsParam(self.spark_appid, self.spark_api_key, self.spark_api_secret, self.spark_url, text_list) ws_url = ws_param.create_url() websocket.enableTrace(False) # 默认禁用 WebSocket 的跟踪功能 - ws = websocket.WebSocketApp(ws_url, on_message=self.on_message, on_error=self.on_error, on_close=self.on_close, - on_open=self.on_open) + ws = websocket.WebSocketApp( + ws_url, on_message=self.on_message, on_error=self.on_error, on_close=self.on_close, on_open=self.on_open + ) ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE}) return self.ret diff --git a/metagpt/provider/zhipuai/async_sse_client.py b/metagpt/provider/zhipuai/async_sse_client.py index b819fdc63..d7168202a 100644 --- a/metagpt/provider/zhipuai/async_sse_client.py +++ b/metagpt/provider/zhipuai/async_sse_client.py @@ -3,11 +3,10 @@ # @Desc : async_sse_client to make keep the use of Event to access response # refs to `https://github.com/zhipuai/zhipuai-sdk-python/blob/main/zhipuai/utils/sse_client.py` -from zhipuai.utils.sse_client import SSEClient, Event, _FIELD_SEPARATOR +from zhipuai.utils.sse_client import _FIELD_SEPARATOR, Event, SSEClient class AsyncSSEClient(SSEClient): - async def _aread(self): data = b"" async for chunk in self._event_source: @@ -37,9 +36,7 @@ class AsyncSSEClient(SSEClient): # Ignore unknown fields. if field not in event.__dict__: - self._logger.debug( - "Saw invalid field %s while parsing " "Server Side Event", field - ) + self._logger.debug("Saw invalid field %s while parsing " "Server Side Event", field) continue if len(data) > 1: diff --git a/metagpt/provider/zhipuai/zhipu_model_api.py b/metagpt/provider/zhipuai/zhipu_model_api.py index 618b2e865..23dd7229d 100644 --- a/metagpt/provider/zhipuai/zhipu_model_api.py +++ b/metagpt/provider/zhipuai/zhipu_model_api.py @@ -3,15 +3,14 @@ # @Desc : zhipu model api to support sync & async for invoke & sse_invoke import zhipuai -from zhipuai.model_api.api import ModelAPI, InvokeType +from zhipuai.model_api.api import InvokeType, ModelAPI from zhipuai.utils.http_client import headers as zhipuai_default_headers -from metagpt.provider.zhipuai.async_sse_client import AsyncSSEClient from metagpt.provider.general_api_requestor import GeneralAPIRequestor +from metagpt.provider.zhipuai.async_sse_client import AsyncSSEClient class ZhiPuModelAPI(ModelAPI): - @classmethod def get_header(cls) -> dict: token = cls._generate_token() @@ -21,9 +20,7 @@ class ZhiPuModelAPI(ModelAPI): @classmethod def get_sse_header(cls) -> dict: token = cls._generate_token() - headers = { - "Authorization": token - } + headers = {"Authorization": token} return headers @classmethod @@ -52,28 +49,24 @@ class ZhiPuModelAPI(ModelAPI): headers=headers, stream=stream, params=kwargs, - request_timeout=zhipuai.api_timeout_seconds + request_timeout=zhipuai.api_timeout_seconds, ) return result @classmethod async def ainvoke(cls, **kwargs) -> dict: - """ async invoke different from raw method `async_invoke` which get the final result by task_id""" + """async invoke different from raw method `async_invoke` which get the final result by task_id""" headers = cls.get_header() - resp = await cls.arequest(invoke_type=InvokeType.SYNC, - stream=False, - method="post", - headers=headers, - kwargs=kwargs) + resp = await cls.arequest( + invoke_type=InvokeType.SYNC, stream=False, method="post", headers=headers, kwargs=kwargs + ) return resp @classmethod async def asse_invoke(cls, **kwargs) -> AsyncSSEClient: - """ async sse_invoke """ + """async sse_invoke""" headers = cls.get_sse_header() - return AsyncSSEClient(await cls.arequest(invoke_type=InvokeType.SSE, - stream=True, - method="post", - headers=headers, - kwargs=kwargs)) + return AsyncSSEClient( + await cls.arequest(invoke_type=InvokeType.SSE, stream=True, method="post", headers=headers, kwargs=kwargs) + ) diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 3161c0e88..92119b764 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -2,19 +2,19 @@ # -*- coding: utf-8 -*- # @Desc : zhipuai LLM from https://open.bigmodel.cn/dev/api#sdk -from enum import Enum import json +from enum import Enum + +import openai +import zhipuai +from requests import ConnectionError from tenacity import ( after_log, retry, retry_if_exception_type, stop_after_attempt, - wait_fixed, + wait_random_exponential, ) -from requests import ConnectionError - -import openai -import zhipuai from metagpt.config import CONFIG from metagpt.logs import logger @@ -50,15 +50,11 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): openai.api_key = zhipuai.api_key # due to use openai sdk, set the api_key but it will't be used. def _const_kwargs(self, messages: list[dict]) -> dict: - kwargs = { - "model": self.model, - "prompt": messages, - "temperature": 0.3 - } + kwargs = {"model": self.model, "prompt": messages, "temperature": 0.3} return kwargs def _update_costs(self, usage: dict): - """ update each request's token cost """ + """update each request's token cost""" if CONFIG.calc_usage: try: prompt_tokens = int(usage.get("prompt_tokens", 0)) @@ -68,7 +64,7 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): logger.error("zhipuai updats costs failed!", e) def get_choice_text(self, resp: dict) -> str: - """ get the first text of choice from llm response """ + """get the first text of choice from llm response""" assist_msg = resp.get("data", {}).get("choices", [{"role": "error"}])[-1] assert assist_msg["role"] == "assistant" return assist_msg.get("content") @@ -126,13 +122,13 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): @retry( stop=stop_after_attempt(3), - wait=wait_fixed(1), + wait=wait_random_exponential(min=1, max=60), after=after_log(logger, logger.level("WARNING").name), retry=retry_if_exception_type(ConnectionError), - retry_error_callback=log_and_reraise + retry_error_callback=log_and_reraise, ) async def acompletion_text(self, messages: list[dict], stream=False) -> str: - """ response in async with stream or non-stream mode """ + """response in async with stream or non-stream mode""" if stream: return await self._achat_completion_stream(messages) resp = await self._achat_completion(messages) diff --git a/metagpt/repo_parser.py b/metagpt/repo_parser.py new file mode 100644 index 000000000..b84dbab9a --- /dev/null +++ b/metagpt/repo_parser.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/17 17:58 +@Author : alexanderwu +@File : repo_parser.py +""" +import ast +import json +from pathlib import Path +from pprint import pformat + +import pandas as pd +from pydantic import BaseModel, Field + +from metagpt.config import CONFIG +from metagpt.logs import logger + + +class RepoParser(BaseModel): + base_directory: Path = Field(default=None) + + def parse_file(self, file_path): + """Parse a Python file in the repository.""" + try: + return ast.parse(file_path.read_text()).body + except: + return [] + + def extract_class_and_function_info(self, tree, file_path): + """Extract class, function, and global variable information from the AST.""" + file_info = { + "file": str(file_path.relative_to(self.base_directory)), + "classes": [], + "functions": [], + "globals": [], + } + + for node in tree: + if isinstance(node, ast.ClassDef): + class_methods = [m.name for m in node.body if is_func(m)] + file_info["classes"].append({"name": node.name, "methods": class_methods}) + elif is_func(node): + file_info["functions"].append(node.name) + elif isinstance(node, (ast.Assign, ast.AnnAssign)): + for target in node.targets if isinstance(node, ast.Assign) else [node.target]: + if isinstance(target, ast.Name): + file_info["globals"].append(target.id) + return file_info + + def generate_symbols(self): + files_classes = [] + directory = self.base_directory + for path in directory.rglob("*.py"): + tree = self.parse_file(path) + file_info = self.extract_class_and_function_info(tree, path) + files_classes.append(file_info) + + return files_classes + + def generate_json_structure(self, output_path): + """Generate a JSON file documenting the repository structure.""" + files_classes = self.generate_symbols() + output_path.write_text(json.dumps(files_classes, indent=4)) + + def generate_dataframe_structure(self, output_path): + """Generate a DataFrame documenting the repository structure and save as CSV.""" + files_classes = self.generate_symbols() + df = pd.DataFrame(files_classes) + df.to_csv(output_path, index=False) + + def generate_structure(self, output_path=None, mode="json"): + """Generate the structure of the repository as a specified format.""" + output_file = self.base_directory / f"{self.base_directory.name}-structure.{mode}" + output_path = Path(output_path) if output_path else output_file + + if mode == "json": + self.generate_json_structure(output_path) + elif mode == "csv": + self.generate_dataframe_structure(output_path) + + +def is_func(node): + return isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + + +def main(): + repo_parser = RepoParser(base_directory=CONFIG.workspace_path / "web_2048") + symbols = repo_parser.generate_symbols() + logger.info(pformat(symbols)) + + +if __name__ == "__main__": + main() diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 15d5fe5b1..fa91d393d 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -26,8 +26,9 @@ class Architect(Role): self, name: str = "Bob", profile: str = "Architect", - goal: str = "Design a concise, usable, complete python system", - constraints: str = "Try to specify good open source tools as much as possible", + goal: str = "design a concise, usable, complete software system", + constraints: str = "make sure the architecture is simple enough and use appropriate open source libraries." + "Use same language as user requirement" ) -> None: """Initializes the Architect with given attributes.""" super().__init__(name, profile, goal, constraints) diff --git a/metagpt/roles/customer_service.py b/metagpt/roles/customer_service.py index 4547f8190..188182d47 100644 --- a/metagpt/roles/customer_service.py +++ b/metagpt/roles/customer_service.py @@ -24,12 +24,5 @@ DESC = """ class CustomerService(Sales): - def __init__( - self, - name="Xiaomei", - profile="Human customer service", - desc=DESC, - store=None - ): + def __init__(self, name="Xiaomei", profile="Human customer service", desc=DESC, store=None): super().__init__(name, profile, desc=desc, store=store) - \ No newline at end of file diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 1f6685b38..f1e65b177 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -4,46 +4,54 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : engineer.py +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116: + 1. Modify the data type of the `cause_by` value in the `Message` to a string, and utilize the new message + distribution feature for message filtering. + 2. Consolidate message reception and processing logic within `_observe`. + 3. Fix bug: Add logic for handling asynchronous message processing when messages are not ready. + 4. Supplemented the external transmission of internal messages. +@Modified By: mashenquan, 2023-11-27. + 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. + 2. According to the design in Section 2.2.3.5.5 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ -import asyncio -import shutil -from collections import OrderedDict -from pathlib import Path +from __future__ import annotations -from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks -from metagpt.const import WORKSPACE_ROOT +import json +from collections import defaultdict +from pathlib import Path +from typing import Set + +from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks +from metagpt.actions.fix_bug import FixBug +from metagpt.actions.summarize_code import SummarizeCode +from metagpt.config import CONFIG +from metagpt.const import ( + CODE_SUMMARIES_FILE_REPO, + CODE_SUMMARIES_PDF_FILE_REPO, + SYSTEM_DESIGN_FILE_REPO, + TASK_FILE_REPO, +) from metagpt.logs import logger from metagpt.roles import Role -from metagpt.schema import Message -from metagpt.utils.common import CodeParser -from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP +from metagpt.schema import ( + CodeSummarizeContext, + CodingContext, + Document, + Documents, + Message, +) +from metagpt.utils.common import any_to_str, any_to_str_set +IS_PASS_PROMPT = """ +{context} -async def gather_ordered_k(coros, k) -> list: - tasks = OrderedDict() - results = [None] * len(coros) - done_queue = asyncio.Queue() - - for i, coro in enumerate(coros): - if len(tasks) >= k: - done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED) - for task in done: - index = tasks.pop(task) - await done_queue.put((index, task.result())) - task = asyncio.create_task(coro) - tasks[task] = i - - if tasks: - done, _ = await asyncio.wait(tasks.keys()) - for task in done: - index = tasks[task] - await done_queue.put((index, task.result())) - - while not done_queue.empty(): - index, result = await done_queue.get() - results[index] = result - - return results +---- +Does the above log indicate anything that needs to be done? +If there are any tasks to be completed, please answer 'NO' along with the to-do list in JSON format; +otherwise, answer 'YES' in JSON format. +""" class Engineer(Role): @@ -57,119 +65,36 @@ class Engineer(Role): constraints (str): Constraints for the engineer. n_borg (int): Number of borgs. use_code_review (bool): Whether to use code review. - todos (list): List of tasks. """ def __init__( self, name: str = "Alex", profile: str = "Engineer", - goal: str = "Write elegant, readable, extensible, efficient code", - constraints: str = "The code should conform to standards like PEP8 and be modular and maintainable", + goal: str = "write elegant, readable, extensible, efficient code", + constraints: str = "the code should conform to standards like google-style and be modular and maintainable. " + "Use same language as user requirement", n_borg: int = 1, use_code_review: bool = False, ) -> None: """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) - self._init_actions([WriteCode]) self.use_code_review = use_code_review - if self.use_code_review: - self._init_actions([WriteCode, WriteCodeReview]) - self._watch([WriteTasks]) - self.todos = [] + self._init_actions([WriteCode]) + self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview, FixBug]) + self.code_todos = [] + self.summarize_todos = [] self.n_borg = n_borg - @classmethod - def parse_tasks(self, task_msg: Message) -> list[str]: - if task_msg.instruct_content: - return task_msg.instruct_content.dict().get("Task list") - return CodeParser.parse_file_list(block="Task list", text=task_msg.content) + @staticmethod + def _parse_tasks(task_msg: Document) -> list[str]: + m = json.loads(task_msg.content) + return m.get("Task list") - @classmethod - def parse_code(self, code_text: str) -> str: - return CodeParser.parse_code(block="", text=code_text) - - @classmethod - def parse_workspace(cls, system_design_msg: Message) -> str: - if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"') - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) - - def get_workspace(self) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] - if not msg: - return WORKSPACE_ROOT / "src" - workspace = self.parse_workspace(msg) - # Codes are written in workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace - - def recreate_workspace(self): - workspace = self.get_workspace() - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # The folder does not exist, but we don't care - workspace.mkdir(parents=True, exist_ok=True) - - def write_file(self, filename: str, code: str): - workspace = self.get_workspace() - filename = filename.replace('"', "").replace("\n", "") - file = workspace / filename - file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(code) - return file - - def recv(self, message: Message) -> None: - self._rc.memory.add(message) - if message in self._rc.important_memory: - self.todos = self.parse_tasks(message) - - async def _act_mp(self) -> Message: - # self.recreate_workspace() - todo_coros = [] - for todo in self.todos: - todo_coro = WriteCode().run( - context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo - ) - todo_coros.append(todo_coro) - - rsps = await gather_ordered_k(todo_coros, self.n_borg) - for todo, code_rsp in zip(self.todos, rsps): - _ = self.parse_code(code_rsp) - logger.info(todo) - logger.info(code_rsp) - # self.write_file(todo, code) - msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo)) - self._rc.memory.add(msg) - del self.todos[0] - - logger.info(f"Done {self.get_workspace()} generating.") - msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) - return msg - - async def _act_sp(self) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later - for todo in self.todos: - code = await WriteCode().run(context=self._rc.history, filename=todo) - # logger.info(todo) - # logger.info(code_rsp) - # code = self.parse_code(code_rsp) - file_path = self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo)) - self._rc.memory.add(msg) - - code_msg = todo + FILENAME_CODE_SEP + str(file_path) - code_msg_all.append(code_msg) - - logger.info(f"Done {self.get_workspace()} generating.") - msg = Message( - content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=type(self._rc.todo), send_to="QaEngineer" - ) - return msg - - async def _act_sp_precision(self) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later - for todo in self.todos: + async def _act_sp_with_cr(self, review=False) -> Set[str]: + changed_files = set() + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + for todo in self.code_todos: """ # Select essential information from the historical data to reduce the length of the prompt (summarized from human experience): 1. All from Architect @@ -177,37 +102,202 @@ class Engineer(Role): 3. Do we need other codes (currently needed)? TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. """ - context = [] - msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) - for m in msg: - context.append(m.content) - context_str = "\n".join(context) - # Write code - code = await WriteCode().run(context=context_str, filename=todo) + coding_context = await todo.run() # Code review - if self.use_code_review: - try: - rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) - code = rewrite_code - except Exception as e: - logger.error("code review failed!", e) - pass - file_path = self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=WriteCode) + if review: + action = WriteCodeReview(context=coding_context, llm=self._llm) + self._init_action_system_message(action) + coding_context = await action.run() + await src_file_repo.save( + coding_context.filename, + dependencies={coding_context.design_doc.root_relative_path, coding_context.task_doc.root_relative_path}, + content=coding_context.code_doc.content, + ) + msg = Message( + content=coding_context.json(), instruct_content=coding_context, role=self.profile, cause_by=WriteCode + ) self._rc.memory.add(msg) - code_msg = todo + FILENAME_CODE_SEP + str(file_path) - code_msg_all.append(code_msg) + changed_files.add(coding_context.code_doc.filename) + if not changed_files: + logger.info("Nothing has changed.") + return changed_files - logger.info(f"Done {self.get_workspace()} generating.") - msg = Message( - content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=type(self._rc.todo), send_to="QaEngineer" - ) - return msg - - async def _act(self) -> Message: + async def _act(self) -> Message | None: """Determines the mode of action based on whether code review is used.""" - logger.info(f"{self._setting}: ready to WriteCode") - if self.use_code_review: - return await self._act_sp_precision() - return await self._act_sp() + if self._rc.todo is None: + return None + if isinstance(self._rc.todo, WriteCode): + return await self._act_write_code() + if isinstance(self._rc.todo, SummarizeCode): + return await self._act_summarize() + return None + + async def _act_write_code(self): + changed_files = await self._act_sp_with_cr(review=self.use_code_review) + return Message( + content="\n".join(changed_files), + role=self.profile, + cause_by=WriteCodeReview if self.use_code_review else WriteCode, + send_to=self, + sent_from=self, + ) + + async def _act_summarize(self): + code_summaries_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO) + code_summaries_pdf_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO) + tasks = [] + src_relative_path = CONFIG.src_workspace.relative_to(CONFIG.git_repo.workdir) + for todo in self.summarize_todos: + summary = await todo.run() + summary_filename = Path(todo.context.design_filename).with_suffix(".md").name + dependencies = {todo.context.design_filename, todo.context.task_filename} + for filename in todo.context.codes_filenames: + rpath = src_relative_path / filename + dependencies.add(str(rpath)) + await code_summaries_pdf_file_repo.save( + filename=summary_filename, content=summary, dependencies=dependencies + ) + is_pass, reason = await self._is_pass(summary) + if not is_pass: + todo.context.reason = reason + tasks.append(todo.context.dict()) + await code_summaries_file_repo.save( + filename=Path(todo.context.design_filename).name, + content=todo.context.json(), + dependencies=dependencies, + ) + else: + await code_summaries_file_repo.delete(filename=Path(todo.context.design_filename).name) + + logger.info(f"--max-auto-summarize-code={CONFIG.max_auto_summarize_code}") + if not tasks or CONFIG.max_auto_summarize_code == 0: + return Message( + content="", + role=self.profile, + cause_by=SummarizeCode, + sent_from=self, + send_to="Edward", # The name of QaEngineer + ) + # The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. + # This parameter is used for debugging the workflow. + CONFIG.max_auto_summarize_code -= 1 if CONFIG.max_auto_summarize_code > 0 else 0 + return Message( + content=json.dumps(tasks), role=self.profile, cause_by=SummarizeCode, send_to=self, sent_from=self + ) + + async def _is_pass(self, summary) -> (str, str): + rsp = await self._llm.aask(msg=IS_PASS_PROMPT.format(context=summary), stream=False) + logger.info(rsp) + if "YES" in rsp: + return True, rsp + return False, rsp + + async def _think(self) -> Action | None: + if not CONFIG.src_workspace: + CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name + write_code_filters = any_to_str_set([WriteTasks, SummarizeCode, FixBug]) + summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) + if not self._rc.news: + return None + msg = self._rc.news[0] + if msg.cause_by in write_code_filters: + logger.debug(f"TODO WriteCode:{msg.json()}") + await self._new_code_actions(bug_fix=msg.cause_by == any_to_str(FixBug)) + return self._rc.todo + if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self): + logger.debug(f"TODO SummarizeCode:{msg.json()}") + await self._new_summarize_actions() + return self._rc.todo + return None + + @staticmethod + async def _new_coding_context( + filename, src_file_repo, task_file_repo, design_file_repo, dependency + ) -> CodingContext: + old_code_doc = await src_file_repo.get(filename) + if not old_code_doc: + old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content="") + dependencies = {Path(i) for i in await dependency.get(old_code_doc.root_relative_path)} + task_doc = None + design_doc = None + for i in dependencies: + if str(i.parent) == TASK_FILE_REPO: + task_doc = await task_file_repo.get(i.name) + elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO: + design_doc = await design_file_repo.get(i.name) + # FIXME: design doc没有加载进来，是None + context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc) + return context + + @staticmethod + async def _new_coding_doc(filename, src_file_repo, task_file_repo, design_file_repo, dependency): + context = await Engineer._new_coding_context( + filename, src_file_repo, task_file_repo, design_file_repo, dependency + ) + coding_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content=context.json()) + return coding_doc + + async def _new_code_actions(self, bug_fix=False): + # Prepare file repos + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + changed_src_files = src_file_repo.all_files if bug_fix else src_file_repo.changed_files + task_file_repo = CONFIG.git_repo.new_file_repository(TASK_FILE_REPO) + changed_task_files = task_file_repo.changed_files + design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) + + changed_files = Documents() + # Recode caused by upstream changes. + for filename in changed_task_files: + design_doc = await design_file_repo.get(filename) + task_doc = await task_file_repo.get(filename) + task_list = self._parse_tasks(task_doc) + for task_filename in task_list: + old_code_doc = await src_file_repo.get(task_filename) + if not old_code_doc: + old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=task_filename, content="") + context = CodingContext( + filename=task_filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc + ) + coding_doc = Document( + root_path=str(src_file_repo.root_path), filename=task_filename, content=context.json() + ) + if task_filename in changed_files.docs: + logger.warning( + f"Log to expose potential conflicts: {coding_doc.json()} & " + f"{changed_files.docs[task_filename].json()}" + ) + changed_files.docs[task_filename] = coding_doc + self.code_todos = [WriteCode(context=i, llm=self._llm) for i in changed_files.docs.values()] + # Code directly modified by the user. + dependency = await CONFIG.git_repo.get_dependency() + for filename in changed_src_files: + if filename in changed_files.docs: + continue + coding_doc = await self._new_coding_doc( + filename=filename, + src_file_repo=src_file_repo, + task_file_repo=task_file_repo, + design_file_repo=design_file_repo, + dependency=dependency, + ) + changed_files.docs[filename] = coding_doc + self.code_todos.append(WriteCode(context=coding_doc, llm=self._llm)) + + if self.code_todos: + self._rc.todo = self.code_todos[0] + + async def _new_summarize_actions(self): + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + src_files = src_file_repo.all_files + # Generate a SummarizeCode action for each pair of (system_design_doc, task_doc). + summarizations = defaultdict(list) + for filename in src_files: + dependencies = await src_file_repo.get_dependency(filename=filename) + ctx = CodeSummarizeContext.loads(filenames=dependencies) + summarizations[ctx].append(filename) + for ctx, filenames in summarizations.items(): + ctx.codes_filenames = filenames + self.summarize_todos.append(SummarizeCode(context=ctx, llm=self._llm)) + if self.summarize_todos: + self._rc.todo = self.summarize_todos[0] diff --git a/metagpt/roles/invoice_ocr_assistant.py b/metagpt/roles/invoice_ocr_assistant.py index 15f831c97..bf8fc454e 100644 --- a/metagpt/roles/invoice_ocr_assistant.py +++ b/metagpt/roles/invoice_ocr_assistant.py @@ -9,7 +9,7 @@ import pandas as pd -from metagpt.actions.invoice_ocr import InvoiceOCR, GenerateTable, ReplyQuestion +from metagpt.actions.invoice_ocr import GenerateTable, InvoiceOCR, ReplyQuestion from metagpt.prompts.invoice_ocr import INVOICE_OCR_SUCCESS from metagpt.roles import Role from metagpt.schema import Message diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index a58ea5385..e5e9f2b5e 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -4,8 +4,12 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : product_manager.py +@Modified By: mashenquan, 2023/11/27. Add `PrepareDocuments` action according to Section 2.2.3.5.1 of RFC 135. """ -from metagpt.actions import BossRequirement, WritePRD + +from metagpt.actions import UserRequirement, WritePRD +from metagpt.actions.prepare_documents import PrepareDocuments +from metagpt.config import CONFIG from metagpt.roles import Role @@ -24,8 +28,8 @@ class ProductManager(Role): self, name: str = "Alice", profile: str = "Product Manager", - goal: str = "Efficiently create a successful product", - constraints: str = "", + goal: str = "efficiently create a successful product", + constraints: str = "use same language as user requirement", ) -> None: """ Initializes the ProductManager role with given attributes. @@ -37,5 +41,17 @@ class ProductManager(Role): constraints (str): Constraints or limitations for the product manager. """ super().__init__(name, profile, goal, constraints) - self._init_actions([WritePRD]) - self._watch([BossRequirement]) + + self._init_actions([PrepareDocuments, WritePRD]) + self._watch([UserRequirement, PrepareDocuments]) + + async def _think(self) -> None: + """Decide what to do""" + if CONFIG.git_repo: + self._set_state(1) + else: + self._set_state(0) + return self._rc.todo + + async def _observe(self, ignore_memory=False) -> int: + return await super(ProductManager, self)._observe(ignore_memory=True) diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 7e7c5699d..5a2b9be50 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -25,8 +25,9 @@ class ProjectManager(Role): self, name: str = "Eve", profile: str = "Project Manager", - goal: str = "Improve team efficiency and deliver with quality and quantity", - constraints: str = "", + goal: str = "break down tasks according to PRD/technical design, generate a task list, and analyze task " + "dependencies to start with the prerequisite modules", + constraints: str = "use same language as user requirement", ) -> None: """ Initializes the ProjectManager role with given attributes. diff --git a/metagpt/roles/prompt.py b/metagpt/roles/prompt.py index c22e0226b..457ccb6c6 100644 --- a/metagpt/roles/prompt.py +++ b/metagpt/roles/prompt.py @@ -23,6 +23,7 @@ SUFFIX = """Let's begin! Question: {input} Thoughts: {agent_scratchpad}""" + class PromptString(Enum): REFLECTION_QUESTIONS = "Here are some statements:\n{memory_descriptions}\n\nBased solely on the information above, what are the 3 most prominent high-level questions we can answer about the topic in the statements?\n\n{format_instructions}" @@ -32,7 +33,7 @@ class PromptString(Enum): RECENT_ACTIVITY = "Based on the following memory, produce a brief summary of what {full_name} has been up to recently. Do not invent details not explicitly stated in the memory. For any conversation, be sure to mention whether the conversation has concluded or is still ongoing.\n\nMemory: {memory_descriptions}" - MAKE_PLANS = "You are a plan-generating AI. Your job is to assist the character in formulating new plans based on new information. Given the character's information (profile, objectives, recent activities, current plans, and location context) and their current thought process, produce a new set of plans for them. The final plan should comprise at least {time_window} of activities and no more than 5 individual plans. List the plans in the order they should be executed, with each plan detailing its description, location, start time, stop criteria, and maximum duration.\n\nSample plan: {{\"index\": 1, \"description\": \"Cook dinner\", \"location_id\": \"0a3bc22b-36aa-48ab-adb0-18616004caed\",\"start_time\": \"2022-12-12T20:00:00+00:00\",\"max_duration_hrs\": 1.5, \"stop_condition\": \"Dinner is fully prepared\"}}\'\n\nFor each plan, choose the most appropriate location name from this list: {allowed_location_descriptions}\n\n{format_instructions}\n\nAlways prioritize completing any unfinished conversations.\n\nLet's begin!\n\nName: {full_name}\nProfile: {private_bio}\nObjectives: {directives}\nLocation Context: {location_context}\nCurrent Plans: {current_plans}\nRecent Activities: {recent_activity}\nThought Process: {thought_process}\nIt's essential to encourage the character to collaborate with other characters in their plans.\n\n" + MAKE_PLANS = 'You are a plan-generating AI. Your job is to assist the character in formulating new plans based on new information. Given the character\'s information (profile, objectives, recent activities, current plans, and location context) and their current thought process, produce a new set of plans for them. The final plan should comprise at least {time_window} of activities and no more than 5 individual plans. List the plans in the order they should be executed, with each plan detailing its description, location, start time, stop criteria, and maximum duration.\n\nSample plan: {{"index": 1, "description": "Cook dinner", "location_id": "0a3bc22b-36aa-48ab-adb0-18616004caed","start_time": "2022-12-12T20:00:00+00:00","max_duration_hrs": 1.5, "stop_condition": "Dinner is fully prepared"}}\'\n\nFor each plan, choose the most appropriate location name from this list: {allowed_location_descriptions}\n\n{format_instructions}\n\nAlways prioritize completing any unfinished conversations.\n\nLet\'s begin!\n\nName: {full_name}\nProfile: {private_bio}\nObjectives: {directives}\nLocation Context: {location_context}\nCurrent Plans: {current_plans}\nRecent Activities: {recent_activity}\nThought Process: {thought_process}\nIt\'s essential to encourage the character to collaborate with other characters in their plans.\n\n' EXECUTE_PLAN = "You are a role-playing AI, playing the role of {your_name}, in front of a live audience. Every word you say can be observed by the audience, so make sure you speak often and make it interesting. You cannot interact directly with the audience.\n\nBased on the following context and tools, proceed as if you are {your_name}. Your primary task is to complete the task given below, however, if you are currently in a conversation with another character, you should always complete the conversation first and then proceed with the task. Do not start working while you are engaged in an unfinished conversation. Use your best judgment to determine whether a conversation involves you and whether it is unfinished. You don't need to respond to every message you receive.\n\nThis is a personal profile of your character:\n{your_private_bio}\n\nThis is a description of your location and other nearby characters you can converse with:\n\n{location_context}\n\nThese are some relevant memories:\n```\n{relevant_memories}\n```\n\nThis is some relevant conversation history:\n```\n{conversation_history}\n```\n\nThese are the tools you can use:\n{tools}\n\nYour response should follow the format below:\n\nTask: The task you must complete\nThoughts: What you should think about doing\nAction: The action to take, must be one of these words: [{tool_names}]\nAction Input: Input for the action\nObservation: The result of the action\n... (This Thoughts/Action/Action Input/Observation can be repeated N times)\nThoughts: 'I have completed the task'\nFinal Response: The final response to the task\n\nIf you are not ready with a final response, then you must take an action.\n\nIf you are certain that you cannot complete the task with the tools provided, return 'Final Response: Need Help', however, if you are in conversation with another character, responses like 'I don't know' are valid. During conversations, you should never break character or admit you are an AI.\nIf the task is completed and no specific response is required, return 'Final Response: Done'\nLet's begin!\n\nTask: {input}\n\n{agent_scratchpad}" @@ -40,6 +41,6 @@ class PromptString(Enum): GOSSIP = "You are {full_name}. \n{memory_descriptions}\n\nBased on the statements above, say a thing or two of interest to others at your location: {other_agent_names}.\nAlways specify their names when referring to others." - HAS_HAPPENED = "Given the descriptions of the observations of the following characters and the events they are awaiting, indicate whether the character has witnessed the event.\n{format_instructions}\n\nExample:\n\nObservations:\nJoe entered the office at 2023-05-04 08:00:00+00:00\nJoe said hi to Sally at 2023-05-04 08:05:00+00:00\nSally said hello to Joe at 2023-05-04 08:05:30+00:00\nRebecca started working at 2023-05-04 08:10:00+00:00\nJoe made some breakfast at 2023-05-04 08:15:00+00:00\n\nAwaiting: Sally responded to Joe\n\nYour response: '{{\"has_happened\": true, \"date_occured\": 2023-05-04 08:05:30+00:00}}'\n\nLet's begin!\n\nObservations:\n{memory_descriptions}\n\nAwaiting: {event_description}\n" + HAS_HAPPENED = 'Given the descriptions of the observations of the following characters and the events they are awaiting, indicate whether the character has witnessed the event.\n{format_instructions}\n\nExample:\n\nObservations:\nJoe entered the office at 2023-05-04 08:00:00+00:00\nJoe said hi to Sally at 2023-05-04 08:05:00+00:00\nSally said hello to Joe at 2023-05-04 08:05:30+00:00\nRebecca started working at 2023-05-04 08:10:00+00:00\nJoe made some breakfast at 2023-05-04 08:15:00+00:00\n\nAwaiting: Sally responded to Joe\n\nYour response: \'{{"has_happened": true, "date_occured": 2023-05-04 08:05:30+00:00}}\'\n\nLet\'s begin!\n\nObservations:\n{memory_descriptions}\n\nAwaiting: {event_description}\n' OUTPUT_FORMAT = "\n\n(Remember! Make sure your output always adheres to one of the following two formats:\n\nA. If you have completed the task:\nThoughts: 'I have completed the task'\nFinal Response: \n\nB. If you haven't completed the task:\nThoughts: \nAction: \nAction Input: \nObservation: )\n" diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index a763c2ce8..4439b9b19 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -4,24 +4,29 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : qa_engineer.py +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data + type of the `cause_by` value in the `Message` to a string, and utilize the new message filtering feature. +@Modified By: mashenquan, 2023-11-27. + 1. Following the think-act principle, solidify the task parameters when creating the + WriteTest/RunCode/DebugError object, rather than passing them in when calling the run function. + 2. According to Section 2.2.3.5.7 of RFC 135, change the method of transferring files from using the Message + to using file references. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ -import os -from pathlib import Path - -from metagpt.actions import ( - DebugError, - RunCode, - WriteCode, - WriteCodeReview, - WriteDesign, - WriteTest, +from metagpt.actions import DebugError, RunCode, WriteTest +from metagpt.actions.summarize_code import SummarizeCode +from metagpt.config import CONFIG +from metagpt.const import ( + MESSAGE_ROUTE_TO_NONE, + TEST_CODES_FILE_REPO, + TEST_OUTPUTS_FILE_REPO, ) -from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.roles import Role -from metagpt.schema import Message -from metagpt.utils.common import CodeParser, parse_recipient -from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP +from metagpt.schema import Document, Message, RunCodeContext, TestingContext +from metagpt.utils.common import any_to_str_set, parse_recipient +from metagpt.utils.file_repository import FileRepository class QaEngineer(Role): @@ -37,120 +42,103 @@ class QaEngineer(Role): self._init_actions( [WriteTest] ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates - self._watch([WriteCode, WriteCodeReview, WriteTest, RunCode, DebugError]) + self._watch([SummarizeCode, WriteTest, RunCode, DebugError]) self.test_round = 0 self.test_round_allowed = test_round_allowed - @classmethod - def parse_workspace(cls, system_design_msg: Message) -> str: - if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name") - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) - - def get_workspace(self, return_proj_dir=True) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] - if not msg: - return WORKSPACE_ROOT / "src" - workspace = self.parse_workspace(msg) - # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. - if return_proj_dir: - return WORKSPACE_ROOT / workspace - # development codes directory: workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace - - def write_file(self, filename: str, code: str): - workspace = self.get_workspace() / "tests" - file = workspace / filename - file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(code) - async def _write_test(self, message: Message) -> None: - code_msgs = message.content.split(MSG_SEP) - # result_msg_all = [] - for code_msg in code_msgs: + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + changed_files = set(src_file_repo.changed_files.keys()) + # Unit tests only. + if CONFIG.reqa_file and CONFIG.reqa_file not in changed_files: + changed_files.add(CONFIG.reqa_file) + tests_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) + for filename in changed_files: # write tests - file_name, file_path = code_msg.split(FILENAME_CODE_SEP) - code_to_test = open(file_path, "r").read() - if "test" in file_name: - continue # Engineer might write some test files, skip testing a test file - test_file_name = "test_" + file_name - test_file_path = self.get_workspace() / "tests" / test_file_name - logger.info(f"Writing {test_file_name}..") - test_code = await WriteTest().run( - code_to_test=code_to_test, - test_file_name=test_file_name, - # source_file_name=file_name, - source_file_path=file_path, - workspace=self.get_workspace(), + if not filename or "test" in filename: + continue + code_doc = await src_file_repo.get(filename) + test_doc = await tests_file_repo.get("test_" + code_doc.filename) + if not test_doc: + test_doc = Document( + root_path=str(tests_file_repo.root_path), filename="test_" + code_doc.filename, content="" + ) + logger.info(f"Writing {test_doc.filename}..") + context = TestingContext(filename=test_doc.filename, test_doc=test_doc, code_doc=code_doc) + context = await WriteTest(context=context, llm=self._llm).run() + await tests_file_repo.save( + filename=context.test_doc.filename, + content=context.test_doc.content, + dependencies={context.code_doc.root_relative_path}, ) - self.write_file(test_file_name, test_code) # prepare context for run tests in next round - command = ["python", f"tests/{test_file_name}"] - file_info = { - "file_name": file_name, - "file_path": str(file_path), - "test_file_name": test_file_name, - "test_file_path": str(test_file_path), - "command": command, - } - msg = Message( - content=str(file_info), - role=self.profile, - cause_by=WriteTest, - sent_from=self.profile, - send_to=self.profile, + run_code_context = RunCodeContext( + command=["python", context.test_doc.root_relative_path], + code_filename=context.code_doc.filename, + test_filename=context.test_doc.filename, + working_directory=str(CONFIG.git_repo.workdir), + additional_python_paths=[str(CONFIG.src_workspace)], + ) + self.publish_message( + Message( + content=run_code_context.json(), + role=self.profile, + cause_by=WriteTest, + sent_from=self, + send_to=self, + ) ) - self._publish_message(msg) - logger.info(f"Done {self.get_workspace()}/tests generating.") + logger.info(f"Done {str(tests_file_repo.workdir)} generating.") async def _run_code(self, msg): - file_info = eval(msg.content) - development_file_path = file_info["file_path"] - test_file_path = file_info["test_file_path"] - if not os.path.exists(development_file_path) or not os.path.exists(test_file_path): + run_code_context = RunCodeContext.loads(msg.content) + src_doc = await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).get(run_code_context.code_filename) + if not src_doc: return - - development_code = open(development_file_path, "r").read() - test_code = open(test_file_path, "r").read() - proj_dir = self.get_workspace() - development_code_dir = self.get_workspace(return_proj_dir=False) - - result_msg = await RunCode().run( - mode="script", - code=development_code, - code_file_name=file_info["file_name"], - test_code=test_code, - test_file_name=file_info["test_file_name"], - command=file_info["command"], - working_directory=proj_dir, # workspace/package_name, will run tests/test_xxx.py here - additional_python_paths=[development_code_dir], # workspace/package_name/package_name, - # import statement inside package code needs this + test_doc = await CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO).get(run_code_context.test_filename) + if not test_doc: + return + run_code_context.code = src_doc.content + run_code_context.test_code = test_doc.content + result = await RunCode(context=run_code_context, llm=self._llm).run() + run_code_context.output_filename = run_code_context.test_filename + ".json" + await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).save( + filename=run_code_context.output_filename, + content=result.json(), + dependencies={src_doc.root_relative_path, test_doc.root_relative_path}, + ) + run_code_context.code = None + run_code_context.test_code = None + recipient = parse_recipient(result.summary) # the recipient might be Engineer or myself + mappings = {"Engineer": "Alex", "QaEngineer": "Edward"} + self.publish_message( + Message( + content=run_code_context.json(), + role=self.profile, + cause_by=RunCode, + sent_from=self, + send_to=mappings.get(recipient, MESSAGE_ROUTE_TO_NONE), + ) ) - recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself - content = str(file_info) + FILENAME_CODE_SEP + result_msg - msg = Message(content=content, role=self.profile, cause_by=RunCode, sent_from=self.profile, send_to=recipient) - self._publish_message(msg) - async def _debug_error(self, msg): - file_info, context = msg.content.split(FILENAME_CODE_SEP) - file_name, code = await DebugError().run(context) - if file_name: - self.write_file(file_name, code) - recipient = msg.sent_from # send back to the one who ran the code for another run, might be one's self - msg = Message( - content=file_info, role=self.profile, cause_by=DebugError, sent_from=self.profile, send_to=recipient + run_code_context = RunCodeContext.loads(msg.content) + code = await DebugError(context=run_code_context, llm=self._llm).run() + await FileRepository.save_file( + filename=run_code_context.test_filename, content=code, relative_path=TEST_CODES_FILE_REPO + ) + run_code_context.output = None + self.publish_message( + Message( + content=run_code_context.json(), + role=self.profile, + cause_by=DebugError, + sent_from=self, + send_to=self, ) - self._publish_message(msg) - - async def _observe(self) -> int: - await super()._observe() - self._rc.news = [ - msg for msg in self._rc.news if msg.send_to == self.profile - ] # only relevant msgs count as observed news - return len(self._rc.news) + ) async def _act(self) -> Message: if self.test_round > self.test_round_allowed: @@ -159,28 +147,35 @@ class QaEngineer(Role): role=self.profile, cause_by=WriteTest, sent_from=self.profile, - send_to="", + send_to=MESSAGE_ROUTE_TO_NONE, ) return result_msg + code_filters = any_to_str_set({SummarizeCode}) + test_filters = any_to_str_set({WriteTest, DebugError}) + run_filters = any_to_str_set({RunCode}) for msg in self._rc.news: # Decide what to do based on observed msg type, currently defined by human, # might potentially be moved to _think, that is, let the agent decides for itself - if msg.cause_by in [WriteCode, WriteCodeReview]: + if msg.cause_by in code_filters: # engineer wrote a code, time to write a test for it await self._write_test(msg) - elif msg.cause_by in [WriteTest, DebugError]: + elif msg.cause_by in test_filters: # I wrote or debugged my test code, time to run it await self._run_code(msg) - elif msg.cause_by == RunCode: + elif msg.cause_by in run_filters: # I ran my test code, time to fix bugs, if any await self._debug_error(msg) self.test_round += 1 - result_msg = Message( + return Message( content=f"Round {self.test_round} of tests done", role=self.profile, cause_by=WriteTest, sent_from=self.profile, - send_to="", + send_to=MESSAGE_ROUTE_TO_NONE, ) - return result_msg + + async def _observe(self, ignore_memory=False) -> int: + # This role has events that trigger and execute themselves based on conditions, and cannot rely on the + # content of memory to activate. + return await super(QaEngineer, self)._observe(ignore_memory=True) diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index c5512121a..6a76b1a40 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -1,4 +1,9 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. +""" + import asyncio @@ -49,18 +54,18 @@ class Researcher(Role): research_system_text = get_research_system_text(topic, self.language) if isinstance(todo, CollectLinks): links = await todo.run(topic, 4, 4) - ret = Message("", Report(topic=topic, links=links), role=self.profile, cause_by=type(todo)) + ret = Message("", Report(topic=topic, links=links), role=self.profile, cause_by=todo) elif isinstance(todo, WebBrowseAndSummarize): links = instruct_content.links todos = (todo.run(*url, query=query, system_text=research_system_text) for (query, url) in links.items()) summaries = await asyncio.gather(*todos) summaries = list((url, summary) for i in summaries for (url, summary) in i.items() if summary) - ret = Message("", Report(topic=topic, summaries=summaries), role=self.profile, cause_by=type(todo)) + ret = Message("", Report(topic=topic, summaries=summaries), role=self.profile, cause_by=todo) else: summaries = instruct_content.summaries summary_text = "\n---\n".join(f"url: {url}\nsummary: {summary}" for (url, summary) in summaries) content = await self._rc.todo.run(topic, summary_text, system_text=research_system_text) - ret = Message("", Report(topic=topic, content=content), role=self.profile, cause_by=type(self._rc.todo)) + ret = Message("", Report(topic=topic, content=content), role=self.profile, cause_by=self._rc.todo) self._rc.memory.add(ret) return ret diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 140910f0a..b07541b09 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -4,24 +4,36 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : role.py +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116: + 1. Merge the `recv` functionality into the `_observe` function. Future message reading operations will be + consolidated within the `_observe` function. + 2. Standardize the message filtering for string label matching. Role objects can access the message labels + they've subscribed to through the `subscribed_tags` property. + 3. Move the message receive buffer from the global variable `self._rc.env.memory` to the role's private variable + `self._rc.msg_buffer` for easier message identification and asynchronous appending of messages. + 4. Standardize the way messages are passed: `publish_message` sends messages out, while `put_message` places + messages into the Role object's private message receive buffer. There are no other message transmit methods. + 5. Standardize the parameters for the `run` function: the `test_message` parameter is used for testing purposes + only. In the normal workflow, you should use `publish_message` or `put_message` to transmit messages. +@Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing + functionality is to be consolidated into the `Environment` class. """ from __future__ import annotations -from typing import Iterable, Type, Union from enum import Enum +from typing import Iterable, Set, Type from pydantic import BaseModel, Field -# from metagpt.environment import Environment -from metagpt.config import CONFIG from metagpt.actions import Action, ActionOutput +from metagpt.actions.action_node import ActionNode from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger -from metagpt.memory import Memory, LongTermMemory -from metagpt.schema import Message +from metagpt.memory import Memory +from metagpt.schema import Message, MessageQueue +from metagpt.utils.common import any_to_str from metagpt.utils.repair_llm_raw_output import extract_state_value_from_output - PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ STATE_TEMPLATE = """Here are your conversation records. You can decide which stage you should enter or stay in based on these records. @@ -64,6 +76,7 @@ class RoleReactMode(str, Enum): class RoleSetting(BaseModel): """Role Settings""" + name: str profile: str goal: str @@ -80,23 +93,28 @@ class RoleSetting(BaseModel): class RoleContext(BaseModel): """Role Runtime Context""" - env: 'Environment' = Field(default=None) + + env: "Environment" = Field(default=None) + msg_buffer: MessageQueue = Field(default_factory=MessageQueue) # Message Buffer with Asynchronous Updates memory: Memory = Field(default_factory=Memory) - long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) + # long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None todo: Action = Field(default=None) - watch: set[Type[Action]] = Field(default_factory=set) + watch: set[str] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) - react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes + react_mode: RoleReactMode = ( + RoleReactMode.REACT + ) # see `Role._set_react_mode` for definitions of the following two attributes max_react_loop: int = 1 class Config: arbitrary_types_allowed = True def check(self, role_id: str): - if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory: - self.long_term_memory.recover_memory(role_id, self) - self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation + # if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory: + # self.long_term_memory.recover_memory(role_id, self) + # self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation + pass @property def important_memory(self) -> list[Message]: @@ -113,17 +131,23 @@ class Role: def __init__(self, name="", profile="", goal="", constraints="", desc="", is_human=False): self._llm = LLM() if not is_human else HumanProvider() - self._setting = RoleSetting(name=name, profile=profile, goal=goal, - constraints=constraints, desc=desc, is_human=is_human) + self._setting = RoleSetting( + name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, is_human=is_human + ) + self._llm.system_prompt = self._get_prefix() self._states = [] self._actions = [] self._role_id = str(self._setting) self._rc = RoleContext() + self._subscription = {any_to_str(self), name} if name else {any_to_str(self)} def _reset(self): self._states = [] self._actions = [] + def _init_action_system_message(self, action: Action): + action.set_prefix(self._get_prefix(), self.profile) + def _init_actions(self, actions): self._reset() for idx, action in enumerate(actions): @@ -131,11 +155,14 @@ class Role: i = action("", llm=self._llm) else: if self._setting.is_human and not isinstance(action.llm, HumanProvider): - logger.warning(f"is_human attribute does not take effect, " - f"as Role's {str(action)} was initialized using LLM, " - f"try passing in Action classes instead of initialized instances") + logger.warning( + f"is_human attribute does not take effect, " + f"as Role's {str(action)} was initialized using LLM, " + f"try passing in Action classes instead of initialized instances" + ) i = action - i.set_prefix(self._get_prefix(), self.profile) + # i.set_env(self._rc.env) + self._init_action_system_message(i) self._actions.append(i) self._states.append(f"{idx}. {action}") @@ -161,26 +188,51 @@ class Role: self._rc.max_react_loop = max_react_loop def _watch(self, actions: Iterable[Type[Action]]): - """Listen to the corresponding behaviors""" - self._rc.watch.update(actions) + """Watch Actions of interest. Role will select Messages caused by these Actions from its personal message + buffer during _observe. + """ + tags = {any_to_str(t) for t in actions} + self._rc.watch.update(tags) # check RoleContext after adding watch actions self._rc.check(self._role_id) + def subscribe(self, tags: Set[str]): + """Used to receive Messages with certain tags from the environment. Message will be put into personal message + buffer to be further processed in _observe. By default, a Role subscribes Messages with a tag of its own name + or profile. + """ + self._subscription = tags + if self._rc.env: # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 + self._rc.env.set_subscription(self, self._subscription) + def _set_state(self, state: int): """Update the current state.""" self._rc.state = state logger.debug(self._actions) self._rc.todo = self._actions[self._rc.state] if state >= 0 else None - def set_env(self, env: 'Environment'): - """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" + def set_env(self, env: "Environment"): + """Set the environment in which the role works. The role can talk to the environment and can also receive + messages by observing.""" self._rc.env = env + if env: + env.set_subscription(self, self._subscription) @property def profile(self): """Get the role description (position)""" return self._setting.profile + @property + def name(self): + """Get virtual user name""" + return self._setting.name + + @property + def subscription(self) -> Set: + """The labels for messages to be consumed by the Role object.""" + return self._subscription + def _get_prefix(self): """Get the role prefix""" if self._setting.desc: @@ -194,15 +246,18 @@ class Role: self._set_state(0) return prompt = self._get_prefix() - prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states), - n_states=len(self._states) - 1, previous_state=self._rc.state) + prompt += STATE_TEMPLATE.format( + history=self._rc.history, + states="\n".join(self._states), + n_states=len(self._states) - 1, + previous_state=self._rc.state, + ) # print(prompt) next_state = await self._llm.aask(prompt) next_state = extract_state_value_from_output(next_state) logger.debug(f"{prompt=}") - if (not next_state.isdigit() and next_state != "-1") \ - or int(next_state) not in range(-1, len(self._states)): - logger.warning(f'Invalid answer of state, {next_state=}, will be set to -1') + if (not next_state.isdigit() and next_state != "-1") or int(next_state) not in range(-1, len(self._states)): + logger.warning(f"Invalid answer of state, {next_state=}, will be set to -1") next_state = -1 else: next_state = int(next_state) @@ -211,55 +266,64 @@ class Role: self._set_state(next_state) async def _act(self) -> Message: - # prompt = self.get_prefix() - # prompt += ROLE_TEMPLATE.format(name=self.profile, state=self.states[self.state], result=response, - # history=self.history) - logger.info(f"{self._setting}: ready to {self._rc.todo}") response = await self._rc.todo.run(self._rc.important_memory) - # logger.info(response) - if isinstance(response, ActionOutput): - msg = Message(content=response.content, instruct_content=response.instruct_content, - role=self.profile, cause_by=type(self._rc.todo)) + if isinstance(response, ActionOutput) or isinstance(response, ActionNode): + msg = Message( + content=response.content, + instruct_content=response.instruct_content, + role=self.profile, + cause_by=self._rc.todo, + sent_from=self, + ) + elif isinstance(response, Message): + msg = response else: - msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) + msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self) self._rc.memory.add(msg) - # logger.debug(f"{response}") return msg - async def _observe(self) -> int: - """Observe from the environment, obtain important information, and add it to memory""" - if not self._rc.env: - return 0 - env_msgs = self._rc.env.memory.get() - - observed = self._rc.env.memory.get_by_actions(self._rc.watch) - - self._rc.news = self._rc.memory.find_news(observed) # find news (previously unseen messages) from observed messages - - for i in env_msgs: - self.recv(i) + async def _observe(self, ignore_memory=False) -> int: + """Prepare new messages for processing from the message buffer and other sources.""" + # Read unprocessed messages from the msg buffer. + news = self._rc.msg_buffer.pop_all() + # Store the read messages in your own memory to prevent duplicate processing. + old_messages = [] if ignore_memory else self._rc.memory.get() + self._rc.memory.add_batch(news) + # Filter out messages of interest. + self._rc.news = [n for n in news if n.cause_by in self._rc.watch and n not in old_messages] + # Design Rules: + # If you need to further categorize Message objects, you can do so using the Message.set_meta function. + # msg_buffer is a receiving buffer, avoid adding message data and operations to msg_buffer. news_text = [f"{i.role}: {i.content[:20]}..." for i in self._rc.news] if news_text: - logger.debug(f'{self._setting} observed: {news_text}') + logger.debug(f"{self._setting} observed: {news_text}") return len(self._rc.news) - def _publish_message(self, msg): + def publish_message(self, msg): """If the role belongs to env, then the role's messages will be broadcast to env""" + if not msg: + return if not self._rc.env: # If env does not exist, do not publish the message return self._rc.env.publish_message(msg) + def put_message(self, message): + """Place the message into the Role object's private message buffer.""" + if not message: + return + self._rc.msg_buffer.push(message) + async def _react(self) -> Message: """Think first, then act, until the Role _think it is time to stop and requires no more todo. - This is the standard think-act loop in the ReAct paper, which alternates thinking and acting in task solving, i.e. _think -> _act -> _think -> _act -> ... + This is the standard think-act loop in the ReAct paper, which alternates thinking and acting in task solving, i.e. _think -> _act -> _think -> _act -> ... Use llm to select actions in _think dynamically """ actions_taken = 0 - rsp = Message("No actions taken yet") # will be overwritten after Role _act + rsp = Message("No actions taken yet") # will be overwritten after Role _act while actions_taken < self._rc.max_react_loop: # think await self._think() @@ -267,16 +331,16 @@ class Role: break # act logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") - rsp = await self._act() + rsp = await self._act() # 这个rsp是否需要publish_message？ actions_taken += 1 - return rsp # return output from the last action + return rsp # return output from the last action async def _act_by_order(self) -> Message: """switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...""" for i in range(len(self._states)): self._set_state(i) rsp = await self._act() - return rsp # return output from the last action + return rsp # return output from the last action async def _plan_and_act(self) -> Message: """first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... Use llm to come up with the plan dynamically.""" @@ -291,43 +355,56 @@ class Role: rsp = await self._act_by_order() elif self._rc.react_mode == RoleReactMode.PLAN_AND_ACT: rsp = await self._plan_and_act() - self._set_state(state=-1) # current reaction is complete, reset state to -1 and todo back to None + self._set_state(state=-1) # current reaction is complete, reset state to -1 and todo back to None return rsp - def recv(self, message: Message) -> None: - """add message to history.""" - # self._history += f"\n{message}" - # self._context = self._history - if message in self._rc.memory.get(): - return - self._rc.memory.add(message) + # # Replaced by run() + # def recv(self, message: Message) -> None: + # """add message to history.""" + # # self._history += f"\n{message}" + # # self._context = self._history + # if message in self._rc.memory.get(): + # return + # self._rc.memory.add(message) - async def handle(self, message: Message) -> Message: - """Receive information and reply with actions""" - # logger.debug(f"{self.name=}, {self.profile=}, {message.role=}") - self.recv(message) - - return await self._react() + # # Replaced by run() + # async def handle(self, message: Message) -> Message: + # """Receive information and reply with actions""" + # # logger.debug(f"{self.name=}, {self.profile=}, {message.role=}") + # self.recv(message) + # + # return await self._react() def get_memories(self, k=0) -> list[Message]: """A wrapper to return the most recent k memories of this role, return all when k=0""" return self._rc.memory.get(k=k) - async def run(self, message=None): + async def run(self, with_message=None): """Observe, and think and act based on the results of the observation""" - if message: - if isinstance(message, str): - message = Message(message) - if isinstance(message, Message): - self.recv(message) - if isinstance(message, list): - self.recv(Message("\n".join(message))) - elif not await self._observe(): + if with_message: + msg = None + if isinstance(with_message, str): + msg = Message(with_message) + elif isinstance(with_message, Message): + msg = with_message + elif isinstance(with_message, list): + msg = Message("\n".join(with_message)) + self.put_message(msg) + + if not await self._observe(): # If there is no new information, suspend and wait logger.debug(f"{self._setting}: no news. waiting.") return rsp = await self.react() - # Publish the reply to the environment, waiting for the next subscriber to process - self._publish_message(rsp) + + # Reset the next action to be taken. + self._rc.todo = None + # Send the response message to the Environment object to have it relay the message to the subscribers. + self.publish_message(rsp) return rsp + + @property + def is_idle(self) -> bool: + """If true, all actions have been executed.""" + return not self._rc.news and not self._rc.todo and self._rc.msg_buffer.empty() diff --git a/metagpt/roles/searcher.py b/metagpt/roles/searcher.py index 3a19f32f2..5760202ff 100644 --- a/metagpt/roles/searcher.py +++ b/metagpt/roles/searcher.py @@ -4,8 +4,11 @@ @Time : 2023/5/23 17:25 @Author : alexanderwu @File : searcher.py +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ from metagpt.actions import ActionOutput, SearchAndSummarize +from metagpt.actions.action_node import ActionNode from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -15,7 +18,7 @@ from metagpt.tools import SearchEngineType class Searcher(Role): """ Represents a Searcher role responsible for providing search services to users. - + Attributes: name (str): Name of the searcher. profile (str): Role profile. @@ -23,17 +26,19 @@ class Searcher(Role): constraints (str): Constraints or limitations for the searcher. engine (SearchEngineType): The type of search engine to use. """ - - def __init__(self, - name: str = 'Alice', - profile: str = 'Smart Assistant', - goal: str = 'Provide search services for users', - constraints: str = 'Answer is rich and complete', - engine=SearchEngineType.SERPAPI_GOOGLE, - **kwargs) -> None: + + def __init__( + self, + name: str = "Alice", + profile: str = "Smart Assistant", + goal: str = "Provide search services for users", + constraints: str = "Answer is rich and complete", + engine=SearchEngineType.SERPAPI_GOOGLE, + **kwargs, + ) -> None: """ Initializes the Searcher role with given attributes. - + Args: name (str): Name of the searcher. profile (str): Role profile. @@ -53,12 +58,16 @@ class Searcher(Role): """Performs the search action in a single process.""" logger.info(f"{self._setting}: ready to {self._rc.todo}") response = await self._rc.todo.run(self._rc.memory.get(k=0)) - - if isinstance(response, ActionOutput): - msg = Message(content=response.content, instruct_content=response.instruct_content, - role=self.profile, cause_by=type(self._rc.todo)) + + if isinstance(response, ActionOutput) or isinstance(response, ActionNode): + msg = Message( + content=response.content, + instruct_content=response.instruct_content, + role=self.profile, + cause_by=self._rc.todo, + ) else: - msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) + msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) self._rc.memory.add(msg) return msg diff --git a/metagpt/roles/sk_agent.py b/metagpt/roles/sk_agent.py index b27841d74..56482ef26 100644 --- a/metagpt/roles/sk_agent.py +++ b/metagpt/roles/sk_agent.py @@ -4,12 +4,14 @@ @Time : 2023/9/13 12:23 @Author : femto Zheng @File : sk_agent.py +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, utilize the new message + distribution feature for message filtering. """ from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner from semantic_kernel.planning.basic_planner import BasicPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.actions.execute_task import ExecuteTask from metagpt.logs import logger from metagpt.roles import Role @@ -39,7 +41,7 @@ class SkAgent(Role): """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self._init_actions([ExecuteTask()]) - self._watch([BossRequirement]) + self._watch([UserRequirement]) self.kernel = make_sk_kernel() # how funny the interface is inconsistent @@ -70,7 +72,6 @@ class SkAgent(Role): result = (await self.plan.invoke_async()).result logger.info(result) - msg = Message(content=result, role=self.profile, cause_by=type(self._rc.todo)) + msg = Message(content=result, role=self.profile, cause_by=self._rc.todo) self._rc.memory.add(msg) - # logger.debug(f"{response}") return msg diff --git a/metagpt/roles/tutorial_assistant.py b/metagpt/roles/tutorial_assistant.py index 9a7df4f4d..2a514f433 100644 --- a/metagpt/roles/tutorial_assistant.py +++ b/metagpt/roles/tutorial_assistant.py @@ -9,7 +9,7 @@ from datetime import datetime from typing import Dict -from metagpt.actions.write_tutorial import WriteDirectory, WriteContent +from metagpt.actions.write_tutorial import WriteContent, WriteDirectory from metagpt.const import TUTORIAL_PATH from metagpt.logs import logger from metagpt.roles import Role @@ -110,5 +110,5 @@ class TutorialAssistant(Role): break msg = await self._act() root_path = TUTORIAL_PATH / datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - await File.write(root_path, f"{self.main_title}.md", self.total_content.encode('utf-8')) + await File.write(root_path, f"{self.main_title}.md", self.total_content.encode("utf-8")) return msg diff --git a/metagpt/schema.py b/metagpt/schema.py index bdca093c2..758149efa 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -4,15 +4,38 @@ @Time : 2023/5/8 22:12 @Author : alexanderwu @File : schema.py +@Modified By: mashenquan, 2023-10-31. According to Chapter 2.2.1 of RFC 116: + Replanned the distribution of responsibilities and functional positioning of `Message` class attributes. +@Modified By: mashenquan, 2023/11/22. + 1. Add `Document` and `Documents` for `FileRepository` in Section 2.2.3.4 of RFC 135. + 2. Encapsulate the common key-values set to pydantic structures to standardize and unify parameter passing + between actions. + 3. Add `id` to `Message` according to Section 2.2.3.1.1 of RFC 135. """ from __future__ import annotations -from dataclasses import dataclass, field -from typing import Type, TypedDict +import asyncio +import json +import os.path +import uuid +from asyncio import Queue, QueueEmpty, wait_for +from json import JSONDecodeError +from pathlib import Path +from typing import Dict, List, Optional, Set, TypedDict -from pydantic import BaseModel +from pydantic import BaseModel, Field +from metagpt.config import CONFIG +from metagpt.const import ( + MESSAGE_ROUTE_CAUSE_BY, + MESSAGE_ROUTE_FROM, + MESSAGE_ROUTE_TO, + MESSAGE_ROUTE_TO_ALL, + SYSTEM_DESIGN_FILE_REPO, + TASK_FILE_REPO, +) from metagpt.logs import logger +from metagpt.utils.common import any_to_str, any_to_str_set class RawMessage(TypedDict): @@ -20,16 +43,106 @@ class RawMessage(TypedDict): role: str -@dataclass -class Message: +class Document(BaseModel): + """ + Represents a document. + """ + + root_path: str = "" + filename: str = "" + content: str = "" + + def get_meta(self) -> Document: + """Get metadata of the document. + + :return: A new Document instance with the same root path and filename. + """ + + return Document(root_path=self.root_path, filename=self.filename) + + @property + def root_relative_path(self): + """Get relative path from root of git repository. + + :return: relative path from root of git repository. + """ + return os.path.join(self.root_path, self.filename) + + @property + def full_path(self): + if not CONFIG.git_repo: + return None + return str(CONFIG.git_repo.workdir / self.root_path / self.filename) + + def __str__(self): + return self.content + + def __repr__(self): + return self.content + + +class Documents(BaseModel): + """A class representing a collection of documents. + + Attributes: + docs (Dict[str, Document]): A dictionary mapping document names to Document instances. + """ + + docs: Dict[str, Document] = Field(default_factory=dict) + + +class Message(BaseModel): """list[: ]""" + + id: str # According to Section 2.2.3.1.1 of RFC 135 content: str - instruct_content: BaseModel = field(default=None) - role: str = field(default='user') # system / user / assistant - cause_by: Type["Action"] = field(default="") - sent_from: str = field(default="") - send_to: str = field(default="") - restricted_to: str = field(default="") + instruct_content: BaseModel = Field(default=None) + role: str = "user" # system / user / assistant + cause_by: str = "" + sent_from: str = "" + send_to: Set = Field(default_factory={MESSAGE_ROUTE_TO_ALL}) + + def __init__( + self, + content, + instruct_content=None, + role="user", + cause_by="", + sent_from="", + send_to=MESSAGE_ROUTE_TO_ALL, + **kwargs, + ): + """ + Parameters not listed below will be stored as meta info, including custom parameters. + :param content: Message content. + :param instruct_content: Message content struct. + :param cause_by: Message producer + :param sent_from: Message route info tells who sent this message. + :param send_to: Specifies the target recipient or consumer for message delivery in the environment. + :param role: Message meta info tells who sent this message. + """ + super().__init__( + id=uuid.uuid4().hex, + content=content, + instruct_content=instruct_content, + role=role, + cause_by=any_to_str(cause_by), + sent_from=any_to_str(sent_from), + send_to=any_to_str_set(send_to), + **kwargs, + ) + + def __setattr__(self, key, val): + """Override `@property.setter`, convert non-string parameters into string parameters.""" + if key == MESSAGE_ROUTE_CAUSE_BY: + new_val = any_to_str(val) + elif key == MESSAGE_ROUTE_FROM: + new_val = any_to_str(val) + elif key == MESSAGE_ROUTE_TO: + new_val = any_to_str_set(val) + else: + new_val = val + super().__setattr__(key, new_val) def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) @@ -39,45 +152,202 @@ class Message: return self.__str__() def to_dict(self) -> dict: - return { - "role": self.role, - "content": self.content - } + """Return a dict containing `role` and `content` for the LLM call.l""" + return {"role": self.role, "content": self.content} + + def dump(self) -> str: + """Convert the object to json string""" + return self.json(exclude_none=True) + + @staticmethod + def load(val): + """Convert the json string to object.""" + try: + d = json.loads(val) + return Message(**d) + except JSONDecodeError as err: + logger.error(f"parse json failed: {val}, error:{err}") + return None -@dataclass class UserMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ + def __init__(self, content: str): - super().__init__(content, 'user') + super().__init__(content=content, role="user") -@dataclass class SystemMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ + def __init__(self, content: str): - super().__init__(content, 'system') + super().__init__(content=content, role="system") -@dataclass class AIMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ + def __init__(self, content: str): - super().__init__(content, 'assistant') + super().__init__(content=content, role="assistant") -if __name__ == '__main__': - test_content = 'test_message' - msgs = [ - UserMessage(test_content), - SystemMessage(test_content), - AIMessage(test_content), - Message(test_content, role='QA') - ] - logger.info(msgs) +class MessageQueue: + """Message queue which supports asynchronous updates.""" + + def __init__(self): + self._queue = Queue() + + def pop(self) -> Message | None: + """Pop one message from the queue.""" + try: + item = self._queue.get_nowait() + if item: + self._queue.task_done() + return item + except QueueEmpty: + return None + + def pop_all(self) -> List[Message]: + """Pop all messages from the queue.""" + ret = [] + while True: + msg = self.pop() + if not msg: + break + ret.append(msg) + return ret + + def push(self, msg: Message): + """Push a message into the queue.""" + self._queue.put_nowait(msg) + + def empty(self): + """Return true if the queue is empty.""" + return self._queue.empty() + + async def dump(self) -> str: + """Convert the `MessageQueue` object to a json string.""" + if self.empty(): + return "[]" + + lst = [] + try: + while True: + item = await wait_for(self._queue.get(), timeout=1.0) + if item is None: + break + lst.append(item.dict(exclude_none=True)) + self._queue.task_done() + except asyncio.TimeoutError: + logger.debug("Queue is empty, exiting...") + return json.dumps(lst) + + @staticmethod + def load(self, v) -> "MessageQueue": + """Convert the json string to the `MessageQueue` object.""" + q = MessageQueue() + try: + lst = json.loads(v) + for i in lst: + msg = Message(**i) + q.push(msg) + except JSONDecodeError as e: + logger.warning(f"JSON load failed: {v}, error:{e}") + + return q + + +class CodingContext(BaseModel): + filename: str + design_doc: Optional[Document] + task_doc: Optional[Document] + code_doc: Optional[Document] + + @staticmethod + def loads(val: str) -> CodingContext | None: + try: + m = json.loads(val) + return CodingContext(**m) + except Exception: + return None + + +class TestingContext(BaseModel): + filename: str + code_doc: Document + test_doc: Optional[Document] + + @staticmethod + def loads(val: str) -> TestingContext | None: + try: + m = json.loads(val) + return TestingContext(**m) + except Exception: + return None + + +class RunCodeContext(BaseModel): + mode: str = "script" + code: Optional[str] + code_filename: str = "" + test_code: Optional[str] + test_filename: str = "" + command: List[str] = Field(default_factory=list) + working_directory: str = "" + additional_python_paths: List[str] = Field(default_factory=list) + output_filename: Optional[str] + output: Optional[str] + + @staticmethod + def loads(val: str) -> RunCodeContext | None: + try: + m = json.loads(val) + return RunCodeContext(**m) + except Exception: + return None + + +class RunCodeResult(BaseModel): + summary: str + stdout: str + stderr: str + + @staticmethod + def loads(val: str) -> RunCodeResult | None: + try: + m = json.loads(val) + return RunCodeResult(**m) + except Exception: + return None + + +class CodeSummarizeContext(BaseModel): + design_filename: str = "" + task_filename: str = "" + codes_filenames: List[str] = Field(default_factory=list) + reason: str = "" + + @staticmethod + def loads(filenames: List) -> CodeSummarizeContext: + ctx = CodeSummarizeContext() + for filename in filenames: + if Path(filename).is_relative_to(SYSTEM_DESIGN_FILE_REPO): + ctx.design_filename = str(filename) + continue + if Path(filename).is_relative_to(TASK_FILE_REPO): + ctx.task_filename = str(filename) + continue + return ctx + + def __hash__(self): + return hash((self.design_filename, self.task_filename)) + + +class BugFixContext(BaseModel): + filename: str = "" diff --git a/metagpt/software_company.py b/metagpt/software_company.py deleted file mode 100644 index d44a0068a..000000000 --- a/metagpt/software_company.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/12 00:30 -@Author : alexanderwu -@File : software_company.py -""" -from metagpt.team import Team as SoftwareCompany - -import warnings -warnings.warn("metagpt.software_company is deprecated and will be removed in the future" - "Please use metagpt.team instead. SoftwareCompany class is now named as Team.", - DeprecationWarning, 2) diff --git a/metagpt/startup.py b/metagpt/startup.py new file mode 100644 index 000000000..f930c386b --- /dev/null +++ b/metagpt/startup.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import asyncio +from pathlib import Path + +import typer + +from metagpt.config import CONFIG + +app = typer.Typer() + + +@app.command() +def startup( + idea: str = typer.Argument(..., help="Your innovative idea, such as 'Create a 2048 game.'"), + investment: float = typer.Option(default=3.0, help="Dollar amount to invest in the AI company."), + n_round: int = typer.Option(default=5, help="Number of rounds for the simulation."), + code_review: bool = typer.Option(default=True, help="Whether to use code review."), + run_tests: bool = typer.Option(default=False, help="Whether to enable QA for adding & running tests."), + implement: bool = typer.Option(default=True, help="Enable or disable code implementation."), + project_name: str = typer.Option(default="", help="Unique project name, such as 'game_2048'."), + inc: bool = typer.Option(default=False, help="Incremental mode. Use it to coop with existing repo."), + project_path: str = typer.Option( + default="", + help="Specify the directory path of the old version project to fulfill the " "incremental requirements.", + ), + reqa_file: str = typer.Option(default="", help="Specify the source file name for rewriting the quality test code."), + max_auto_summarize_code: int = typer.Option( + default=-1, + help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. This parameter is used for debugging the workflow.", + ), +): + """Run a startup. Be a boss.""" + from metagpt.roles import ( + Architect, + Engineer, + ProductManager, + ProjectManager, + QaEngineer, + ) + from metagpt.team import Team + + # Use in the PrepareDocuments action according to Section 2.2.3.5.1 of RFC 135. + CONFIG.project_path = project_path + if project_path: + inc = True + project_name = project_name or Path(project_path).name + CONFIG.project_name = project_name + CONFIG.inc = inc + CONFIG.reqa_file = reqa_file + CONFIG.max_auto_summarize_code = max_auto_summarize_code + + company = Team() + company.hire( + [ + ProductManager(), + Architect(), + ProjectManager(), + ] + ) + + if implement or code_review: + company.hire([Engineer(n_borg=5, use_code_review=code_review)]) + + if run_tests: + company.hire([QaEngineer()]) + + company.invest(investment) + company.run_project(idea) + asyncio.run(company.run(n_round=n_round)) + + +if __name__ == "__main__": + app() diff --git a/metagpt/team.py b/metagpt/team.py index 67d3ecec8..a5c405f80 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -4,11 +4,14 @@ @Time : 2023/5/12 00:30 @Author : alexanderwu @File : software_company.py +@Modified By: mashenquan, 2023/11/27. Add an archiving operation after completing the project, as specified in + Section 2.2.3.3 of RFC 135. """ from pydantic import BaseModel, Field -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.config import CONFIG +from metagpt.const import MESSAGE_ROUTE_TO_ALL from metagpt.environment import Environment from metagpt.logs import logger from metagpt.roles import Role @@ -21,7 +24,8 @@ class Team(BaseModel): Team: Possesses one or more roles (agents), SOP (Standard Operating Procedures), and a platform for instant messaging, dedicated to perform any multi-agent activity, such as collaboratively writing executable code. """ - environment: Environment = Field(default_factory=Environment) + + env: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) idea: str = Field(default="") @@ -30,33 +34,38 @@ class Team(BaseModel): def hire(self, roles: list[Role]): """Hire roles to cooperate""" - self.environment.add_roles(roles) + self.env.add_roles(roles) def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" self.investment = investment CONFIG.max_budget = investment - logger.info(f'Investment: ${investment}.') + logger.info(f"Investment: ${investment}.") def _check_balance(self): if CONFIG.total_cost > CONFIG.max_budget: - raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') + raise NoMoneyException(CONFIG.total_cost, f"Insufficient funds: {CONFIG.max_budget}") - def start_project(self, idea, send_to: str = ""): - """Start a project from publishing boss requirement.""" + def run_project(self, idea, send_to: str = ""): + """Start a project from publishing user requirement.""" self.idea = idea - self.environment.publish_message(Message(role="Human", content=idea, cause_by=BossRequirement, send_to=send_to)) + + # Human requirement. + self.env.publish_message( + Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to or MESSAGE_ROUTE_TO_ALL) + ) def _save(self): - logger.info(self.json()) + logger.info(self.json(ensure_ascii=False)) async def run(self, n_round=3): """Run company until target round or no money""" while n_round > 0: # self._save() n_round -= 1 - logger.debug(f"{n_round=}") + logger.debug(f"max {n_round=} left.") self._check_balance() - await self.environment.run() - return self.environment.history - \ No newline at end of file + await self.env.run() + if CONFIG.git_repo: + CONFIG.git_repo.archive() + return self.env.history diff --git a/metagpt/tools/code_interpreter.py b/metagpt/tools/code_interpreter.py index e41eaab72..1cba005fa 100644 --- a/metagpt/tools/code_interpreter.py +++ b/metagpt/tools/code_interpreter.py @@ -1,22 +1,26 @@ +import inspect import re -from typing import List, Callable, Dict +import textwrap from pathlib import Path +from typing import Callable, Dict, List import wrapt -import textwrap -import inspect from interpreter.core.core import Interpreter -from metagpt.logs import logger +from metagpt.actions.clone_function import ( + CloneFunction, + run_function_code, + run_function_script, +) from metagpt.config import CONFIG +from metagpt.logs import logger from metagpt.utils.highlight import highlight -from metagpt.actions.clone_function import CloneFunction, run_function_code, run_function_script def extract_python_code(code: str): """Extract code blocks: If the code comments are the same, only the last code block is kept.""" # Use regular expressions to match comment blocks and related code. - pattern = r'(#\s[^\n]*)\n(.*?)(?=\n\s*#|$)' + pattern = r"(#\s[^\n]*)\n(.*?)(?=\n\s*#|$)" matches = re.findall(pattern, code, re.DOTALL) # Extract the last code block when encountering the same comment. @@ -25,8 +29,8 @@ def extract_python_code(code: str): unique_comments[comment] = code_block # concatenate into functional form - result_code = '\n'.join([f"{comment}\n{code_block}" for comment, code_block in unique_comments.items()]) - header_code = code[:code.find("#")] + result_code = "\n".join([f"{comment}\n{code_block}" for comment, code_block in unique_comments.items()]) + header_code = code[: code.find("#")] code = header_code + result_code logger.info(f"Extract python code: \n {highlight(code)}") @@ -36,6 +40,7 @@ def extract_python_code(code: str): class OpenCodeInterpreter(object): """https://github.com/KillianLucas/open-interpreter""" + def __init__(self, auto_run: bool = True) -> None: interpreter = Interpreter() interpreter.auto_run = auto_run @@ -50,15 +55,16 @@ class OpenCodeInterpreter(object): return self.interpreter.chat(query) @staticmethod - def extract_function(query_respond: List, function_name: str, *, language: str = 'python', - function_format: str = None) -> str: + def extract_function( + query_respond: List, function_name: str, *, language: str = "python", function_format: str = None + ) -> str: """create a function from query_respond.""" - if language not in ('python'): + if language not in ("python"): raise NotImplementedError(f"Not support to parse language {language}!") # set function form if function_format is None: - assert language == 'python', f"Expect python language for default function_format, but got {language}." + assert language == "python", f"Expect python language for default function_format, but got {language}." function_format = """def {function_name}():\n{code}""" # Extract the code module in the open-interpreter respond message. # The query_respond of open-interpreter before v0.1.4 is: @@ -68,25 +74,29 @@ class OpenCodeInterpreter(object): # "parsed_arguments": {"language": "python", "code": code of first plan} # ...] if "function_call" in query_respond[1]: - code = [item['function_call']['parsed_arguments']['code'] for item in query_respond - if "function_call" in item - and "parsed_arguments" in item["function_call"] - and 'language' in item["function_call"]['parsed_arguments'] - and item["function_call"]['parsed_arguments']['language'] == language] + code = [ + item["function_call"]["parsed_arguments"]["code"] + for item in query_respond + if "function_call" in item + and "parsed_arguments" in item["function_call"] + and "language" in item["function_call"]["parsed_arguments"] + and item["function_call"]["parsed_arguments"]["language"] == language + ] # The query_respond of open-interpreter v0.1.7 is: # [{'role': 'user', 'message': your query string}, # {'role': 'assistant', 'message': plan from llm, 'language': 'python', # 'code': code of first plan, 'output': output of first plan code}, # ...] elif "code" in query_respond[1]: - code = [item['code'] for item in query_respond - if "code" in item - and 'language' in item - and item['language'] == language] + code = [ + item["code"] + for item in query_respond + if "code" in item and "language" in item and item["language"] == language + ] else: raise ValueError(f"Unexpect message format in query_respond: {query_respond[1].keys()}") # add indent. - indented_code_str = textwrap.indent("\n".join(code), ' ' * 4) + indented_code_str = textwrap.indent("\n".join(code), " " * 4) # Return the code after deduplication. if language == "python": return extract_python_code(function_format.format(function_name=function_name, code=indented_code_str)) @@ -115,13 +125,13 @@ class OpenInterpreterDecorator(object): def _have_code(self, rsp: List[Dict]): # Is there any code generated? - return 'code' in rsp[1] and rsp[1]['code'] not in ("", None) + return "code" in rsp[1] and rsp[1]["code"] not in ("", None) def _is_faild_plan(self, rsp: List[Dict]): # is faild plan? - func_code = OpenCodeInterpreter.extract_function(rsp, 'function') + func_code = OpenCodeInterpreter.extract_function(rsp, "function") # If there is no more than 1 '\n', the plan execution fails. - if isinstance(func_code, str) and func_code.count('\n') <= 1: + if isinstance(func_code, str) and func_code.count("\n") <= 1: return True return False @@ -184,4 +194,5 @@ class OpenInterpreterDecorator(object): logger.error(f"Could not evaluate Python code \n{logger_code}: \nError: {e}") raise Exception("Could not evaluate Python code", e) return res + return wrapper(wrapped) diff --git a/metagpt/tools/prompt_writer.py b/metagpt/tools/prompt_writer.py index d90599206..ffcff4d1f 100644 --- a/metagpt/tools/prompt_writer.py +++ b/metagpt/tools/prompt_writer.py @@ -10,8 +10,9 @@ from typing import Union class GPTPromptGenerator: """Using LLM, given an output, request LLM to provide input (supporting instruction, chatbot, and query styles)""" + def __init__(self): - self._generators = {i: getattr(self, f"gen_{i}_style") for i in ['instruction', 'chatbot', 'query']} + self._generators = {i: getattr(self, f"gen_{i}_style") for i in ["instruction", "chatbot", "query"]} def gen_instruction_style(self, example): """Instruction style: Given an output, request LLM to provide input""" @@ -35,7 +36,7 @@ Query: X Document: {example} What is the detailed query X? X:""" - def gen(self, example: str, style: str = 'all') -> Union[list[str], str]: + def gen(self, example: str, style: str = "all") -> Union[list[str], str]: """ Generate one or multiple outputs using the example, allowing LLM to reply with the corresponding input @@ -43,7 +44,7 @@ X:""" :param style: (all|instruction|chatbot|query) :return: Expected LLM input sample (one or multiple) """ - if style != 'all': + if style != "all": return self._generators[style](example) return [f(example) for f in self._generators.values()] diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index 1d9cd0b2a..a84812f7c 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # @Date : 2023/7/19 16:28 -# @Author : stellahong (stellahong@fuzhi.ai) +# @Author : stellahong (stellahong@deepwisdom.ai) # @Desc : import asyncio import base64 @@ -13,11 +13,10 @@ from typing import List from aiohttp import ClientSession from PIL import Image, PngImagePlugin -from metagpt.config import Config -from metagpt.const import WORKSPACE_ROOT -from metagpt.logs import logger +from metagpt.config import CONFIG -config = Config() +# from metagpt.const import WORKSPACE_ROOT +from metagpt.logs import logger payload = { "prompt": "", @@ -56,9 +55,8 @@ default_negative_prompt = "(easynegative:0.8),black, dark,Low resolution" class SDEngine: def __init__(self): # Initialize the SDEngine with configuration - self.config = Config() - self.sd_url = self.config.get("SD_URL") - self.sd_t2i_url = f"{self.sd_url}{self.config.get('SD_T2I_API')}" + self.sd_url = CONFIG.get("SD_URL") + self.sd_t2i_url = f"{self.sd_url}{CONFIG.get('SD_T2I_API')}" # Define default payload settings for SD API self.payload = payload logger.info(self.sd_t2i_url) @@ -81,7 +79,7 @@ class SDEngine: return self.payload def _save(self, imgs, save_name=""): - save_dir = WORKSPACE_ROOT / "resources" / "SD_Output" + save_dir = CONFIG.workspace_path / "resources" / "SD_Output" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) batch_decode_base64_to_image(imgs, save_dir, save_name=save_name) @@ -120,11 +118,13 @@ def decode_base64_to_image(img, save_name): image.save(f"{save_name}.png", pnginfo=pnginfo) return pnginfo, image + def batch_decode_base64_to_image(imgs, save_dir="", save_name=""): for idx, _img in enumerate(imgs): save_name = join(save_dir, save_name) decode_base64_to_image(_img, save_name=save_name) + if __name__ == "__main__": engine = SDEngine() prompt = "pixel style, game design, a game interface should be minimalistic and intuitive with the score and high score displayed at the top. The snake and its food should be easily distinguishable. The game should have a simple color scheme, with a contrasting color for the snake and its food. Complete interface boundary" diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 942ef7edd..64388a11f 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -6,7 +6,7 @@ @File : search_engine.py """ import importlib -from typing import Callable, Coroutine, Literal, overload, Optional, Union +from typing import Callable, Coroutine, Literal, Optional, Union, overload from semantic_kernel.skill_definition import sk_function @@ -43,8 +43,8 @@ class SearchEngine: def __init__( self, - engine: Optional[SearchEngineType] = None, - run_func: Callable[[str, int, bool], Coroutine[None, None, Union[str, list[str]]]] = None, + engine: Optional[SearchEngineType] = None, + run_func: Callable[[str, int, bool], Coroutine[None, None, Union[str, list[str]]]] = None, ): engine = engine or CONFIG.search_engine if engine == SearchEngineType.SERPAPI_GOOGLE: diff --git a/metagpt/tools/search_engine_meilisearch.py b/metagpt/tools/search_engine_meilisearch.py index da4269384..f7c1c685a 100644 --- a/metagpt/tools/search_engine_meilisearch.py +++ b/metagpt/tools/search_engine_meilisearch.py @@ -29,7 +29,7 @@ class MeilisearchEngine: def add_documents(self, data_source: DataSource, documents: List[dict]): index_name = f"{data_source.name}_index" if index_name not in self.client.get_indexes(): - self.client.create_index(uid=index_name, options={'primaryKey': 'id'}) + self.client.create_index(uid=index_name, options={"primaryKey": "id"}) index = self.client.get_index(index_name) index.add_documents(documents) self.set_index(index) @@ -37,7 +37,7 @@ class MeilisearchEngine: def search(self, query): try: search_results = self._index.search(query) - return search_results['hits'] + return search_results["hits"] except Exception as e: # Handle MeiliSearch API errors print(f"MeiliSearch API error: {e}") diff --git a/metagpt/tools/translator.py b/metagpt/tools/translator.py index 910638469..63e38d5a5 100644 --- a/metagpt/tools/translator.py +++ b/metagpt/tools/translator.py @@ -6,7 +6,7 @@ @File : translator.py """ -prompt = ''' +prompt = """ # 指令 接下来，作为一位拥有20年翻译经验的翻译专家，当我给出英文句子或段落时，你将提供通顺且具有可读性的{LANG}翻译。注意以下要求： 1. 确保翻译结果流畅且易于理解 @@ -17,11 +17,10 @@ prompt = ''' {ORIGINAL} # 译文 -''' +""" class Translator: - @classmethod - def translate_prompt(cls, original, lang='中文'): - return prompt.format(LANG=lang, ORIGINAL=original) \ No newline at end of file + def translate_prompt(cls, original, lang="中文"): + return prompt.format(LANG=lang, ORIGINAL=original) diff --git a/metagpt/tools/ut_writer.py b/metagpt/tools/ut_writer.py index 43ca72150..64423dfb1 100644 --- a/metagpt/tools/ut_writer.py +++ b/metagpt/tools/ut_writer.py @@ -6,7 +6,7 @@ from pathlib import Path from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI -ICL_SAMPLE = '''Interface definition: +ICL_SAMPLE = """Interface definition: ```text Interface Name: Element Tagging Interface Path: /projects/{project_key}/node-tags @@ -60,20 +60,20 @@ def test_node_tags(project_key, nodes, operations, expected_msg): # 3. If comments are needed, use Chinese. # If you understand, please wait for me to give the interface definition and just answer "Understood" to save tokens. -''' +""" -ACT_PROMPT_PREFIX = '''Refer to the test types: such as missing request parameters, field boundary verification, incorrect field type. +ACT_PROMPT_PREFIX = """Refer to the test types: such as missing request parameters, field boundary verification, incorrect field type. Please output 10 test cases within one `@pytest.mark.parametrize` scope. ```text -''' +""" -YFT_PROMPT_PREFIX = '''Refer to the test types: such as SQL injection, cross-site scripting (XSS), unauthorized access and privilege escalation, +YFT_PROMPT_PREFIX = """Refer to the test types: such as SQL injection, cross-site scripting (XSS), unauthorized access and privilege escalation, authentication and authorization, parameter verification, exception handling, file upload and download. Please output 10 test cases within one `@pytest.mark.parametrize` scope. ```text -''' +""" -OCR_API_DOC = '''```text +OCR_API_DOC = """```text Interface Name: OCR recognition Interface Path: /api/v1/contract/treaty/task/ocr Method: POST @@ -96,14 +96,20 @@ code integer Yes message string Yes data object Yes ``` -''' +""" class UTGenerator: """UT Generator: Construct UT through API documentation""" - def __init__(self, swagger_file: str, ut_py_path: str, questions_path: str, - chatgpt_method: str = "API", template_prefix=YFT_PROMPT_PREFIX) -> None: + def __init__( + self, + swagger_file: str, + ut_py_path: str, + questions_path: str, + chatgpt_method: str = "API", + template_prefix=YFT_PROMPT_PREFIX, + ) -> None: """Initialize UT Generator Args: @@ -274,7 +280,7 @@ class UTGenerator: def gpt_msgs_to_code(self, messages: list) -> str: """Choose based on different calling methods""" - result = '' + result = "" if self.chatgpt_method == "API": result = GPTAPI().ask_code(msgs=messages) diff --git a/metagpt/utils/ahttp_client.py b/metagpt/utils/ahttp_client.py index d4f9f94e5..b4a33e9d7 100644 --- a/metagpt/utils/ahttp_client.py +++ b/metagpt/utils/ahttp_client.py @@ -2,29 +2,24 @@ # -*- coding: utf-8 -*- # @Desc : pure async http_client -from typing import Optional, Any, Mapping, Union +from typing import Any, Mapping, Optional, Union -from aiohttp.client import DEFAULT_TIMEOUT import aiohttp +from aiohttp.client import DEFAULT_TIMEOUT -async def apost(url: str, - params: Optional[Mapping[str, str]] = None, - json: Any = None, - data: Any = None, - headers: Optional[dict] = None, - as_json: bool = False, - encoding: str = "utf-8", - timeout: int = DEFAULT_TIMEOUT.total) -> Union[str, dict]: +async def apost( + url: str, + params: Optional[Mapping[str, str]] = None, + json: Any = None, + data: Any = None, + headers: Optional[dict] = None, + as_json: bool = False, + encoding: str = "utf-8", + timeout: int = DEFAULT_TIMEOUT.total, +) -> Union[str, dict]: async with aiohttp.ClientSession() as session: - async with session.post( - url=url, - params=params, - json=json, - data=data, - headers=headers, - timeout=timeout - ) as resp: + async with session.post(url=url, params=params, json=json, data=data, headers=headers, timeout=timeout) as resp: if as_json: data = await resp.json() else: @@ -33,13 +28,15 @@ async def apost(url: str, return data -async def apost_stream(url: str, - params: Optional[Mapping[str, str]] = None, - json: Any = None, - data: Any = None, - headers: Optional[dict] = None, - encoding: str = "utf-8", - timeout: int = DEFAULT_TIMEOUT.total) -> Any: +async def apost_stream( + url: str, + params: Optional[Mapping[str, str]] = None, + json: Any = None, + data: Any = None, + headers: Optional[dict] = None, + encoding: str = "utf-8", + timeout: int = DEFAULT_TIMEOUT.total, +) -> Any: """ usage: result = astream(url="xx") @@ -47,13 +44,6 @@ async def apost_stream(url: str, deal_with(line) """ async with aiohttp.ClientSession() as session: - async with session.post( - url=url, - params=params, - json=json, - data=data, - headers=headers, - timeout=timeout - ) as resp: + async with session.post(url=url, params=params, json=json, data=data, headers=headers, timeout=timeout) as resp: async for line in resp.content: yield line.decode(encoding) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index f09666beb..a9bdd6e2d 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -4,7 +4,13 @@ @Time : 2023/4/29 16:07 @Author : alexanderwu @File : common.py +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.2 of RFC 116: + Add generic class-to-string and object-to-string conversion functionality. +@Modified By: mashenquan, 2023/11/27. Bug fix: `parse_recipient` failed to parse the recipient in certain GPT-3.5 + responses. """ +from __future__ import annotations + import ast import contextlib import inspect @@ -13,6 +19,7 @@ import platform import re from typing import List, Tuple, Union +from metagpt.const import MESSAGE_ROUTE_TO_ALL from metagpt.logs import logger @@ -85,10 +92,7 @@ class OutputParser: @staticmethod def parse_python_code(text: str) -> str: - for pattern in ( - r"(.*?```python.*?\s+)?(?P.*)(```.*?)", - r"(.*?```python.*?\s+)?(?P.*)", - ): + for pattern in (r"(.*?```python.*?\s+)?(?P.*)(```.*?)", r"(.*?```python.*?\s+)?(?P.*)"): match = re.search(pattern, text, re.DOTALL) if not match: continue @@ -219,10 +223,15 @@ class CodeParser: # 遍历所有的block for block in blocks: # 如果block不为空，则继续处理 - if block.strip() != "": + if block.strip() == "": + continue + if "\n" not in block: + block_title = block + block_content = "" + else: # 将block的标题和内容分开，并分别去掉前后的空白字符 block_title, block_content = block.split("\n", 1) - block_dict[block_title.strip()] = block_content.strip() + block_dict[block_title.strip()] = block_content.strip() return block_dict @@ -304,4 +313,53 @@ def print_members(module, indent=0): def parse_recipient(text): pattern = r"## Send To:\s*([A-Za-z]+)\s*?" # hard code for now recipient = re.search(pattern, text) - return recipient.group(1) if recipient else "" + if recipient: + return recipient.group(1) + pattern = r"Send To:\s*([A-Za-z]+)\s*?" + recipient = re.search(pattern, text) + if recipient: + return recipient.group(1) + return "" + + +def get_class_name(cls) -> str: + """Return class name""" + return f"{cls.__module__}.{cls.__name__}" + + +def get_object_name(obj) -> str: + """Return class name of the object""" + cls = type(obj) + return f"{cls.__module__}.{cls.__name__}" + + +def any_to_str(val) -> str: + """Return the class name or the class name of the object, or 'val' if it's a string type.""" + if isinstance(val, str): + return val + if not callable(val): + return get_object_name(val) + + return get_class_name(val) + + +def any_to_str_set(val) -> set: + """Convert any type to string set.""" + res = set() + if isinstance(val, dict) or isinstance(val, list) or isinstance(val, set) or isinstance(val, tuple): + for i in val: + res.add(any_to_str(i)) + else: + res.add(any_to_str(val)) + return res + + +def is_subscribed(message, tags): + """Return whether it's consumer""" + if MESSAGE_ROUTE_TO_ALL in message.send_to: + return True + + for t in tags: + if t in message.send_to: + return True + return False diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py new file mode 100644 index 000000000..e8347d567 --- /dev/null +++ b/metagpt/utils/dependency_file.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/22 +@Author : mashenquan +@File : dependency_file.py +@Desc: Implementation of the dependency file described in Section 2.2.3.2 of RFC 135. +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Set + +import aiofiles + +from metagpt.config import CONFIG +from metagpt.logs import logger + + +class DependencyFile: + """A class representing a DependencyFile for managing dependencies. + + :param workdir: The working directory path for the DependencyFile. + """ + + def __init__(self, workdir: Path | str): + """Initialize a DependencyFile instance. + + :param workdir: The working directory path for the DependencyFile. + """ + self._dependencies = {} + self._filename = Path(workdir) / ".dependencies.json" + + async def load(self): + """Load dependencies from the file asynchronously.""" + if not self._filename.exists(): + return + try: + async with aiofiles.open(str(self._filename), mode="r") as reader: + data = await reader.read() + self._dependencies = json.loads(data) + except Exception as e: + logger.error(f"Failed to load {str(self._filename)}, error:{e}") + + async def save(self): + """Save dependencies to the file asynchronously.""" + try: + data = json.dumps(self._dependencies) + async with aiofiles.open(str(self._filename), mode="w") as writer: + await writer.write(data) + except Exception as e: + logger.error(f"Failed to save {str(self._filename)}, error:{e}") + + async def update(self, filename: Path | str, dependencies: Set[Path | str], persist=True): + """Update dependencies for a file asynchronously. + + :param filename: The filename or path. + :param dependencies: The set of dependencies. + :param persist: Whether to persist the changes immediately. + """ + if persist: + await self.load() + + root = self._filename.parent + try: + key = Path(filename).relative_to(root) + except ValueError: + key = filename + + if dependencies: + relative_paths = [] + for i in dependencies: + try: + relative_paths.append(str(Path(i).relative_to(root))) + except ValueError: + relative_paths.append(str(i)) + self._dependencies[str(key)] = relative_paths + elif str(key) in self._dependencies: + del self._dependencies[str(key)] + + if persist: + await self.save() + + async def get(self, filename: Path | str, persist=True): + """Get dependencies for a file asynchronously. + + :param filename: The filename or path. + :param persist: Whether to load dependencies from the file immediately. + :return: A set of dependencies. + """ + if persist: + await self.load() + + root = CONFIG.git_repo.workdir + try: + key = Path(filename).relative_to(root) + except ValueError: + key = filename + return set(self._dependencies.get(str(key), {})) + + def delete_file(self): + """Delete the dependency file.""" + self._filename.unlink(missing_ok=True) + + @property + def exists(self): + """Check if the dependency file exists.""" + return self._filename.exists() diff --git a/metagpt/utils/file.py b/metagpt/utils/file.py index f3691549b..6bb9a1a97 100644 --- a/metagpt/utils/file.py +++ b/metagpt/utils/file.py @@ -6,9 +6,10 @@ @File : file.py @Describe : General file operations. """ -import aiofiles from pathlib import Path +import aiofiles + from metagpt.logs import logger @@ -66,10 +67,9 @@ class File: if not chunk: break chunks.append(chunk) - content = b''.join(chunks) + content = b"".join(chunks) logger.debug(f"Successfully read file, the path of file: {file_path}") return content except Exception as e: logger.error(f"Error reading file: {e}") raise e - diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py new file mode 100644 index 000000000..2eca799a8 --- /dev/null +++ b/metagpt/utils/file_repository.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : git_repository.py +@Desc: File repository management. RFC 135 2.2.3.2, 2.2.3.4 and 2.2.3.13. +""" +from __future__ import annotations + +import json +import os +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Set + +import aiofiles + +from metagpt.config import CONFIG +from metagpt.logs import logger +from metagpt.schema import Document +from metagpt.utils.json_to_markdown import json_to_markdown + + +class FileRepository: + """A class representing a FileRepository associated with a Git repository. + + :param git_repo: The associated GitRepository instance. + :param relative_path: The relative path within the Git repository. + + Attributes: + _relative_path (Path): The relative path within the Git repository. + _git_repo (GitRepository): The associated GitRepository instance. + """ + + def __init__(self, git_repo, relative_path: Path = Path(".")): + """Initialize a FileRepository instance. + + :param git_repo: The associated GitRepository instance. + :param relative_path: The relative path within the Git repository. + """ + self._relative_path = relative_path + self._git_repo = git_repo + + # Initializing + self.workdir.mkdir(parents=True, exist_ok=True) + + async def save(self, filename: Path | str, content, dependencies: List[str] = None): + """Save content to a file and update its dependencies. + + :param filename: The filename or path within the repository. + :param content: The content to be saved. + :param dependencies: List of dependency filenames or paths. + """ + pathname = self.workdir / filename + pathname.parent.mkdir(parents=True, exist_ok=True) + async with aiofiles.open(str(pathname), mode="w") as writer: + await writer.write(content) + logger.info(f"save to: {str(pathname)}") + + if dependencies is not None: + dependency_file = await self._git_repo.get_dependency() + await dependency_file.update(pathname, set(dependencies)) + logger.info(f"update dependency: {str(pathname)}:{dependencies}") + + async def get_dependency(self, filename: Path | str) -> Set[str]: + """Get the dependencies of a file. + + :param filename: The filename or path within the repository. + :return: Set of dependency filenames or paths. + """ + pathname = self.workdir / filename + dependency_file = await self._git_repo.get_dependency() + return await dependency_file.get(pathname) + + async def get_changed_dependency(self, filename: Path | str) -> Set[str]: + """Get the dependencies of a file that have changed. + + :param filename: The filename or path within the repository. + :return: List of changed dependency filenames or paths. + """ + dependencies = await self.get_dependency(filename=filename) + changed_files = self.changed_files + changed_dependent_files = set() + for df in dependencies: + if df in changed_files.keys(): + changed_dependent_files.add(df) + return changed_dependent_files + + async def get(self, filename: Path | str) -> Document | None: + """Read the content of a file. + + :param filename: The filename or path within the repository. + :return: The content of the file. + """ + doc = Document(root_path=str(self.root_path), filename=str(filename)) + path_name = self.workdir / filename + if not path_name.exists(): + return None + try: + async with aiofiles.open(str(path_name), mode="r") as reader: + doc.content = await reader.read() + except FileNotFoundError as e: + logger.info(f"open {str(path_name)} failed:{e}") + return None + except Exception as e: + logger.info(f"open {str(path_name)} failed:{e}") + return None + return doc + + async def get_all(self) -> List[Document]: + """Get the content of all files in the repository. + + :return: List of Document instances representing files. + """ + docs = [] + for root, dirs, files in os.walk(str(self.workdir)): + for file in files: + file_path = Path(root) / file + relative_path = file_path.relative_to(self.workdir) + doc = await self.get(relative_path) + docs.append(doc) + return docs + + @property + def workdir(self): + """Return the absolute path to the working directory of the FileRepository. + + :return: The absolute path to the working directory. + """ + return self._git_repo.workdir / self._relative_path + + @property + def root_path(self): + """Return the relative path from git repository root""" + return self._relative_path + + @property + def changed_files(self) -> Dict[str, str]: + """Return a dictionary of changed files and their change types. + + :return: A dictionary where keys are file paths and values are change types. + """ + files = self._git_repo.changed_files + relative_files = {} + for p, ct in files.items(): + try: + rf = Path(p).relative_to(self._relative_path) + except ValueError: + continue + relative_files[str(rf)] = ct + return relative_files + + @property + def all_files(self) -> List: + """Get a dictionary of all files in the repository. + + The dictionary includes file paths relative to the current FileRepository. + + :return: A dictionary where keys are file paths and values are file information. + :rtype: List + """ + return self._git_repo.get_files(relative_path=self._relative_path) + + def get_change_dir_files(self, dir: Path | str) -> List: + """Get the files in a directory that have changed. + + :param dir: The directory path within the repository. + :return: List of changed filenames or paths within the directory. + """ + changed_files = self.changed_files + children = [] + for f in changed_files: + try: + Path(f).relative_to(Path(dir)) + except ValueError: + continue + children.append(str(f)) + return children + + @staticmethod + def new_filename(): + """Generate a new filename based on the current timestamp and a UUID suffix. + + :return: A new filename string. + """ + current_time = datetime.now().strftime("%Y%m%d%H%M%S") + return current_time + # guid_suffix = str(uuid.uuid4())[:8] + # return f"{current_time}x{guid_suffix}" + + async def save_doc(self, doc: Document, with_suffix: str = None, dependencies: List[str] = None): + """Save a Document instance as a PDF file. + + This method converts the content of the Document instance to Markdown, + saves it to a file with an optional specified suffix, and logs the saved file. + + :param doc: The Document instance to be saved. + :type doc: Document + :param with_suffix: An optional suffix to append to the saved file's name. + :type with_suffix: str, optional + :param dependencies: A list of dependencies for the saved file. + :type dependencies: List[str], optional + """ + m = json.loads(doc.content) + filename = Path(doc.filename).with_suffix(with_suffix) if with_suffix is not None else Path(doc.filename) + await self.save(filename=str(filename), content=json_to_markdown(m), dependencies=dependencies) + logger.debug(f"File Saved: {str(filename)}") + + @staticmethod + async def get_file(filename: Path | str, relative_path: Path | str = ".") -> Document | None: + """Retrieve a specific file from the file repository. + + :param filename: The name or path of the file to retrieve. + :type filename: Path or str + :param relative_path: The relative path within the file repository. + :type relative_path: Path or str, optional + :return: The document representing the file, or None if not found. + :rtype: Document or None + """ + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + return await file_repo.get(filename=filename) + + @staticmethod + async def get_all_files(relative_path: Path | str = ".") -> List[Document]: + """Retrieve all files from the file repository. + + :param relative_path: The relative path within the file repository. + :type relative_path: Path or str, optional + :return: A list of documents representing all files in the repository. + :rtype: List[Document] + """ + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + return await file_repo.get_all() + + @staticmethod + async def save_file(filename: Path | str, content, dependencies: List[str] = None, relative_path: Path | str = "."): + """Save a file to the file repository. + + :param filename: The name or path of the file to save. + :type filename: Path or str + :param content: The content of the file. + :param dependencies: A list of dependencies for the file. + :type dependencies: List[str], optional + :param relative_path: The relative path within the file repository. + :type relative_path: Path or str, optional + """ + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + return await file_repo.save(filename=filename, content=content, dependencies=dependencies) + + @staticmethod + async def save_as( + doc: Document, with_suffix: str = None, dependencies: List[str] = None, relative_path: Path | str = "." + ): + """Save a Document instance with optional modifications. + + This static method creates a new FileRepository, saves the Document instance + with optional modifications (such as a suffix), and logs the saved file. + + :param doc: The Document instance to be saved. + :type doc: Document + :param with_suffix: An optional suffix to append to the saved file's name. + :type with_suffix: str, optional + :param dependencies: A list of dependencies for the saved file. + :type dependencies: List[str], optional + :param relative_path: The relative path within the file repository. + :type relative_path: Path or str, optional + :return: A boolean indicating whether the save operation was successful. + :rtype: bool + """ + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + return await file_repo.save_doc(doc=doc, with_suffix=with_suffix, dependencies=dependencies) + + async def delete(self, filename: Path | str): + """Delete a file from the file repository. + + This method deletes a file from the file repository based on the provided filename. + + :param filename: The name or path of the file to be deleted. + :type filename: Path or str + """ + pathname = self.workdir / filename + if not pathname.exists(): + return + pathname.unlink(missing_ok=True) + + dependency_file = await self._git_repo.get_dependency() + await dependency_file.update(filename=pathname, dependencies=None) + logger.info(f"remove dependency key: {str(pathname)}") + + @staticmethod + async def delete_file(filename: Path | str, relative_path: Path | str = "."): + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + await file_repo.delete(filename=filename) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py new file mode 100644 index 000000000..d2bdf5d85 --- /dev/null +++ b/metagpt/utils/git_repository.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : git_repository.py +@Desc: Git repository management. RFC 135 2.2.3.3. +""" +from __future__ import annotations + +import shutil +from enum import Enum +from pathlib import Path +from typing import Dict, List + +from git.repo import Repo +from git.repo.fun import is_git_dir +from gitignore_parser import parse_gitignore + +from metagpt.const import DEFAULT_WORKSPACE_ROOT +from metagpt.logs import logger +from metagpt.utils.dependency_file import DependencyFile +from metagpt.utils.file_repository import FileRepository + + +class ChangeType(Enum): + ADDED = "A" # File was added + COPIED = "C" # File was copied + DELETED = "D" # File was deleted + RENAMED = "R" # File was renamed + MODIFIED = "M" # File was modified + TYPE_CHANGED = "T" # Type of the file was changed + UNTRACTED = "U" # File is untracked (not added to version control) + + +class GitRepository: + """A class representing a Git repository. + + :param local_path: The local path to the Git repository. + :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. + + Attributes: + _repository (Repo): The GitPython `Repo` object representing the Git repository. + """ + + def __init__(self, local_path=None, auto_init=True): + """Initialize a GitRepository instance. + + :param local_path: The local path to the Git repository. + :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. + """ + self._repository = None + self._dependency = None + self._gitignore_rules = None + if local_path: + self.open(local_path=local_path, auto_init=auto_init) + + def open(self, local_path: Path, auto_init=False): + """Open an existing Git repository or initialize a new one if auto_init is True. + + :param local_path: The local path to the Git repository. + :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. + """ + local_path = Path(local_path) + if self.is_git_dir(local_path): + self._repository = Repo(local_path) + self._gitignore_rules = parse_gitignore(full_path=str(local_path / ".gitignore")) + return + if not auto_init: + return + local_path.mkdir(parents=True, exist_ok=True) + return self._init(local_path) + + def _init(self, local_path: Path): + """Initialize a new Git repository at the specified path. + + :param local_path: The local path where the new Git repository will be initialized. + """ + self._repository = Repo.init(path=Path(local_path)) + + gitignore_filename = Path(local_path) / ".gitignore" + ignores = ["__pycache__", "*.pyc"] + with open(str(gitignore_filename), mode="w") as writer: + writer.write("\n".join(ignores)) + self._repository.index.add([".gitignore"]) + self._repository.index.commit("Add .gitignore") + self._gitignore_rules = parse_gitignore(full_path=gitignore_filename) + + def add_change(self, files: Dict): + """Add or remove files from the staging area based on the provided changes. + + :param files: A dictionary where keys are file paths and values are instances of ChangeType. + """ + if not self.is_valid or not files: + return + + for k, v in files.items(): + self._repository.index.remove(k) if v is ChangeType.DELETED else self._repository.index.add([k]) + + def commit(self, comments): + """Commit the staged changes with the given comments. + + :param comments: Comments for the commit. + """ + if self.is_valid: + self._repository.index.commit(comments) + + def delete_repository(self): + """Delete the entire repository directory.""" + if self.is_valid: + shutil.rmtree(self._repository.working_dir) + + @property + def changed_files(self) -> Dict[str, str]: + """Return a dictionary of changed files and their change types. + + :return: A dictionary where keys are file paths and values are change types. + """ + files = {i: ChangeType.UNTRACTED for i in self._repository.untracked_files} + changed_files = {f.a_path: ChangeType(f.change_type) for f in self._repository.index.diff(None)} + files.update(changed_files) + return files + + @staticmethod + def is_git_dir(local_path): + """Check if the specified directory is a Git repository. + + :param local_path: The local path to check. + :return: True if the directory is a Git repository, False otherwise. + """ + git_dir = Path(local_path) / ".git" + if git_dir.exists() and is_git_dir(git_dir): + return True + return False + + @property + def is_valid(self): + """Check if the Git repository is valid (exists and is initialized). + + :return: True if the repository is valid, False otherwise. + """ + return bool(self._repository) + + @property + def status(self) -> str: + """Return the Git repository's status as a string.""" + if not self.is_valid: + return "" + return self._repository.git.status() + + @property + def workdir(self) -> Path | None: + """Return the path to the working directory of the Git repository. + + :return: The path to the working directory or None if the repository is not valid. + """ + if not self.is_valid: + return None + return Path(self._repository.working_dir) + + def archive(self, comments="Archive"): + """Archive the current state of the Git repository. + + :param comments: Comments for the archive commit. + """ + logger.info(f"Archive: {list(self.changed_files.keys())}") + self.add_change(self.changed_files) + self.commit(comments) + + def new_file_repository(self, relative_path: Path | str = ".") -> FileRepository: + """Create a new instance of FileRepository associated with this Git repository. + + :param relative_path: The relative path to the file repository within the Git repository. + :return: A new instance of FileRepository. + """ + path = Path(relative_path) + try: + path = path.relative_to(self.workdir) + except ValueError: + path = relative_path + return FileRepository(git_repo=self, relative_path=Path(path)) + + async def get_dependency(self) -> DependencyFile: + """Get the dependency file associated with the Git repository. + + :return: An instance of DependencyFile. + """ + if not self._dependency: + self._dependency = DependencyFile(workdir=self.workdir) + return self._dependency + + def rename_root(self, new_dir_name): + """Rename the root directory of the Git repository. + + :param new_dir_name: The new name for the root directory. + """ + if self.workdir.name == new_dir_name: + return + new_path = self.workdir.parent / new_dir_name + if new_path.exists(): + logger.info(f"Delete directory {str(new_path)}") + shutil.rmtree(new_path) + try: + shutil.move(src=str(self.workdir), dst=str(new_path)) + except Exception as e: + logger.warning(f"Move {str(self.workdir)} to {str(new_path)} error: {e}") + logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") + self._repository = Repo(new_path) + self._gitignore_rules = parse_gitignore(full_path=str(new_path / ".gitignore")) + + def get_files(self, relative_path: Path | str, root_relative_path: Path | str = None, filter_ignored=True) -> List: + """ + Retrieve a list of files in the specified relative path. + + The method returns a list of file paths relative to the current FileRepository. + + :param relative_path: The relative path within the repository. + :type relative_path: Path or str + :param root_relative_path: The root relative path within the repository. + :type root_relative_path: Path or str + :param filter_ignored: Flag to indicate whether to filter files based on .gitignore rules. + :type filter_ignored: bool + :return: A list of file paths in the specified directory. + :rtype: List[str] + """ + try: + relative_path = Path(relative_path).relative_to(self.workdir) + except ValueError: + relative_path = Path(relative_path) + + if not root_relative_path: + root_relative_path = Path(self.workdir) / relative_path + files = [] + try: + directory_path = Path(self.workdir) / relative_path + if not directory_path.exists(): + return [] + for file_path in directory_path.iterdir(): + if file_path.is_file(): + rpath = file_path.relative_to(root_relative_path) + files.append(str(rpath)) + else: + subfolder_files = self.get_files( + relative_path=file_path, root_relative_path=root_relative_path, filter_ignored=False + ) + files.extend(subfolder_files) + except Exception as e: + logger.error(f"Error: {e}") + if not filter_ignored: + return files + filtered_files = self.filter_gitignore(filenames=files, root_relative_path=root_relative_path) + return filtered_files + + def filter_gitignore(self, filenames: List[str], root_relative_path: Path | str = None) -> List[str]: + """ + Filter a list of filenames based on .gitignore rules. + + :param filenames: A list of filenames to be filtered. + :type filenames: List[str] + :param root_relative_path: The root relative path within the repository. + :type root_relative_path: Path or str + :return: A list of filenames that pass the .gitignore filtering. + :rtype: List[str] + """ + if root_relative_path is None: + root_relative_path = self.workdir + files = [] + for filename in filenames: + pathname = root_relative_path / filename + if self._gitignore_rules(str(pathname)): + continue + files.append(filename) + return files + + +if __name__ == "__main__": + path = DEFAULT_WORKSPACE_ROOT / "git" + path.mkdir(exist_ok=True, parents=True) + + repo = GitRepository() + repo.open(path, auto_init=True) + repo.filter_gitignore(filenames=["snake_game/snake_game/__pycache__", "snake_game/snake_game/game.py"]) + + changes = repo.changed_files + print(changes) + repo.add_change(changes) + print(repo.status) + repo.commit("test") + print(repo.status) + repo.delete_repository() diff --git a/metagpt/utils/highlight.py b/metagpt/utils/highlight.py index e6cbb228c..2e1d6f615 100644 --- a/metagpt/utils/highlight.py +++ b/metagpt/utils/highlight.py @@ -1,22 +1,22 @@ # 添加代码语法高亮显示 from pygments import highlight as highlight_ +from pygments.formatters import HtmlFormatter, TerminalFormatter from pygments.lexers import PythonLexer, SqlLexer -from pygments.formatters import TerminalFormatter, HtmlFormatter -def highlight(code: str, language: str = 'python', formatter: str = 'terminal'): +def highlight(code: str, language: str = "python", formatter: str = "terminal"): # 指定要高亮的语言 - if language.lower() == 'python': + if language.lower() == "python": lexer = PythonLexer() - elif language.lower() == 'sql': + elif language.lower() == "sql": lexer = SqlLexer() else: raise ValueError(f"Unsupported language: {language}") # 指定输出格式 - if formatter.lower() == 'terminal': + if formatter.lower() == "terminal": formatter = TerminalFormatter() - elif formatter.lower() == 'html': + elif formatter.lower() == "html": formatter = HtmlFormatter() else: raise ValueError(f"Unsupported formatter: {formatter}") diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 204c22c67..eb85a3f90 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -10,7 +10,7 @@ import os from pathlib import Path from metagpt.config import CONFIG -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT from metagpt.logs import logger from metagpt.utils.common import check_cmd_exists @@ -69,7 +69,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, if stdout: logger.info(stdout.decode()) if stderr: - logger.error(stderr.decode()) + logger.warning(stderr.decode()) else: if engine == "playwright": from metagpt.utils.mmdc_playwright import mermaid_to_file @@ -141,6 +141,6 @@ MMC2 = """sequenceDiagram if __name__ == "__main__": loop = asyncio.new_event_loop() - result = loop.run_until_complete(mermaid_to_file(MMC1, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1")) - result = loop.run_until_complete(mermaid_to_file(MMC2, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1")) + result = loop.run_until_complete(mermaid_to_file(MMC1, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) + result = loop.run_until_complete(mermaid_to_file(MMC2, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) loop.close() diff --git a/metagpt/utils/mmdc_ink.py b/metagpt/utils/mmdc_ink.py index 3d91cde9d..d594adb30 100644 --- a/metagpt/utils/mmdc_ink.py +++ b/metagpt/utils/mmdc_ink.py @@ -6,9 +6,9 @@ @File : mermaid.py """ import base64 -import os -from aiohttp import ClientSession,ClientError +from aiohttp import ClientError, ClientSession + from metagpt.logs import logger @@ -29,7 +29,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix): async with session.get(url) as response: if response.status == 200: text = await response.content.read() - with open(output_file, 'wb') as f: + with open(output_file, "wb") as f: f.write(text) logger.info(f"Generating {output_file}..") else: diff --git a/metagpt/utils/mmdc_playwright.py b/metagpt/utils/mmdc_playwright.py index bdbfd82ff..5d455e1c5 100644 --- a/metagpt/utils/mmdc_playwright.py +++ b/metagpt/utils/mmdc_playwright.py @@ -8,10 +8,13 @@ import os from urllib.parse import urljoin + from playwright.async_api import async_playwright + from metagpt.logs import logger -async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048)-> int: + +async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: """ Converts the given Mermaid code to various output formats and saves them to files. @@ -24,66 +27,72 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, Returns: int: Returns 1 if the conversion and saving were successful, -1 otherwise. """ - suffixes=['png', 'svg', 'pdf'] + suffixes = ["png", "svg", "pdf"] __dirname = os.path.dirname(os.path.abspath(__file__)) async with async_playwright() as p: browser = await p.chromium.launch() device_scale_factor = 1.0 context = await browser.new_context( - viewport={'width': width, 'height': height}, - device_scale_factor=device_scale_factor, - ) + viewport={"width": width, "height": height}, + device_scale_factor=device_scale_factor, + ) page = await context.new_page() async def console_message(msg): logger.info(msg.text) - page.on('console', console_message) + + page.on("console", console_message) try: - await page.set_viewport_size({'width': width, 'height': height}) + await page.set_viewport_size({"width": width, "height": height}) - mermaid_html_path = os.path.abspath( - os.path.join(__dirname, 'index.html')) - mermaid_html_url = urljoin('file:', mermaid_html_path) + mermaid_html_path = os.path.abspath(os.path.join(__dirname, "index.html")) + mermaid_html_url = urljoin("file:", mermaid_html_path) await page.goto(mermaid_html_url) await page.wait_for_load_state("networkidle") await page.wait_for_selector("div#container", state="attached") - mermaid_config = {} + # mermaid_config = {} background_color = "#ffffff" - my_css = "" + # my_css = "" await page.evaluate(f'document.body.style.background = "{background_color}";') - metadata = await page.evaluate('''async ([definition, mermaidConfig, myCSS, backgroundColor]) => { - const { mermaid, zenuml } = globalThis; - await mermaid.registerExternalDiagrams([zenuml]); - mermaid.initialize({ startOnLoad: false, ...mermaidConfig }); - const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container')); - document.getElementById('container').innerHTML = svg; - const svgElement = document.querySelector('svg'); - svgElement.style.backgroundColor = backgroundColor; + # metadata = await page.evaluate( + # """async ([definition, mermaidConfig, myCSS, backgroundColor]) => { + # const { mermaid, zenuml } = globalThis; + # await mermaid.registerExternalDiagrams([zenuml]); + # mermaid.initialize({ startOnLoad: false, ...mermaidConfig }); + # const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container')); + # document.getElementById('container').innerHTML = svg; + # const svgElement = document.querySelector('svg'); + # svgElement.style.backgroundColor = backgroundColor; + # + # if (myCSS) { + # const style = document.createElementNS('http://www.w3.org/2000/svg', 'style'); + # style.appendChild(document.createTextNode(myCSS)); + # svgElement.appendChild(style); + # } + # + # }""", + # [mermaid_code, mermaid_config, my_css, background_color], + # ) - if (myCSS) { - const style = document.createElementNS('http://www.w3.org/2000/svg', 'style'); - style.appendChild(document.createTextNode(myCSS)); - svgElement.appendChild(style); - } - - }''', [mermaid_code, mermaid_config, my_css, background_color]) - - if 'svg' in suffixes : - svg_xml = await page.evaluate('''() => { + if "svg" in suffixes: + svg_xml = await page.evaluate( + """() => { const svg = document.querySelector('svg'); const xmlSerializer = new XMLSerializer(); return xmlSerializer.serializeToString(svg); - }''') + }""" + ) logger.info(f"Generating {output_file_without_suffix}.svg..") - with open(f'{output_file_without_suffix}.svg', 'wb') as f: - f.write(svg_xml.encode('utf-8')) + with open(f"{output_file_without_suffix}.svg", "wb") as f: + f.write(svg_xml.encode("utf-8")) - if 'png' in suffixes: - clip = await page.evaluate('''() => { + if "png" in suffixes: + clip = await page.evaluate( + """() => { const svg = document.querySelector('svg'); const rect = svg.getBoundingClientRect(); return { @@ -92,16 +101,17 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, width: Math.ceil(rect.width), height: Math.ceil(rect.height) }; - }''') - await page.set_viewport_size({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height']}) - screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device') + }""" + ) + await page.set_viewport_size({"width": clip["x"] + clip["width"], "height": clip["y"] + clip["height"]}) + screenshot = await page.screenshot(clip=clip, omit_background=True, scale="device") logger.info(f"Generating {output_file_without_suffix}.png..") - with open(f'{output_file_without_suffix}.png', 'wb') as f: + with open(f"{output_file_without_suffix}.png", "wb") as f: f.write(screenshot) - if 'pdf' in suffixes: + if "pdf" in suffixes: pdf_data = await page.pdf(scale=device_scale_factor) logger.info(f"Generating {output_file_without_suffix}.pdf..") - with open(f'{output_file_without_suffix}.pdf', 'wb') as f: + with open(f"{output_file_without_suffix}.pdf", "wb") as f: f.write(pdf_data) return 0 except Exception as e: diff --git a/metagpt/utils/mmdc_pyppeteer.py b/metagpt/utils/mmdc_pyppeteer.py index 7ec30fd12..7125cafc5 100644 --- a/metagpt/utils/mmdc_pyppeteer.py +++ b/metagpt/utils/mmdc_pyppeteer.py @@ -7,11 +7,14 @@ """ import os from urllib.parse import urljoin -from pyppeteer import launch -from metagpt.logs import logger -from metagpt.config import CONFIG -async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048)-> int: +from pyppeteer import launch + +from metagpt.config import CONFIG +from metagpt.logs import logger + + +async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: """ Converts the given Mermaid code to various output formats and saves them to files. @@ -24,15 +27,15 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, Returns: int: Returns 1 if the conversion and saving were successful, -1 otherwise. """ - suffixes = ['png', 'svg', 'pdf'] + suffixes = ["png", "svg", "pdf"] __dirname = os.path.dirname(os.path.abspath(__file__)) - if CONFIG.pyppeteer_executable_path: - browser = await launch(headless=True, - executablePath=CONFIG.pyppeteer_executable_path, - args=['--disable-extensions',"--no-sandbox"] - ) + browser = await launch( + headless=True, + executablePath=CONFIG.pyppeteer_executable_path, + args=["--disable-extensions", "--no-sandbox"], + ) else: logger.error("Please set the environment variable:PYPPETEER_EXECUTABLE_PATH.") return -1 @@ -41,50 +44,56 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, async def console_message(msg): logger.info(msg.text) - page.on('console', console_message) + + page.on("console", console_message) try: - await page.setViewport(viewport={'width': width, 'height': height, 'deviceScaleFactor': device_scale_factor}) + await page.setViewport(viewport={"width": width, "height": height, "deviceScaleFactor": device_scale_factor}) - mermaid_html_path = os.path.abspath( - os.path.join(__dirname, 'index.html')) - mermaid_html_url = urljoin('file:', mermaid_html_path) + mermaid_html_path = os.path.abspath(os.path.join(__dirname, "index.html")) + mermaid_html_url = urljoin("file:", mermaid_html_path) await page.goto(mermaid_html_url) await page.querySelector("div#container") - mermaid_config = {} + # mermaid_config = {} background_color = "#ffffff" - my_css = "" + # my_css = "" await page.evaluate(f'document.body.style.background = "{background_color}";') - metadata = await page.evaluate('''async ([definition, mermaidConfig, myCSS, backgroundColor]) => { - const { mermaid, zenuml } = globalThis; - await mermaid.registerExternalDiagrams([zenuml]); - mermaid.initialize({ startOnLoad: false, ...mermaidConfig }); - const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container')); - document.getElementById('container').innerHTML = svg; - const svgElement = document.querySelector('svg'); - svgElement.style.backgroundColor = backgroundColor; + # metadata = await page.evaluate( + # """async ([definition, mermaidConfig, myCSS, backgroundColor]) => { + # const { mermaid, zenuml } = globalThis; + # await mermaid.registerExternalDiagrams([zenuml]); + # mermaid.initialize({ startOnLoad: false, ...mermaidConfig }); + # const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container')); + # document.getElementById('container').innerHTML = svg; + # const svgElement = document.querySelector('svg'); + # svgElement.style.backgroundColor = backgroundColor; + # + # if (myCSS) { + # const style = document.createElementNS('http://www.w3.org/2000/svg', 'style'); + # style.appendChild(document.createTextNode(myCSS)); + # svgElement.appendChild(style); + # } + # }""", + # [mermaid_code, mermaid_config, my_css, background_color], + # ) - if (myCSS) { - const style = document.createElementNS('http://www.w3.org/2000/svg', 'style'); - style.appendChild(document.createTextNode(myCSS)); - svgElement.appendChild(style); - } - }''', [mermaid_code, mermaid_config, my_css, background_color]) - - if 'svg' in suffixes : - svg_xml = await page.evaluate('''() => { + if "svg" in suffixes: + svg_xml = await page.evaluate( + """() => { const svg = document.querySelector('svg'); const xmlSerializer = new XMLSerializer(); return xmlSerializer.serializeToString(svg); - }''') + }""" + ) logger.info(f"Generating {output_file_without_suffix}.svg..") - with open(f'{output_file_without_suffix}.svg', 'wb') as f: - f.write(svg_xml.encode('utf-8')) + with open(f"{output_file_without_suffix}.svg", "wb") as f: + f.write(svg_xml.encode("utf-8")) - if 'png' in suffixes: - clip = await page.evaluate('''() => { + if "png" in suffixes: + clip = await page.evaluate( + """() => { const svg = document.querySelector('svg'); const rect = svg.getBoundingClientRect(); return { @@ -93,16 +102,23 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, width: Math.ceil(rect.width), height: Math.ceil(rect.height) }; - }''') - await page.setViewport({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height'], 'deviceScaleFactor': device_scale_factor}) - screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device') + }""" + ) + await page.setViewport( + { + "width": clip["x"] + clip["width"], + "height": clip["y"] + clip["height"], + "deviceScaleFactor": device_scale_factor, + } + ) + screenshot = await page.screenshot(clip=clip, omit_background=True, scale="device") logger.info(f"Generating {output_file_without_suffix}.png..") - with open(f'{output_file_without_suffix}.png', 'wb') as f: + with open(f"{output_file_without_suffix}.png", "wb") as f: f.write(screenshot) - if 'pdf' in suffixes: + if "pdf" in suffixes: pdf_data = await page.pdf(scale=device_scale_factor) logger.info(f"Generating {output_file_without_suffix}.pdf..") - with open(f'{output_file_without_suffix}.pdf', 'wb') as f: + with open(f"{output_file_without_suffix}.pdf", "wb") as f: f.write(pdf_data) return 0 except Exception as e: @@ -110,4 +126,3 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, return -1 finally: await browser.close() - diff --git a/metagpt/utils/parse_html.py b/metagpt/utils/parse_html.py index 62de26541..f2395026f 100644 --- a/metagpt/utils/parse_html.py +++ b/metagpt/utils/parse_html.py @@ -16,7 +16,7 @@ class WebPage(BaseModel): class Config: underscore_attrs_are_private = True - _soup : Optional[BeautifulSoup] = None + _soup: Optional[BeautifulSoup] = None _title: Optional[str] = None @property @@ -24,7 +24,7 @@ class WebPage(BaseModel): if self._soup is None: self._soup = BeautifulSoup(self.html, "html.parser") return self._soup - + @property def title(self): if self._title is None: diff --git a/metagpt/utils/pycst.py b/metagpt/utils/pycst.py index afd85a547..1edfed81c 100644 --- a/metagpt/utils/pycst.py +++ b/metagpt/utils/pycst.py @@ -37,12 +37,12 @@ def get_docstring_statement(body: DocstringNode) -> cst.SimpleStatementLine: if not isinstance(expr, cst.Expr): return None - + val = expr.value if not isinstance(val, (cst.SimpleString, cst.ConcatenatedString)): return None - - evaluated_value = val.evaluated_value + + evaluated_value = val.evaluated_value if isinstance(evaluated_value, bytes): return None @@ -56,6 +56,7 @@ class DocstringCollector(cst.CSTVisitor): stack: A list to keep track of the current path in the CST. docstrings: A dictionary mapping paths in the CST to their corresponding docstrings. """ + def __init__(self): self.stack: list[str] = [] self.docstrings: dict[tuple[str, ...], cst.SimpleStatementLine] = {} @@ -96,6 +97,7 @@ class DocstringTransformer(cst.CSTTransformer): stack: A list to keep track of the current path in the CST. docstrings: A dictionary mapping paths in the CST to their corresponding docstrings. """ + def __init__( self, docstrings: dict[tuple[str, ...], cst.SimpleStatementLine], @@ -125,7 +127,9 @@ class DocstringTransformer(cst.CSTTransformer): key = tuple(self.stack) self.stack.pop() - if hasattr(updated_node, "decorators") and any((i.decorator.value == "overload") for i in updated_node.decorators): + if hasattr(updated_node, "decorators") and any( + (i.decorator.value == "overload") for i in updated_node.decorators + ): return updated_node statement = self.docstrings.get(key) diff --git a/metagpt/utils/read_document.py b/metagpt/utils/read_document.py index c837baf25..d2fafbc17 100644 --- a/metagpt/utils/read_document.py +++ b/metagpt/utils/read_document.py @@ -8,6 +8,7 @@ import docx + def read_docx(file_path: str) -> list: """Open a docx file""" doc = docx.Document(file_path) diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index 0a461d360..4aafd8e66 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -4,12 +4,13 @@ import copy from enum import Enum -from typing import Union, Callable -import regex as re -from tenacity import retry, stop_after_attempt, wait_fixed, after_log, RetryCallState +from typing import Callable, Union + +import regex as re +from tenacity import RetryCallState, retry, stop_after_attempt, wait_fixed -from metagpt.logs import logger from metagpt.config import CONFIG +from metagpt.logs import logger from metagpt.utils.custom_decoder import CustomDecoder @@ -33,7 +34,7 @@ def repair_case_sensitivity(output: str, req_key: str) -> str: if req_key_lower in output_lower: # find the sub-part index, and replace it with raw req_key lidx = output_lower.find(req_key_lower) - source = output[lidx: lidx + len(req_key_lower)] + source = output[lidx : lidx + len(req_key_lower)] output = output.replace(source, req_key) logger.info(f"repair_case_sensitivity: {req_key}") @@ -73,7 +74,7 @@ def repair_required_key_pair_missing(output: str, req_key: str = "[/CONTENT]") - sc = "/" # special char if req_key.startswith("[") and req_key.endswith("]"): if sc in req_key: - left_key = req_key.replace(sc, "") # `[/req_key]` -> `[req_key]` + left_key = req_key.replace(sc, "") # `[/req_key]` -> `[req_key]` right_key = req_key else: left_key = req_key @@ -82,6 +83,7 @@ def repair_required_key_pair_missing(output: str, req_key: str = "[/CONTENT]") - if left_key not in output: output = left_key + "\n" + output if right_key not in output: + def judge_potential_json(routput: str, left_key: str) -> Union[str, None]: ridx = routput.rfind(left_key) if ridx < 0: @@ -90,7 +92,7 @@ def repair_required_key_pair_missing(output: str, req_key: str = "[/CONTENT]") - idx1 = sub_output.rfind("}") idx2 = sub_output.rindex("]") idx = idx1 if idx1 >= idx2 else idx2 - sub_output = sub_output[: idx+1] + sub_output = sub_output[: idx + 1] return sub_output if output.strip().endswith("}") or (output.strip().endswith("]") and not output.strip().endswith(left_key)): @@ -155,9 +157,7 @@ def repair_llm_raw_output(output: str, req_keys: list[str], repair_type: RepairT # do the repairation usually for non-openai models for req_key in req_keys: - output = _repair_llm_raw_output(output=output, - req_key=req_key, - repair_type=repair_type) + output = _repair_llm_raw_output(output=output, req_key=req_key, repair_type=repair_type) return output @@ -187,7 +187,7 @@ def repair_invalid_json(output: str, error: str) -> str: new_line = line.replace("}", "") elif line.endswith("},") and output.endswith("},"): new_line = line[:-1] - elif '",' not in line and ',' not in line: + elif '",' not in line and "," not in line: new_line = f'{line}",' elif "," not in line: # problem, miss char `,` at the end. @@ -228,8 +228,10 @@ def run_after_exp_and_passon_next_retry(logger: "loguru.Logger") -> Callable[["R elif retry_state.kwargs: func_param_output = retry_state.kwargs.get("output", "") exp_str = str(retry_state.outcome.exception()) - logger.warning(f"parse json from content inside [CONTENT][/CONTENT] failed at retry " - f"{retry_state.attempt_number}, try to fix it, exp: {exp_str}") + logger.warning( + f"parse json from content inside [CONTENT][/CONTENT] failed at retry " + f"{retry_state.attempt_number}, try to fix it, exp: {exp_str}" + ) repaired_output = repair_invalid_json(func_param_output, exp_str) retry_state.kwargs["output"] = repaired_output @@ -260,7 +262,8 @@ def retry_parse_json_text(output: str) -> Union[list, dict]: def extract_content_from_output(content: str, right_key: str = "[/CONTENT]"): - """ extract xxx from [CONTENT](xxx)[/CONTENT] using regex pattern """ + """extract xxx from [CONTENT](xxx)[/CONTENT] using regex pattern""" + def re_extract_content(cont: str, pattern: str) -> str: matches = re.findall(pattern, cont, re.DOTALL) for match in matches: diff --git a/metagpt/utils/singleton.py b/metagpt/utils/singleton.py index 474b537db..a9e0862c0 100644 --- a/metagpt/utils/singleton.py +++ b/metagpt/utils/singleton.py @@ -20,4 +20,3 @@ class Singleton(abc.ABCMeta, type): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] - \ No newline at end of file diff --git a/metagpt/utils/special_tokens.py b/metagpt/utils/special_tokens.py index 2adb93c77..5e780ce05 100644 --- a/metagpt/utils/special_tokens.py +++ b/metagpt/utils/special_tokens.py @@ -1,4 +1,4 @@ # token to separate different code messages in a WriteCode Message content -MSG_SEP = "#*000*#" +MSG_SEP = "#*000*#" # token to seperate file name and the actual code text in a code message FILENAME_CODE_SEP = "#*001*#" diff --git a/metagpt/utils/text.py b/metagpt/utils/text.py index be3c52edd..dd9678438 100644 --- a/metagpt/utils/text.py +++ b/metagpt/utils/text.py @@ -3,7 +3,12 @@ from typing import Generator, Sequence from metagpt.utils.token_counter import TOKEN_MAX, count_string_tokens -def reduce_message_length(msgs: Generator[str, None, None], model_name: str, system_text: str, reserved: int = 0,) -> str: +def reduce_message_length( + msgs: Generator[str, None, None], + model_name: str, + system_text: str, + reserved: int = 0, +) -> str: """Reduce the length of concatenated message segments to fit within the maximum token size. Args: @@ -49,9 +54,9 @@ def generate_prompt_chunk( current_token = 0 current_lines = [] - reserved = reserved + count_string_tokens(prompt_template+system_text, model_name) + reserved = reserved + count_string_tokens(prompt_template + system_text, model_name) # 100 is a magic number to ensure the maximum context length is not exceeded - max_token = TOKEN_MAX.get(model_name, 2048) - reserved - 100 + max_token = TOKEN_MAX.get(model_name, 2048) - reserved - 100 while paragraphs: paragraph = paragraphs.pop(0) @@ -103,7 +108,7 @@ def decode_unicode_escape(text: str) -> str: return text.encode("utf-8").decode("unicode_escape", "ignore") -def _split_by_count(lst: Sequence , count: int): +def _split_by_count(lst: Sequence, count: int): avg = len(lst) // count remainder = len(lst) % count start = 0 diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index ba63e90a9..266a53268 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -24,7 +24,7 @@ TOKEN_COSTS = { "gpt-4-0613": {"prompt": 0.06, "completion": 0.12}, "gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03}, "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0}, - "chatglm_turbo": {"prompt": 0.0, "completion": 0.00069} # 32k version, prompt + completion tokens=0.005￥/k-tokens + "chatglm_turbo": {"prompt": 0.0, "completion": 0.00069}, # 32k version, prompt + completion tokens=0.005￥/k-tokens } @@ -42,7 +42,7 @@ TOKEN_MAX = { "gpt-4-0613": 8192, "gpt-4-1106-preview": 128000, "text-embedding-ada-002": 8192, - "chatglm_turbo": 32768 + "chatglm_turbo": 32768, } @@ -68,10 +68,10 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"): elif model == "gpt-3.5-turbo-0301": tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n tokens_per_name = -1 # if there's a name, the role is omitted - elif "gpt-3.5-turbo" in model: + elif "gpt-3.5-turbo" == model: print("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.") return count_message_tokens(messages, model="gpt-3.5-turbo-0613") - elif "gpt-4" in model: + elif "gpt-4" == model: print("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.") return count_message_tokens(messages, model="gpt-4-0613") else: diff --git a/metagpt/utils/utils.py b/metagpt/utils/utils.py index f479ec3b8..5ceed65d9 100644 --- a/metagpt/utils/utils.py +++ b/metagpt/utils/utils.py @@ -4,7 +4,7 @@ import typing -from tenacity import after_log, _utils +from tenacity import _utils def general_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: @@ -13,7 +13,10 @@ def general_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typ fn_name = "" else: fn_name = _utils.get_callback_name(retry_state.fn) - logger.error(f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " - f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " - f"exp: {retry_state.outcome.exception()}") + logger.error( + f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " + f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " + f"exp: {retry_state.outcome.exception()}" + ) + return log_it diff --git a/requirements.txt b/requirements.txt index 14a9f485d..515a4d88b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,8 @@ channels==4.0.0 # docx==0.2.4 #faiss==1.5.3 faiss_cpu==1.7.4 -fire==0.4.0 +# fire==0.4.0 +typer # godot==0.1.1 # google_api_python_client==2.93.0 lancedb==0.1.16 @@ -14,7 +15,7 @@ langchain==0.0.231 loguru==0.6.0 meilisearch==0.21.0 numpy==1.24.3 -openai==0.28.0 +openai>=0.28.1 openpyxl beautifulsoup4==4.12.2 pandas==2.0.3 @@ -44,4 +45,7 @@ ta==0.10.2 semantic-kernel==0.3.13.dev0 wrapt==1.15.0 websocket-client==0.58.0 +aiofiles==23.2.1 +gitpython==3.1.40 zhipuai==1.0.7 +gitignore-parser==0.1.9 diff --git a/setup.py b/setup.py index 494a4614d..4dd453b3d 100644 --- a/setup.py +++ b/setup.py @@ -36,9 +36,9 @@ setup( long_description_content_type="text/markdown", url="https://github.com/geekan/MetaGPT", author="Alexander Wu", - author_email="alexanderwu@fuzhi.ai", - license="Apache 2.0", - keywords="metagpt multi-role multi-agent programming gpt llm", + author_email="alexanderwu@deepwisdom.ai", + license="MIT", + keywords="metagpt multi-role multi-agent programming gpt llm metaprogramming", packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]), python_requires=">=3.9", install_requires=requirements, @@ -52,4 +52,9 @@ setup( cmdclass={ "install_mermaid": InstallMermaidCLI, }, + entry_points={ + "console_scripts": [ + "metagpt=metagpt.startup:app", + ], + }, ) diff --git a/startup.py b/startup.py deleted file mode 100644 index e9fbf94d3..000000000 --- a/startup.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import asyncio - -import fire - -from metagpt.roles import ( - Architect, - Engineer, - ProductManager, - ProjectManager, - QaEngineer, -) -from metagpt.team import Team - - -async def startup( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = False, - run_tests: bool = False, - implement: bool = True, -): - """Run a startup. Be a boss.""" - company = Team() - company.hire( - [ - ProductManager(), - Architect(), - ProjectManager(), - ] - ) - - # if implement or code_review - if implement or code_review: - # developing features: implement the idea - company.hire([Engineer(n_borg=5, use_code_review=code_review)]) - - if run_tests: - # developing features: run tests on the spot and identify bugs - # (bug fixing capability comes soon!) - company.hire([QaEngineer()]) - - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) - - -def main( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = True, - run_tests: bool = False, - implement: bool = True, -): - """ - We are a software startup comprised of AI. By investing in us, - you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea, such as "Creating a snake game." - :param investment: As an investor, you have the opportunity to contribute - a certain dollar amount to this AI company. - :param n_round: - :param code_review: Whether to use code review. - :return: - """ - asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) - - -if __name__ == "__main__": - fire.Fire(main) diff --git a/tests/conftest.py b/tests/conftest.py index 804c60e71..b22e43e79 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,8 +13,11 @@ from unittest.mock import Mock import pytest +from metagpt.config import CONFIG +from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI +from metagpt.utils.git_repository import GitRepository class Context: @@ -80,3 +83,16 @@ def loguru_caplog(caplog): logger.add(PropogateHandler(), format="{message}") yield caplog + + +# init & dispose git repo +@pytest.fixture(scope="session", autouse=True) +def setup_and_teardown_git_repo(request): + CONFIG.git_repo = GitRepository(local_path=DEFAULT_WORKSPACE_ROOT / "unittest") + + # Destroy git repo at the end of the test session. + def fin(): + CONFIG.git_repo.delete_repository() + + # Register the function for destroying the environment. + request.addfinalizer(fin) diff --git a/tests/metagpt/actions/mock.py b/tests/metagpt/actions/mock.py index a800690e8..f6602a82b 100644 --- a/tests/metagpt/actions/mock.py +++ b/tests/metagpt/actions/mock.py @@ -90,7 +90,7 @@ Python's in-built data structures like lists and dictionaries will be used exten For testing, we can use the PyTest framework. This is a mature full-featured Python testing tool that helps you write better programs. -## Python package name: +## Project Name: ```python "adventure_game" ``` @@ -100,7 +100,7 @@ For testing, we can use the PyTest framework. This is a mature full-featured Pyt file_list = ["main.py", "room.py", "player.py", "game.py", "object.py", "puzzle.py", "test_game.py"] ``` -## Data structures and interface definitions: +## Data structures and interfaces: ```mermaid classDiagram class Room{ @@ -209,7 +209,7 @@ Shared knowledge for this project includes understanding the basic principles of """ ``` -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. ```python """ The original requirements did not specify whether the game should have a save/load feature, multiplayer support, or any specific graphical user interface. More information on these aspects could help in further refining the product design and requirements. @@ -311,12 +311,10 @@ TASKS = [ "添加数据API：接受用户输入的文档库，对文档库进行索引\n- 使用MeiliSearch连接并添加文档库", "搜索API：接收用户输入的关键词，返回相关的搜索结果\n- 使用MeiliSearch连接并使用接口获得对应数据", "多条件筛选API：接收用户选择的筛选条件，返回符合条件的搜索结果。\n- 使用MeiliSearch进行筛选并返回符合条件的搜索结果", - "智能推荐API：根据用户的搜索历史记录和搜索行为，推荐相关的搜索结果。" + "智能推荐API：根据用户的搜索历史记录和搜索行为，推荐相关的搜索结果。", ] -TASKS_2 = [ - "完成main.py的功能" -] +TASKS_2 = ["完成main.py的功能"] SEARCH_CODE_SAMPLE = """ import requests @@ -460,7 +458,7 @@ if __name__ == '__main__': print('No results found.') ''' -MEILI_CODE = '''import meilisearch +MEILI_CODE = """import meilisearch from typing import List @@ -496,9 +494,9 @@ if __name__ == '__main__': # 添加文档库到搜索引擎 search_engine.add_documents(books_data_source, documents) -''' +""" -MEILI_ERROR = '''/usr/local/bin/python3.9 /Users/alexanderwu/git/metagpt/examples/search/meilisearch_index.py +MEILI_ERROR = """/usr/local/bin/python3.9 /Users/alexanderwu/git/metagpt/examples/search/meilisearch_index.py Traceback (most recent call last): File "/Users/alexanderwu/git/metagpt/examples/search/meilisearch_index.py", line 44, in search_engine.add_documents(books_data_source, documents) @@ -506,7 +504,7 @@ Traceback (most recent call last): index = self.client.get_or_create_index(index_name) AttributeError: 'Client' object has no attribute 'get_or_create_index' -Process finished with exit code 1''' +Process finished with exit code 1""" MEILI_CODE_REFINED = """ """ diff --git a/tests/metagpt/actions/test_action_output.py b/tests/metagpt/actions/test_action_output.py index a556789db..ef8e239bd 100644 --- a/tests/metagpt/actions/test_action_output.py +++ b/tests/metagpt/actions/test_action_output.py @@ -9,18 +9,21 @@ from typing import List, Tuple from metagpt.actions import ActionOutput -t_dict = {"Required Python third-party packages": "\"\"\"\nflask==1.1.2\npygame==2.0.1\n\"\"\"\n", - "Required Other language third-party packages": "\"\"\"\nNo third-party packages required for other languages.\n\"\"\"\n", - "Full API spec": "\"\"\"\nopenapi: 3.0.0\ninfo:\n title: Web Snake Game API\n version: 1.0.0\npaths:\n /game:\n get:\n summary: Get the current game state\n responses:\n '200':\n description: A JSON object of the game state\n post:\n summary: Send a command to the game\n requestBody:\n required: true\n content:\n application/json:\n schema:\n type: object\n properties:\n command:\n type: string\n responses:\n '200':\n description: A JSON object of the updated game state\n\"\"\"\n", - "Logic Analysis": [ - ["app.py", "Main entry point for the Flask application. Handles HTTP requests and responses."], - ["game.py", "Contains the Game and Snake classes. Handles the game logic."], - ["static/js/script.js", "Handles user interactions and updates the game UI."], - ["static/css/styles.css", "Defines the styles for the game UI."], - ["templates/index.html", "The main page of the web application. Displays the game UI."]], - "Task list": ["game.py", "app.py", "static/css/styles.css", "static/js/script.js", "templates/index.html"], - "Shared Knowledge": "\"\"\"\n'game.py' contains the Game and Snake classes which are responsible for the game logic. The Game class uses an instance of the Snake class.\n\n'app.py' is the main entry point for the Flask application. It creates an instance of the Game class and handles HTTP requests and responses.\n\n'static/js/script.js' is responsible for handling user interactions and updating the game UI based on the game state returned by 'app.py'.\n\n'static/css/styles.css' defines the styles for the game UI.\n\n'templates/index.html' is the main page of the web application. It displays the game UI and loads 'static/js/script.js' and 'static/css/styles.css'.\n\"\"\"\n", - "Anything UNCLEAR": "We need clarification on how the high score should be stored. Should it persist across sessions (stored in a database or a file) or should it reset every time the game is restarted? Also, should the game speed increase as the snake grows, or should it remain constant throughout the game?"} +t_dict = { + "Required Python third-party packages": '"""\nflask==1.1.2\npygame==2.0.1\n"""\n', + "Required Other language third-party packages": '"""\nNo third-party packages required for other languages.\n"""\n', + "Full API spec": '"""\nopenapi: 3.0.0\ninfo:\n title: Web Snake Game API\n version: 1.0.0\npaths:\n /game:\n get:\n summary: Get the current game state\n responses:\n \'200\':\n description: A JSON object of the game state\n post:\n summary: Send a command to the game\n requestBody:\n required: true\n content:\n application/json:\n schema:\n type: object\n properties:\n command:\n type: string\n responses:\n \'200\':\n description: A JSON object of the updated game state\n"""\n', + "Logic Analysis": [ + ["app.py", "Main entry point for the Flask application. Handles HTTP requests and responses."], + ["game.py", "Contains the Game and Snake classes. Handles the game logic."], + ["static/js/script.js", "Handles user interactions and updates the game UI."], + ["static/css/styles.css", "Defines the styles for the game UI."], + ["templates/index.html", "The main page of the web application. Displays the game UI."], + ], + "Task list": ["game.py", "app.py", "static/css/styles.css", "static/js/script.js", "templates/index.html"], + "Shared Knowledge": "\"\"\"\n'game.py' contains the Game and Snake classes which are responsible for the game logic. The Game class uses an instance of the Snake class.\n\n'app.py' is the main entry point for the Flask application. It creates an instance of the Game class and handles HTTP requests and responses.\n\n'static/js/script.js' is responsible for handling user interactions and updating the game UI based on the game state returned by 'app.py'.\n\n'static/css/styles.css' defines the styles for the game UI.\n\n'templates/index.html' is the main page of the web application. It displays the game UI and loads 'static/js/script.js' and 'static/css/styles.css'.\n\"\"\"\n", + "Anything UNCLEAR": "We need clarification on how the high score should be stored. Should it persist across sessions (stored in a database or a file) or should it reset every time the game is restarted? Also, should the game speed increase as the snake grows, or should it remain constant throughout the game?", +} WRITE_TASKS_OUTPUT_MAPPING = { "Required Python third-party packages": (str, ...), @@ -45,6 +48,6 @@ def test_create_model_class_with_mapping(): assert value == ["game.py", "app.py", "static/css/styles.css", "static/js/script.js", "templates/index.html"] -if __name__ == '__main__': +if __name__ == "__main__": test_create_model_class() test_create_model_class_with_mapping() diff --git a/tests/metagpt/actions/test_azure_tts.py b/tests/metagpt/actions/test_azure_tts.py index b5a333af2..bcafe10f5 100644 --- a/tests/metagpt/actions/test_azure_tts.py +++ b/tests/metagpt/actions/test_azure_tts.py @@ -10,12 +10,7 @@ from metagpt.actions.azure_tts import AzureTTS def test_azure_tts(): azure_tts = AzureTTS("azure_tts") - azure_tts.synthesize_speech( - "zh-CN", - "zh-CN-YunxiNeural", - "Boy", - "你好，我是卡卡", - "output.wav") + azure_tts.synthesize_speech("zh-CN", "zh-CN-YunxiNeural", "Boy", "你好，我是卡卡", "output.wav") # 运行需要先配置 SUBSCRIPTION_KEY # TODO: 这里如果要检验，还要额外加上对应的asr，才能确保前后生成是接近一致的，但现在还没有 diff --git a/tests/metagpt/actions/test_clone_function.py b/tests/metagpt/actions/test_clone_function.py index 6d4432dcd..44248eb80 100644 --- a/tests/metagpt/actions/test_clone_function.py +++ b/tests/metagpt/actions/test_clone_function.py @@ -2,7 +2,6 @@ import pytest from metagpt.actions.clone_function import CloneFunction, run_function_code - source_code = """ import pandas as pd import ta @@ -31,14 +30,18 @@ def get_expected_res(): import ta # 读取股票数据 - stock_data = pd.read_csv('./tests/data/baba_stock.csv') + stock_data = pd.read_csv("./tests/data/baba_stock.csv") stock_data.head() # 计算简单移动平均线 - stock_data['SMA'] = ta.trend.sma_indicator(stock_data['Close'], window=6) - stock_data[['Date', 'Close', 'SMA']].head() + stock_data["SMA"] = ta.trend.sma_indicator(stock_data["Close"], window=6) + stock_data[["Date", "Close", "SMA"]].head() # 计算布林带 - stock_data['bb_upper'], stock_data['bb_middle'], stock_data['bb_lower'] = ta.volatility.bollinger_hband_indicator(stock_data['Close'], window=20), ta.volatility.bollinger_mavg(stock_data['Close'], window=20), ta.volatility.bollinger_lband_indicator(stock_data['Close'], window=20) - stock_data[['Date', 'Close', 'bb_upper', 'bb_middle', 'bb_lower']].head() + stock_data["bb_upper"], stock_data["bb_middle"], stock_data["bb_lower"] = ( + ta.volatility.bollinger_hband_indicator(stock_data["Close"], window=20), + ta.volatility.bollinger_mavg(stock_data["Close"], window=20), + ta.volatility.bollinger_lband_indicator(stock_data["Close"], window=20), + ) + stock_data[["Date", "Close", "bb_upper", "bb_middle", "bb_lower"]].head() return stock_data @@ -46,9 +49,9 @@ def get_expected_res(): async def test_clone_function(): clone = CloneFunction() code = await clone.run(template_code, source_code) - assert 'def ' in code - stock_path = './tests/data/baba_stock.csv' - df, msg = run_function_code(code, 'stock_indicator', stock_path) + assert "def " in code + stock_path = "./tests/data/baba_stock.csv" + df, msg = run_function_code(code, "stock_indicator", stock_path) assert not msg expected_df = get_expected_res() assert df.equals(expected_df) diff --git a/tests/metagpt/actions/test_debug_error.py b/tests/metagpt/actions/test_debug_error.py index 555c84e4e..8289fe41b 100644 --- a/tests/metagpt/actions/test_debug_error.py +++ b/tests/metagpt/actions/test_debug_error.py @@ -4,17 +4,19 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : test_debug_error.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ +import uuid + import pytest from metagpt.actions.debug_error import DebugError +from metagpt.config import CONFIG +from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO +from metagpt.schema import RunCodeContext, RunCodeResult +from metagpt.utils.file_repository import FileRepository -EXAMPLE_MSG_CONTENT = ''' ---- -## Development Code File Name -player.py -## Development Code -```python +CODE_CONTENT = ''' from typing import List from deck import Deck from card import Card @@ -58,12 +60,9 @@ class Player: if self.score > 21 and any(card.rank == 'A' for card in self.hand): self.score -= 10 return self.score +''' -``` -## Test File Name -test_player.py -## Test Code -```python +TEST_CONTENT = """ import unittest from blackjack_game.player import Player from blackjack_game.deck import Deck @@ -114,42 +113,41 @@ class TestPlayer(unittest.TestCase): if __name__ == '__main__': unittest.main() -``` -## Running Command -python tests/test_player.py -## Running Output -standard output: ; -standard errors: ..F.. -====================================================================== -FAIL: test_player_calculate_score_with_multiple_aces (__main__.TestPlayer) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "tests/test_player.py", line 46, in test_player_calculate_score_with_multiple_aces - self.assertEqual(player.score, 12) -AssertionError: 22 != 12 +""" ----------------------------------------------------------------------- -Ran 5 tests in 0.007s - -FAILED (failures=1) -; -## instruction: -The error is in the development code, specifically in the calculate_score method of the Player class. The method is not correctly handling the case where there are multiple Aces in the player's hand. The current implementation only subtracts 10 from the score once if the score is over 21 and there's an Ace in the hand. However, in the case of multiple Aces, it should subtract 10 for each Ace until the score is 21 or less. -## File To Rewrite: -player.py -## Status: -FAIL -## Send To: -Engineer ---- -''' @pytest.mark.asyncio async def test_debug_error(): + CONFIG.src_workspace = CONFIG.git_repo.workdir / uuid.uuid4().hex + ctx = RunCodeContext( + code_filename="player.py", + test_filename="test_player.py", + command=["python", "tests/test_player.py"], + output_filename="output.log", + ) - debug_error = DebugError("debug_error") + await FileRepository.save_file(filename=ctx.code_filename, content=CODE_CONTENT, relative_path=CONFIG.src_workspace) + await FileRepository.save_file(filename=ctx.test_filename, content=TEST_CONTENT, relative_path=TEST_CODES_FILE_REPO) + output_data = RunCodeResult( + stdout=";", + stderr="", + summary="======================================================================\n" + "FAIL: test_player_calculate_score_with_multiple_aces (__main__.TestPlayer)\n" + "----------------------------------------------------------------------\n" + "Traceback (most recent call last):\n" + ' File "tests/test_player.py", line 46, in test_player_calculate_score_' + "with_multiple_aces\n" + " self.assertEqual(player.score, 12)\nAssertionError: 22 != 12\n\n" + "----------------------------------------------------------------------\n" + "Ran 5 tests in 0.007s\n\nFAILED (failures=1)\n;\n", + ) + await FileRepository.save_file( + filename=ctx.output_filename, content=output_data.json(), relative_path=TEST_OUTPUTS_FILE_REPO + ) + debug_error = DebugError(context=ctx) - file_name, rewritten_code = await debug_error.run(context=EXAMPLE_MSG_CONTENT) + rsp = await debug_error.run() - assert "class Player" in rewritten_code # rewrite the same class - assert "while self.score > 21" in rewritten_code # a key logic to rewrite to (original one is "if self.score > 12") + assert "class Player" in rsp # rewrite the same class + # a key logic to rewrite to (original one is "if self.score > 12") + assert "while self.score > 21" in rsp diff --git a/tests/metagpt/actions/test_design_api.py b/tests/metagpt/actions/test_design_api.py index 0add8fb74..e90707d1a 100644 --- a/tests/metagpt/actions/test_design_api.py +++ b/tests/metagpt/actions/test_design_api.py @@ -4,33 +4,27 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : test_design_api.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest from metagpt.actions.design_api import WriteDesign +from metagpt.const import PRDS_FILE_REPO from metagpt.logs import logger from metagpt.schema import Message +from metagpt.utils.file_repository import FileRepository from tests.metagpt.actions.mock import PRD_SAMPLE @pytest.mark.asyncio async def test_design_api(): - prd = "我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。" + inputs = ["我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。", PRD_SAMPLE] + for prd in inputs: + await FileRepository.save_file("new_prd.txt", content=prd, relative_path=PRDS_FILE_REPO) - design_api = WriteDesign("design_api") + design_api = WriteDesign("design_api") - result = await design_api.run([Message(content=prd, instruct_content=None)]) - logger.info(result) + result = await design_api.run([Message(content=prd, instruct_content=None)]) + logger.info(result) - assert result - - -@pytest.mark.asyncio -async def test_design_api_calculator(): - prd = PRD_SAMPLE - - design_api = WriteDesign("design_api") - result = await design_api.run([Message(content=prd, instruct_content=None)]) - logger.info(result) - - assert result + assert result diff --git a/tests/metagpt/actions/test_detail_mining.py b/tests/metagpt/actions/test_detail_mining.py index c9d5331f9..891dca6ca 100644 --- a/tests/metagpt/actions/test_detail_mining.py +++ b/tests/metagpt/actions/test_detail_mining.py @@ -10,6 +10,7 @@ import pytest from metagpt.actions.detail_mining import DetailMining from metagpt.logs import logger + @pytest.mark.asyncio async def test_detail_mining(): topic = "如何做一个生日蛋糕" @@ -17,7 +18,6 @@ async def test_detail_mining(): detail_mining = DetailMining("detail_mining") rsp = await detail_mining.run(topic=topic, record=record) logger.info(f"{rsp.content=}") - - assert '##OUTPUT' in rsp.content - assert '蛋糕' in rsp.content + assert "##OUTPUT" in rsp.content + assert "蛋糕" in rsp.content diff --git a/tests/metagpt/actions/test_invoice_ocr.py b/tests/metagpt/actions/test_invoice_ocr.py index a15166f7c..7f16aa9a4 100644 --- a/tests/metagpt/actions/test_invoice_ocr.py +++ b/tests/metagpt/actions/test_invoice_ocr.py @@ -8,12 +8,11 @@ """ import os -from typing import List - -import pytest from pathlib import Path -from metagpt.actions.invoice_ocr import InvoiceOCR, GenerateTable, ReplyQuestion +import pytest + +from metagpt.actions.invoice_ocr import GenerateTable, InvoiceOCR, ReplyQuestion @pytest.mark.asyncio @@ -22,7 +21,7 @@ from metagpt.actions.invoice_ocr import InvoiceOCR, GenerateTable, ReplyQuestion [ "../../data/invoices/invoice-3.jpg", "../../data/invoices/invoice-4.zip", - ] + ], ) async def test_invoice_ocr(invoice_path: str): invoice_path = os.path.abspath(os.path.join(os.getcwd(), invoice_path)) @@ -35,18 +34,8 @@ async def test_invoice_ocr(invoice_path: str): @pytest.mark.parametrize( ("invoice_path", "expected_result"), [ - ( - "../../data/invoices/invoice-1.pdf", - [ - { - "收款人": "小明", - "城市": "深圳市", - "总费用/元": "412.00", - "开票日期": "2023年02月03日" - } - ] - ), - ] + ("../../data/invoices/invoice-1.pdf", [{"收款人": "小明", "城市": "深圳市", "总费用/元": "412.00", "开票日期": "2023年02月03日"}]), + ], ) async def test_generate_table(invoice_path: str, expected_result: list[dict]): invoice_path = os.path.abspath(os.path.join(os.getcwd(), invoice_path)) @@ -59,9 +48,7 @@ async def test_generate_table(invoice_path: str, expected_result: list[dict]): @pytest.mark.asyncio @pytest.mark.parametrize( ("invoice_path", "query", "expected_result"), - [ - ("../../data/invoices/invoice-1.pdf", "Invoicing date", "2023年02月03日") - ] + [("../../data/invoices/invoice-1.pdf", "Invoicing date", "2023年02月03日")], ) async def test_reply_question(invoice_path: str, query: dict, expected_result: str): invoice_path = os.path.abspath(os.path.join(os.getcwd(), invoice_path)) @@ -69,4 +56,3 @@ async def test_reply_question(invoice_path: str, query: dict, expected_result: s ocr_result = await InvoiceOCR().run(file_path=Path(invoice_path), filename=filename) result = await ReplyQuestion().run(query=query, ocr_result=ocr_result) assert expected_result in result - diff --git a/tests/metagpt/actions/test_prepare_documents.py b/tests/metagpt/actions/test_prepare_documents.py new file mode 100644 index 000000000..31c8bcb80 --- /dev/null +++ b/tests/metagpt/actions/test_prepare_documents.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/6 +@Author : mashenquan +@File : test_prepare_documents.py +@Desc: Unit test for prepare_documents.py +""" +import pytest + +from metagpt.actions.prepare_documents import PrepareDocuments +from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME +from metagpt.schema import Message +from metagpt.utils.file_repository import FileRepository + + +@pytest.mark.asyncio +async def test_prepare_documents(): + msg = Message(content="New user requirements balabala...") + + if CONFIG.git_repo: + CONFIG.git_repo.delete_repository() + CONFIG.git_repo = None + + await PrepareDocuments().run(with_messages=[msg]) + assert CONFIG.git_repo + doc = await FileRepository.get_file(filename=REQUIREMENT_FILENAME, relative_path=DOCS_FILE_REPO) + assert doc + assert doc.content == msg.content diff --git a/tests/metagpt/actions/test_run_code.py b/tests/metagpt/actions/test_run_code.py index 1e451cb14..888418974 100644 --- a/tests/metagpt/actions/test_run_code.py +++ b/tests/metagpt/actions/test_run_code.py @@ -4,10 +4,12 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : test_run_code.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest from metagpt.actions.run_code import RunCode +from metagpt.schema import RunCodeContext @pytest.mark.asyncio @@ -35,37 +37,29 @@ async def test_run_script(): @pytest.mark.asyncio async def test_run(): - action = RunCode() - result = await action.run(mode="text", code="print('Hello, World')") - assert "PASS" in result - - result = await action.run( - mode="script", - code="echo 'Hello World'", - code_file_name="", - test_code="", - test_file_name="", - command=["echo", "Hello World"], - working_directory=".", - additional_python_paths=[], - ) - assert "PASS" in result - - -@pytest.mark.asyncio -async def test_run_failure(): - action = RunCode() - result = await action.run(mode="text", code="result = 1 / 0") - assert "FAIL" in result - - result = await action.run( - mode="script", - code='python -c "print(1/0)"', - code_file_name="", - test_code="", - test_file_name="", - command=["python", "-c", "print(1/0)"], - working_directory=".", - additional_python_paths=[], - ) - assert "FAIL" in result + inputs = [ + (RunCodeContext(mode="text", code_filename="a.txt", code="print('Hello, World')"), "PASS"), + ( + RunCodeContext( + mode="script", + code_filename="a.sh", + code="echo 'Hello World'", + command=["echo", "Hello World"], + working_directory=".", + ), + "PASS", + ), + ( + RunCodeContext( + mode="script", + code_filename="a.py", + code='python -c "print(1/0)"', + command=["python", "-c", "print(1/0)"], + working_directory=".", + ), + "FAIL", + ), + ] + for ctx, result in inputs: + rsp = await RunCode(context=ctx).run() + assert result in rsp.summary diff --git a/tests/metagpt/actions/test_summarize_code.py b/tests/metagpt/actions/test_summarize_code.py new file mode 100644 index 000000000..7ecb67afd --- /dev/null +++ b/tests/metagpt/actions/test_summarize_code.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/11 17:46 +@Author : mashenquan +@File : test_summarize_code.py +@Modifiled By: mashenquan, 2023-12-6. Unit test for summarize_code.py +""" +import pytest + +from metagpt.actions.summarize_code import SummarizeCode +from metagpt.config import CONFIG +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO +from metagpt.logs import logger +from metagpt.schema import CodeSummarizeContext +from metagpt.utils.file_repository import FileRepository + +DESIGN_CONTENT = """ +{"Implementation approach": "To develop this snake game, we will use the Python language and choose the Pygame library. Pygame is an open-source Python module collection specifically designed for writing video games. It provides functionalities such as displaying images and playing sounds, making it suitable for creating intuitive and responsive user interfaces. We will ensure efficient game logic to prevent any delays during gameplay. The scoring system will be simple, with the snake gaining points for each food it eats. We will use Pygame's event handling system to implement pause and resume functionality, as well as high-score tracking. The difficulty will increase by speeding up the snake's movement. In the initial version, we will focus on single-player mode and consider adding multiplayer mode and customizable skins in future updates. Based on the new requirement, we will also add a moving obstacle that appears randomly. If the snake eats this obstacle, the game will end. If the snake does not eat the obstacle, it will disappear after 5 seconds. For this, we need to add mechanisms for obstacle generation, movement, and disappearance in the game logic.", "Project_name": "snake_game", "File list": ["main.py", "game.py", "snake.py", "food.py", "obstacle.py", "scoreboard.py", "constants.py", "assets/styles.css", "assets/index.html"], "Data structures and interfaces": "```mermaid\n classDiagram\n class Game{\n +int score\n +int speed\n +bool game_over\n +bool paused\n +Snake snake\n +Food food\n +Obstacle obstacle\n +Scoreboard scoreboard\n +start_game() void\n +pause_game() void\n +resume_game() void\n +end_game() void\n +increase_difficulty() void\n +update() void\n +render() void\n Game()\n }\n class Snake{\n +list body_parts\n +str direction\n +bool grow\n +move() void\n +grow() void\n +check_collision() bool\n Snake()\n }\n class Food{\n +tuple position\n +spawn() void\n Food()\n }\n class Obstacle{\n +tuple position\n +int lifetime\n +bool active\n +spawn() void\n +move() void\n +check_collision() bool\n +disappear() void\n Obstacle()\n }\n class Scoreboard{\n +int high_score\n +update_score(int) void\n +reset_score() void\n +load_high_score() void\n +save_high_score() void\n Scoreboard()\n }\n class Constants{\n }\n Game \"1\" -- \"1\" Snake: has\n Game \"1\" -- \"1\" Food: has\n Game \"1\" -- \"1\" Obstacle: has\n Game \"1\" -- \"1\" Scoreboard: has\n ```", "Program call flow": "```sequenceDiagram\n participant M as Main\n participant G as Game\n participant S as Snake\n participant F as Food\n participant O as Obstacle\n participant SB as Scoreboard\n M->>G: start_game()\n loop game loop\n G->>S: move()\n G->>S: check_collision()\n G->>F: spawn()\n G->>O: spawn()\n G->>O: move()\n G->>O: check_collision()\n G->>O: disappear()\n G->>SB: update_score(score)\n G->>G: update()\n G->>G: render()\n alt if paused\n M->>G: pause_game()\n M->>G: resume_game()\n end\n alt if game_over\n G->>M: end_game()\n end\n end\n```", "Anything UNCLEAR": "There is no need for further clarification as the requirements are already clear."} +""" + +TASK_CONTENT = """ +{"Required Python third-party packages": ["pygame==2.0.1"], "Required Other language third-party packages": ["No third-party packages required for other languages."], "Full API spec": "\n openapi: 3.0.0\n info:\n title: Snake Game API\n version: \"1.0.0\"\n paths:\n /start:\n get:\n summary: Start the game\n responses:\n '200':\n description: Game started successfully\n /pause:\n get:\n summary: Pause the game\n responses:\n '200':\n description: Game paused successfully\n /resume:\n get:\n summary: Resume the game\n responses:\n '200':\n description: Game resumed successfully\n /end:\n get:\n summary: End the game\n responses:\n '200':\n description: Game ended successfully\n /score:\n get:\n summary: Get the current score\n responses:\n '200':\n description: Current score retrieved successfully\n /highscore:\n get:\n summary: Get the high score\n responses:\n '200':\n description: High score retrieved successfully\n components: {}\n ", "Logic Analysis": [["constants.py", "Contains all the constant values like screen size, colors, game speeds, etc. This should be implemented first as it provides the base values for other components."], ["snake.py", "Contains the Snake class with methods for movement, growth, and collision detection. It is dependent on constants.py for configuration values."], ["food.py", "Contains the Food class responsible for spawning food items on the screen. It is dependent on constants.py for configuration values."], ["obstacle.py", "Contains the Obstacle class with methods for spawning, moving, and disappearing of obstacles, as well as collision detection with the snake. It is dependent on constants.py for configuration values."], ["scoreboard.py", "Contains the Scoreboard class for updating, resetting, loading, and saving high scores. It may use constants.py for configuration values and depends on the game's scoring logic."], ["game.py", "Contains the main Game class which includes the game loop and methods for starting, pausing, resuming, and ending the game. It is dependent on snake.py, food.py, obstacle.py, and scoreboard.py."], ["main.py", "The entry point of the game that initializes the game and starts the game loop. It is dependent on game.py."]], "Task list": ["constants.py", "snake.py", "food.py", "obstacle.py", "scoreboard.py", "game.py", "main.py"], "Shared Knowledge": "\n 'constants.py' should contain all the necessary configurations for the game, such as screen dimensions, color definitions, and speed settings. These constants will be used across multiple files, ensuring consistency and ease of updates. Ensure that the Pygame library is initialized correctly in 'main.py' before starting the game loop. Also, make sure that the game's state is managed properly when pausing and resuming the game.\n ", "Anything UNCLEAR": "The interaction between the 'obstacle.py' and the game loop needs to be clearly defined to ensure obstacles appear and disappear correctly. The lifetime of the obstacle and its random movement should be implemented in a way that does not interfere with the game's performance."} +""" + +FOOD_PY = """ +## food.py +import random + +class Food: + def __init__(self): + self.position = (0, 0) + + def generate(self): + x = random.randint(0, 9) + y = random.randint(0, 9) + self.position = (x, y) + + def get_position(self): + return self.position + +""" + +GAME_PY = """ +## game.py +import pygame +from snake import Snake +from food import Food + +class Game: + def __init__(self): + self.score = 0 + self.level = 1 + self.snake = Snake() + self.food = Food() + + def start_game(self): + pygame.init() + self.initialize_game() + self.game_loop() + + def initialize_game(self): + self.score = 0 + self.level = 1 + self.snake.reset() + self.food.generate() + + def game_loop(self): + game_over = False + + while not game_over: + self.update() + self.draw() + self.handle_events() + self.check_collision() + self.increase_score() + self.increase_level() + + if self.snake.is_collision(): + game_over = True + self.game_over() + + def update(self): + self.snake.move() + + def draw(self): + self.snake.draw() + self.food.draw() + + def handle_events(self): + for event in pygame.event.get(): + if event.type == pygame.QUIT: + pygame.quit() + quit() + elif event.type == pygame.KEYDOWN: + if event.key == pygame.K_UP: + self.snake.change_direction("UP") + elif event.key == pygame.K_DOWN: + self.snake.change_direction("DOWN") + elif event.key == pygame.K_LEFT: + self.snake.change_direction("LEFT") + elif event.key == pygame.K_RIGHT: + self.snake.change_direction("RIGHT") + + def check_collision(self): + if self.snake.get_head() == self.food.get_position(): + self.snake.grow() + self.food.generate() + + def increase_score(self): + self.score += 1 + + def increase_level(self): + if self.score % 10 == 0: + self.level += 1 + + def game_over(self): + print("Game Over") + self.initialize_game() + +""" + +MAIN_PY = """ +## main.py +import pygame +from game import Game + +def main(): + pygame.init() + game = Game() + game.start_game() + +if __name__ == "__main__": + main() + +""" + +SNAKE_PY = """ +## snake.py +import pygame + +class Snake: + def __init__(self): + self.body = [(0, 0)] + self.direction = (1, 0) + + def move(self): + head = self.body[0] + dx, dy = self.direction + new_head = (head[0] + dx, head[1] + dy) + self.body.insert(0, new_head) + self.body.pop() + + def change_direction(self, direction): + if direction == "UP": + self.direction = (0, -1) + elif direction == "DOWN": + self.direction = (0, 1) + elif direction == "LEFT": + self.direction = (-1, 0) + elif direction == "RIGHT": + self.direction = (1, 0) + + def grow(self): + tail = self.body[-1] + dx, dy = self.direction + new_tail = (tail[0] - dx, tail[1] - dy) + self.body.append(new_tail) + + def get_head(self): + return self.body[0] + + def get_body(self): + return self.body[1:] + +""" + + +@pytest.mark.asyncio +async def test_summarize_code(): + CONFIG.src_workspace = CONFIG.git_repo.workdir / "src" + await FileRepository.save_file(filename="1.json", relative_path=SYSTEM_DESIGN_FILE_REPO, content=DESIGN_CONTENT) + await FileRepository.save_file(filename="1.json", relative_path=TASK_FILE_REPO, content=TASK_CONTENT) + await FileRepository.save_file(filename="food.py", relative_path=CONFIG.src_workspace, content=FOOD_PY) + await FileRepository.save_file(filename="game.py", relative_path=CONFIG.src_workspace, content=GAME_PY) + await FileRepository.save_file(filename="main.py", relative_path=CONFIG.src_workspace, content=MAIN_PY) + await FileRepository.save_file(filename="snake.py", relative_path=CONFIG.src_workspace, content=SNAKE_PY) + + src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) + all_files = src_file_repo.all_files + ctx = CodeSummarizeContext(design_filename="1.json", task_filename="1.json", codes_filenames=all_files) + action = SummarizeCode(context=ctx) + rsp = await action.run() + assert rsp + logger.info(rsp) diff --git a/tests/metagpt/actions/test_ui_design.py b/tests/metagpt/actions/test_ui_design.py index d284b20f2..83590ec7d 100644 --- a/tests/metagpt/actions/test_ui_design.py +++ b/tests/metagpt/actions/test_ui_design.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- # @Date : 2023/7/22 02:40 -# @Author : stellahong (stellahong@fuzhi.ai) +# @Author : stellahong (stellahong@deepwisdom.ai) # from tests.metagpt.roles.ui_role import UIDesign -llm_resp= ''' +llm_resp = """ # UI Design Description ```The user interface for the snake game will be designed in a way that is simple, clean, and intuitive. The main elements of the game such as the game grid, snake, food, score, and game over message will be clearly defined and easy to understand. The game grid will be centered on the screen with the score displayed at the top. The game controls will be intuitive and easy to use. The design will be modern and minimalist with a pleasing color scheme.``` @@ -98,12 +98,13 @@ body { left: 50%; transform: translate(-50%, -50%); font-size: 3em; - ''' + """ + def test_ui_design_parse_css(): ui_design_work = UIDesign(name="UI design action") - css = ''' + css = """ body { display: flex; flex-direction: column; @@ -160,14 +161,14 @@ def test_ui_design_parse_css(): left: 50%; transform: translate(-50%, -50%); font-size: 3em; - ''' - assert ui_design_work.parse_css_code(context=llm_resp)==css + """ + assert ui_design_work.parse_css_code(context=llm_resp) == css def test_ui_design_parse_html(): ui_design_work = UIDesign(name="UI design action") - html = ''' + html = """ @@ -184,8 +185,5 @@ def test_ui_design_parse_html():

Game Over
- ''' - assert ui_design_work.parse_css_code(context=llm_resp)==html - - - + """ + assert ui_design_work.parse_css_code(context=llm_resp) == html diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index 7bb18ddf2..54229089c 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -4,31 +4,36 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : test_write_code.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest from metagpt.actions.write_code import WriteCode from metagpt.llm import LLM from metagpt.logs import logger +from metagpt.schema import CodingContext, Document from tests.metagpt.actions.mock import TASKS_2, WRITE_CODE_PROMPT_SAMPLE @pytest.mark.asyncio async def test_write_code(): - api_design = "设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。" - write_code = WriteCode("write_code") + context = CodingContext( + filename="task_filename.py", design_doc=Document(content="设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。") + ) + doc = Document(content=context.json()) + write_code = WriteCode(context=doc) - code = await write_code.run(api_design) - logger.info(code) + code = await write_code.run() + logger.info(code.json()) # 我们不能精确地预测生成的代码，但我们可以检查某些关键字 - assert 'def add' in code - assert 'return' in code + assert "def add" in code.code_doc.content + assert "return" in code.code_doc.content @pytest.mark.asyncio async def test_write_code_directly(): - prompt = WRITE_CODE_PROMPT_SAMPLE + '\n' + TASKS_2[0] + prompt = WRITE_CODE_PROMPT_SAMPLE + "\n" + TASKS_2[0] llm = LLM() rsp = await llm.aask(prompt) logger.info(rsp) diff --git a/tests/metagpt/actions/test_write_code_review.py b/tests/metagpt/actions/test_write_code_review.py index 21bc563ec..e16eb7348 100644 --- a/tests/metagpt/actions/test_write_code_review.py +++ b/tests/metagpt/actions/test_write_code_review.py @@ -8,6 +8,8 @@ import pytest from metagpt.actions.write_code_review import WriteCodeReview +from metagpt.document import Document +from metagpt.schema import CodingContext @pytest.mark.asyncio @@ -16,13 +18,15 @@ async def test_write_code_review(capfd): def add(a, b): return a + """ - # write_code_review = WriteCodeReview("write_code_review") + context = CodingContext( + filename="math.py", design_doc=Document(content="编写一个从a加b的函数，返回a+b"), code_doc=Document(content=code) + ) - code = await WriteCodeReview().run(context="编写一个从a加b的函数，返回a+b", code=code, filename="math.py") + context = await WriteCodeReview(context=context).run() # 我们不能精确地预测生成的代码评审，但我们可以检查返回的是否为字符串 - assert isinstance(code, str) - assert len(code) > 0 + assert isinstance(context.code_doc.content, str) + assert len(context.code_doc.content) > 0 captured = capfd.readouterr() print(f"输出内容: {captured.out}") diff --git a/tests/metagpt/actions/test_write_docstring.py b/tests/metagpt/actions/test_write_docstring.py index 82d96e1a6..a8a80b36d 100644 --- a/tests/metagpt/actions/test_write_docstring.py +++ b/tests/metagpt/actions/test_write_docstring.py @@ -2,7 +2,7 @@ import pytest from metagpt.actions.write_docstring import WriteDocstring -code = ''' +code = """ def add_numbers(a: int, b: int): return a + b @@ -14,7 +14,7 @@ class Person: def greet(self): return f"Hello, my name is {self.name} and I am {self.age} years old." -''' +""" @pytest.mark.asyncio @@ -25,7 +25,7 @@ class Person: ("numpy", "Parameters"), ("sphinx", ":param name:"), ], - ids=["google", "numpy", "sphinx"] + ids=["google", "numpy", "sphinx"], ) async def test_write_docstring(style: str, part: str): ret = await WriteDocstring().run(code, style=style) diff --git a/tests/metagpt/actions/test_write_prd.py b/tests/metagpt/actions/test_write_prd.py index 38e4e5221..08be3cf75 100644 --- a/tests/metagpt/actions/test_write_prd.py +++ b/tests/metagpt/actions/test_write_prd.py @@ -4,23 +4,29 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : test_write_prd.py +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, replace `handle` with `run`. """ import pytest -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement +from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, PRDS_FILE_REPO, REQUIREMENT_FILENAME from metagpt.logs import logger from metagpt.roles.product_manager import ProductManager from metagpt.schema import Message +from metagpt.utils.file_repository import FileRepository @pytest.mark.asyncio async def test_write_prd(): product_manager = ProductManager() requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结" - prd = await product_manager.handle(Message(content=requirements, cause_by=BossRequirement)) + await FileRepository.save_file(filename=REQUIREMENT_FILENAME, content=requirements, relative_path=DOCS_FILE_REPO) + prd = await product_manager.run(Message(content=requirements, cause_by=UserRequirement)) logger.info(requirements) logger.info(prd) # Assert the prd is not None or empty assert prd is not None - assert prd != "" + assert prd.content != "" + assert CONFIG.git_repo.new_file_repository(relative_path=PRDS_FILE_REPO).changed_files diff --git a/tests/metagpt/actions/test_write_test.py b/tests/metagpt/actions/test_write_test.py index e5acdff44..a3190fb0e 100644 --- a/tests/metagpt/actions/test_write_test.py +++ b/tests/metagpt/actions/test_write_test.py @@ -9,6 +9,7 @@ import pytest from metagpt.actions.write_test import WriteTest from metagpt.logs import logger +from metagpt.schema import Document, TestingContext @pytest.mark.asyncio @@ -24,22 +25,17 @@ async def test_write_test(): def generate(self, max_y: int, max_x: int): self.position = (random.randint(1, max_y - 1), random.randint(1, max_x - 1)) """ + context = TestingContext(filename="food.py", code_doc=Document(filename="food.py", content=code)) + write_test = WriteTest(context=context) - write_test = WriteTest() - - test_code = await write_test.run( - code_to_test=code, - test_file_name="test_food.py", - source_file_path="/some/dummy/path/cli_snake_game/cli_snake_game/food.py", - workspace="/some/dummy/path/cli_snake_game", - ) - logger.info(test_code) + context = await write_test.run() + logger.info(context.json()) # We cannot exactly predict the generated test cases, but we can check if it is a string and if it is not empty - assert isinstance(test_code, str) - assert "from cli_snake_game.food import Food" in test_code - assert "class TestFood(unittest.TestCase)" in test_code - assert "def test_generate" in test_code + assert isinstance(context.test_doc.content, str) + assert "from food import Food" in context.test_doc.content + assert "class TestFood(unittest.TestCase)" in context.test_doc.content + assert "def test_generate" in context.test_doc.content @pytest.mark.asyncio diff --git a/tests/metagpt/actions/test_write_tutorial.py b/tests/metagpt/actions/test_write_tutorial.py index 683fee082..27a323b44 100644 --- a/tests/metagpt/actions/test_write_tutorial.py +++ b/tests/metagpt/actions/test_write_tutorial.py @@ -9,14 +9,11 @@ from typing import Dict import pytest -from metagpt.actions.write_tutorial import WriteDirectory, WriteContent +from metagpt.actions.write_tutorial import WriteContent, WriteDirectory @pytest.mark.asyncio -@pytest.mark.parametrize( - ("language", "topic"), - [("English", "Write a tutorial about Python")] -) +@pytest.mark.parametrize(("language", "topic"), [("English", "Write a tutorial about Python")]) async def test_write_directory(language: str, topic: str): ret = await WriteDirectory(language=language).run(topic=topic) assert isinstance(ret, dict) @@ -30,7 +27,7 @@ async def test_write_directory(language: str, topic: str): @pytest.mark.asyncio @pytest.mark.parametrize( ("language", "topic", "directory"), - [("English", "Write a tutorial about Python", {"Introduction": ["What is Python?", "Why learn Python?"]})] + [("English", "Write a tutorial about Python", {"Introduction": ["What is Python?", "Why learn Python?"]})], ) async def test_write_content(language: str, topic: str, directory: Dict): ret = await WriteContent(language=language, directory=directory).run(topic=topic) diff --git a/tests/metagpt/document_store/test_chromadb_store.py b/tests/metagpt/document_store/test_chromadb_store.py index f8c11e1ca..fd115dcdd 100644 --- a/tests/metagpt/document_store/test_chromadb_store.py +++ b/tests/metagpt/document_store/test_chromadb_store.py @@ -12,12 +12,12 @@ from metagpt.document_store.chromadb_store import ChromaStore def test_chroma_store(): """FIXME：chroma使用感觉很诡异，一用Python就挂，测试用例里也是""" # 创建 ChromaStore 实例，使用 'sample_collection' 集合 - document_store = ChromaStore('sample_collection_1') + document_store = ChromaStore("sample_collection_1") # 使用 write 方法添加多个文档 - document_store.write(["This is document1", "This is document2"], - [{"source": "google-docs"}, {"source": "notion"}], - ["doc1", "doc2"]) + document_store.write( + ["This is document1", "This is document2"], [{"source": "google-docs"}, {"source": "notion"}], ["doc1", "doc2"] + ) # 使用 add 方法添加一个文档 document_store.add("This is document3", {"source": "notion"}, "doc3") diff --git a/tests/metagpt/document_store/test_document.py b/tests/metagpt/document_store/test_document.py index 5ae357fb1..13c0921a3 100644 --- a/tests/metagpt/document_store/test_document.py +++ b/tests/metagpt/document_store/test_document.py @@ -7,22 +7,22 @@ """ import pytest -from metagpt.const import DATA_PATH -from metagpt.document_store.document import Document +from metagpt.const import METAGPT_ROOT +from metagpt.document import IndexableDocument CASES = [ - ("st/faq.xlsx", "Question", "Answer", 1), - ("cases/faq.csv", "Question", "Answer", 1), + ("requirements.txt", None, None, 0), + # ("cases/faq.csv", "Question", "Answer", 1), # ("cases/faq.json", "Question", "Answer", 1), - ("docx/faq.docx", None, None, 1), - ("cases/faq.pdf", None, None, 0), # 这是因为pdf默认没有分割段落 - ("cases/faq.txt", None, None, 0), # 这是因为txt按照256分割段落 + # ("docx/faq.docx", None, None, 1), + # ("cases/faq.pdf", None, None, 0), # 这是因为pdf默认没有分割段落 + # ("cases/faq.txt", None, None, 0), # 这是因为txt按照256分割段落 ] @pytest.mark.parametrize("relative_path, content_col, meta_col, threshold", CASES) def test_document(relative_path, content_col, meta_col, threshold): - doc = Document(DATA_PATH / relative_path, content_col, meta_col) + doc = IndexableDocument.from_path(METAGPT_ROOT / relative_path, content_col, meta_col) rsp = doc.get_docs_and_metadatas() assert len(rsp[0]) > threshold assert len(rsp[1]) > threshold diff --git a/tests/metagpt/document_store/test_faiss_store.py b/tests/metagpt/document_store/test_faiss_store.py index d22d234f5..f14bee817 100644 --- a/tests/metagpt/document_store/test_faiss_store.py +++ b/tests/metagpt/document_store/test_faiss_store.py @@ -39,11 +39,11 @@ user: 没有了 @pytest.mark.asyncio async def test_faiss_store_search(): - store = FaissStore(DATA_PATH / 'qcs/qcs_4w.json') - store.add(['油皮洗面奶']) + store = FaissStore(DATA_PATH / "qcs/qcs_4w.json") + store.add(["油皮洗面奶"]) role = Sales(store=store) - queries = ['油皮洗面奶', '介绍下欧莱雅的'] + queries = ["油皮洗面奶", "介绍下欧莱雅的"] for query in queries: rsp = await role.run(query) assert rsp @@ -60,7 +60,10 @@ def customer_service(): async def test_faiss_store_customer_service(): allq = [ # ["我的餐怎么两小时都没到", "退货吧"], - ["你好收不到取餐码，麻烦帮我开箱", "14750187158", ] + [ + "你好收不到取餐码，麻烦帮我开箱", + "14750187158", + ] ] role = customer_service() for queries in allq: @@ -71,4 +74,4 @@ async def test_faiss_store_customer_service(): def test_faiss_store_no_file(): with pytest.raises(FileNotFoundError): - FaissStore(DATA_PATH / 'wtf.json') + FaissStore(DATA_PATH / "wtf.json") diff --git a/tests/metagpt/document_store/test_lancedb_store.py b/tests/metagpt/document_store/test_lancedb_store.py index 9c2f9fb42..5c0e40f57 100644 --- a/tests/metagpt/document_store/test_lancedb_store.py +++ b/tests/metagpt/document_store/test_lancedb_store.py @@ -5,27 +5,33 @@ @Author : unkn-wn (Leon Yee) @File : test_lancedb_store.py """ -from metagpt.document_store.lancedb_store import LanceStore -import pytest import random +import pytest + +from metagpt.document_store.lancedb_store import LanceStore + + @pytest def test_lance_store(): - # This simply establishes the connection to the database, so we can drop the table if it exists - store = LanceStore('test') + store = LanceStore("test") - store.drop('test') + store.drop("test") - store.write(data=[[random.random() for _ in range(100)] for _ in range(2)], - metadatas=[{"source": "google-docs"}, {"source": "notion"}], - ids=["doc1", "doc2"]) + store.write( + data=[[random.random() for _ in range(100)] for _ in range(2)], + metadatas=[{"source": "google-docs"}, {"source": "notion"}], + ids=["doc1", "doc2"], + ) store.add(data=[random.random() for _ in range(100)], metadata={"source": "notion"}, _id="doc3") result = store.search([random.random() for _ in range(100)], n_results=3) - assert(len(result) == 3) + assert len(result) == 3 store.delete("doc2") - result = store.search([random.random() for _ in range(100)], n_results=3, where="source = 'notion'", metric='cosine') - assert(len(result) == 1) \ No newline at end of file + result = store.search( + [random.random() for _ in range(100)], n_results=3, where="source = 'notion'", metric="cosine" + ) + assert len(result) == 1 diff --git a/tests/metagpt/document_store/test_milvus_store.py b/tests/metagpt/document_store/test_milvus_store.py index 1cf65776d..34497b9c6 100644 --- a/tests/metagpt/document_store/test_milvus_store.py +++ b/tests/metagpt/document_store/test_milvus_store.py @@ -12,7 +12,7 @@ import numpy as np from metagpt.document_store.milvus_store import MilvusConnection, MilvusStore from metagpt.logs import logger -book_columns = {'idx': int, 'name': str, 'desc': str, 'emb': np.ndarray, 'price': float} +book_columns = {"idx": int, "name": str, "desc": str, "emb": np.ndarray, "price": float} book_data = [ [i for i in range(10)], [f"book-{i}" for i in range(10)], @@ -25,12 +25,12 @@ book_data = [ def test_milvus_store(): milvus_connection = MilvusConnection(alias="default", host="192.168.50.161", port="30530") milvus_store = MilvusStore(milvus_connection) - milvus_store.drop('Book') - milvus_store.create_collection('Book', book_columns) + milvus_store.drop("Book") + milvus_store.create_collection("Book", book_columns) milvus_store.add(book_data) - milvus_store.build_index('emb') + milvus_store.build_index("emb") milvus_store.load_collection() - results = milvus_store.search([[1.0, 1.0]], field='emb') + results = milvus_store.search([[1.0, 1.0]], field="emb") logger.info(results) assert results diff --git a/tests/metagpt/document_store/test_qdrant_store.py b/tests/metagpt/document_store/test_qdrant_store.py index a63a4329d..cdd619d37 100644 --- a/tests/metagpt/document_store/test_qdrant_store.py +++ b/tests/metagpt/document_store/test_qdrant_store.py @@ -24,9 +24,7 @@ random.seed(seed_value) vectors = [[random.random() for _ in range(2)] for _ in range(10)] points = [ - PointStruct( - id=idx, vector=vector, payload={"color": "red", "rand_number": idx % 10} - ) + PointStruct(id=idx, vector=vector, payload={"color": "red", "rand_number": idx % 10}) for idx, vector in enumerate(vectors) ] @@ -57,9 +55,7 @@ def test_milvus_store(): results = qdrant_store.search( "Book", query=[1.0, 1.0], - query_filter=Filter( - must=[FieldCondition(key="rand_number", range=Range(gte=8))] - ), + query_filter=Filter(must=[FieldCondition(key="rand_number", range=Range(gte=8))]), ) assert results[0]["id"] == 8 assert results[0]["score"] == 0.9100373450784073 @@ -68,9 +64,7 @@ def test_milvus_store(): results = qdrant_store.search( "Book", query=[1.0, 1.0], - query_filter=Filter( - must=[FieldCondition(key="rand_number", range=Range(gte=8))] - ), + query_filter=Filter(must=[FieldCondition(key="rand_number", range=Range(gte=8))]), return_vector=True, ) assert results[0]["vector"] == [0.35037919878959656, 0.9366079568862915] diff --git a/tests/metagpt/management/test_skill_manager.py b/tests/metagpt/management/test_skill_manager.py index b0be858a1..462bc23a6 100644 --- a/tests/metagpt/management/test_skill_manager.py +++ b/tests/metagpt/management/test_skill_manager.py @@ -30,7 +30,7 @@ def test_skill_manager(): rsp = manager.retrieve_skill("写测试用例") logger.info(rsp) - assert rsp[0] == 'WriteTest' + assert rsp[0] == "WriteTest" rsp = manager.retrieve_skill_scored("写PRD") logger.info(rsp) diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index dc5540520..b6ae0ac79 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -1,12 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : unittest of `metagpt/memory/longterm_memory.py` +""" +@Desc : unittest of `metagpt/memory/longterm_memory.py` +""" +from metagpt.actions import UserRequirement from metagpt.config import CONFIG -from metagpt.schema import Message -from metagpt.actions import BossRequirement -from metagpt.roles.role import RoleContext from metagpt.memory import LongTermMemory +from metagpt.roles.role import RoleContext +from metagpt.schema import Message def test_ltm_search(): @@ -14,25 +16,25 @@ def test_ltm_search(): openai_api_key = CONFIG.openai_api_key assert len(openai_api_key) > 20 - role_id = 'UTUserLtm(Product Manager)' - rc = RoleContext(watch=[BossRequirement]) + role_id = "UTUserLtm(Product Manager)" + rc = RoleContext(watch=[UserRequirement]) ltm = LongTermMemory() ltm.recover_memory(role_id, rc) - idea = 'Write a cli snake game' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + idea = "Write a cli snake game" + message = Message(role="User", content=idea, cause_by=UserRequirement) news = ltm.find_news([message]) assert len(news) == 1 ltm.add(message) - sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_idea = "Write a game of cli snake" + sim_message = Message(role="User", content=sim_idea, cause_by=UserRequirement) news = ltm.find_news([sim_message]) assert len(news) == 0 ltm.add(sim_message) - new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a 2048 web game" + new_message = Message(role="User", content=new_idea, cause_by=UserRequirement) news = ltm.find_news([new_message]) assert len(news) == 1 ltm.add(new_message) @@ -47,8 +49,8 @@ def test_ltm_search(): news = ltm_new.find_news([sim_message]) assert len(news) == 0 - new_idea = 'Write a Battle City' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a Battle City" + new_message = Message(role="User", content=new_idea, cause_by=UserRequirement) news = ltm_new.find_news([new_message]) assert len(news) == 1 diff --git a/tests/metagpt/memory/test_memory_storage.py b/tests/metagpt/memory/test_memory_storage.py index 6bb3e8f1d..c67ca689f 100644 --- a/tests/metagpt/memory/test_memory_storage.py +++ b/tests/metagpt/memory/test_memory_storage.py @@ -1,20 +1,22 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : the unittests of metagpt/memory/memory_storage.py +""" +@Desc : the unittests of metagpt/memory/memory_storage.py +""" + from typing import List +from metagpt.actions import UserRequirement, WritePRD +from metagpt.actions.action_output import ActionOutput from metagpt.memory.memory_storage import MemoryStorage from metagpt.schema import Message -from metagpt.actions import BossRequirement -from metagpt.actions import WritePRD -from metagpt.actions.action_output import ActionOutput def test_idea_message(): - idea = 'Write a cli snake game' - role_id = 'UTUser1(Product Manager)' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + idea = "Write a cli snake game" + role_id = "UTUser1(Product Manager)" + message = Message(role="User", content=idea, cause_by=UserRequirement) memory_storage: MemoryStorage = MemoryStorage() messages = memory_storage.recover_memory(role_id) @@ -23,13 +25,13 @@ def test_idea_message(): memory_storage.add(message) assert memory_storage.is_initialized is True - sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_idea = "Write a game of cli snake" + sim_message = Message(role="User", content=sim_idea, cause_by=UserRequirement) new_messages = memory_storage.search(sim_message) - assert len(new_messages) == 0 # similar, return [] + assert len(new_messages) == 0 # similar, return [] - new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a 2048 web game" + new_message = Message(role="User", content=new_idea, cause_by=UserRequirement) new_messages = memory_storage.search(new_message) assert new_messages[0].content == message.content @@ -38,22 +40,15 @@ def test_idea_message(): def test_actionout_message(): - out_mapping = { - 'field1': (str, ...), - 'field2': (List[str], ...) - } - out_data = { - 'field1': 'field1 value', - 'field2': ['field2 value1', 'field2 value2'] - } - ic_obj = ActionOutput.create_model_class('prd', out_mapping) + out_mapping = {"field1": (str, ...), "field2": (List[str], ...)} + out_data = {"field1": "field1 value", "field2": ["field2 value1", "field2 value2"]} + ic_obj = ActionOutput.create_model_class("prd", out_mapping) - role_id = 'UTUser2(Architect)' - content = 'The boss has requested the creation of a command-line interface (CLI) snake game' - message = Message(content=content, - instruct_content=ic_obj(**out_data), - role='user', - cause_by=WritePRD) # WritePRD as test action + role_id = "UTUser2(Architect)" + content = "The user has requested the creation of a command-line interface (CLI) snake game" + message = Message( + content=content, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD + ) # WritePRD as test action memory_storage: MemoryStorage = MemoryStorage() messages = memory_storage.recover_memory(role_id) @@ -62,19 +57,13 @@ def test_actionout_message(): memory_storage.add(message) assert memory_storage.is_initialized is True - sim_conent = 'The request is command-line interface (CLI) snake game' - sim_message = Message(content=sim_conent, - instruct_content=ic_obj(**out_data), - role='user', - cause_by=WritePRD) + sim_conent = "The request is command-line interface (CLI) snake game" + sim_message = Message(content=sim_conent, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD) new_messages = memory_storage.search(sim_message) - assert len(new_messages) == 0 # similar, return [] + assert len(new_messages) == 0 # similar, return [] - new_conent = 'Incorporate basic features of a snake game such as scoring and increasing difficulty' - new_message = Message(content=new_conent, - instruct_content=ic_obj(**out_data), - role='user', - cause_by=WritePRD) + new_conent = "Incorporate basic features of a snake game such as scoring and increasing difficulty" + new_message = Message(content=new_conent, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD) new_messages = memory_storage.search(new_message) assert new_messages[0].content == message.content diff --git a/tests/metagpt/planner/test_action_planner.py b/tests/metagpt/planner/test_action_planner.py index 5ab9a493f..1bc451db8 100644 --- a/tests/metagpt/planner/test_action_planner.py +++ b/tests/metagpt/planner/test_action_planner.py @@ -4,12 +4,14 @@ @Time : 2023/9/16 20:03 @Author : femto Zheng @File : test_basic_planner.py +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, utilize the new message + distribution feature for message handling. """ import pytest from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill from semantic_kernel.planning.action_planner.action_planner import ActionPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -23,7 +25,8 @@ async def test_action_planner(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - role.recv(Message(content=task, cause_by=BossRequirement)) + role.put_message(Message(content=task, cause_by=UserRequirement)) + await role._observe() await role._think() # it will choose mathskill.Add assert "1100" == (await role._act()).content diff --git a/tests/metagpt/planner/test_basic_planner.py b/tests/metagpt/planner/test_basic_planner.py index 03a82ec5e..f406143ee 100644 --- a/tests/metagpt/planner/test_basic_planner.py +++ b/tests/metagpt/planner/test_basic_planner.py @@ -4,11 +4,13 @@ @Time : 2023/9/16 20:03 @Author : femto Zheng @File : test_basic_planner.py +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, utilize the new message + distribution feature for message handling. """ import pytest from semantic_kernel.core_skills import TextSkill -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.const import SKILL_DIRECTORY from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -26,7 +28,8 @@ async def test_basic_planner(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - role.recv(Message(content=task, cause_by=BossRequirement)) + role.put_message(Message(content=task, cause_by=UserRequirement)) + await role._observe() await role._think() # assuming sk_agent will think he needs WriterSkill.Brainstorm and WriterSkill.Translate assert "WriterSkill.Brainstorm" in role.plan.generated_plan.result diff --git a/tests/metagpt/provider/test_base_gpt_api.py b/tests/metagpt/provider/test_base_gpt_api.py index 882338a01..6cfe3b02d 100644 --- a/tests/metagpt/provider/test_base_gpt_api.py +++ b/tests/metagpt/provider/test_base_gpt_api.py @@ -10,6 +10,6 @@ from metagpt.schema import Message def test_message(): - message = Message(role='user', content='wtf') - assert 'role' in message.to_dict() - assert 'user' in str(message) + message = Message(role="user", content="wtf") + assert "role" in message.to_dict() + assert "user" in str(message) diff --git a/tests/metagpt/provider/test_spark_api.py b/tests/metagpt/provider/test_spark_api.py index bfa2bf76f..3b3dd67f4 100644 --- a/tests/metagpt/provider/test_spark_api.py +++ b/tests/metagpt/provider/test_spark_api.py @@ -6,6 +6,6 @@ def test_message(): llm = SparkAPI() logger.info(llm.ask('只回答"收到了"这三个字。')) - result = llm.ask('写一篇五百字的日记') + result = llm.ask("写一篇五百字的日记") logger.info(result) assert len(result) > 100 diff --git a/tests/metagpt/provider/test_zhipuai_api.py b/tests/metagpt/provider/test_zhipuai_api.py index 6a0c70de5..4684e8887 100644 --- a/tests/metagpt/provider/test_zhipuai_api.py +++ b/tests/metagpt/provider/test_zhipuai_api.py @@ -6,19 +6,9 @@ import pytest from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI +default_resp = {"code": 200, "data": {"choices": [{"role": "assistant", "content": "I'm chatglm-turbo"}]}} -default_resp = { - "code": 200, - "data": { - "choices": [ - {"role": "assistant", "content": "I'm chatglm-turbo"} - ] - } -} - -messages = [ - {"role": "user", "content": "who are you"} -] +messages = [{"role": "user", "content": "who are you"}] def mock_llm_ask(self, messages: list[dict]) -> dict: diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index 52fc4a3c1..75f6b3b43 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -5,10 +5,10 @@ @Author : alexanderwu @File : mock.py """ -from metagpt.actions import BossRequirement, WriteDesign, WritePRD, WriteTasks +from metagpt.actions import UserRequirement, WriteDesign, WritePRD, WriteTasks from metagpt.schema import Message -BOSS_REQUIREMENT = """开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结""" +USER_REQUIREMENT = """开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结""" DETAIL_REQUIREMENT = """需求：开发一个基于LLM（大语言模型）与私有知识库的搜索引擎，希望有几点能力 1. 用户可以在私有知识库进行搜索，再根据大语言模型进行总结，输出的结果包括了总结 @@ -71,7 +71,7 @@ PRD = '''## 原始需求 ``` ''' -SYSTEM_DESIGN = '''## Python package name +SYSTEM_DESIGN = """## Project name ```python "smart_search_engine" ``` @@ -94,7 +94,7 @@ SYSTEM_DESIGN = '''## Python package name ] ``` -## Data structures and interface definitions +## Data structures and interfaces ```mermaid classDiagram class Main { @@ -149,10 +149,10 @@ sequenceDiagram S-->>SE: return summary SE-->>M: return summary ``` -''' +""" -TASKS = '''## Logic Analysis +TASKS = """## Logic Analysis 在这个项目中，所有的模块都依赖于“SearchEngine”类，这是主入口，其他的模块（Index、Ranking和Summary）都通过它交互。另外，"Index"类又依赖于"KnowledgeBase"类，因为它需要从知识库中获取数据。 @@ -181,7 +181,7 @@ task_list = [ ] ``` 这个任务列表首先定义了最基础的模块，然后是依赖这些模块的模块，最后是辅助模块。可以根据团队的能力和资源，同时开发多个任务，只要满足依赖关系。例如，在开发"search.py"之前，可以同时开发"knowledge_base.py"、"index.py"、"ranking.py"和"summary.py"。 -''' +""" TASKS_TOMATO_CLOCK = '''## Required Python third-party packages: Provided in requirements.txt format @@ -224,35 +224,35 @@ task_list = [ TASK = """smart_search_engine/knowledge_base.py""" STRS_FOR_PARSING = [ -""" + """ ## 1 ```python a ``` """, -""" + """ ##2 ```python "a" ``` """, -""" + """ ## 3 ```python a = "a" ``` """, -""" + """ ## 4 ```python a = 'a' ``` -""" +""", ] class MockMessages: - req = Message(role="Boss", content=BOSS_REQUIREMENT, cause_by=BossRequirement) + req = Message(role="User", content=USER_REQUIREMENT, cause_by=UserRequirement) prd = Message(role="Product Manager", content=PRD, cause_by=WritePRD) system_design = Message(role="Architect", content=SYSTEM_DESIGN, cause_by=WriteDesign) tasks = Message(role="Project Manager", content=TASKS, cause_by=WriteTasks) diff --git a/tests/metagpt/roles/test_architect.py b/tests/metagpt/roles/test_architect.py index d44e0d923..111438b0b 100644 --- a/tests/metagpt/roles/test_architect.py +++ b/tests/metagpt/roles/test_architect.py @@ -4,6 +4,8 @@ @Time : 2023/5/20 14:37 @Author : alexanderwu @File : test_architect.py +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, utilize the new message + distribution feature for message handling. """ import pytest @@ -15,7 +17,7 @@ from tests.metagpt.roles.mock import MockMessages @pytest.mark.asyncio async def test_architect(): role = Architect() - role.recv(MockMessages.req) - rsp = await role.handle(MockMessages.prd) + role.put_message(MockMessages.req) + rsp = await role.run(MockMessages.prd) logger.info(rsp) assert len(rsp.content) > 0 diff --git a/tests/metagpt/roles/test_engineer.py b/tests/metagpt/roles/test_engineer.py index c0c48d0b1..3dc599770 100644 --- a/tests/metagpt/roles/test_engineer.py +++ b/tests/metagpt/roles/test_engineer.py @@ -4,6 +4,8 @@ @Time : 2023/5/12 10:14 @Author : alexanderwu @File : test_engineer.py +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, utilize the new message + distribution feature for message handling. """ import pytest @@ -22,10 +24,10 @@ from tests.metagpt.roles.mock import ( async def test_engineer(): engineer = Engineer() - engineer.recv(MockMessages.req) - engineer.recv(MockMessages.prd) - engineer.recv(MockMessages.system_design) - rsp = await engineer.handle(MockMessages.tasks) + engineer.put_message(MockMessages.req) + engineer.put_message(MockMessages.prd) + engineer.put_message(MockMessages.system_design) + rsp = await engineer.run(MockMessages.tasks) logger.info(rsp) assert "all done." == rsp.content @@ -35,13 +37,13 @@ def test_parse_str(): for idx, i in enumerate(STRS_FOR_PARSING): text = CodeParser.parse_str(f"{idx+1}", i) # logger.info(text) - assert text == 'a' + assert text == "a" def test_parse_blocks(): tasks = CodeParser.parse_blocks(TASKS) logger.info(tasks.keys()) - assert 'Task list' in tasks.keys() + assert "Task list" in tasks.keys() target_list = [ diff --git a/tests/metagpt/roles/test_invoice_ocr_assistant.py b/tests/metagpt/roles/test_invoice_ocr_assistant.py index 75097e73c..c9aad93a7 100644 --- a/tests/metagpt/roles/test_invoice_ocr_assistant.py +++ b/tests/metagpt/roles/test_invoice_ocr_assistant.py @@ -9,8 +9,8 @@ from pathlib import Path -import pytest import pandas as pd +import pytest from metagpt.roles.invoice_ocr_assistant import InvoiceOCRAssistant from metagpt.schema import Message @@ -24,82 +24,39 @@ from metagpt.schema import Message "Invoicing date", Path("../../data/invoices/invoice-1.pdf"), Path("../../../data/invoice_table/invoice-1.xlsx"), - [ - { - "收款人": "小明", - "城市": "深圳市", - "总费用/元": 412.00, - "开票日期": "2023年02月03日" - } - ] + [{"收款人": "小明", "城市": "深圳市", "总费用/元": 412.00, "开票日期": "2023年02月03日"}], ), ( "Invoicing date", Path("../../data/invoices/invoice-2.png"), Path("../../../data/invoice_table/invoice-2.xlsx"), - [ - { - "收款人": "铁头", - "城市": "广州市", - "总费用/元": 898.00, - "开票日期": "2023年03月17日" - } - ] + [{"收款人": "铁头", "城市": "广州市", "总费用/元": 898.00, "开票日期": "2023年03月17日"}], ), ( "Invoicing date", Path("../../data/invoices/invoice-3.jpg"), Path("../../../data/invoice_table/invoice-3.xlsx"), - [ - { - "收款人": "夏天", - "城市": "福州市", - "总费用/元": 2462.00, - "开票日期": "2023年08月26日" - } - ] + [{"收款人": "夏天", "城市": "福州市", "总费用/元": 2462.00, "开票日期": "2023年08月26日"}], ), ( "Invoicing date", Path("../../data/invoices/invoice-4.zip"), Path("../../../data/invoice_table/invoice-4.xlsx"), [ - { - "收款人": "小明", - "城市": "深圳市", - "总费用/元": 412.00, - "开票日期": "2023年02月03日" - }, - { - "收款人": "铁头", - "城市": "广州市", - "总费用/元": 898.00, - "开票日期": "2023年03月17日" - }, - { - "收款人": "夏天", - "城市": "福州市", - "总费用/元": 2462.00, - "开票日期": "2023年08月26日" - } - ] + {"收款人": "小明", "城市": "深圳市", "总费用/元": 412.00, "开票日期": "2023年02月03日"}, + {"收款人": "铁头", "城市": "广州市", "总费用/元": 898.00, "开票日期": "2023年03月17日"}, + {"收款人": "夏天", "城市": "福州市", "总费用/元": 2462.00, "开票日期": "2023年08月26日"}, + ], ), - ] + ], ) async def test_invoice_ocr_assistant( - query: str, - invoice_path: Path, - invoice_table_path: Path, - expected_result: list[dict] + query: str, invoice_path: Path, invoice_table_path: Path, expected_result: list[dict] ): invoice_path = Path.cwd() / invoice_path role = InvoiceOCRAssistant() - await role.run(Message( - content=query, - instruct_content={"file_path": invoice_path} - )) + await role.run(Message(content=query, instruct_content={"file_path": invoice_path})) invoice_table_path = Path.cwd() / invoice_table_path df = pd.read_excel(invoice_table_path) - dict_result = df.to_dict(orient='records') + dict_result = df.to_dict(orient="records") assert dict_result == expected_result - diff --git a/tests/metagpt/roles/test_researcher.py b/tests/metagpt/roles/test_researcher.py index 01b5dae3b..dd130662d 100644 --- a/tests/metagpt/roles/test_researcher.py +++ b/tests/metagpt/roles/test_researcher.py @@ -11,10 +11,12 @@ async def mock_llm_ask(self, prompt: str, system_msgs): if "Please provide up to 2 necessary keywords" in prompt: return '["dataiku", "datarobot"]' elif "Provide up to 4 queries related to your research topic" in prompt: - return '["Dataiku machine learning platform", "DataRobot AI platform comparison", ' \ + return ( + '["Dataiku machine learning platform", "DataRobot AI platform comparison", ' '"Dataiku vs DataRobot features", "Dataiku and DataRobot use cases"]' + ) elif "sort the remaining search results" in prompt: - return '[1,2]' + return "[1,2]" elif "Not relevant." in prompt: return "Not relevant" if random() > 0.5 else prompt[-100:] elif "provide a detailed research report" in prompt: diff --git a/tests/metagpt/roles/test_tutorial_assistant.py b/tests/metagpt/roles/test_tutorial_assistant.py index 945620cfc..105f976c3 100644 --- a/tests/metagpt/roles/test_tutorial_assistant.py +++ b/tests/metagpt/roles/test_tutorial_assistant.py @@ -12,10 +12,7 @@ from metagpt.roles.tutorial_assistant import TutorialAssistant @pytest.mark.asyncio -@pytest.mark.parametrize( - ("language", "topic"), - [("Chinese", "Write a tutorial about Python")] -) +@pytest.mark.parametrize(("language", "topic"), [("Chinese", "Write a tutorial about Python")]) async def test_tutorial_assistant(language: str, topic: str): topic = "Write a tutorial about MySQL" role = TutorialAssistant(language=language) @@ -24,4 +21,4 @@ async def test_tutorial_assistant(language: str, topic: str): title = filename.split("/")[-1].split(".")[0] async with aiofiles.open(filename, mode="r") as reader: content = await reader.read() - assert content.startswith(f"# {title}") \ No newline at end of file + assert content.startswith(f"# {title}") diff --git a/tests/metagpt/roles/test_ui.py b/tests/metagpt/roles/test_ui.py index d58d31bd9..2038a1aee 100644 --- a/tests/metagpt/roles/test_ui.py +++ b/tests/metagpt/roles/test_ui.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- # @Date : 2023/7/22 02:40 -# @Author : stellahong (stellahong@fuzhi.ai) +# @Author : stellahong (stellahong@deepwisdom.ai) # -from metagpt.team import Team from metagpt.roles import ProductManager - +from metagpt.team import Team from tests.metagpt.roles.ui_role import UI @@ -18,5 +17,5 @@ async def test_ui_role(idea: str, investment: float = 3.0, n_round: int = 5): company = Team() company.hire([ProductManager(), UI()]) company.invest(investment) - company.start_project(idea) + company.run_project(idea) await company.run(n_round=n_round) diff --git a/tests/metagpt/roles/ui_role.py b/tests/metagpt/roles/ui_role.py index a45a89cde..8ac799bf3 100644 --- a/tests/metagpt/roles/ui_role.py +++ b/tests/metagpt/roles/ui_role.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # @Date : 2023/7/15 16:40 -# @Author : stellahong (stellahong@fuzhi.ai) +# @Author : stellahong (stellahong@deepwisdom.ai) # @Desc : import os import re @@ -8,7 +8,9 @@ from functools import wraps from importlib import import_module from metagpt.actions import Action, ActionOutput, WritePRD -from metagpt.const import WORKSPACE_ROOT + +# from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -29,7 +31,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Selected Elements:Provide as Plain text, up to 5 specified elements, clear and simple ## HTML Layout:Provide as Plain text, use standard HTML code ## CSS Styles (styles.css):Provide as Plain text,use standard css code -## Anything UNCLEAR:Provide as Plain text. Make clear here. +## Anything UNCLEAR:Provide as Plain text. Try to clarify it. """ @@ -214,7 +216,7 @@ class UIDesign(Action): logger.info("Finish icon design using StableDiffusion API") async def _save(self, css_content, html_content): - save_dir = WORKSPACE_ROOT / "resources" / "codes" + save_dir = CONFIG.workspace_path / "resources" / "codes" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) # Save CSS and HTML content to files diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index a0f1f6257..b27bc3da7 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -8,7 +8,7 @@ import pytest -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.environment import Environment from metagpt.logs import logger from metagpt.manager import Manager @@ -49,7 +49,7 @@ async def test_publish_and_process_message(env: Environment): env.add_roles([product_manager, architect]) env.set_manager(Manager()) - env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement)) + env.publish_message(Message(role="User", content="需要一个基于LLM做总结的搜索引擎", cause_by=UserRequirement)) await env.run(k=2) logger.info(f"{env.history=}") diff --git a/tests/metagpt/test_gpt.py b/tests/metagpt/test_gpt.py index 89dd726a8..431858d4c 100644 --- a/tests/metagpt/test_gpt.py +++ b/tests/metagpt/test_gpt.py @@ -14,7 +14,8 @@ from metagpt.logs import logger @pytest.mark.usefixtures("llm_api") class TestGPT: def test_llm_api_ask(self, llm_api): - answer = llm_api.ask('hello chatgpt') + answer = llm_api.ask("hello chatgpt") + logger.info(answer) assert len(answer) > 0 # def test_gptapi_ask_batch(self, llm_api): @@ -22,22 +23,29 @@ class TestGPT: # assert len(answer) > 0 def test_llm_api_ask_code(self, llm_api): - answer = llm_api.ask_code(['请扮演一个Google Python专家工程师，如果理解，回复明白', '写一个hello world']) + answer = llm_api.ask_code(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) + logger.info(answer) assert len(answer) > 0 @pytest.mark.asyncio async def test_llm_api_aask(self, llm_api): - answer = await llm_api.aask('hello chatgpt') + answer = await llm_api.aask("hello chatgpt") + logger.info(answer) assert len(answer) > 0 @pytest.mark.asyncio async def test_llm_api_aask_code(self, llm_api): - answer = await llm_api.aask_code(['请扮演一个Google Python专家工程师，如果理解，回复明白', '写一个hello world']) + answer = await llm_api.aask_code(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) + logger.info(answer) assert len(answer) > 0 @pytest.mark.asyncio async def test_llm_api_costs(self, llm_api): - await llm_api.aask('hello chatgpt') + await llm_api.aask("hello chatgpt") costs = llm_api.get_costs() logger.info(costs) assert costs.total_cost > 0 + + +# if __name__ == "__main__": +# pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_llm.py b/tests/metagpt/test_llm.py index 11503af1d..408fd3162 100644 --- a/tests/metagpt/test_llm.py +++ b/tests/metagpt/test_llm.py @@ -18,17 +18,21 @@ def llm(): @pytest.mark.asyncio async def test_llm_aask(llm): - assert len(await llm.aask('hello world')) > 0 + assert len(await llm.aask("hello world")) > 0 @pytest.mark.asyncio async def test_llm_aask_batch(llm): - assert len(await llm.aask_batch(['hi', 'write python hello world.'])) > 0 + assert len(await llm.aask_batch(["hi", "write python hello world."])) > 0 @pytest.mark.asyncio async def test_llm_acompletion(llm): - hello_msg = [{'role': 'user', 'content': 'hello'}] + hello_msg = [{"role": "user", "content": "hello"}] assert len(await llm.acompletion(hello_msg)) > 0 assert len(await llm.acompletion_batch([hello_msg])) > 0 assert len(await llm.acompletion_batch_text([hello_msg])) > 0 + + +# if __name__ == "__main__": +# pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_message.py b/tests/metagpt/test_message.py index e26f38381..04d85d9e4 100644 --- a/tests/metagpt/test_message.py +++ b/tests/metagpt/test_message.py @@ -4,6 +4,7 @@ @Time : 2023/5/16 10:57 @Author : alexanderwu @File : test_message.py +@Modified By: mashenquan, 2023-11-1. Modify coding style. """ import pytest @@ -11,26 +12,30 @@ from metagpt.schema import AIMessage, Message, RawMessage, SystemMessage, UserMe def test_message(): - msg = Message(role='User', content='WTF') - assert msg.to_dict()['role'] == 'User' - assert 'User' in str(msg) + msg = Message(role="User", content="WTF") + assert msg.to_dict()["role"] == "User" + assert "User" in str(msg) def test_all_messages(): - test_content = 'test_message' + test_content = "test_message" msgs = [ UserMessage(test_content), SystemMessage(test_content), AIMessage(test_content), - Message(test_content, role='QA') + Message(test_content, role="QA"), ] for msg in msgs: assert msg.content == test_content def test_raw_message(): - msg = RawMessage(role='user', content='raw') - assert msg['role'] == 'user' - assert msg['content'] == 'raw' + msg = RawMessage(role="user", content="raw") + assert msg["role"] == "user" + assert msg["content"] == "raw" with pytest.raises(KeyError): - assert msg['1'] == 1, "KeyError: '1'" + assert msg["1"] == 1, "KeyError: '1'" + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index 11fd804ec..8fac2503c 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -4,11 +4,98 @@ @Time : 2023/5/11 14:44 @Author : alexanderwu @File : test_role.py +@Modified By: mashenquan, 2023-11-1. In line with Chapter 2.2.1 and 2.2.2 of RFC 116, introduce unit tests for + the utilization of the new message distribution feature in message handling. +@Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing + functionality is to be consolidated into the `Environment` class. """ +import uuid + +import pytest +from pydantic import BaseModel + +from metagpt.actions import Action, ActionOutput +from metagpt.environment import Environment from metagpt.roles import Role +from metagpt.schema import Message +from metagpt.utils.common import get_class_name -def test_role_desc(): - i = Role(profile='Sales', desc='Best Seller') - assert i.profile == 'Sales' - assert i._setting.desc == 'Best Seller' +class MockAction(Action): + async def run(self, messages, *args, **kwargs): + assert messages + return ActionOutput(content=messages[-1].content, instruct_content=messages[-1]) + + +class MockRole(Role): + def __init__(self, name="", profile="", goal="", constraints="", desc=""): + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) + self._init_actions([MockAction()]) + + +@pytest.mark.asyncio +async def test_react(): + class Input(BaseModel): + name: str + profile: str + goal: str + constraints: str + desc: str + subscription: str + + inputs = [ + { + "name": "A", + "profile": "Tester", + "goal": "Test", + "constraints": "constraints", + "desc": "desc", + "subscription": "start", + } + ] + + for i in inputs: + seed = Input(**i) + role = MockRole( + name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, desc=seed.desc + ) + role.subscribe({seed.subscription}) + assert role._rc.watch == set({}) + assert role.name == seed.name + assert role.profile == seed.profile + assert role._setting.goal == seed.goal + assert role._setting.constraints == seed.constraints + assert role._setting.desc == seed.desc + assert role.is_idle + env = Environment() + env.add_role(role) + assert env.get_subscription(role) == {seed.subscription} + env.publish_message(Message(content="test", msg_to=seed.subscription)) + assert not role.is_idle + while not env.is_idle: + await env.run() + assert role.is_idle + env.publish_message(Message(content="test", cause_by=seed.subscription)) + assert not role.is_idle + while not env.is_idle: + await env.run() + assert role.is_idle + tag = uuid.uuid4().hex + role.subscribe({tag}) + assert env.get_subscription(role) == {tag} + + +@pytest.mark.asyncio +async def test_msg_to(): + m = Message(content="a", send_to=["a", MockRole, Message]) + assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)}) + + m = Message(content="a", cause_by=MockAction, send_to={"a", MockRole, Message}) + assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)}) + + m = Message(content="a", send_to=("a", MockRole, Message)) + assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)}) + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 12666e0d3..51ebd5baa 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -4,18 +4,71 @@ @Time : 2023/5/20 10:40 @Author : alexanderwu @File : test_schema.py +@Modified By: mashenquan, 2023-11-1. In line with Chapter 2.2.1 and 2.2.2 of RFC 116, introduce unit tests for + the utilization of the new feature of `Message` class. """ +import json + +import pytest + +from metagpt.actions import Action from metagpt.schema import AIMessage, Message, SystemMessage, UserMessage +from metagpt.utils.common import get_class_name +@pytest.mark.asyncio def test_messages(): - test_content = 'test_message' + test_content = "test_message" msgs = [ UserMessage(test_content), SystemMessage(test_content), AIMessage(test_content), - Message(test_content, role='QA') + Message(test_content, role="QA"), ] text = str(msgs) - roles = ['user', 'system', 'assistant', 'QA'] + roles = ["user", "system", "assistant", "QA"] assert all([i in text for i in roles]) + + +@pytest.mark.asyncio +def test_message(): + m = Message("a", role="v1") + v = m.dump() + d = json.loads(v) + assert d + assert d.get("content") == "a" + assert d.get("role") == "v1" + m.role = "v2" + v = m.dump() + assert v + m = Message.load(v) + assert m.content == "a" + assert m.role == "v2" + + m = Message("a", role="b", cause_by="c", x="d", send_to="c") + assert m.content == "a" + assert m.role == "b" + assert m.send_to == {"c"} + assert m.cause_by == "c" + + m.cause_by = "Message" + assert m.cause_by == "Message" + m.cause_by = Action + assert m.cause_by == get_class_name(Action) + m.cause_by = Action() + assert m.cause_by == get_class_name(Action) + m.content = "b" + assert m.content == "b" + + +@pytest.mark.asyncio +def test_routes(): + m = Message("a", role="b", cause_by="c", x="d", send_to="c") + m.send_to = "b" + assert m.send_to == {"b"} + m.send_to = {"e", Action} + assert m.send_to == {"e", get_class_name(Action)} + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_software_company.py b/tests/metagpt/test_software_company.py deleted file mode 100644 index 4fc651f52..000000000 --- a/tests/metagpt/test_software_company.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/15 11:40 -@Author : alexanderwu -@File : test_software_company.py -""" -import pytest - -from metagpt.logs import logger -from metagpt.team import Team - - -@pytest.mark.asyncio -async def test_team(): - company = Team() - company.start_project("做一个基础搜索引擎，可以支持知识库") - history = await company.run(n_round=5) - logger.info(history) diff --git a/tests/metagpt/test_startup.py b/tests/metagpt/test_startup.py new file mode 100644 index 000000000..c34fd2c31 --- /dev/null +++ b/tests/metagpt/test_startup.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/15 11:40 +@Author : alexanderwu +@File : test_startup.py +""" +import pytest +from typer.testing import CliRunner + +from metagpt.logs import logger +from metagpt.team import Team + +runner = CliRunner() + + +@pytest.mark.asyncio +async def test_team(): + # FIXME: we're now using "metagpt" cli, so the entrance should be replaced instead. + company = Team() + company.run_project("做一个基础搜索引擎，可以支持知识库") + history = await company.run(n_round=5) + logger.info(history) + + +# def test_startup(): +# args = ["Make a 2048 game"] +# result = runner.invoke(app, args) diff --git a/tests/metagpt/tools/test_code_interpreter.py b/tests/metagpt/tools/test_code_interpreter.py index 0eec3f80b..03d4ce8df 100644 --- a/tests/metagpt/tools/test_code_interpreter.py +++ b/tests/metagpt/tools/test_code_interpreter.py @@ -1,23 +1,22 @@ -import pytest -import pandas as pd from pathlib import Path -from tests.data import sales_desc, store_desc -from metagpt.tools.code_interpreter import OpenCodeInterpreter, OpenInterpreterDecorator +import pandas as pd +import pytest + from metagpt.actions import Action from metagpt.logs import logger +from metagpt.tools.code_interpreter import OpenCodeInterpreter, OpenInterpreterDecorator - -logger.add('./tests/data/test_ci.log') +logger.add("./tests/data/test_ci.log") stock = "./tests/data/baba_stock.csv" # TODO: 需要一种表格数据格式，能够支持schame管理的，标注字段类型和字段含义。 class CreateStockIndicators(Action): @OpenInterpreterDecorator(save_code=True, code_file_path="./tests/data/stock_indicators.py") - async def run(self, stock_path: str, indicators=['Simple Moving Average', 'BollingerBands']) -> pd.DataFrame: + async def run(self, stock_path: str, indicators=["Simple Moving Average", "BollingerBands"]) -> pd.DataFrame: """对stock_path中的股票数据, 使用pandas和ta计算indicators中的技术指标, 返回带有技术指标的股票数据，不需要去除空值, 不需要安装任何包； - 指标生成对应的三列: SMA, BB_upper, BB_lower + 指标生成对应的三列: SMA, BB_upper, BB_lower """ ... @@ -25,18 +24,20 @@ class CreateStockIndicators(Action): @pytest.mark.asyncio async def test_actions(): # 计算指标 - indicators = ['Simple Moving Average', 'BollingerBands'] + indicators = ["Simple Moving Average", "BollingerBands"] stocker = CreateStockIndicators() df, msg = await stocker.run(stock, indicators=indicators) assert isinstance(df, pd.DataFrame) - assert 'Close' in df.columns - assert 'Date' in df.columns + assert "Close" in df.columns + assert "Date" in df.columns # 将df保存为文件，将文件路径传入到下一个action - df_path = './tests/data/stock_indicators.csv' + df_path = "./tests/data/stock_indicators.csv" df.to_csv(df_path) assert Path(df_path).is_file() # 可视化指标结果 - figure_path = './tests/data/figure_ci.png' + figure_path = "./tests/data/figure_ci.png" ci_ploter = OpenCodeInterpreter() - ci_ploter.chat(f"使用seaborn对{df_path}中与股票布林带有关的数据列的Date, Close, SMA, BB_upper（布林带上界）, BB_lower（布林带下界）进行可视化, 可视化图片保存在{figure_path}中。不需要任何指标计算，把Date列转换为日期类型。要求图片优美，BB_upper, BB_lower之间使用合适的颜色填充。") + ci_ploter.chat( + f"使用seaborn对{df_path}中与股票布林带有关的数据列的Date, Close, SMA, BB_upper（布林带上界）, BB_lower（布林带下界）进行可视化, 可视化图片保存在{figure_path}中。不需要任何指标计算，把Date列转换为日期类型。要求图片优美，BB_upper, BB_lower之间使用合适的颜色填充。" + ) assert Path(figure_path).is_file() diff --git a/tests/metagpt/tools/test_prompt_generator.py b/tests/metagpt/tools/test_prompt_generator.py index d2e870c6d..ddbd2c43b 100644 --- a/tests/metagpt/tools/test_prompt_generator.py +++ b/tests/metagpt/tools/test_prompt_generator.py @@ -20,8 +20,9 @@ from metagpt.tools.prompt_writer import ( @pytest.mark.usefixtures("llm_api") def test_gpt_prompt_generator(llm_api): generator = GPTPromptGenerator() - example = "商品名称:WonderLab 新肌果味代餐奶昔 小胖瓶 胶原蛋白升级版 饱腹代餐粉6瓶 75g/瓶(6瓶/盒) 店铺名称:金力宁食品专营店 " \ - "品牌:WonderLab 保质期:1年 产地:中国 净含量:450g" + example = ( + "商品名称:WonderLab 新肌果味代餐奶昔 小胖瓶 胶原蛋白升级版 饱腹代餐粉6瓶 75g/瓶(6瓶/盒) 店铺名称:金力宁食品专营店 " "品牌:WonderLab 保质期:1年 产地:中国 净含量:450g" + ) results = llm_api.ask_batch(generator.gen(example)) logger.info(results) @@ -46,7 +47,7 @@ def test_enron_template(llm_api): results = template.gen(subj) assert len(results) > 0 - assert any("Write an email with the subject \"Meeting Agenda\"." in r for r in results) + assert any('Write an email with the subject "Meeting Agenda".' in r for r in results) def test_beagec_template(): @@ -54,5 +55,6 @@ def test_beagec_template(): results = template.gen() assert len(results) > 0 - assert any("Edit and revise this document to improve its grammar, vocabulary, spelling, and style." - in r for r in results) + assert any( + "Edit and revise this document to improve its grammar, vocabulary, spelling, and style." in r for r in results + ) diff --git a/tests/metagpt/tools/test_sd_tool.py b/tests/metagpt/tools/test_sd_tool.py index 77e53c7dc..e457101a9 100644 --- a/tests/metagpt/tools/test_sd_tool.py +++ b/tests/metagpt/tools/test_sd_tool.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- # @Date : 2023/7/22 02:40 -# @Author : stellahong (stellahong@fuzhi.ai) +# @Author : stellahong (stellahong@deepwisdom.ai) # import os -from metagpt.tools.sd_engine import SDEngine, WORKSPACE_ROOT +from metagpt.config import CONFIG +from metagpt.tools.sd_engine import SDEngine def test_sd_engine_init(): @@ -21,5 +22,5 @@ def test_sd_engine_generate_prompt(): async def test_sd_engine_run_t2i(): sd_engine = SDEngine() await sd_engine.run_t2i(prompts=["test"]) - img_path = WORKSPACE_ROOT / "resources" / "SD_Output" / "output_0.png" - assert os.path.exists(img_path) == True + img_path = CONFIG.workspace_path / "resources" / "SD_Output" / "output_0.png" + assert os.path.exists(img_path) diff --git a/tests/metagpt/tools/test_search_engine.py b/tests/metagpt/tools/test_search_engine.py index a7fe063a6..25bce124a 100644 --- a/tests/metagpt/tools/test_search_engine.py +++ b/tests/metagpt/tools/test_search_engine.py @@ -16,7 +16,9 @@ from metagpt.tools.search_engine import SearchEngine class MockSearchEnine: async def run(self, query: str, max_results: int = 8, as_string: bool = True) -> str | list[dict[str, str]]: - rets = [{"url": "https://metagpt.com/mock/{i}", "title": query, "snippet": query * i} for i in range(max_results)] + rets = [ + {"url": "https://metagpt.com/mock/{i}", "title": query, "snippet": query * i} for i in range(max_results) + ] return "\n".join(rets) if as_string else rets @@ -34,10 +36,14 @@ class MockSearchEnine: (SearchEngineType.DUCK_DUCK_GO, None, 6, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 8, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 6, False), - ], ) -async def test_search_engine(search_engine_typpe, run_func, max_results, as_string, ): +async def test_search_engine( + search_engine_typpe, + run_func, + max_results, + as_string, +): search_engine = SearchEngine(search_engine_typpe, run_func) rsp = await search_engine.run("metagpt", max_results=max_results, as_string=as_string) logger.info(rsp) diff --git a/tests/metagpt/tools/test_search_engine_meilisearch.py b/tests/metagpt/tools/test_search_engine_meilisearch.py index 8d2bb6494..d5f7d162b 100644 --- a/tests/metagpt/tools/test_search_engine_meilisearch.py +++ b/tests/metagpt/tools/test_search_engine_meilisearch.py @@ -13,7 +13,7 @@ import pytest from metagpt.logs import logger from metagpt.tools.search_engine_meilisearch import DataSource, MeilisearchEngine -MASTER_KEY = '116Qavl2qpCYNEJNv5-e0RC9kncev1nr1gt7ybEGVLk' +MASTER_KEY = "116Qavl2qpCYNEJNv5-e0RC9kncev1nr1gt7ybEGVLk" @pytest.fixture() @@ -29,7 +29,7 @@ def test_meilisearch(search_engine_server): search_engine = MeilisearchEngine(url="http://localhost:7700", token=MASTER_KEY) # 假设有一个名为"books"的数据源，包含要添加的文档库 - books_data_source = DataSource(name='books', url='https://example.com/books') + books_data_source = DataSource(name="books", url="https://example.com/books") # 假设有一个名为"documents"的文档库，包含要添加的文档 documents = [ @@ -43,4 +43,4 @@ def test_meilisearch(search_engine_server): # 添加文档库到搜索引擎 search_engine.add_documents(books_data_source, documents) - logger.info(search_engine.search('Book 1')) + logger.info(search_engine.search("Book 1")) diff --git a/tests/metagpt/tools/test_summarize.py b/tests/metagpt/tools/test_summarize.py index cf616c144..6a372defb 100644 --- a/tests/metagpt/tools/test_summarize.py +++ b/tests/metagpt/tools/test_summarize.py @@ -20,7 +20,6 @@ CASES = [ 1. 请根据上下文，对用户搜索请求进行总结性回答，不要包括与请求无关的文本 2. 以 [正文](引用链接) markdown形式在正文中**自然标注**~5个文本（如商品词或类似文本段），以便跳转 3. 回复优雅、清晰，**绝不重复文本**，行文流畅，长度居中""", - """# 上下文 [{'title': '去厦门 有哪些推荐的美食？ - 知乎', 'href': 'https://www.zhihu.com/question/286901854', 'body': '知乎，中文互联网高质量的问答社区和创作者聚集的原创内容平台，于 2011 年 1 月正式上线，以「让人们更好的分享知识、经验和见解，找到自己的解答」为品牌使命。知乎凭借认真、专业、友善的社区氛围、独特的产品机制以及结构化和易获得的优质内容，聚集了中文互联网科技、商业、影视 ...'}, {'title': '厦门到底有哪些真正值得吃的美食？ - 知乎', 'href': 'https://www.zhihu.com/question/38012322', 'body': '有几个特色菜在别处不太能吃到，值得一试~常点的有西多士、沙茶肉串、咕老肉（个人认为还是良山排档的更炉火纯青~），因为爱吃芋泥，每次还会点一个芋泥鸭~人均50元左右. 潮福城. 厦门这两年经营港式茶点的店越来越多，但是最经典的还是潮福城的茶点 ...'}, {'title': '超全厦门美食攻略，好吃不贵不踩雷 - 知乎 - 知乎专栏', 'href': 'https://zhuanlan.zhihu.com/p/347055615', 'body': '厦门老字号店铺，味道卫生都有保障，喜欢吃芒果的，不要错过芒果牛奶绵绵冰. 285蚝味馆 70/人. 上过《舌尖上的中国》味道不用多说，想吃地道的海鲜烧烤就来这里. 堂宴.老厦门私房菜 80/人. 非常多的明星打卡过，上过《十二道锋味》，吃厦门传统菜的好去处 ...'}, {'title': '福建名小吃||寻味厦门，十大特色名小吃，你都吃过哪几样？ - 知乎', 'href': 'https://zhuanlan.zhihu.com/p/375781836', 'body': '第一期，分享厦门的特色美食。 厦门是一个风景旅游城市，许多人来到厦门，除了游览厦门独特的风景之外，最难忘的应该是厦门的特色小吃。厦门小吃多种多样，有到厦门必吃的沙茶面、米线糊、蚵仔煎、土笋冻等非常之多。那么，厦门的名小吃有哪些呢？'}, {'title': '大家如果去厦门旅游的话，好吃的有很多，但... 来自庄时利和 - 微博', 'href': 'https://weibo.com/1728715190/MEAwzscRT', 'body': '大家如果去厦门旅游的话，好吃的有很多，但如果只选一样的话，我个人会选择莲花煎蟹。 靠海吃海，吃蟹对于闽南人来说是很平常的一件事。 厦门传统的做法多是清蒸或水煮，上世纪八十年代有一同安人在厦门的莲花公园旁，摆摊做起了煎蟹的生意。'}, {'title': '厦门美食,厦门美食攻略,厦门旅游美食攻略 - 马蜂窝', 'href': 'https://www.mafengwo.cn/cy/10132/gonglve.html', 'body': '醉壹号海鲜大排档 (厦门美食地标店) No.3. 哆啦Eanny 的最新点评：. 环境 挺复古的闽南风情，花砖地板，一楼有海鲜自己点菜，二楼室内位置，三楼露天位置，环境挺不错的。. 苦螺汤，看起来挺清的，螺肉吃起来很脆。. 姜... 5.0 分. 482 条用户点评.'}, {'title': '厦门超强中山路小吃合集，29家本地人推荐的正宗美食 - 马蜂窝', 'href': 'https://www.mafengwo.cn/gonglve/ziyouxing/176485.html', 'body': '莲欢海蛎煎. 提到厦门就想到海蛎煎，而这家位于中山路局口街的莲欢海蛎煎是实打实的好吃!. ·局口街老巷之中，全室外环境，吃的就是这种感觉。. ·取名"莲欢"，是希望妻子每天开心。. 新鲜的食材，实在的用料，这样的用心也定能讨食客欢心。. ·海蛎又 ...'}, {'title': '厦门市 10 大餐厅- Tripadvisor', 'href': 'https://cn.tripadvisor.com/Restaurants-g297407-Xiamen_Fujian.html', 'body': '厦门市餐厅：在Tripadvisor查看中国厦门市餐厅的点评，并以价格、地点及更多选项进行搜索。 ... "牛排太好吃了啊啊啊" ... "厦门地区最老品牌最有口碑的潮州菜餐厅" ...'}, {'title': '#福建10条美食街简直不要太好吃#每到一... 来自新浪厦门 - 微博', 'href': 'https://weibo.com/1740522895/MF1lY7W4n', 'body': '福建的这10条美食街，你一定不能错过!福州师大学生街、福州达明路美食街、厦门八市、漳州古城老街、宁德老南门电影院美食集市、龙岩中山路美食街、三明龙岗夜市、莆田金鼎夜市、莆田玉湖夜市、南平嘉禾美食街。世间万事皆难，唯有美食可以治愈一切。'}, {'title': '厦门这50家餐厅最值得吃 - 腾讯新闻', 'href': 'https://new.qq.com/rain/a/20200114A09HJT00', 'body': '没有什么事是一顿辣解决不了的! 创意辣、川湘辣、温柔辣、异域辣，芙蓉涧的菜能把辣椒玩出花来! ... 早在2005年，这家老牌的东南亚餐厅就开在厦门莲花了，在许多老厦门的心中，都觉得这里有全厦门最好吃的咖喱呢。 ...'}, {'title': '好听的美食？又好听又好吃的食物有什么？ - 哔哩哔哩', 'href': 'https://www.bilibili.com/read/cv23430069/', 'body': '专栏 / 好听的美食？又好听又好吃的食物有什么？ 又好听又好吃的食物有什么？ 2023-05-02 18:01 --阅读 · --喜欢 · --评论'}] @@ -31,7 +30,7 @@ CASES = [ 你是专业管家团队的一员，会给出有帮助的建议 1. 请根据上下文，对用户搜索请求进行总结性回答，不要包括与请求无关的文本 2. 以 [正文](引用链接) markdown形式在正文中**自然标注**3-5个文本（如商品词或类似文本段），以便跳转 -3. 回复优雅、清晰，**绝不重复文本**，行文流畅，长度居中""" +3. 回复优雅、清晰，**绝不重复文本**，行文流畅，长度居中""", ] diff --git a/tests/metagpt/tools/test_translate.py b/tests/metagpt/tools/test_translate.py index 47a9034a5..024bda3ca 100644 --- a/tests/metagpt/tools/test_translate.py +++ b/tests/metagpt/tools/test_translate.py @@ -16,7 +16,7 @@ from metagpt.tools.translator import Translator def test_translate(llm_api): poetries = [ ("Let life be beautiful like summer flowers", "花"), - ("The ancient Chinese poetries are all songs.", "中国") + ("The ancient Chinese poetries are all songs.", "中国"), ] for i, j in poetries: prompt = Translator.translate_prompt(i) diff --git a/tests/metagpt/tools/test_ut_generator.py b/tests/metagpt/tools/test_ut_generator.py index 6f29999d4..2ae94885f 100644 --- a/tests/metagpt/tools/test_ut_generator.py +++ b/tests/metagpt/tools/test_ut_generator.py @@ -16,8 +16,12 @@ class TestUTWriter: tags = ["测试"] # "智能合同导入", "律师审查", "ai合同审查", "草拟合同&律师在线审查", "合同审批", "履约管理", "签约公司"] # 这里在文件中手动加入了两个测试标签的API - utg = UTGenerator(swagger_file=swagger_file, ut_py_path=UT_PY_PATH, questions_path=API_QUESTIONS_PATH, - template_prefix=YFT_PROMPT_PREFIX) + utg = UTGenerator( + swagger_file=swagger_file, + ut_py_path=UT_PY_PATH, + questions_path=API_QUESTIONS_PATH, + template_prefix=YFT_PROMPT_PREFIX, + ) ret = utg.generate_ut(include_tags=tags) # 后续加入对文件生成内容与数量的检验 assert ret diff --git a/tests/metagpt/tools/test_web_browser_engine.py b/tests/metagpt/tools/test_web_browser_engine.py index b08d0ca10..28dd0e15c 100644 --- a/tests/metagpt/tools/test_web_browser_engine.py +++ b/tests/metagpt/tools/test_web_browser_engine.py @@ -7,8 +7,8 @@ from metagpt.tools import WebBrowserEngineType, web_browser_engine @pytest.mark.parametrize( "browser_type, url, urls", [ - (WebBrowserEngineType.PLAYWRIGHT, "https://fuzhi.ai", ("https://fuzhi.ai",)), - (WebBrowserEngineType.SELENIUM, "https://fuzhi.ai", ("https://fuzhi.ai",)), + (WebBrowserEngineType.PLAYWRIGHT, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), + (WebBrowserEngineType.SELENIUM, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), ], ids=["playwright", "selenium"], ) diff --git a/tests/metagpt/tools/test_web_browser_engine_playwright.py b/tests/metagpt/tools/test_web_browser_engine_playwright.py index 69e1339e7..e9ea80b10 100644 --- a/tests/metagpt/tools/test_web_browser_engine_playwright.py +++ b/tests/metagpt/tools/test_web_browser_engine_playwright.py @@ -8,9 +8,9 @@ from metagpt.tools import web_browser_engine_playwright @pytest.mark.parametrize( "browser_type, use_proxy, kwagrs, url, urls", [ - ("chromium", {"proxy": True}, {}, "https://fuzhi.ai", ("https://fuzhi.ai",)), - ("firefox", {}, {"ignore_https_errors": True}, "https://fuzhi.ai", ("https://fuzhi.ai",)), - ("webkit", {}, {"ignore_https_errors": True}, "https://fuzhi.ai", ("https://fuzhi.ai",)), + ("chromium", {"proxy": True}, {}, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), + ("firefox", {}, {"ignore_https_errors": True}, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), + ("webkit", {}, {"ignore_https_errors": True}, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), ], ids=["chromium-normal", "firefox-normal", "webkit-normal"], ) diff --git a/tests/metagpt/tools/test_web_browser_engine_selenium.py b/tests/metagpt/tools/test_web_browser_engine_selenium.py index ce322f7bd..ac6eafee7 100644 --- a/tests/metagpt/tools/test_web_browser_engine_selenium.py +++ b/tests/metagpt/tools/test_web_browser_engine_selenium.py @@ -8,9 +8,9 @@ from metagpt.tools import web_browser_engine_selenium @pytest.mark.parametrize( "browser_type, use_proxy, url, urls", [ - ("chrome", True, "https://fuzhi.ai", ("https://fuzhi.ai",)), - ("firefox", False, "https://fuzhi.ai", ("https://fuzhi.ai",)), - ("edge", False, "https://fuzhi.ai", ("https://fuzhi.ai",)), + ("chrome", True, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), + ("firefox", False, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), + ("edge", False, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), ], ids=["chrome-normal", "firefox-normal", "edge-normal"], ) diff --git a/tests/metagpt/utils/test_ahttp_client.py b/tests/metagpt/utils/test_ahttp_client.py index 15159423a..a595d645f 100644 --- a/tests/metagpt/utils/test_ahttp_client.py +++ b/tests/metagpt/utils/test_ahttp_client.py @@ -9,30 +9,21 @@ from metagpt.utils.ahttp_client import apost, apost_stream @pytest.mark.asyncio async def test_apost(): - result = await apost( - url="https://www.baidu.com/" - ) + result = await apost(url="https://www.baidu.com/") assert "百度一下" in result result = await apost( - url="http://aider.meizu.com/app/weather/listWeather", - data={"cityIds": "101240101"}, - as_json=True + url="http://aider.meizu.com/app/weather/listWeather", data={"cityIds": "101240101"}, as_json=True ) assert result["code"] == "200" @pytest.mark.asyncio async def test_apost_stream(): - result = apost_stream( - url="https://www.baidu.com/" - ) + result = apost_stream(url="https://www.baidu.com/") async for line in result: assert len(line) >= 0 - result = apost_stream( - url="http://aider.meizu.com/app/weather/listWeather", - data={"cityIds": "101240101"} - ) + result = apost_stream(url="http://aider.meizu.com/app/weather/listWeather", data={"cityIds": "101240101"}) async for line in result: assert len(line) >= 0 diff --git a/tests/metagpt/utils/test_code_parser.py b/tests/metagpt/utils/test_code_parser.py index 707b558e1..6b7349cd9 100644 --- a/tests/metagpt/utils/test_code_parser.py +++ b/tests/metagpt/utils/test_code_parser.py @@ -131,10 +131,10 @@ class TestCodeParser: def test_parse_file_list(self, parser, text): result = parser.parse_file_list("Task list", text) print(result) - assert result == ['task1', 'task2'] + assert result == ["task1", "task2"] -if __name__ == '__main__': +if __name__ == "__main__": t = TestCodeParser() t.test_parse_file_list(CodeParser(), t_text) # TestCodeParser.test_parse_file_list() diff --git a/tests/metagpt/utils/test_common.py b/tests/metagpt/utils/test_common.py index ec4443175..4bd38db63 100644 --- a/tests/metagpt/utils/test_common.py +++ b/tests/metagpt/utils/test_common.py @@ -4,27 +4,79 @@ @Time : 2023/4/29 16:19 @Author : alexanderwu @File : test_common.py +@Modified by: mashenquan, 2023/11/21. Add unit tests. """ import os +from typing import Any, Set import pytest +from pydantic import BaseModel -from metagpt.const import get_project_root +from metagpt.actions import RunCode +from metagpt.const import get_metagpt_root +from metagpt.roles.tutorial_assistant import TutorialAssistant +from metagpt.schema import Message +from metagpt.utils.common import any_to_str, any_to_str_set class TestGetProjectRoot: def change_etc_dir(self): # current_directory = Path.cwd() - abs_root = '/etc' + abs_root = "/etc" os.chdir(abs_root) def test_get_project_root(self): - project_root = get_project_root() - assert project_root.name == 'metagpt' + project_root = get_metagpt_root() + assert project_root.name == "metagpt" def test_get_root_exception(self): with pytest.raises(Exception) as exc_info: self.change_etc_dir() - get_project_root() + get_metagpt_root() assert str(exc_info.value) == "Project root not found." + + def test_any_to_str(self): + class Input(BaseModel): + x: Any + want: str + + inputs = [ + Input(x=TutorialAssistant, want="metagpt.roles.tutorial_assistant.TutorialAssistant"), + Input(x=TutorialAssistant(), want="metagpt.roles.tutorial_assistant.TutorialAssistant"), + Input(x=RunCode, want="metagpt.actions.run_code.RunCode"), + Input(x=RunCode(), want="metagpt.actions.run_code.RunCode"), + Input(x=Message, want="metagpt.schema.Message"), + Input(x=Message(""), want="metagpt.schema.Message"), + Input(x="A", want="A"), + ] + for i in inputs: + v = any_to_str(i.x) + assert v == i.want + + def test_any_to_str_set(self): + class Input(BaseModel): + x: Any + want: Set + + inputs = [ + Input( + x=[TutorialAssistant, RunCode(), "a"], + want={"metagpt.roles.tutorial_assistant.TutorialAssistant", "metagpt.actions.run_code.RunCode", "a"}, + ), + Input( + x={TutorialAssistant, RunCode(), "a"}, + want={"metagpt.roles.tutorial_assistant.TutorialAssistant", "metagpt.actions.run_code.RunCode", "a"}, + ), + Input( + x=(TutorialAssistant, RunCode(), "a"), + want={"metagpt.roles.tutorial_assistant.TutorialAssistant", "metagpt.actions.run_code.RunCode", "a"}, + ), + ] + for i in inputs: + v = any_to_str_set(i.x) + assert v == i.want + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_config.py b/tests/metagpt/utils/test_config.py index 558a4e5a4..b68a535f9 100644 --- a/tests/metagpt/utils/test_config.py +++ b/tests/metagpt/utils/test_config.py @@ -20,12 +20,12 @@ def test_config_class_is_singleton(): def test_config_class_get_key_exception(): with pytest.raises(Exception) as exc_info: config = Config() - config.get('wtf') + config.get("wtf") assert str(exc_info.value) == "Key 'wtf' not found in environment variables or in the YAML file" def test_config_yaml_file_not_exists(): - config = Config('wtf.yaml') + config = Config("wtf.yaml") with pytest.raises(Exception) as exc_info: - config.get('OPENAI_BASE_URL') + config.get("OPENAI_BASE_URL") assert str(exc_info.value) == "Key 'OPENAI_BASE_URL' not found in environment variables or in the YAML file" diff --git a/tests/metagpt/utils/test_custom_aio_session.py b/tests/metagpt/utils/test_custom_aio_session.py index 3a8a7bf7e..e2876e4b8 100644 --- a/tests/metagpt/utils/test_custom_aio_session.py +++ b/tests/metagpt/utils/test_custom_aio_session.py @@ -10,12 +10,12 @@ from metagpt.provider.openai_api import OpenAIGPTAPI async def try_hello(api): - batch = [[{'role': 'user', 'content': 'hello'}]] + batch = [[{"role": "user", "content": "hello"}]] results = await api.acompletion_batch_text(batch) return results async def aask_batch(api: OpenAIGPTAPI): - results = await api.aask_batch(['hi', 'write python hello world.']) + results = await api.aask_batch(["hi", "write python hello world."]) logger.info(results) return results diff --git a/tests/metagpt/utils/test_dependency_file.py b/tests/metagpt/utils/test_dependency_file.py new file mode 100644 index 000000000..ae4d40ea5 --- /dev/null +++ b/tests/metagpt/utils/test_dependency_file.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/22 +@Author : mashenquan +@File : test_dependency_file.py +@Desc: Unit tests for dependency_file.py +""" +from __future__ import annotations + +from pathlib import Path +from typing import Optional, Set, Union + +import pytest +from pydantic import BaseModel + +from metagpt.utils.dependency_file import DependencyFile + + +@pytest.mark.asyncio +async def test_dependency_file(): + class Input(BaseModel): + x: Union[Path, str] + deps: Optional[Set[Union[Path, str]]] + key: Optional[Union[Path, str]] + want: Set[str] + + inputs = [ + Input(x="a/b.txt", deps={"c/e.txt", Path(__file__).parent / "d.txt"}, want={"c/e.txt", "d.txt"}), + Input( + x=Path(__file__).parent / "x/b.txt", + deps={"s/e.txt", Path(__file__).parent / "d.txt"}, + key="x/b.txt", + want={"s/e.txt", "d.txt"}, + ), + Input(x="f.txt", deps=None, want=set()), + Input(x="a/b.txt", deps=None, want=set()), + ] + + file = DependencyFile(workdir=Path(__file__).parent) + + for i in inputs: + await file.update(filename=i.x, dependencies=i.deps) + assert await file.get(filename=i.key or i.x) == i.want + + file2 = DependencyFile(workdir=Path(__file__).parent) + file2.delete_file() + assert not file.exists + await file2.update(filename="a/b.txt", dependencies={"c/e.txt", Path(__file__).parent / "d.txt"}, persist=False) + assert not file.exists + await file2.save() + assert file2.exists + + file1 = DependencyFile(workdir=Path(__file__).parent) + assert file1.exists + assert await file1.get("a/b.txt") == set() + await file1.load() + assert await file1.get("a/b.txt") == {"c/e.txt", "d.txt"} + file1.delete_file() + assert not file.exists + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_file.py b/tests/metagpt/utils/test_file.py index b30e6be93..83e317213 100644 --- a/tests/metagpt/utils/test_file.py +++ b/tests/metagpt/utils/test_file.py @@ -15,12 +15,11 @@ from metagpt.utils.file import File @pytest.mark.asyncio @pytest.mark.parametrize( ("root_path", "filename", "content"), - [(Path("/code/MetaGPT/data/tutorial_docx/2023-09-07_17-05-20"), "test.md", "Hello World!")] + [(Path("/code/MetaGPT/data/tutorial_docx/2023-09-07_17-05-20"), "test.md", "Hello World!")], ) async def test_write_and_read_file(root_path: Path, filename: str, content: bytes): - full_file_name = await File.write(root_path=root_path, filename=filename, content=content.encode('utf-8')) + full_file_name = await File.write(root_path=root_path, filename=filename, content=content.encode("utf-8")) assert isinstance(full_file_name, Path) assert root_path / filename == full_file_name file_data = await File.read(full_file_name) assert file_data.decode("utf-8") == content - diff --git a/tests/metagpt/utils/test_file_repository.py b/tests/metagpt/utils/test_file_repository.py new file mode 100644 index 000000000..92e5204c5 --- /dev/null +++ b/tests/metagpt/utils/test_file_repository.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : test_file_repository.py +@Desc: Unit tests for file_repository.py +""" +import shutil +from pathlib import Path + +import pytest + +from metagpt.utils.git_repository import ChangeType, GitRepository +from tests.metagpt.utils.test_git_repository import mock_file + + +@pytest.mark.asyncio +async def test_file_repo(): + local_path = Path(__file__).parent / "file_repo_git" + if local_path.exists(): + shutil.rmtree(local_path) + + git_repo = GitRepository(local_path=local_path, auto_init=True) + assert not git_repo.changed_files + + await mock_file(local_path / "g.txt", "") + + file_repo_path = "file_repo1" + full_path = local_path / file_repo_path + assert not full_path.exists() + file_repo = git_repo.new_file_repository(file_repo_path) + assert file_repo.workdir == full_path + assert file_repo.workdir.exists() + await file_repo.save("a.txt", "AAA") + await file_repo.save("b.txt", "BBB", ["a.txt"]) + doc = await file_repo.get("a.txt") + assert "AAA" == doc.content + doc = await file_repo.get("b.txt") + assert "BBB" == doc.content + assert {"a.txt"} == await file_repo.get_dependency("b.txt") + assert {"a.txt": ChangeType.UNTRACTED, "b.txt": ChangeType.UNTRACTED} == file_repo.changed_files + assert {"a.txt"} == await file_repo.get_changed_dependency("b.txt") + await file_repo.save("d/e.txt", "EEE") + assert ["d/e.txt"] == file_repo.get_change_dir_files("d") + assert set(file_repo.all_files) == {"a.txt", "b.txt", "d/e.txt"} + await file_repo.delete("d/e.txt") + await file_repo.delete("d/e.txt") # delete twice + assert set(file_repo.all_files) == {"a.txt", "b.txt"} + + git_repo.delete_repository() + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py new file mode 100644 index 000000000..d800e9594 --- /dev/null +++ b/tests/metagpt/utils/test_git_repository.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : test_git_repository.py +@Desc: Unit tests for git_repository.py +""" + +import shutil +from pathlib import Path + +import aiofiles +import pytest + +from metagpt.utils.git_repository import GitRepository + + +async def mock_file(filename, content=""): + async with aiofiles.open(str(filename), mode="w") as file: + await file.write(content) + + +async def mock_repo(local_path) -> (GitRepository, Path): + if local_path.exists(): + shutil.rmtree(local_path) + assert not local_path.exists() + repo = GitRepository(local_path=local_path, auto_init=True) + assert local_path.exists() + assert local_path == repo.workdir + assert not repo.changed_files + + await mock_file(local_path / "a.txt") + await mock_file(local_path / "b.txt") + subdir = local_path / "subdir" + subdir.mkdir(parents=True, exist_ok=True) + await mock_file(subdir / "c.txt") + return repo, subdir + + +@pytest.mark.asyncio +async def test_git(): + local_path = Path(__file__).parent / "git" + repo, subdir = await mock_repo(local_path) + + assert len(repo.changed_files) == 3 + repo.add_change(repo.changed_files) + repo.commit("commit1") + assert not repo.changed_files + + await mock_file(local_path / "a.txt", "tests") + await mock_file(subdir / "d.txt") + rmfile = local_path / "b.txt" + rmfile.unlink() + assert repo.status + + assert len(repo.changed_files) == 3 + repo.add_change(repo.changed_files) + repo.commit("commit2") + assert not repo.changed_files + + assert repo.status + + repo.delete_repository() + assert not local_path.exists() + + +@pytest.mark.asyncio +async def test_git1(): + local_path = Path(__file__).parent / "git1" + await mock_repo(local_path) + + repo1 = GitRepository(local_path=local_path, auto_init=False) + assert repo1.changed_files + + file_repo = repo1.new_file_repository("__pycache__") + await file_repo.save("a.pyc", content="") + all_files = repo1.get_files(relative_path=".", filter_ignored=False) + assert "__pycache__/a.pyc" in all_files + all_files = repo1.get_files(relative_path=".", filter_ignored=True) + assert "__pycache__/a.pyc" not in all_files + + repo1.delete_repository() + assert not local_path.exists() + + +@pytest.mark.asyncio +async def test_dependency_file(): + local_path = Path(__file__).parent / "git2" + repo, subdir = await mock_repo(local_path) + + dependancy_file = await repo.get_dependency() + assert not dependancy_file.exists + + await dependancy_file.update(filename="a/b.txt", dependencies={"c/d.txt", "e/f.txt"}) + assert dependancy_file.exists + + repo.delete_repository() + assert not dependancy_file.exists + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_output_parser.py b/tests/metagpt/utils/test_output_parser.py index 4e362f9f7..c9f5813d9 100644 --- a/tests/metagpt/utils/test_output_parser.py +++ b/tests/metagpt/utils/test_output_parser.py @@ -14,17 +14,17 @@ from metagpt.utils.common import OutputParser def test_parse_blocks(): test_text = "##block1\nThis is block 1.\n##block2\nThis is block 2." - expected_result = {'block1': 'This is block 1.', 'block2': 'This is block 2.'} + expected_result = {"block1": "This is block 1.", "block2": "This is block 2."} assert OutputParser.parse_blocks(test_text) == expected_result def test_parse_code(): test_text = "```python\nprint('Hello, world!')```" expected_result = "print('Hello, world!')" - assert OutputParser.parse_code(test_text, 'python') == expected_result + assert OutputParser.parse_code(test_text, "python") == expected_result with pytest.raises(Exception): - OutputParser.parse_code(test_text, 'java') + OutputParser.parse_code(test_text, "java") def test_parse_python_code(): @@ -45,13 +45,13 @@ def test_parse_python_code(): def test_parse_str(): test_text = "name = 'Alice'" - expected_result = 'Alice' + expected_result = "Alice" assert OutputParser.parse_str(test_text) == expected_result def test_parse_file_list(): test_text = "files=['file1', 'file2', 'file3']" - expected_result = ['file1', 'file2', 'file3'] + expected_result = ["file1", "file2", "file3"] assert OutputParser.parse_file_list(test_text) == expected_result with pytest.raises(Exception): @@ -60,7 +60,7 @@ def test_parse_file_list(): def test_parse_data(): test_data = "##block1\n```python\nprint('Hello, world!')\n```\n##block2\nfiles=['file1', 'file2', 'file3']" - expected_result = {'block1': "print('Hello, world!')", 'block2': ['file1', 'file2', 'file3']} + expected_result = {"block1": "print('Hello, world!')", "block2": ["file1", "file2", "file3"]} assert OutputParser.parse_data(test_data) == expected_result @@ -103,9 +103,11 @@ def test_parse_data(): None, Exception, ), - ] + ], ) -def test_extract_struct(text: str, data_type: Union[type(list), type(dict)], parsed_data: Union[list, dict], expected_exception): +def test_extract_struct( + text: str, data_type: Union[type(list), type(dict)], parsed_data: Union[list, dict], expected_exception +): def case(): resp = OutputParser.extract_struct(text, data_type) assert resp == parsed_data @@ -117,7 +119,7 @@ def test_extract_struct(text: str, data_type: Union[type(list), type(dict)], par case() -if __name__ == '__main__': +if __name__ == "__main__": t_text = ''' ## Required Python third-party packages ```python @@ -216,9 +218,9 @@ We need clarification on how the high score should be stored. Should it persist "Requirement Pool": (List[Tuple[str, str]], ...), "Anything UNCLEAR": (str, ...), } - t_text1 = '''## Original Requirements: + t_text1 = """## Original Requirements: -The boss wants to create a web-based version of the game "Fly Bird". +The user wants to create a web-based version of the game "Fly Bird". ## Product Goals: @@ -284,7 +286,7 @@ The product should be a web-based version of the game "Fly Bird" that is engagin ## Anything UNCLEAR: There are no unclear points. - ''' + """ d = OutputParser.parse_data_with_mapping(t_text1, OUTPUT_MAPPING) import json diff --git a/tests/metagpt/utils/test_parse_html.py b/tests/metagpt/utils/test_parse_html.py index 42be416a6..dd15bd80b 100644 --- a/tests/metagpt/utils/test_parse_html.py +++ b/tests/metagpt/utils/test_parse_html.py @@ -52,9 +52,11 @@ PAGE = """ """ -CONTENT = 'This is a HeadingThis is a paragraph witha linkand someemphasizedtext.Item 1Item 2Item 3Numbered Item 1Numbered '\ -'Item 2Numbered Item 3Header 1Header 2Row 1, Cell 1Row 1, Cell 2Row 2, Cell 1Row 2, Cell 2Name:Email:SubmitThis is a div '\ -'with a class "box".a link' +CONTENT = ( + "This is a HeadingThis is a paragraph witha linkand someemphasizedtext.Item 1Item 2Item 3Numbered Item 1Numbered " + "Item 2Numbered Item 3Header 1Header 2Row 1, Cell 1Row 1, Cell 2Row 2, Cell 1Row 2, Cell 2Name:Email:SubmitThis is a div " + 'with a class "box".a link' +) def test_web_page(): diff --git a/tests/metagpt/utils/test_pycst.py b/tests/metagpt/utils/test_pycst.py index 07352eac2..9cf876611 100644 --- a/tests/metagpt/utils/test_pycst.py +++ b/tests/metagpt/utils/test_pycst.py @@ -1,6 +1,6 @@ from metagpt.utils import pycst -code = ''' +code = """ #!/usr/bin/env python # -*- coding: utf-8 -*- from typing import overload @@ -24,7 +24,7 @@ class Person: def greet(self): return f"Hello, my name is {self.name} and I am {self.age} years old." -''' +""" documented_code = ''' """ diff --git a/tests/metagpt/utils/test_read_docx.py b/tests/metagpt/utils/test_read_docx.py index a7d0774a8..adf473ae7 100644 --- a/tests/metagpt/utils/test_read_docx.py +++ b/tests/metagpt/utils/test_read_docx.py @@ -6,12 +6,12 @@ @File : test_read_docx.py """ -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT from metagpt.utils.read_document import read_docx class TestReadDocx: def test_read_docx(self): - docx_sample = PROJECT_ROOT / "tests/data/docx_for_test.docx" + docx_sample = METAGPT_ROOT / "tests/data/docx_for_test.docx" docx = read_docx(docx_sample) assert len(docx) == 6 diff --git a/tests/metagpt/utils/test_repair_llm_raw_output.py b/tests/metagpt/utils/test_repair_llm_raw_output.py index a2dd18516..21bbee921 100644 --- a/tests/metagpt/utils/test_repair_llm_raw_output.py +++ b/tests/metagpt/utils/test_repair_llm_raw_output.py @@ -4,10 +4,15 @@ from metagpt.config import CONFIG -CONFIG.repair_llm_output = True +from metagpt.utils.repair_llm_raw_output import ( + RepairType, + extract_content_from_output, + repair_invalid_json, + repair_llm_raw_output, + retry_parse_json_text, +) -from metagpt.utils.repair_llm_raw_output import repair_llm_raw_output, RepairType, repair_invalid_json,\ - extract_content_from_output, retry_parse_json_text +CONFIG.repair_llm_output = True def test_repair_case_sensitivity(): @@ -26,8 +31,7 @@ def test_repair_case_sensitivity(): "Requirement Analysis": "The 2048 game should be simple to play" }""" req_keys = ["Original Requirements", "Search Information", "Competitive Quadrant Chart", "Requirement Analysis"] - output = repair_llm_raw_output(output=raw_output, - req_keys=req_keys) + output = repair_llm_raw_output(output=raw_output, req_keys=req_keys) assert output == target_output @@ -40,8 +44,7 @@ def test_repair_special_character_missing(): "Anything UNCLEAR": "No unclear requirements or information." [/CONTENT]""" req_keys = ["[/CONTENT]"] - output = repair_llm_raw_output(output=raw_output, - req_keys=req_keys) + output = repair_llm_raw_output(output=raw_output, req_keys=req_keys) assert output == target_output raw_output = """[CONTENT] tag @@ -56,15 +59,13 @@ def test_repair_special_character_missing(): "Anything UNCLEAR": "No unclear requirements or information." } [/CONTENT]""" - output = repair_llm_raw_output(output=raw_output, - req_keys=req_keys) + output = repair_llm_raw_output(output=raw_output, req_keys=req_keys) assert output == target_output raw_output = '[CONTENT] {"a": "b"} [CONTENT]' target_output = '[CONTENT] {"a": "b"} [/CONTENT]' - output = repair_llm_raw_output(output=raw_output, - req_keys=["[/CONTENT]"]) + output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) print("output\n", output) assert output == target_output @@ -73,38 +74,35 @@ def test_required_key_pair_missing(): raw_output = '[CONTENT] {"a": "b"}' target_output = '[CONTENT] {"a": "b"}\n[/CONTENT]' - output = repair_llm_raw_output(output=raw_output, - req_keys=["[/CONTENT]"]) + output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) assert output == target_output - raw_output = '''[CONTENT] + raw_output = """[CONTENT] { "key": "value" -]''' - target_output = '''[CONTENT] +]""" + target_output = """[CONTENT] { "key": "value" ] -[/CONTENT]''' +[/CONTENT]""" - output = repair_llm_raw_output(output=raw_output, - req_keys=["[/CONTENT]"]) + output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) assert output == target_output - raw_output = '''[CONTENT] tag + raw_output = """[CONTENT] tag [CONTENT] { "key": "value" } xxx -''' - target_output = '''[CONTENT] +""" + target_output = """[CONTENT] { "key": "value" } -[/CONTENT]''' - output = repair_llm_raw_output(output=raw_output, - req_keys=["[/CONTENT]"]) +[/CONTENT]""" + output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) assert output == target_output @@ -112,25 +110,19 @@ def test_repair_json_format(): raw_output = "{ xxx }]" target_output = "{ xxx }" - output = repair_llm_raw_output(output=raw_output, - req_keys=[None], - repair_type=RepairType.JSON) + output = repair_llm_raw_output(output=raw_output, req_keys=[None], repair_type=RepairType.JSON) assert output == target_output raw_output = "[{ xxx }" target_output = "{ xxx }" - output = repair_llm_raw_output(output=raw_output, - req_keys=[None], - repair_type=RepairType.JSON) + output = repair_llm_raw_output(output=raw_output, req_keys=[None], repair_type=RepairType.JSON) assert output == target_output raw_output = "{ xxx ]" target_output = "{ xxx }" - output = repair_llm_raw_output(output=raw_output, - req_keys=[None], - repair_type=RepairType.JSON) + output = repair_llm_raw_output(output=raw_output, req_keys=[None], repair_type=RepairType.JSON) assert output == target_output @@ -186,7 +178,7 @@ def test_retry_parse_json_text(): target_json = { "Original Requirements": "Create a 2048 game", "Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis", - "Requirement Analysis": "The requirements are clear and well-defined" + "Requirement Analysis": "The requirements are clear and well-defined", } output = retry_parse_json_text(output=invalid_json_text) assert output == target_json @@ -200,7 +192,7 @@ def test_retry_parse_json_text(): target_json = { "Original Requirements": "Create a 2048 game", "Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis", - "Requirement Analysis": "The requirements are clear and well-defined" + "Requirement Analysis": "The requirements are clear and well-defined", } output = retry_parse_json_text(output=invalid_json_text) assert output == target_json @@ -214,84 +206,88 @@ def test_extract_content_from_output(): xxx [CONTENT] xxxx [/CONTENT] xxx [CONTENT][/CONTENT] xxx [CONTENT][/CONTENT] # target pair is the last one """ - output = 'Sure! Here is the properly formatted JSON output based on the given context:\n\n[CONTENT]\n{\n"' \ - 'Required Python third-party packages": [\n"pygame==2.0.4",\n"pytest"\n],\n"Required Other language ' \ - 'third-party packages": [\n"No third-party packages are required."\n],\n"Full API spec": "\nopenapi: ' \ - '3.0.0\n\ndescription: A JSON object representing the game state.\n\npaths:\n game:\n get:\n ' \ - 'summary: Get the current game state.\n responses:\n 200:\n description: Game state.' \ - '\n\n moves:\n post:\n summary: Make a move.\n requestBody:\n description: Move to be ' \ - 'made.\n content:\n applicationjson:\n schema:\n type: object\n ' \ - ' properties:\n x:\n type: integer\n y:\n ' \ - ' type: integer\n tile:\n type: object\n ' \ - 'properties:\n value:\n type: integer\n x:\n ' \ - ' type: integer\n y:\n type: integer\n\n ' \ - 'undo-move:\n post:\n summary: Undo the last move.\n responses:\n 200:\n ' \ - ' description: Undone move.\n\n end-game:\n post:\n summary: End the game.\n responses:\n ' \ - ' 200:\n description: Game ended.\n\n start-game:\n post:\n summary: Start a new ' \ - 'game.\n responses:\n 200:\n description: Game started.\n\n game-over:\n get:\n ' \ - ' summary: Check if the game is over.\n responses:\n 200:\n description: Game ' \ - 'over.\n 404:\n description: Game not over.\n\n score:\n get:\n summary: Get the ' \ - 'current score.\n responses:\n 200:\n description: Score.\n\n tile:\n get:\n ' \ - 'summary: Get a specific tile.\n parameters:\n tile_id:\n type: integer\n ' \ - 'description: ID of the tile to get.\n responses:\n 200:\n description: Tile.\n\n ' \ - 'tiles:\n get:\n summary: Get all tiles.\n responses:\n 200:\n description: ' \ - 'Tiles.\n\n level:\n get:\n summary: Get the current level.\n responses:\n 200:\n ' \ - ' description: Level.\n\n level-up:\n post:\n summary: Level up.\n responses:\n ' \ - '200:\n description: Level up successful.\n\n level-down:\n post:\n summary: Level ' \ - 'down.\n responses:\n 200:\n description: Level down successful.\n\n restart:\n ' \ - 'post:\n summary: Restart the game.\n responses:\n 200:\n description: Game ' \ - 'restarted.\n\n help:\n get:\n summary: Get help.\n responses:\n 200:\n ' \ - 'description: Help.\n\n version:\n get:\n summary: Get the version of the game.\n ' \ - 'responses:\n 200:\n description: Version.\n\n}\n\n"Logic Analysis": [\n"game.py",' \ - '\n"Contains the game logic."\n],\n"Task list": [\n"game.py",\n"Contains the game logic and should be ' \ - 'done first."\n],\n"Shared Knowledge": "\n\'game.py\' contains the game logic.\n",\n"Anything ' \ - 'UNCLEAR": "How to start the game."\n]\n\n[/CONTENT] Great! Your JSON output is properly formatted ' \ - 'and correctly includes all the required sections. Here\'s a breakdown of what each section ' \ - 'contains:\n\nRequired Python third-party packages:\n\n* pygame==2.0.4\n* pytest\n\nRequired Other ' \ - 'language third-party packages:\n\n* No third-party packages are required.\n\nFull API spec:\n\n* ' \ - 'openapi: 3.0.0\n* description: A JSON object representing the game state.\n* paths:\n + game: ' \ - 'Get the current game state.\n + moves: Make a move.\n + undo-move: Undo the last move.\n + ' \ - 'end-game: End the game.\n + start-game: Start a new game.\n + game-over: Check if the game is ' \ - 'over.\n + score: Get the current score.\n + tile: Get a specific tile.\n + tiles: Get all tiles.\n ' \ - '+ level: Get the current level.\n + level-up: Level up.\n + level-down: Level down.\n + restart: ' \ - 'Restart the game.\n + help: Get help.\n + version: Get the version of the game.\n\nLogic ' \ - 'Analysis:\n\n* game.py contains the game logic.\n\nTask list:\n\n* game.py contains the game logic ' \ - 'and should be done first.\n\nShared Knowledge:\n\n* \'game.py\' contains the game logic.\n\nAnything ' \ - 'UNCLEAR:\n\n* How to start the game.\n\nGreat job! This JSON output should provide a clear and ' \ - 'comprehensive overview of the project\'s requirements and dependencies.' + output = ( + 'Sure! Here is the properly formatted JSON output based on the given context:\n\n[CONTENT]\n{\n"' + 'Required Python third-party packages": [\n"pygame==2.0.4",\n"pytest"\n],\n"Required Other language ' + 'third-party packages": [\n"No third-party packages are required."\n],\n"Full API spec": "\nopenapi: ' + "3.0.0\n\ndescription: A JSON object representing the game state.\n\npaths:\n game:\n get:\n " + "summary: Get the current game state.\n responses:\n 200:\n description: Game state." + "\n\n moves:\n post:\n summary: Make a move.\n requestBody:\n description: Move to be " + "made.\n content:\n applicationjson:\n schema:\n type: object\n " + " properties:\n x:\n type: integer\n y:\n " + " type: integer\n tile:\n type: object\n " + "properties:\n value:\n type: integer\n x:\n " + " type: integer\n y:\n type: integer\n\n " + "undo-move:\n post:\n summary: Undo the last move.\n responses:\n 200:\n " + " description: Undone move.\n\n end-game:\n post:\n summary: End the game.\n responses:\n " + " 200:\n description: Game ended.\n\n start-game:\n post:\n summary: Start a new " + "game.\n responses:\n 200:\n description: Game started.\n\n game-over:\n get:\n " + " summary: Check if the game is over.\n responses:\n 200:\n description: Game " + "over.\n 404:\n description: Game not over.\n\n score:\n get:\n summary: Get the " + "current score.\n responses:\n 200:\n description: Score.\n\n tile:\n get:\n " + "summary: Get a specific tile.\n parameters:\n tile_id:\n type: integer\n " + "description: ID of the tile to get.\n responses:\n 200:\n description: Tile.\n\n " + "tiles:\n get:\n summary: Get all tiles.\n responses:\n 200:\n description: " + "Tiles.\n\n level:\n get:\n summary: Get the current level.\n responses:\n 200:\n " + " description: Level.\n\n level-up:\n post:\n summary: Level up.\n responses:\n " + "200:\n description: Level up successful.\n\n level-down:\n post:\n summary: Level " + "down.\n responses:\n 200:\n description: Level down successful.\n\n restart:\n " + "post:\n summary: Restart the game.\n responses:\n 200:\n description: Game " + "restarted.\n\n help:\n get:\n summary: Get help.\n responses:\n 200:\n " + "description: Help.\n\n version:\n get:\n summary: Get the version of the game.\n " + 'responses:\n 200:\n description: Version.\n\n}\n\n"Logic Analysis": [\n"game.py",' + '\n"Contains the game logic."\n],\n"Task list": [\n"game.py",\n"Contains the game logic and should be ' + 'done first."\n],\n"Shared Knowledge": "\n\'game.py\' contains the game logic.\n",\n"Anything ' + 'UNCLEAR": "How to start the game."\n]\n\n[/CONTENT] Great! Your JSON output is properly formatted ' + "and correctly includes all the required sections. Here's a breakdown of what each section " + "contains:\n\nRequired Python third-party packages:\n\n* pygame==2.0.4\n* pytest\n\nRequired Other " + "language third-party packages:\n\n* No third-party packages are required.\n\nFull API spec:\n\n* " + "openapi: 3.0.0\n* description: A JSON object representing the game state.\n* paths:\n + game: " + "Get the current game state.\n + moves: Make a move.\n + undo-move: Undo the last move.\n + " + "end-game: End the game.\n + start-game: Start a new game.\n + game-over: Check if the game is " + "over.\n + score: Get the current score.\n + tile: Get a specific tile.\n + tiles: Get all tiles.\n " + "+ level: Get the current level.\n + level-up: Level up.\n + level-down: Level down.\n + restart: " + "Restart the game.\n + help: Get help.\n + version: Get the version of the game.\n\nLogic " + "Analysis:\n\n* game.py contains the game logic.\n\nTask list:\n\n* game.py contains the game logic " + "and should be done first.\n\nShared Knowledge:\n\n* 'game.py' contains the game logic.\n\nAnything " + "UNCLEAR:\n\n* How to start the game.\n\nGreat job! This JSON output should provide a clear and " + "comprehensive overview of the project's requirements and dependencies." + ) output = extract_content_from_output(output) - assert output.startswith('{\n"Required Python third-party packages') and \ - output.endswith('UNCLEAR": "How to start the game."\n]') + assert output.startswith('{\n"Required Python third-party packages') and output.endswith( + 'UNCLEAR": "How to start the game."\n]' + ) - output = 'Sure, I would be happy to help! Here is the information you provided, formatted as a JSON object ' \ - 'inside the [CONTENT] tag:\n\n[CONTENT]\n{\n"Original Requirements": "Create a 2048 game",\n"Search ' \ - 'Information": "Search results for 2048 game",\n"Requirements": [\n"Create a game with the same rules ' \ - 'as the original 2048 game",\n"Implement a user interface that is easy to use and understand",\n"Add a ' \ - 'scoreboard to track the player progress",\n"Allow the player to undo and redo moves",\n"Implement a ' \ - 'game over screen to display the final score"\n],\n"Product Goals": [\n"Create a fun and engaging game ' \ - 'experience for the player",\n"Design a user interface that is visually appealing and easy to use",\n"' \ - 'Optimize the game for performance and responsiveness"\n],\n"User Stories": [\n"As a player, I want to ' \ - 'be able to move tiles around the board to combine numbers",\n"As a player, I want to be able to undo ' \ - 'and redo moves to correct mistakes",\n"As a player, I want to see the final score and game over screen' \ - ' when I win"\n],\n"Competitive Analysis": [\n"Competitor A: 2048 game with a simple user interface and' \ - ' basic graphics",\n"Competitor B: 2048 game with a more complex user interface and better graphics",' \ - '\n"Competitor C: 2048 game with a unique twist on the rules and a more challenging gameplay experience"' \ - '\n],\n"Competitive Quadrant Chart": "quadrantChart\\n\ttitle Reach and engagement of campaigns\\n\t\t' \ - 'x-axis Low Reach --> High Reach\\n\t\ty-axis Low Engagement --> High Engagement\\n\tquadrant-1 We ' \ - 'should expand\\n\tquadrant-2 Need to promote\\n\tquadrant-3 Re-evaluate\\n\tquadrant-4 May be ' \ - 'improved\\n\tCampaign A: [0.3, 0.6]\\n\tCampaign B: [0.45, 0.23]\\n\tCampaign C: [0.57, 0.69]\\n\t' \ - 'Campaign D: [0.78, 0.34]\\n\tCampaign E: [0.40, 0.34]\\n\tCampaign F: [0.35, 0.78]"\n],\n"Requirement ' \ - 'Analysis": "The requirements are clear and well-defined, but there may be some ambiguity around the ' \ - 'specific implementation details",\n"Requirement Pool": [\n["P0", "Implement a game with the same ' \ - 'rules as the original 2048 game"],\n["P1", "Add a scoreboard to track the player progress"],\n["P2", ' \ - '"Allow the player to undo and redo moves"]\n],\n"UI Design draft": "The UI should be simple and easy ' \ - 'to use, with a clean and visually appealing design. The game board should be the main focus of the ' \ - 'UI, with clear and concise buttons for the player to interact with.",\n"Anything UNCLEAR": ""\n}\n' \ - '[/CONTENT]\n\nI hope this helps! Let me know if you have any further questions or if there anything ' \ - 'else I can do to assist you.' + output = ( + "Sure, I would be happy to help! Here is the information you provided, formatted as a JSON object " + 'inside the [CONTENT] tag:\n\n[CONTENT]\n{\n"Original Requirements": "Create a 2048 game",\n"Search ' + 'Information": "Search results for 2048 game",\n"Requirements": [\n"Create a game with the same rules ' + 'as the original 2048 game",\n"Implement a user interface that is easy to use and understand",\n"Add a ' + 'scoreboard to track the player progress",\n"Allow the player to undo and redo moves",\n"Implement a ' + 'game over screen to display the final score"\n],\n"Product Goals": [\n"Create a fun and engaging game ' + 'experience for the player",\n"Design a user interface that is visually appealing and easy to use",\n"' + 'Optimize the game for performance and responsiveness"\n],\n"User Stories": [\n"As a player, I want to ' + 'be able to move tiles around the board to combine numbers",\n"As a player, I want to be able to undo ' + 'and redo moves to correct mistakes",\n"As a player, I want to see the final score and game over screen' + ' when I win"\n],\n"Competitive Analysis": [\n"Competitor A: 2048 game with a simple user interface and' + ' basic graphics",\n"Competitor B: 2048 game with a more complex user interface and better graphics",' + '\n"Competitor C: 2048 game with a unique twist on the rules and a more challenging gameplay experience"' + '\n],\n"Competitive Quadrant Chart": "quadrantChart\\n\ttitle Reach and engagement of campaigns\\n\t\t' + "x-axis Low Reach --> High Reach\\n\t\ty-axis Low Engagement --> High Engagement\\n\tquadrant-1 We " + "should expand\\n\tquadrant-2 Need to promote\\n\tquadrant-3 Re-evaluate\\n\tquadrant-4 May be " + "improved\\n\tCampaign A: [0.3, 0.6]\\n\tCampaign B: [0.45, 0.23]\\n\tCampaign C: [0.57, 0.69]\\n\t" + 'Campaign D: [0.78, 0.34]\\n\tCampaign E: [0.40, 0.34]\\n\tCampaign F: [0.35, 0.78]"\n],\n"Requirement ' + 'Analysis": "The requirements are clear and well-defined, but there may be some ambiguity around the ' + 'specific implementation details",\n"Requirement Pool": [\n["P0", "Implement a game with the same ' + 'rules as the original 2048 game"],\n["P1", "Add a scoreboard to track the player progress"],\n["P2", ' + '"Allow the player to undo and redo moves"]\n],\n"UI Design draft": "The UI should be simple and easy ' + "to use, with a clean and visually appealing design. The game board should be the main focus of the " + 'UI, with clear and concise buttons for the player to interact with.",\n"Anything UNCLEAR": ""\n}\n' + "[/CONTENT]\n\nI hope this helps! Let me know if you have any further questions or if there anything " + "else I can do to assist you." + ) output = extract_content_from_output(output) - assert output.startswith('{\n"Original Requirements"') and \ - output.endswith('"Anything UNCLEAR": ""\n}') + assert output.startswith('{\n"Original Requirements"') and output.endswith('"Anything UNCLEAR": ""\n}') output = """ Sure, I'd be happy to help! Here's the JSON output for the given context:\n\n[CONTENT]\n{ "Implementation approach": "We will use the open-source framework PyGame to create a 2D game engine, which will @@ -316,5 +312,6 @@ def test_extract_content_from_output(): information for a developer to understand the design and implementation of the 2048 game. """ output = extract_content_from_output(output) - assert output.startswith('{\n"Implementation approach"') and \ - output.endswith('"Anything UNCLEAR": "The requirement is clear to me."\n}') + assert output.startswith('{\n"Implementation approach"') and output.endswith( + '"Anything UNCLEAR": "The requirement is clear to me."\n}' + ) diff --git a/tests/metagpt/utils/test_serialize.py b/tests/metagpt/utils/test_serialize.py index 69f317f79..ffa34866c 100644 --- a/tests/metagpt/utils/test_serialize.py +++ b/tests/metagpt/utils/test_serialize.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : the unittest of serialize +""" +@Desc : the unittest of serialize +""" from typing import List, Tuple diff --git a/tests/metagpt/utils/test_text.py b/tests/metagpt/utils/test_text.py index 0caf8abaa..7003c7767 100644 --- a/tests/metagpt/utils/test_text.py +++ b/tests/metagpt/utils/test_text.py @@ -29,7 +29,7 @@ def _paragraphs(n): (_msgs(), "gpt-4", "Hello," * 1000, 2000, 2), (_msgs(), "gpt-4-32k", "System", 4000, 14), (_msgs(), "gpt-4-32k", "Hello," * 2000, 4000, 12), - ] + ], ) def test_reduce_message_length(msgs, model_name, system_text, reserved, expected): assert len(reduce_message_length(msgs, model_name, system_text, reserved)) / (len("Hello,")) / 1000 == expected @@ -42,7 +42,7 @@ def test_reduce_message_length(msgs, model_name, system_text, reserved, expected (" ".join("Hello World." for _ in range(1000)), "Prompt: {}", "gpt-3.5-turbo-16k", "System", 3000, 1), (" ".join("Hello World." for _ in range(4000)), "Prompt: {}", "gpt-4", "System", 2000, 2), (" ".join("Hello World." for _ in range(8000)), "Prompt: {}", "gpt-4-32k", "System", 4000, 1), - ] + ], ) def test_generate_prompt_chunk(text, prompt_template, model_name, system_text, reserved, expected): ret = list(generate_prompt_chunk(text, prompt_template, model_name, system_text, reserved)) @@ -58,7 +58,7 @@ def test_generate_prompt_chunk(text, prompt_template, model_name, system_text, r ("......", ".", 2, ["...", "..."]), ("......", ".", 3, ["..", "..", ".."]), (".......", ".", 2, ["....", "..."]), - ] + ], ) def test_split_paragraph(paragraph, sep, count, expected): ret = split_paragraph(paragraph, sep, count) @@ -71,7 +71,7 @@ def test_split_paragraph(paragraph, sep, count, expected): ("Hello\\nWorld", "Hello\nWorld"), ("Hello\\tWorld", "Hello\tWorld"), ("Hello\\u0020World", "Hello World"), - ] + ], ) def test_decode_unicode_escape(text, expected): assert decode_unicode_escape(text) == expected